From 9dc0065df72952ea2189c754d475225a3bf23832 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:39:06 +0400 Subject: [PATCH 01/18] Add the reporting window and command line for telemetry error issues The new script reports one US Eastern calendar day, from midnight to midnight, so a run at 00:45 Eastern never falls in the hour the clocks skip or repeat. Days when the clocks change last 23 or 25 hours. --report-date selects a different day; an empty value means the previous day, as on scheduled runs. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 89 +++++++++++ tests/unit/test_telemetry_error_issues.py | 141 ++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 .github/scripts/sync_telemetry_error_issues.py create mode 100644 tests/unit/test_telemetry_error_issues.py diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py new file mode 100644 index 000000000..678711ca7 --- /dev/null +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -0,0 +1,89 @@ +"""Report each day's telemetry error groups as GitHub issues. + +Each run reads one US Eastern calendar day of error records from the central +Cloud Logging bucket through its linked BigQuery dataset. It groups the records +by a keyed fingerprint, measures each group in production and staging, and +creates or updates one GitHub issue per production error group. + +Workflow logs, job summaries and issues in this repository are public. The +script therefore publishes only fingerprints, counts, code-defined names and a +Logs Explorer link, never error messages, stack text or identifiers. +""" + +from __future__ import annotations + +import argparse +from datetime import UTC, date, datetime, time, timedelta +from typing import NamedTuple +from zoneinfo import ZoneInfo + +REPORTING_TIME_ZONE = ZoneInfo("America/New_York") + + +class ReportingWindow(NamedTuple): + """One Eastern calendar day, as the half-open interval [start, end).""" + + start: datetime + end: datetime + + @property + def hours(self) -> float: + """Return the elapsed hours: 23 or 25 on the days the clocks change.""" + elapsed = self.end.astimezone(UTC) - self.start.astimezone(UTC) + return elapsed / timedelta(hours=1) + + +def reporting_window(report_date: date) -> ReportingWindow: + """Return the window from midnight Eastern on ``report_date`` to the next.""" + next_date = report_date + timedelta(days=1) + return ReportingWindow( + start=datetime.combine(report_date, time(0), tzinfo=REPORTING_TIME_ZONE), + end=datetime.combine(next_date, time(0), tzinfo=REPORTING_TIME_ZONE), + ) + + +def default_report_date(now: datetime) -> date: + """Return the most recent Eastern calendar day that ended before ``now``.""" + return now.astimezone(REPORTING_TIME_ZONE).date() - timedelta(days=1) + + +def _report_date_argument(value: str) -> date | None: + if not value: + return None + try: + return date.fromisoformat(value) + except ValueError as error: + msg = "expected an Eastern calendar date as YYYY-MM-DD" + raise argparse.ArgumentTypeError(msg) from error + + +def parse_arguments(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the command line; argparse exits with code 2 on invalid input.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--project-id", + required=True, + help="Google Cloud project that holds the linked BigQuery dataset", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="query and group errors, but make no changes to GitHub issues", + ) + parser.add_argument( + "--report-date", + type=_report_date_argument, + default=None, + help="Eastern calendar day to report (default: the previous day)", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + """Run one report and return the process exit code.""" + parse_arguments(argv) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py new file mode 100644 index 000000000..dd83b6897 --- /dev/null +++ b/tests/unit/test_telemetry_error_issues.py @@ -0,0 +1,141 @@ +"""Tests for .github/scripts/sync_telemetry_error_issues.py.""" + +from __future__ import annotations + +import functools +import importlib.util +from datetime import UTC, date, datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING +from zoneinfo import ZoneInfo + +import pytest + +if TYPE_CHECKING: + from types import ModuleType + +REPO = Path(__file__).resolve().parents[2] +SCRIPT_PATH = REPO / ".github" / "scripts" / "sync_telemetry_error_issues.py" +EASTERN = ZoneInfo("America/New_York") + + +@functools.cache +def _load_script() -> ModuleType: + spec = importlib.util.spec_from_file_location( + "sync_telemetry_error_issues", SCRIPT_PATH + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def sync_script() -> ModuleType: + return _load_script() + + +# Reporting window and command-line interface + + +@pytest.mark.parametrize( + "run_time", + [ + datetime(2026, 10, 21, 0, 45, tzinfo=EASTERN), + datetime(2026, 10, 21, 3, 30, tzinfo=EASTERN), + datetime(2026, 10, 21, 4, 45, tzinfo=UTC), + ], +) +def test_window_scheduled_run_reports_the_previous_eastern_day( + sync_script: ModuleType, run_time: datetime +) -> None: + window = sync_script.reporting_window(sync_script.default_report_date(run_time)) + + assert window.start == datetime(2026, 10, 20, 0, 0, tzinfo=EASTERN) + assert window.end == datetime(2026, 10, 21, 0, 0, tzinfo=EASTERN) + assert window.hours == 24 + + +def test_window_default_date_follows_eastern_time_not_utc( + sync_script: ModuleType, +) -> None: + # 02:00 UTC on 21 October is still 22:00 on 20 October in New York. + run_time = datetime(2026, 10, 21, 2, 0, tzinfo=UTC) + + assert sync_script.default_report_date(run_time) == date(2026, 10, 19) + + +def test_window_spring_forward_day_lasts_23_hours(sync_script: ModuleType) -> None: + window = sync_script.reporting_window(date(2026, 3, 8)) + + assert window.start.utcoffset() == timedelta(hours=-5) + assert window.end.utcoffset() == timedelta(hours=-4) + assert window.hours == 23 + + +def test_window_fall_back_day_lasts_25_hours(sync_script: ModuleType) -> None: + window = sync_script.reporting_window(date(2026, 11, 1)) + + assert window.start.utcoffset() == timedelta(hours=-4) + assert window.end.utcoffset() == timedelta(hours=-5) + assert window.hours == 25 + + +def test_window_bounds_convert_to_utc_for_queries(sync_script: ModuleType) -> None: + window = sync_script.reporting_window(date(2026, 11, 1)) + + assert window.start.astimezone(UTC) == datetime(2026, 11, 1, 4, 0, tzinfo=UTC) + assert window.end.astimezone(UTC) == datetime(2026, 11, 2, 5, 0, tzinfo=UTC) + + +def test_cli_parses_project_dry_run_and_report_date(sync_script: ModuleType) -> None: + arguments = sync_script.parse_arguments( + [ + "--project-id", + "policyengine-observability", + "--dry-run", + "--report-date", + "2026-10-19", + ] + ) + + assert arguments.project_id == "policyengine-observability" + assert arguments.dry_run is True + assert arguments.report_date == date(2026, 10, 19) + + +def test_cli_defaults_to_a_live_run_of_the_previous_day( + sync_script: ModuleType, +) -> None: + arguments = sync_script.parse_arguments( + ["--project-id", "policyengine-observability"] + ) + + assert arguments.dry_run is False + assert arguments.report_date is None + + +def test_cli_treats_an_empty_report_date_as_unset(sync_script: ModuleType) -> None: + arguments = sync_script.parse_arguments( + ["--project-id", "policyengine-observability", "--report-date", ""] + ) + + assert arguments.report_date is None + + +@pytest.mark.parametrize( + "argv", + [ + [], + ["--dry-run"], + ["--project-id", "policyengine-observability", "--report-date", "20 Oct"], + ], +) +def test_cli_rejects_missing_project_or_invalid_date_with_exit_code_2( + sync_script: ModuleType, argv: list[str] +) -> None: + with pytest.raises(SystemExit) as raised: + sync_script.main(argv) + + assert raised.value.code == 2 From c62a086a24f5e45c802057d553ce62624085d544 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:41:29 +0400 Subject: [PATCH 02/18] Normalize services, environments and stack frames for error grouping - Versioned simulation services (policyengine-simulation-py--) share one family; the Modal gateway's "main" environment counts as production. - FRAME_PATTERN extracts 'path", line N, in function' from standard, chained, exception group and truncated tracebacks. It has one capturing group so BigQuery's REGEXP_EXTRACT_ALL can apply it, and no source line or message text matches it. - Frame paths lose their installation prefix and line number. The fingerprint uses the three innermost frames in the Public API and simulation packages, or the innermost frames when there are none. - Error types, route templates, service families, operation names and HTTP statuses are validated before any public output; anything else is shown as . Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 162 ++++++++- tests/unit/test_telemetry_error_issues.py | 324 ++++++++++++++++++ 2 files changed, 485 insertions(+), 1 deletion(-) diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index 678711ca7..d13b15b6b 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -13,12 +13,60 @@ from __future__ import annotations import argparse +import re from datetime import UTC, date, datetime, time, timedelta -from typing import NamedTuple +from typing import TYPE_CHECKING, NamedTuple from zoneinfo import ZoneInfo +if TYPE_CHECKING: + from collections.abc import Iterable + REPORTING_TIME_ZONE = ZoneInfo("America/New_York") +# Shown in place of a value that is missing, or that fails validation and so +# might not be code-defined. +MISSING = "-" +UNRECOGNIZED = "" + +PRODUCTION_ENVIRONMENT_NAMES = frozenset({"production", "main"}) +STAGING_ENVIRONMENT_NAMES = frozenset({"staging"}) +VERSIONED_SIMULATION_SERVICE = re.compile(r"^(policyengine-simulation-py)\d+-\d+-\d+$") + +# Captures 'path", line N, in function' from each frame header of a Python +# traceback, including indented and "|"-prefixed exception group frames. It has +# one capturing group because BigQuery's REGEXP_EXTRACT_ALL allows only one, +# and the trailing newline skips a header that stack truncation cut short. +FRAME_PATTERN = r'File "([^"\n]+", line [0-9]+, in [^\s]+)\n' +FRAME_PARTS = re.compile(r'^(?P[^"\n]+)", line [0-9]+, in (?P\S+)$') +FINGERPRINT_FRAME_COUNT = 3 +INSTALLED_PACKAGE_PREFIX = re.compile(r"^.*/(?:site|dist)-packages/") +STANDARD_LIBRARY_PREFIX = re.compile(r"^.*/lib/python3(?:\.\d+)?/") +APPLICATION_PACKAGES = ( + r"(?:policyengine_api|policyengine_fastapi" + r"|policyengine_simulation_[a-z0-9_]+|policyengine_stage12_[a-z0-9_]+)/" +) +APPLICATION_PATH = re.compile(rf"^{APPLICATION_PACKAGES}") +APPLICATION_PATH_IN_ABSOLUTE_PATH = re.compile(rf"(?:^|/)({APPLICATION_PACKAGES}.*)$") +FALLBACK_PATH_COMPONENTS = 2 + +ERROR_TYPE_PATTERN = re.compile(r"^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*$", re.ASCII) +MAX_ERROR_TYPE_LENGTH = 120 +UNMATCHED_ROUTE = "" +MAX_ROUTE_LENGTH = 200 +ROUTE_PARAMETER = re.compile( + r"^(?:<(?:[a-z]+:)?[A-Za-z_]\w*>|\{[A-Za-z_]\w*(?::[a-z]+)?\})$", re.ASCII +) +ROUTE_LITERAL = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,39}$") +# Literal route segments are words such as "v2" or "user-policies". A segment +# with more digits than this is more likely a record ID than code. +MAX_ROUTE_LITERAL_DIGITS = 2 +FAMILY_PATTERN = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") +MAX_FAMILY_LENGTH = 63 +OPERATION_PATTERN = re.compile(r"^[A-Za-z_][\w.:/-]*$", re.ASCII) +MAX_OPERATION_LENGTH = 100 +LOWEST_HTTP_STATUS = 100 +HIGHEST_HTTP_STATUS = 599 + class ReportingWindow(NamedTuple): """One Eastern calendar day, as the half-open interval [start, end).""" @@ -47,6 +95,118 @@ def default_report_date(now: datetime) -> date: return now.astimezone(REPORTING_TIME_ZONE).date() - timedelta(days=1) +def service_family(service_name: str) -> str: + """Return the service name, merging versioned simulation services into one.""" + return VERSIONED_SIMULATION_SERVICE.sub(r"\1", service_name) + + +def environment_class(environment_name: str | None) -> str: + """Classify a deployment environment name as production, staging or other. + + The Modal gateway reports its production environment as ``main``. + """ + if environment_name in PRODUCTION_ENVIRONMENT_NAMES: + return "production" + if environment_name in STAGING_ENVIRONMENT_NAMES: + return "staging" + return "other" + + +def normalize_path(path: str) -> str: + """Return a source path without its installation or checkout prefix.""" + installed = INSTALLED_PACKAGE_PREFIX.sub("", path, count=1) + if installed != path: + return installed + application = APPLICATION_PATH_IN_ABSOLUTE_PATH.search(path) + if application: + return application.group(1) + standard_library = STANDARD_LIBRARY_PREFIX.sub("", path, count=1) + if standard_library != path: + return standard_library + return "/".join(path.split("/")[-FALLBACK_PATH_COMPONENTS:]) + + +def frame_key(frame: str) -> str | None: + """Return ``path:function`` for one ``FRAME_PATTERN`` match, without its line.""" + parts = FRAME_PARTS.match(frame) + if parts is None: + return None + return f"{normalize_path(parts['path'])}:{parts['function']}" + + +def fingerprint_frames(frames: Iterable[str] | None) -> tuple[str, ...]: + """Choose the frames that identify where an exception was raised. + + These are the three innermost frames in application packages. Without + application frames they are the three innermost frames of any kind, and + without a stack, a placeholder. + """ + keys = [key for key in map(frame_key, frames or ()) if key is not None] + if not keys: + return (MISSING,) + application_keys = [key for key in keys if APPLICATION_PATH.match(key)] + return tuple((application_keys or keys)[-FINGERPRINT_FRAME_COUNT:]) + + +def _validated(value: str | None, pattern: re.Pattern[str], max_length: int) -> str: + if not value: + return MISSING + if len(value) <= max_length and pattern.fullmatch(value): + return value + return UNRECOGNIZED + + +def public_error_type(value: str | None) -> str: + """Return an exception class name, or a placeholder for anything else.""" + return _validated(value, ERROR_TYPE_PATTERN, MAX_ERROR_TYPE_LENGTH) + + +def public_family(value: str | None) -> str: + """Return a service name or family, or a placeholder for anything else.""" + return _validated(value, FAMILY_PATTERN, MAX_FAMILY_LENGTH) + + +def public_operation(value: str | None) -> str: + """Return a code-defined operation name, or a placeholder for anything else.""" + return _validated(value, OPERATION_PATTERN, MAX_OPERATION_LENGTH) + + +def _route_segment_is_code(segment: str) -> bool: + if ROUTE_PARAMETER.match(segment): + return True + digits = sum(character.isdigit() for character in segment) + return bool(ROUTE_LITERAL.match(segment)) and digits <= MAX_ROUTE_LITERAL_DIGITS + + +def public_route(value: str | None) -> str: + """Return a route template, or a placeholder for anything else. + + A path that holds a record ID, a query string or free text is not a + template, so it never reaches public output. + """ + if not value: + return MISSING + if value == UNMATCHED_ROUTE: + return value + if len(value) > MAX_ROUTE_LENGTH or not value.startswith("/"): + return UNRECOGNIZED + segments = [segment for segment in value.split("/") if segment] + if all(map(_route_segment_is_code, segments)): + return value + return UNRECOGNIZED + + +def public_status(value: object) -> int | None: + """Return an HTTP status code as an integer, or None for anything else.""" + if isinstance(value, bool): + return None + if isinstance(value, str) and value.isascii() and value.isdigit(): + value = int(value) + if isinstance(value, int) and LOWEST_HTTP_STATUS <= value <= HIGHEST_HTTP_STATUS: + return value + return None + + def _report_date_argument(value: str) -> date | None: if not value: return None diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index dd83b6897..b3fff98f3 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -4,6 +4,7 @@ import functools import importlib.util +import re from datetime import UTC, date, datetime, timedelta from pathlib import Path from typing import TYPE_CHECKING @@ -139,3 +140,326 @@ def test_cli_rejects_missing_project_or_invalid_date_with_exit_code_2( sync_script.main(argv) assert raised.value.code == 2 + + +# Normalization, stack frames and public value validation + +STANDARD_STACK = """\ +Traceback (most recent call last): + File "/opt/venv/lib/python3.12/site-packages/flask/app.py", line 880, in full_dispatch_request + rv = self.dispatch_request() + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/policyengine_api/routes/report_routes.py", line 41, in create_report + return service.create(payload) + File "/app/policyengine_api/services/report_service.py", line 88, in create + session.flush() + File "/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 4331, in flush + self._flush(objects) +sqlalchemy.orm.exc.StaleDataError: SENTINEL message about 'reports' +""" + +CHAINED_STACK = """\ +Traceback (most recent call last): + File "/app/policyengine_api/services/cache.py", line 12, in read + return store[key] +KeyError: 'SENTINEL key' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/app/policyengine_api/routes/policy_routes.py", line 30, in get_policy + return service.get(policy_id) + File "/app/policyengine_api/services/policy_service.py", line 55, in get + raise LookupError(policy_id) +LookupError: SENTINEL policy +""" + +EXCEPTION_GROUP_STACK = """\ + + Exception Group Traceback (most recent call last): + | File "/root/policyengine_simulation_gateway/app.py", line 10, in run + | raise ExceptionGroup("SENTINEL group", errors) + | ExceptionGroup: SENTINEL group (1 sub-exception) + +-+---------------- 1 ---------------- + | Traceback (most recent call last): + | File "/root/policyengine_simulation_gateway/worker.py", line 22, in step + | raise ValueError("SENTINEL value") + | ValueError: SENTINEL value + +------------------------------------ +""" + + +def _extract_frames(sync_script: ModuleType, stack: str) -> list[str]: + """Apply the pattern the way BigQuery's REGEXP_EXTRACT_ALL does.""" + return re.findall(sync_script.FRAME_PATTERN, stack) + + +@pytest.mark.parametrize( + ("service_name", "family"), + [ + ("policyengine-simulation-py1-2-3", "policyengine-simulation-py"), + ("policyengine-simulation-py0-412-16", "policyengine-simulation-py"), + ("policyengine-api", "policyengine-api"), + ("policyengine-simulation-gateway", "policyengine-simulation-gateway"), + ("policyengine-stage12-us-worker", "policyengine-stage12-us-worker"), + ], +) +def test_normalize_service_family_merges_simulation_versions( + sync_script: ModuleType, service_name: str, family: str +) -> None: + assert sync_script.service_family(service_name) == family + + +@pytest.mark.parametrize( + ("environment_name", "environment_class"), + [ + ("production", "production"), + ("main", "production"), + ("staging", "staging"), + ("local", "other"), + ("", "other"), + (None, "other"), + ], +) +def test_normalize_environment_maps_modal_main_to_production( + sync_script: ModuleType, environment_name: str | None, environment_class: str +) -> None: + assert sync_script.environment_class(environment_name) == environment_class + + +def test_frames_pattern_has_one_capturing_group_for_bigquery( + sync_script: ModuleType, +) -> None: + assert re.compile(sync_script.FRAME_PATTERN).groups == 1 + + +def test_frames_pattern_extracts_frame_headers_without_source_or_message( + sync_script: ModuleType, +) -> None: + frames = _extract_frames(sync_script, STANDARD_STACK) + + assert [sync_script.frame_key(frame) for frame in frames] == [ + "flask/app.py:full_dispatch_request", + "policyengine_api/routes/report_routes.py:create_report", + "policyengine_api/services/report_service.py:create", + "sqlalchemy/orm/session.py:flush", + ] + assert not any("SENTINEL" in frame or "\n" in frame for frame in frames) + + +def test_frames_pattern_reads_every_block_of_a_chained_traceback( + sync_script: ModuleType, +) -> None: + frames = _extract_frames(sync_script, CHAINED_STACK) + + assert [sync_script.frame_key(frame) for frame in frames] == [ + "policyengine_api/services/cache.py:read", + "policyengine_api/routes/policy_routes.py:get_policy", + "policyengine_api/services/policy_service.py:get", + ] + + +def test_frames_pattern_reads_exception_group_tracebacks( + sync_script: ModuleType, +) -> None: + frames = _extract_frames(sync_script, EXCEPTION_GROUP_STACK) + + assert [sync_script.frame_key(frame) for frame in frames] == [ + "policyengine_simulation_gateway/app.py:run", + "policyengine_simulation_gateway/worker.py:step", + ] + + +def test_frames_pattern_skips_a_frame_header_cut_off_by_truncation( + sync_script: ModuleType, +) -> None: + cut_at = STANDARD_STACK.index("create_report") + len("create_re") + frames = _extract_frames(sync_script, STANDARD_STACK[:cut_at]) + + assert [sync_script.frame_key(frame) for frame in frames] == [ + "flask/app.py:full_dispatch_request", + ] + + +@pytest.mark.parametrize( + ("path", "normalized"), + [ + ( + "/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", + "sqlalchemy/orm/session.py", + ), + ("/usr/lib/python3/dist-packages/yaml/loader.py", "yaml/loader.py"), + ("/usr/local/lib/python3.13/asyncio/tasks.py", "asyncio/tasks.py"), + ("/app/policyengine_api/services/x.py", "policyengine_api/services/x.py"), + ( + "/home/runner/work/api/api/policyengine_api/services/x.py", + "policyengine_api/services/x.py", + ), + ( + "/root/policyengine_simulation_gateway/app.py", + "policyengine_simulation_gateway/app.py", + ), + ("/pkg/synchronicity/combined_types.py", "synchronicity/combined_types.py"), + ("", ""), + ], +) +def test_frames_paths_drop_installation_prefixes( + sync_script: ModuleType, path: str, normalized: str +) -> None: + assert sync_script.normalize_path(path) == normalized + + +def test_frames_keys_ignore_line_numbers(sync_script: ModuleType) -> None: + first = sync_script.frame_key('/app/policyengine_api/x.py", line 10, in handler') + second = sync_script.frame_key('/srv/policyengine_api/x.py", line 99, in handler') + + assert first == second == "policyengine_api/x.py:handler" + + +def test_frames_choose_the_three_innermost_application_frames( + sync_script: ModuleType, +) -> None: + frames = [ + '/app/policyengine_api/routes/a.py", line 1, in outer', + '/app/policyengine_api/services/b.py", line 2, in middle', + '/app/policyengine_api/services/c.py", line 3, in inner', + '/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/d.py", line 4, in lib', + '/app/policyengine_api/services/e.py", line 5, in callback', + '/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/f.py", line 6, in raises', + ] + + assert sync_script.fingerprint_frames(frames) == ( + "policyengine_api/services/b.py:middle", + "policyengine_api/services/c.py:inner", + "policyengine_api/services/e.py:callback", + ) + + +def test_frames_count_simulation_and_stage12_packages_as_application_code( + sync_script: ModuleType, +) -> None: + frames = [ + '/pkg/modal/_runtime.py", line 1, in run', + '/root/policyengine_simulation_executor/run.py", line 2, in execute', + '/root/policyengine_stage12_persistence/store.py", line 3, in save', + '/pkg/synchronicity/combined_types.py", line 4, in __call__', + ] + + assert sync_script.fingerprint_frames(frames) == ( + "policyengine_simulation_executor/run.py:execute", + "policyengine_stage12_persistence/store.py:save", + ) + + +def test_frames_without_application_frames_use_the_innermost_frames( + sync_script: ModuleType, +) -> None: + frames = [ + '/usr/local/lib/python3.13/asyncio/a.py", line 1, in first', + '/pkg/modal/b.py", line 2, in second', + '/pkg/modal/c.py", line 3, in third', + '/pkg/synchronicity/d.py", line 4, in fourth', + ] + + assert sync_script.fingerprint_frames(frames) == ( + "modal/b.py:second", + "modal/c.py:third", + "synchronicity/d.py:fourth", + ) + + +@pytest.mark.parametrize("frames", [[], None, ["not a frame header"]]) +def test_frames_without_a_stack_use_a_placeholder( + sync_script: ModuleType, frames: list[str] | None +) -> None: + assert sync_script.fingerprint_frames(frames) == ("-",) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("StaleDataError", "StaleDataError"), + ("sqlalchemy.orm.exc.StaleDataError", "sqlalchemy.orm.exc.StaleDataError"), + ("SENTINEL message with spaces", ""), + ("Error: 'quoted'", ""), + ("E" * 200, ""), + (None, "-"), + ("", "-"), + ], +) +def test_public_error_type_allows_only_class_names( + sync_script: ModuleType, value: str | None, expected: str +) -> None: + assert sync_script.public_error_type(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ( + "//economy//over/", + "//economy//over/", + ), + ("/v2/user-policies", "/v2/user-policies"), + ("/jobs/{job_id}", "/jobs/{job_id}"), + ("", ""), + ("/us/policy/12345", ""), + ("/v2/households/3f2b1c9e-8d7a-4c1b-9e2f-0a1b2c3d4e5f", ""), + ("/us/report?token=SENTINEL", ""), + ("SENTINEL route", ""), + ("/" + "a" * 300, ""), + (None, "-"), + ("", "-"), + ], +) +def test_public_route_allows_only_route_templates( + sync_script: ModuleType, value: str | None, expected: str +) -> None: + assert sync_script.public_route(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("policyengine-api", "policyengine-api"), + ("policyengine-simulation-py", "policyengine-simulation-py"), + ("SENTINEL family", ""), + ("Policyengine_API", ""), + (None, "-"), + ], +) +def test_public_family_allows_only_service_names( + sync_script: ModuleType, value: str | None, expected: str +) -> None: + assert sync_script.public_family(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("simulation.dispatch", "simulation.dispatch"), + ("stage12/run_comparison", "stage12/run_comparison"), + ("SENTINEL operation", ""), + (None, "-"), + ], +) +def test_public_operation_allows_only_operation_names( + sync_script: ModuleType, value: str | None, expected: str +) -> None: + assert sync_script.public_operation(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (500, 500), + ("503", 503), + (99, None), + (600, None), + ("SENTINEL", None), + (None, None), + ], +) +def test_public_status_allows_only_http_status_codes( + sync_script: ModuleType, value: object, expected: int | None +) -> None: + assert sync_script.public_status(value) == expected From 752de777491d058ef63ed0aa809e4fcf0a4d3a2e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:44:00 +0400 Subject: [PATCH 03/18] Choose one record per failure and fingerprint its error group A failed request can log an exception, an error message and a failed request.completed record. Records of one service that share a request ID (or a trace, without one) are one failure, represented by its exception record, else its message record, else its request.completed record. An unhandled exception is preferred over a handled one. Failures in two services stay separate even when they share a request. Fingerprints are 16 hex characters of HMAC-SHA256 (algorithm 1) over: - exception: service family, exception type and the chosen frames; the route is left out so one shared failure opens one issue; - message: service family, route or operation, message template, and the error_type and failure_category attributes; - failed_request: service family, route and HTTP status. The key must be at least 32 characters. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 150 +++++++++ tests/unit/test_telemetry_error_issues.py | 314 +++++++++++++++++- 2 files changed, 459 insertions(+), 5 deletions(-) diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index d13b15b6b..784ca09a6 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -13,7 +13,11 @@ from __future__ import annotations import argparse +import hashlib +import hmac +import json import re +from dataclasses import dataclass from datetime import UTC, date, datetime, time, timedelta from typing import TYPE_CHECKING, NamedTuple from zoneinfo import ZoneInfo @@ -67,6 +71,14 @@ LOWEST_HTTP_STATUS = 100 HIGHEST_HTTP_STATUS = 599 +REQUEST_COMPLETED_EVENT = "request.completed" +# Changing how fingerprints are computed changes every fingerprint, and the +# job matches existing issues only by the current algorithm. A change to it +# must say how existing issues are carried over. +FINGERPRINT_ALGORITHM = 1 +FINGERPRINT_LENGTH = 16 +MIN_FINGERPRINT_KEY_LENGTH = 32 + class ReportingWindow(NamedTuple): """One Eastern calendar day, as the half-open interval [start, end).""" @@ -207,6 +219,144 @@ def public_status(value: object) -> int | None: return None +@dataclass(frozen=True) +class ErrorRecord: + """One ERROR record from the central log bucket, as the query returns it. + + The query returns identifiers only as SHA-256 hashes, stacks only as frame + headers, and messages only as their first few words. + """ + + timestamp: datetime + service_name: str + environment_name: str | None + event_name: str | None + error_type: str | None + error_handled: bool | None + frames: tuple[str, ...] + stack_truncated: bool + message_template: str | None + attribute_error_type: str | None + failure_category: str | None + route: str | None + status_code: int | None + operation_name: str | None + request_key: str | None + observability_key: str | None + trace_key: str | None + + @property + def family(self) -> str: + """Return the service family that produced the record.""" + return service_family(self.service_name) + + @property + def environment(self) -> str: + """Return production, staging or other.""" + return environment_class(self.environment_name) + + @property + def kind(self) -> str: + """Return exception, message or failed_request. + + A record with an exception type is an exception, even a + ``request.completed`` record. Any other ``request.completed`` record is + a request that failed without a recorded exception. + """ + if self.error_type: + return "exception" + if self.event_name == REQUEST_COMPLETED_EVENT: + return "failed_request" + return "message" + + +def _primary_preference(record: ErrorRecord) -> tuple[int, int, datetime]: + """Sort key: exceptions before messages before request completions. + + Among exceptions, an unhandled one (the cause of an error response) comes + first; otherwise the earliest record does. + """ + if record.event_name == REQUEST_COMPLETED_EVENT: + source_rank = 2 + elif record.error_type: + source_rank = 0 + else: + source_rank = 1 + handled_rank = 0 if record.error_handled is False else 1 + return source_rank, handled_rank, record.timestamp + + +def primary_records(records: Iterable[ErrorRecord]) -> list[ErrorRecord]: + """Return one record per failure, in time order. + + Records of one service that share a request ID, or a trace when there is no + request ID, describe one failure. A record with neither is its own failure. + """ + failures: dict[tuple[str, str], list[ErrorRecord]] = {} + standalone: list[ErrorRecord] = [] + for record in records: + correlation = record.request_key or record.trace_key + if correlation is None: + standalone.append(record) + else: + failures.setdefault((record.family, correlation), []).append(record) + chosen = [min(group, key=_primary_preference) for group in failures.values()] + return sorted([*chosen, *standalone], key=lambda record: record.timestamp) + + +def fingerprint_parts(record: ErrorRecord) -> tuple[str, ...]: + """Return the kind and the values that identify the record's error group. + + An exception's route is left out, so one shared failure doesn't open an + issue for every route that reaches it. + """ + kind = record.kind + if kind == "exception": + return ( + kind, + record.family, + record.error_type or MISSING, + *fingerprint_frames(record.frames), + ) + if kind == "message": + return ( + kind, + record.family, + record.route or record.operation_name or MISSING, + record.message_template or MISSING, + record.attribute_error_type or MISSING, + record.failure_category or MISSING, + ) + status = MISSING if record.status_code is None else str(record.status_code) + return (kind, record.family, record.route or MISSING, status) + + +def fingerprint_key(secret: str) -> bytes: + """Return the HMAC key, rejecting a secret too short to be random.""" + if len(secret) < MIN_FINGERPRINT_KEY_LENGTH: + msg = ( + "the fingerprint key must be at least " + f"{MIN_FINGERPRINT_KEY_LENGTH} characters" + ) + raise ValueError(msg) + return secret.encode() + + +def fingerprint(record: ErrorRecord, key: bytes) -> str: + """Return the record's error group as 16 hex characters of HMAC-SHA256. + + The key keeps anyone from confirming a guessed message or stack frame by + hashing it and comparing the result with a public fingerprint. + """ + payload = json.dumps( + [FINGERPRINT_ALGORITHM, *fingerprint_parts(record)], + ensure_ascii=False, + separators=(",", ":"), + ) + digest = hmac.new(key, payload.encode(), hashlib.sha256).hexdigest() + return digest[:FINGERPRINT_LENGTH] + + def _report_date_argument(value: str) -> date | None: if not value: return None diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index b3fff98f3..2d2fbd256 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -5,6 +5,7 @@ import functools import importlib.util import re +import sys from datetime import UTC, date, datetime, timedelta from pathlib import Path from typing import TYPE_CHECKING @@ -28,6 +29,8 @@ def _load_script() -> ModuleType: assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) + # Dataclasses resolve their annotations through sys.modules. + sys.modules[spec.name] = module spec.loader.exec_module(module) return module @@ -146,14 +149,14 @@ def test_cli_rejects_missing_project_or_invalid_date_with_exit_code_2( STANDARD_STACK = """\ Traceback (most recent call last): - File "/opt/venv/lib/python3.12/site-packages/flask/app.py", line 880, in full_dispatch_request + File "/venv/lib/python3.12/site-packages/flask/app.py", line 880, in dispatch_request rv = self.dispatch_request() ^^^^^^^^^^^^^^^^^^^^^^^ File "/app/policyengine_api/routes/report_routes.py", line 41, in create_report return service.create(payload) File "/app/policyengine_api/services/report_service.py", line 88, in create session.flush() - File "/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 4331, in flush + File "/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 43, in flush self._flush(objects) sqlalchemy.orm.exc.StaleDataError: SENTINEL message about 'reports' """ @@ -238,7 +241,7 @@ def test_frames_pattern_extracts_frame_headers_without_source_or_message( frames = _extract_frames(sync_script, STANDARD_STACK) assert [sync_script.frame_key(frame) for frame in frames] == [ - "flask/app.py:full_dispatch_request", + "flask/app.py:dispatch_request", "policyengine_api/routes/report_routes.py:create_report", "policyengine_api/services/report_service.py:create", "sqlalchemy/orm/session.py:flush", @@ -276,7 +279,7 @@ def test_frames_pattern_skips_a_frame_header_cut_off_by_truncation( frames = _extract_frames(sync_script, STANDARD_STACK[:cut_at]) assert [sync_script.frame_key(frame) for frame in frames] == [ - "flask/app.py:full_dispatch_request", + "flask/app.py:dispatch_request", ] @@ -324,7 +327,7 @@ def test_frames_choose_the_three_innermost_application_frames( '/app/policyengine_api/services/c.py", line 3, in inner', '/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/d.py", line 4, in lib', '/app/policyengine_api/services/e.py", line 5, in callback', - '/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/f.py", line 6, in raises', + '/venv/lib/python3.12/site-packages/sqlalchemy/orm/f.py", line 6, in raises', ] assert sync_script.fingerprint_frames(frames) == ( @@ -463,3 +466,304 @@ def test_public_status_allows_only_http_status_codes( sync_script: ModuleType, value: object, expected: int | None ) -> None: assert sync_script.public_status(value) == expected + + +# Primary record per failure and fingerprints + +FINGERPRINT_KEY = b"k" * 32 +APPLICATION_FRAMES = ( + '/app/policyengine_api/routes/report_routes.py", line 41, in create_report', + '/app/policyengine_api/services/report_service.py", line 88, in create', + '/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 43, in flush', +) + + +def _record(sync_script: ModuleType, **overrides: object) -> object: + fields: dict[str, object] = { + "timestamp": datetime(2026, 10, 20, 16, 0, tzinfo=UTC), + "service_name": "policyengine-api", + "environment_name": "production", + "event_name": None, + "error_type": None, + "error_handled": None, + "frames": (), + "stack_truncated": False, + "message_template": None, + "attribute_error_type": None, + "failure_category": None, + "route": "//report", + "status_code": None, + "operation_name": None, + "request_key": "request-1", + "observability_key": None, + "trace_key": "trace-1", + } + fields.update(overrides) + return sync_script.ErrorRecord(**fields) + + +def _exception_record(sync_script: ModuleType, **overrides: object) -> object: + fields: dict[str, object] = { + "event_name": "exception.recorded", + "error_type": "StaleDataError", + "error_handled": False, + "frames": APPLICATION_FRAMES, + "status_code": 500, + } + fields.update(overrides) + return _record(sync_script, **fields) + + +def _message_record(sync_script: ModuleType, **overrides: object) -> object: + fields: dict[str, object] = {"message_template": "failed to save"} + fields.update(overrides) + return _record(sync_script, **fields) + + +def _failed_request_record(sync_script: ModuleType, **overrides: object) -> object: + fields: dict[str, object] = {"event_name": "request.completed", "status_code": 500} + fields.update(overrides) + return _record(sync_script, **fields) + + +def test_primary_prefers_the_exception_record_of_a_failed_request( + sync_script: ModuleType, +) -> None: + exception = _exception_record(sync_script) + records = [ + _message_record(sync_script), + exception, + _failed_request_record(sync_script), + ] + + assert sync_script.primary_records(records) == [exception] + + +def test_primary_falls_back_to_the_message_record(sync_script: ModuleType) -> None: + message = _message_record(sync_script) + + primary = sync_script.primary_records( + [_failed_request_record(sync_script), message] + ) + + assert primary == [message] + assert primary[0].kind == "message" + + +def test_primary_falls_back_to_the_failed_request_record( + sync_script: ModuleType, +) -> None: + failed_request = _failed_request_record(sync_script) + + primary = sync_script.primary_records([failed_request]) + + assert primary == [failed_request] + assert primary[0].kind == "failed_request" + + +def test_primary_prefers_an_unhandled_exception_over_a_handled_one( + sync_script: ModuleType, +) -> None: + handled = _exception_record(sync_script, error_type="KeyError", error_handled=True) + unhandled = _exception_record( + sync_script, + timestamp=datetime(2026, 10, 20, 16, 0, 1, tzinfo=UTC), + error_type="LookupError", + ) + + assert sync_script.primary_records([handled, unhandled]) == [unhandled] + + +def test_primary_treats_a_request_completed_record_with_error_fields_as_an_exception( + sync_script: ModuleType, +) -> None: + completed = _failed_request_record( + sync_script, error_type="StaleDataError", frames=APPLICATION_FRAMES + ) + + assert completed.kind == "exception" + assert sync_script.primary_records([completed]) == [completed] + + +def test_primary_correlates_by_trace_when_the_request_id_is_missing( + sync_script: ModuleType, +) -> None: + exception = _exception_record(sync_script, request_key=None) + message = _message_record(sync_script, request_key=None) + + assert sync_script.primary_records([message, exception]) == [exception] + + +def test_primary_records_without_request_or_trace_stand_alone( + sync_script: ModuleType, +) -> None: + first = _message_record(sync_script, request_key=None, trace_key=None) + second = _message_record( + sync_script, + request_key=None, + trace_key=None, + timestamp=datetime(2026, 10, 20, 17, 0, tzinfo=UTC), + ) + + assert sync_script.primary_records([second, first]) == [first, second] + + +def test_primary_keeps_failures_of_different_services_separate( + sync_script: ModuleType, +) -> None: + api_failure = _failed_request_record(sync_script) + simulation_failure = _exception_record( + sync_script, + service_name="policyengine-simulation-entry", + frames=(), + ) + + primary = sync_script.primary_records([api_failure, simulation_failure]) + + assert primary == [api_failure, simulation_failure] + + +def test_fingerprint_exception_ignores_lines_path_prefixes_and_routes( + sync_script: ModuleType, +) -> None: + moved_frames = tuple( + frame.replace("/app/", "/srv/checkout/").replace("line 4", "line 9") + for frame in APPLICATION_FRAMES + ) + first = _exception_record(sync_script) + second = _exception_record( + sync_script, + frames=moved_frames, + route="//economy//over/", + ) + + assert sync_script.fingerprint(first, FINGERPRINT_KEY) == sync_script.fingerprint( + second, FINGERPRINT_KEY + ) + + +def test_fingerprint_exception_merges_simulation_service_versions( + sync_script: ModuleType, +) -> None: + first = _exception_record( + sync_script, service_name="policyengine-simulation-py1-2-3" + ) + second = _exception_record( + sync_script, service_name="policyengine-simulation-py1-2-4" + ) + + assert sync_script.fingerprint(first, FINGERPRINT_KEY) == sync_script.fingerprint( + second, FINGERPRINT_KEY + ) + + +@pytest.mark.parametrize( + "change", + [ + {"error_type": "ValueError"}, + {"frames": APPLICATION_FRAMES[:1]}, + {"service_name": "policyengine-simulation-entry"}, + ], +) +def test_fingerprint_exception_differs_by_type_frames_and_service( + sync_script: ModuleType, change: dict[str, object] +) -> None: + original = _exception_record(sync_script) + changed = _exception_record(sync_script, **change) + + assert sync_script.fingerprint( + original, FINGERPRINT_KEY + ) != sync_script.fingerprint(changed, FINGERPRINT_KEY) + + +def test_fingerprint_depends_on_the_key(sync_script: ModuleType) -> None: + record = _exception_record(sync_script) + + assert sync_script.fingerprint(record, FINGERPRINT_KEY) != sync_script.fingerprint( + record, b"j" * 32 + ) + + +def test_fingerprint_is_16_hex_characters(sync_script: ModuleType) -> None: + value = sync_script.fingerprint(_exception_record(sync_script), FINGERPRINT_KEY) + + assert re.fullmatch(r"[0-9a-f]{16}", value) + + +def test_fingerprint_key_parts_for_each_kind(sync_script: ModuleType) -> None: + exception = _exception_record(sync_script) + message = _message_record( + sync_script, + attribute_error_type="TimeoutError", + failure_category="upstream", + ) + operation_message = _message_record( + sync_script, route=None, operation_name="simulation.dispatch" + ) + failed_request = _failed_request_record(sync_script, status_code=502) + + assert sync_script.fingerprint_parts(exception) == ( + "exception", + "policyengine-api", + "StaleDataError", + "policyengine_api/routes/report_routes.py:create_report", + "policyengine_api/services/report_service.py:create", + ) + assert sync_script.fingerprint_parts(message) == ( + "message", + "policyengine-api", + "//report", + "failed to save", + "TimeoutError", + "upstream", + ) + assert sync_script.fingerprint_parts(operation_message)[2] == "simulation.dispatch" + assert sync_script.fingerprint_parts(failed_request) == ( + "failed_request", + "policyengine-api", + "//report", + "502", + ) + + +@pytest.mark.parametrize( + "change", + [ + {"message_template": "failed to load"}, + {"route": "//calculate"}, + {"attribute_error_type": "ValueError"}, + {"failure_category": "validation"}, + ], +) +def test_fingerprint_message_differs_by_each_key_part( + sync_script: ModuleType, change: dict[str, object] +) -> None: + original = _message_record(sync_script) + changed = _message_record(sync_script, **change) + + assert sync_script.fingerprint( + original, FINGERPRINT_KEY + ) != sync_script.fingerprint(changed, FINGERPRINT_KEY) + + +def test_fingerprint_failed_request_differs_by_status(sync_script: ModuleType) -> None: + first = _failed_request_record(sync_script, status_code=500) + second = _failed_request_record(sync_script, status_code=504) + + assert sync_script.fingerprint(first, FINGERPRINT_KEY) != sync_script.fingerprint( + second, FINGERPRINT_KEY + ) + + +@pytest.mark.parametrize("value", ["", "too-short", "x" * 31]) +def test_fingerprint_key_rejects_a_secret_shorter_than_32_characters( + sync_script: ModuleType, value: str +) -> None: + with pytest.raises(ValueError, match="at least 32 characters"): + sync_script.fingerprint_key(value) + + +def test_fingerprint_key_accepts_a_64_character_hex_secret( + sync_script: ModuleType, +) -> None: + assert sync_script.fingerprint_key("ab" * 32) == ("ab" * 32).encode() From 39a73a13c55ce376ff8448d1332ab510fc6ebbe4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:47:05 +0400 Subject: [PATCH 04/18] Measure each telemetry error group in production and staging For each group with a failure in the window, and separately for production and staging: - occurrences, and distinct affected requests (request ID, else observability ID, else trace); - distinct observability IDs; - error rate: affected requests divided by the requests completed on the routes the group failed on, adding all simulation service versions; - occurrences per hour in the window and over the seven days before it, measured in UTC so 23- and 25-hour days are counted correctly; - first and last seen across the eight queried days. Each group also lists up to three groups that share the most traces with it, and is described only by validated values: the exception type and its most frequent production route. Groups seen only in other environments, or only before the window, are not reported. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 263 ++++++++++- tests/unit/test_telemetry_error_issues.py | 423 ++++++++++++++++++ 2 files changed, 684 insertions(+), 2 deletions(-) diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index 784ca09a6..43629d5f2 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -17,6 +17,7 @@ import hmac import json import re +from collections import Counter from dataclasses import dataclass from datetime import UTC, date, datetime, time, timedelta from typing import TYPE_CHECKING, NamedTuple @@ -79,6 +80,12 @@ FINGERPRINT_LENGTH = 16 MIN_FINGERPRINT_KEY_LENGTH = 32 +# Each run also reads the seven days before the window, for comparison rates, +# first-seen times and regressions. +LOOKBACK = timedelta(days=7) +REPORTED_ENVIRONMENTS = ("production", "staging") +MAX_RELATED_FINGERPRINTS = 3 + class ReportingWindow(NamedTuple): """One Eastern calendar day, as the half-open interval [start, end).""" @@ -89,8 +96,23 @@ class ReportingWindow(NamedTuple): @property def hours(self) -> float: """Return the elapsed hours: 23 or 25 on the days the clocks change.""" - elapsed = self.end.astimezone(UTC) - self.start.astimezone(UTC) - return elapsed / timedelta(hours=1) + return _hours_between(self.start, self.end) + + @property + def lookback_start(self) -> datetime: + """Return midnight Eastern seven days before the window starts.""" + return self.start - LOOKBACK + + @property + def lookback_hours(self) -> float: + """Return the elapsed hours of the seven days before the window.""" + return _hours_between(self.lookback_start, self.start) + + +def _hours_between(start: datetime, end: datetime) -> float: + # Subtracting two datetimes in the same zone ignores a clock change, so + # compare them in UTC. + return (end.astimezone(UTC) - start.astimezone(UTC)) / timedelta(hours=1) def reporting_window(report_date: date) -> ReportingWindow: @@ -357,6 +379,243 @@ def fingerprint(record: ErrorRecord, key: bytes) -> str: return digest[:FINGERPRINT_LENGTH] +class RequestCount(NamedTuple): + """Requests that one service completed on one route during the window.""" + + service_name: str + environment_name: str | None + route: str | None + requests: int + + +class Occurrence(NamedTuple): + """One failure in an error group.""" + + timestamp: datetime + environment: str + route: str | None + affected_key: str + observability_key: str | None + trace_key: str | None + + +@dataclass(frozen=True) +class EnvironmentMeasures: + """An error group's measures in one environment. + + Counts and rates cover the window; first and last seen cover the window + and the seven days before it. + """ + + occurrences: int + affected_requests: int + observability_ids: int + route_requests: int + error_rate: float | None + hourly_rate: float + previous_hourly_rate: float + first_seen: datetime | None + last_seen: datetime | None + + +@dataclass(frozen=True) +class ErrorGroup: + """Failures that share a fingerprint, described only by validated values.""" + + fingerprint: str + kind: str + family: str + error_type: str + route: str + status_code: int | None + operation: str + production: EnvironmentMeasures + staging: EnvironmentMeasures + related_fingerprints: tuple[str, ...] + occurrences: tuple[Occurrence, ...] + + def production_since(self, moment: datetime) -> tuple[int, int]: + """Return production occurrences and affected requests from ``moment``.""" + recent = [ + occurrence + for occurrence in self.occurrences + if occurrence.environment == "production" and occurrence.timestamp >= moment + ] + return len(recent), len({occurrence.affected_key for occurrence in recent}) + + +def _occurrence(record: ErrorRecord, position: int) -> Occurrence: + """Describe a failure, identifying what it affected as precisely as possible. + + The affected request is the request ID, else the observability ID, else the + trace. A failure with none of them counts as its own request. + """ + if record.request_key: + affected_key = f"request:{record.request_key}" + elif record.observability_key: + affected_key = f"observability:{record.observability_key}" + elif record.trace_key: + affected_key = f"trace:{record.trace_key}" + else: + affected_key = f"record:{position}" + return Occurrence( + timestamp=record.timestamp, + environment=record.environment, + route=record.route, + affected_key=affected_key, + observability_key=record.observability_key, + trace_key=record.trace_key, + ) + + +class RouteRequests: + """Requests completed in the window, per service family, environment and route.""" + + def __init__(self, request_counts: Iterable[RequestCount]) -> None: + """Add up the request counts, merging versioned simulation services.""" + self._totals: Counter[tuple[str, str, str | None]] = Counter() + for count in request_counts: + environment = environment_class(count.environment_name) + family = service_family(count.service_name) + self._totals[family, environment, count.route] += count.requests + + def total(self, family: str, environment: str, routes: Iterable[str]) -> int: + """Return the requests one service family completed on the given routes.""" + return sum(self._totals[family, environment, route] for route in routes) + + +def _related_fingerprints( + occurrences_by_group: dict[str, list[Occurrence]], +) -> dict[str, tuple[str, ...]]: + """For each group, the groups that share the most traces with it.""" + groups_by_trace: dict[str, set[str]] = {} + for group, occurrences in occurrences_by_group.items(): + for occurrence in occurrences: + if occurrence.trace_key: + groups_by_trace.setdefault(occurrence.trace_key, set()).add(group) + related: dict[str, tuple[str, ...]] = {} + for group, occurrences in occurrences_by_group.items(): + shared_traces: Counter[str] = Counter() + traces = {occurrence.trace_key for occurrence in occurrences} + for trace in traces - {None}: + shared_traces.update(groups_by_trace[trace] - {group}) + ranked = sorted(shared_traces, key=lambda other: (-shared_traces[other], other)) + related[group] = tuple(ranked[:MAX_RELATED_FINGERPRINTS]) + return related + + +def _environment_measures( + occurrences: list[Occurrence], + environment: str, + family: str, + route_requests: RouteRequests, + window: ReportingWindow, +) -> EnvironmentMeasures: + in_environment = [item for item in occurrences if item.environment == environment] + in_window = [item for item in in_environment if item.timestamp >= window.start] + before_window = len(in_environment) - len(in_window) + affected_requests = len({item.affected_key for item in in_window}) + routes = {item.route for item in in_window if item.route} + requests = route_requests.total(family, environment, routes) + timestamps = [item.timestamp for item in in_environment] + return EnvironmentMeasures( + occurrences=len(in_window), + affected_requests=affected_requests, + observability_ids=len( + {item.observability_key for item in in_window if item.observability_key} + ), + route_requests=requests, + error_rate=affected_requests / requests if requests else None, + hourly_rate=len(in_window) / window.hours, + previous_hourly_rate=before_window / window.lookback_hours, + first_seen=min(timestamps, default=None), + last_seen=max(timestamps, default=None), + ) + + +def _most_frequent_route(occurrences: list[Occurrence], window: ReportingWindow) -> str: + """Return the route most failures in the window used, preferring production.""" + in_window = [item for item in occurrences if item.timestamp >= window.start] + production = [item for item in in_window if item.environment == "production"] + routes = Counter(item.route or MISSING for item in production or in_window) + route, _ = min(routes.items(), key=lambda item: (-item[1], item[0])) + return public_route(route) + + +def _error_group( + group_fingerprint: str, + members: list[tuple[ErrorRecord, Occurrence]], + related_fingerprints: tuple[str, ...], + route_requests: RouteRequests, + window: ReportingWindow, +) -> ErrorGroup: + example = members[0][0] + occurrences = [occurrence for _, occurrence in members] + measures = { + environment: _environment_measures( + occurrences, environment, example.family, route_requests, window + ) + for environment in REPORTED_ENVIRONMENTS + } + return ErrorGroup( + fingerprint=group_fingerprint, + kind=example.kind, + family=public_family(example.family), + error_type=public_error_type(example.error_type), + route=_most_frequent_route(occurrences, window), + status_code=public_status(example.status_code), + operation=public_operation(example.operation_name), + production=measures["production"], + staging=measures["staging"], + related_fingerprints=related_fingerprints, + occurrences=tuple(occurrences), + ) + + +def build_groups( + records: Iterable[ErrorRecord], + request_counts: Iterable[RequestCount], + window: ReportingWindow, + key: bytes, +) -> list[ErrorGroup]: + """Group the failures in production and staging and measure each group. + + Only groups with a failure in the window are returned, most production + occurrences first. + """ + in_scope = [ + record + for record in records + if record.environment in REPORTED_ENVIRONMENTS + and window.lookback_start <= record.timestamp < window.end + ] + members_by_group: dict[str, list[tuple[ErrorRecord, Occurrence]]] = {} + for position, record in enumerate(primary_records(in_scope)): + members_by_group.setdefault(fingerprint(record, key), []).append( + (record, _occurrence(record, position)) + ) + related = _related_fingerprints( + { + group: [occurrence for _, occurrence in members] + for group, members in members_by_group.items() + } + ) + route_requests = RouteRequests(request_counts) + groups = [ + _error_group(group, members, related[group], route_requests, window) + for group, members in members_by_group.items() + if any(occurrence.timestamp >= window.start for _, occurrence in members) + ] + return sorted( + groups, + key=lambda group: ( + -group.production.occurrences, + -group.staging.occurrences, + group.fingerprint, + ), + ) + + def _report_date_argument(value: str) -> date | None: if not value: return None diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index 2d2fbd256..ab94545c4 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -767,3 +767,426 @@ def test_fingerprint_key_accepts_a_64_character_hex_secret( sync_script: ModuleType, ) -> None: assert sync_script.fingerprint_key("ab" * 32) == ("ab" * 32).encode() + + +# Measures per group + +REPORT_DATE = date(2026, 10, 20) +# Noon Eastern on the report date: 12 hours after the window starts, and 180 +# hours after the seven days before it start. +WINDOW_NOON = datetime(2026, 10, 20, 16, 0, tzinfo=UTC) + + +def _at(hours_after_window_noon: float) -> datetime: + return WINDOW_NOON + timedelta(hours=hours_after_window_noon) + + +def _request_count( + sync_script: ModuleType, + route: str, + requests: int, + *, + service_name: str = "policyengine-api", + environment_name: str = "production", +) -> object: + return sync_script.RequestCount(service_name, environment_name, route, requests) + + +def _only_group(sync_script: ModuleType, records: list, request_counts=()) -> object: + groups = sync_script.build_groups( + records, + list(request_counts), + sync_script.reporting_window(REPORT_DATE), + FINGERPRINT_KEY, + ) + assert len(groups) == 1 + return groups[0] + + +def test_measures_count_occurrences_requests_and_observability_ids( + sync_script: ModuleType, +) -> None: + records = [ + _exception_record(sync_script, request_key="r1", observability_key="o1"), + _exception_record( + sync_script, request_key="r2", observability_key="o1", timestamp=_at(1) + ), + _exception_record( + sync_script, request_key="r3", observability_key="o2", timestamp=_at(2) + ), + _exception_record( + sync_script, request_key="s1", environment_name="staging", timestamp=_at(3) + ), + ] + + group = _only_group(sync_script, records) + + assert group.production.occurrences == 3 + assert group.production.affected_requests == 3 + assert group.production.observability_ids == 2 + assert group.staging.occurrences == 1 + assert group.staging.affected_requests == 1 + assert group.staging.observability_ids == 0 + + +def test_measures_affected_requests_fall_back_to_observability_id_then_trace( + sync_script: ModuleType, +) -> None: + records = [ + _exception_record( + sync_script, request_key=None, trace_key=None, observability_key="o1" + ), + _exception_record( + sync_script, + request_key=None, + trace_key=None, + observability_key="o1", + timestamp=_at(1), + ), + _exception_record(sync_script, request_key=None, trace_key="t1"), + _exception_record(sync_script, request_key=None, trace_key=None), + _exception_record( + sync_script, request_key=None, trace_key=None, timestamp=_at(2) + ), + ] + + group = _only_group(sync_script, records) + + assert group.production.occurrences == 5 + # o1 once, t1 once, and each record without any identifier on its own. + assert group.production.affected_requests == 4 + + +def test_measures_error_rate_uses_requests_to_the_affected_routes( + sync_script: ModuleType, +) -> None: + records = [ + _exception_record(sync_script, request_key="r1", route="//report"), + _exception_record( + sync_script, + request_key="r2", + route="//calculate", + timestamp=_at(1), + ), + _exception_record( + sync_script, + request_key="r3", + route="//calculate", + timestamp=_at(2), + ), + ] + request_counts = [ + _request_count(sync_script, "//report", 200), + _request_count(sync_script, "//calculate", 100), + _request_count(sync_script, "//metadata", 5000), + _request_count( + sync_script, "//report", 10, environment_name="staging" + ), + _request_count( + sync_script, + "//report", + 7000, + service_name="policyengine-simulation-entry", + ), + ] + + group = _only_group(sync_script, records, request_counts) + + assert group.production.route_requests == 300 + assert group.production.error_rate == pytest.approx(0.01) + assert group.staging.error_rate is None + + +def test_measures_error_rate_counts_every_simulation_service_version( + sync_script: ModuleType, +) -> None: + records = [ + _failed_request_record( + sync_script, + service_name="policyengine-simulation-py1-2-3", + route="/jobs/{job_id}", + ) + ] + request_counts = [ + _request_count( + sync_script, + "/jobs/{job_id}", + 30, + service_name="policyengine-simulation-py1-2-3", + ), + _request_count( + sync_script, + "/jobs/{job_id}", + 70, + service_name="policyengine-simulation-py1-2-4", + ), + ] + + group = _only_group(sync_script, records, request_counts) + + assert group.production.route_requests == 100 + assert group.production.error_rate == pytest.approx(0.01) + + +def test_measures_error_rate_is_unknown_without_route_requests( + sync_script: ModuleType, +) -> None: + group = _only_group( + sync_script, [_message_record(sync_script, route=None, operation_name="sync")] + ) + + assert group.production.route_requests == 0 + assert group.production.error_rate is None + + +def test_measures_hourly_rates_for_the_window_and_the_previous_seven_days( + sync_script: ModuleType, +) -> None: + window_records = [ + _message_record(sync_script, request_key=f"w{index}", timestamp=_at(index)) + for index in range(6) + ] + previous_records = [ + _message_record( + sync_script, request_key=f"p{index}", timestamp=_at(-13 - 4 * index) + ) + for index in range(42) + ] + + group = _only_group(sync_script, window_records + previous_records) + + assert group.production.occurrences == 6 + assert group.production.hourly_rate == pytest.approx(6 / 24) + assert group.production.previous_hourly_rate == pytest.approx(42 / 168) + + +def test_measures_hourly_rate_uses_the_25_hour_fall_back_day( + sync_script: ModuleType, +) -> None: + fall_back_noon = datetime(2026, 11, 1, 17, 0, tzinfo=UTC) + records = [ + _message_record( + sync_script, + request_key=f"w{index}", + timestamp=fall_back_noon + timedelta(hours=index), + ) + for index in range(5) + ] + + groups = sync_script.build_groups( + records, [], sync_script.reporting_window(date(2026, 11, 1)), FINGERPRINT_KEY + ) + + assert groups[0].production.hourly_rate == pytest.approx(5 / 25) + + +def test_measures_first_and_last_seen_cover_the_queried_eight_days( + sync_script: ModuleType, +) -> None: + records = [ + _message_record(sync_script, request_key="early", timestamp=_at(-24 * 6)), + _message_record(sync_script, request_key="latest", timestamp=_at(5)), + _message_record(sync_script, request_key="middle", timestamp=_at(1)), + ] + + group = _only_group(sync_script, records) + + assert group.production.first_seen == _at(-24 * 6) + assert group.production.last_seen == _at(5) + assert group.staging.first_seen is None + assert group.staging.last_seen is None + + +def test_measures_ignore_records_outside_the_queried_days( + sync_script: ModuleType, +) -> None: + records = [ + _message_record(sync_script, request_key="in-window"), + _message_record(sync_script, request_key="too-old", timestamp=_at(-24 * 9)), + _message_record(sync_script, request_key="too-new", timestamp=_at(24)), + ] + + group = _only_group(sync_script, records) + + assert group.production.occurrences == 1 + assert group.production.first_seen == WINDOW_NOON + + +def test_measures_report_only_groups_seen_in_the_window( + sync_script: ModuleType, +) -> None: + records = [ + _message_record(sync_script, request_key="today"), + _message_record( + sync_script, + request_key="last-week", + message_template="something else", + timestamp=_at(-48), + ), + ] + + groups = sync_script.build_groups( + records, [], sync_script.reporting_window(REPORT_DATE), FINGERPRINT_KEY + ) + + assert [group.fingerprint for group in groups] == [ + sync_script.fingerprint(records[0], FINGERPRINT_KEY) + ] + + +def test_measures_ignore_environments_other_than_production_and_staging( + sync_script: ModuleType, +) -> None: + records = [ + _message_record(sync_script, environment_name="local"), + _message_record(sync_script, environment_name=None, request_key="r2"), + ] + + groups = sync_script.build_groups( + records, [], sync_script.reporting_window(REPORT_DATE), FINGERPRINT_KEY + ) + + assert groups == [] + + +def test_measures_describe_the_group_with_its_most_frequent_production_route( + sync_script: ModuleType, +) -> None: + records = [ + _exception_record(sync_script, request_key="r1", route="//report"), + _exception_record( + sync_script, request_key="r2", route="//calculate" + ), + _exception_record( + sync_script, request_key="r3", route="//calculate" + ), + _exception_record( + sync_script, + request_key="s1", + route="//report", + environment_name="staging", + ), + _exception_record( + sync_script, + request_key="s2", + route="//report", + environment_name="staging", + ), + ] + + group = _only_group(sync_script, records) + + assert group.kind == "exception" + assert group.family == "policyengine-api" + assert group.error_type == "StaleDataError" + assert group.route == "//calculate" + + +def test_measures_describe_groups_only_with_validated_values( + sync_script: ModuleType, +) -> None: + group = _only_group( + sync_script, + [ + _exception_record( + sync_script, + error_type="SENTINEL not a class", + route="/us/policy/12345", + ) + ], + ) + + assert group.error_type == "" + assert group.route == "" + + +def test_measures_list_up_to_three_related_fingerprints_by_shared_traces( + sync_script: ModuleType, +) -> None: + base = _exception_record(sync_script, request_key="r1", trace_key="t1") + base_second = _exception_record( + sync_script, request_key="r2", trace_key="t2", timestamp=_at(1) + ) + related_twice = [ + _message_record( + sync_script, + service_name="policyengine-simulation-entry", + request_key=f"x{index}", + trace_key=trace, + ) + for index, trace in enumerate(("t1", "t2")) + ] + related_once = [ + _message_record( + sync_script, + service_name="policyengine-simulation-gateway", + request_key=f"y{index}", + trace_key="t1", + message_template=template, + ) + for index, template in enumerate(("first error", "second error", "third")) + ] + + groups = sync_script.build_groups( + [base, base_second, *related_twice, *related_once], + [], + sync_script.reporting_window(REPORT_DATE), + FINGERPRINT_KEY, + ) + by_fingerprint = {group.fingerprint: group for group in groups} + base_group = by_fingerprint[sync_script.fingerprint(base, FINGERPRINT_KEY)] + twice = sync_script.fingerprint(related_twice[0], FINGERPRINT_KEY) + once = sorted( + sync_script.fingerprint(record, FINGERPRINT_KEY) for record in related_once + ) + + assert base_group.related_fingerprints == (twice, once[0], once[1]) + + +def test_measures_sort_groups_by_production_occurrences( + sync_script: ModuleType, +) -> None: + frequent = [ + _message_record(sync_script, request_key=f"f{index}", message_template="often") + for index in range(3) + ] + staging_only = [ + _message_record( + sync_script, + request_key=f"s{index}", + message_template="staging", + environment_name="staging", + ) + for index in range(5) + ] + rare = [_message_record(sync_script, request_key="rare", message_template="rare")] + + groups = sync_script.build_groups( + staging_only + rare + frequent, + [], + sync_script.reporting_window(REPORT_DATE), + FINGERPRINT_KEY, + ) + + assert [group.production.occurrences for group in groups] == [3, 1, 0] + + +def test_measures_count_production_occurrences_since_a_moment( + sync_script: ModuleType, +) -> None: + records = [ + _exception_record(sync_script, request_key="before", timestamp=_at(-30)), + _exception_record(sync_script, request_key="r1", timestamp=_at(1)), + _exception_record(sync_script, request_key="r2", timestamp=_at(2)), + _exception_record( + sync_script, request_key="r2", trace_key="t9", timestamp=_at(3) + ), + _exception_record( + sync_script, request_key="staging", environment_name="staging" + ), + ] + + group = _only_group(sync_script, records) + + assert group.production_since(_at(-1)) == (2, 2) + assert group.production_since(_at(-48)) == (3, 3) From b3acba53b268b2fa92b7510ae7b48a654c559589 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:51:08 +0400 Subject: [PATCH 05/18] Query telemetry error records and request counts from BigQuery Two standard SQL queries read the linked dataset's _AllLogs view: - error records from the window and the seven days before it: Public API and simulation records (service namespace policyengine.api-v1) at ERROR or above, without the health-check routes, with duplicate deliveries removed; - request.completed counts per service, environment and route in the window, for error rates. Only code-defined or normalized values leave BigQuery. Stacks are read only through FRAME_PATTERN and a truncation flag, messages only as their first three whole words outside quotes, and request IDs, observability IDs and traces only as SHA-256 hashes. The urllib client sends named parameters, a 5 GiB maximumBytesBilled limit and job labels, polls until the job completes, follows page tokens in the job's location and converts values from their string form. An error response raises an exception that holds only the HTTP status and the API's reason code, never the response body. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 421 +++++++++++++++++- tests/unit/test_telemetry_error_issues.py | 416 +++++++++++++++++ 2 files changed, 835 insertions(+), 2 deletions(-) diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index 43629d5f2..bd5d136d5 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -20,11 +20,15 @@ from collections import Counter from dataclasses import dataclass from datetime import UTC, date, datetime, time, timedelta -from typing import TYPE_CHECKING, NamedTuple +from http import HTTPStatus +from typing import TYPE_CHECKING, Any, NamedTuple +from urllib.error import HTTPError +from urllib.parse import quote, urlencode +from urllib.request import Request, urlopen from zoneinfo import ZoneInfo if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Callable, Iterable, Mapping REPORTING_TIME_ZONE = ZoneInfo("America/New_York") @@ -86,6 +90,104 @@ REPORTED_ENVIRONMENTS = ("production", "staging") MAX_RELATED_FINGERPRINTS = 3 +HTTP_TIMEOUT_SECONDS = 60 +# A short code from an API error, such as "invalidQuery" or "Not Found". The +# error's message is never used, because it can repeat values from a record. +ERROR_REASON_PATTERN = re.compile(r"^[A-Za-z][A-Za-z ]{0,59}$") + +BIGQUERY_API = "https://bigquery.googleapis.com/bigquery/v2" +LINKED_DATASET = "observability_logs" +MAXIMUM_BYTES_BILLED = 5 * 1024**3 +QUERY_WAIT_MILLISECONDS = 30_000 +MAX_QUERY_POLLS = 40 +UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=UTC) +HEALTH_CHECK_ROUTES = ( + "/health", + "/simulation-gateway-check", + "/liveness-check", + "/readiness-check", +) + +# Each query reads the linked dataset's _AllLogs view. Only code-defined or +# normalized values leave BigQuery: stacks only as frame headers, messages +# only as their first three whole words outside quotes, and identifiers only +# as SHA-256 hashes. Duplicate deliveries of one log entry are removed. +ERROR_RECORDS_QUERY = r""" +SELECT + timestamp, + JSON_VALUE(json_payload, '$."service.name"') AS service_name, + JSON_VALUE(json_payload, '$."deployment.environment.name"') AS environment_name, + JSON_VALUE(json_payload, '$."event.name"') AS event_name, + JSON_VALUE(json_payload, '$."error.type"') AS error_type, + JSON_VALUE(json_payload, '$."error.handled"') AS error_handled, + REGEXP_EXTRACT_ALL(JSON_VALUE(json_payload, '$."error.stack"'), @frame_pattern) + AS frames, + COALESCE(LENGTH(JSON_VALUE(json_payload, '$."error.stack"')) >= 16384, FALSE) + AS stack_truncated, + ARRAY_TO_STRING( + ARRAY( + SELECT LOWER(word) + FROM UNNEST( + REGEXP_EXTRACT_ALL( + REGEXP_REPLACE( + REGEXP_REPLACE(JSON_VALUE(json_payload, '$.message'), r'"[^"]*"', ''), + r"'[^']*'", + '' + ), + r'\b[A-Za-z]+\b' + ) + ) AS word WITH OFFSET AS position + WHERE position < 3 + ORDER BY position + ), + ' ' + ) AS message_template, + JSON_VALUE(json_payload, '$.attributes.error_type') AS attribute_error_type, + JSON_VALUE(json_payload, '$.attributes.failure_category') AS failure_category, + JSON_VALUE(json_payload, '$."http.route"') AS route, + JSON_VALUE(json_payload, '$."http.response.status_code"') AS status_code, + JSON_VALUE(json_payload, '$."operation.name"') AS operation_name, + TO_HEX(SHA256(JSON_VALUE(json_payload, '$."request.id"'))) AS request_key, + TO_HEX(SHA256(JSON_VALUE(json_payload, '$.attributes.observability_id'))) + AS observability_key, + TO_HEX(SHA256(COALESCE( + JSON_VALUE(json_payload, '$.trace_id'), + REGEXP_EXTRACT(trace, r'/traces/([0-9A-Fa-f]+)$') + ))) AS trace_key +FROM `_AllLogs` +WHERE timestamp >= @query_start AND timestamp < @window_end + AND JSON_VALUE(json_payload, '$.schema_version') = 'policyengine.observability.v2' + AND JSON_VALUE(json_payload, '$."service.namespace"') = 'policyengine.api-v1' + AND severity_number >= 500 + AND COALESCE(JSON_VALUE(json_payload, '$."http.route"'), '') + NOT IN UNNEST(@excluded_routes) +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY insert_id, timestamp ORDER BY receive_timestamp +) = 1 +""" + +REQUEST_COUNTS_QUERY = r""" +WITH completed_requests AS ( + SELECT + JSON_VALUE(json_payload, '$."service.name"') AS service_name, + JSON_VALUE(json_payload, '$."deployment.environment.name"') AS environment_name, + JSON_VALUE(json_payload, '$."http.route"') AS route + FROM `_AllLogs` + WHERE timestamp >= @window_start AND timestamp < @window_end + AND JSON_VALUE(json_payload, '$.schema_version') = 'policyengine.observability.v2' + AND JSON_VALUE(json_payload, '$."service.namespace"') = 'policyengine.api-v1' + AND JSON_VALUE(json_payload, '$."event.name"') = 'request.completed' + AND COALESCE(JSON_VALUE(json_payload, '$."http.route"'), '') + NOT IN UNNEST(@excluded_routes) + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY insert_id, timestamp ORDER BY receive_timestamp + ) = 1 +) +SELECT service_name, environment_name, route, COUNT(*) AS requests +FROM completed_requests +GROUP BY service_name, environment_name, route +""" + class ReportingWindow(NamedTuple): """One Eastern calendar day, as the half-open interval [start, end).""" @@ -616,6 +718,321 @@ def build_groups( ) +class HttpRequest(NamedTuple): + """An HTTP request for a transport to send.""" + + method: str + url: str + headers: dict[str, str] + body: bytes | None + + +class HttpResponse(NamedTuple): + """An HTTP response, including error responses. Header names are lowercase.""" + + status: int + reason: str + headers: Mapping[str, str] + body: bytes + + +def urllib_transport(request: HttpRequest) -> HttpResponse: + """Send a request with the standard library, returning HTTP errors too.""" + outgoing = Request( + request.url, + data=request.body, + headers=request.headers, + method=request.method, + ) + try: + response = urlopen(outgoing, timeout=HTTP_TIMEOUT_SECONDS) + except HTTPError as error: + with error: + return HttpResponse( + status=error.code, + reason=str(error.reason), + headers={name.lower(): value for name, value in error.headers.items()}, + body=error.read(), + ) + with response: + return HttpResponse( + status=response.status, + reason=response.reason, + headers={name.lower(): value for name, value in response.headers.items()}, + body=response.read(), + ) + + +class ApiError(Exception): + """An API request failed. The text holds only code-defined values.""" + + service = "API" + + def __init__( + self, message: str, *, status: int | None = None, reason: str | None = None + ) -> None: + """Keep the HTTP status and reason for the caller's summary.""" + super().__init__(message) + self.status = status + self.reason = reason + + @classmethod + def from_response(cls, response: HttpResponse) -> ApiError: + """Describe an error response by its status and reason, never its body.""" + reason = _error_reason(response) + message = f"{cls.service} request failed: HTTP {response.status} {reason}" + return cls(message, status=response.status, reason=reason) + + +class BigQueryError(ApiError): + """A BigQuery request failed.""" + + service = "BigQuery" + + +def _error_reason(response: HttpResponse) -> str: + """Return a Google API error's reason code, else the HTTP reason phrase.""" + try: + payload = json.loads(response.body) + reason = payload["error"]["errors"][0]["reason"] + except (ValueError, KeyError, IndexError, TypeError): + reason = None + for candidate in (reason, response.reason): + if isinstance(candidate, str) and ERROR_REASON_PATTERN.match(candidate): + return candidate + return "unknown" + + +def _bigquery_boolean(text: str) -> bool: + return text == "true" + + +def _bigquery_timestamp(text: str) -> datetime: + """Convert microseconds since the epoch, as ``useInt64Timestamp`` returns.""" + return UNIX_EPOCH + timedelta(microseconds=int(text)) + + +CELL_CONVERTERS: dict[str, Callable[[str], object]] = { + "INTEGER": int, + "INT64": int, + "FLOAT": float, + "FLOAT64": float, + "BOOLEAN": _bigquery_boolean, + "BOOL": _bigquery_boolean, + "TIMESTAMP": _bigquery_timestamp, +} + + +def _convert_cell(field: Mapping[str, Any], value: object) -> object: + """Convert one BigQuery REST value, which arrives as a string, to Python.""" + if value is None: + return None + if field.get("mode") == "REPEATED" and isinstance(value, list): + item_field = {**field, "mode": "NULLABLE"} + return [_convert_cell(item_field, item["v"]) for item in value] + converter = CELL_CONVERTERS.get(str(field.get("type")), str) + return converter(str(value)) + + +def _convert_rows(response: Mapping[str, Any]) -> list[dict[str, object]]: + fields = response["schema"]["fields"] + return [ + { + field["name"]: _convert_cell(field, cell["v"]) + for field, cell in zip(fields, row["f"], strict=True) + } + for row in response.get("rows", []) + ] + + +class BigQueryClient: + """Runs parameterized standard SQL queries through the BigQuery REST API.""" + + def __init__( + self, + project_id: str, + access_token: str, + transport: Callable[[HttpRequest], HttpResponse] = urllib_transport, + ) -> None: + """Run jobs in ``project_id`` against its linked log dataset.""" + self.project_id = project_id + self._access_token = access_token + self._transport = transport + self._queries_url = ( + f"{BIGQUERY_API}/projects/{quote(project_id, safe='')}/queries" + ) + + def query( + self, sql: str, parameters: list[dict[str, Any]], label: str + ) -> list[dict[str, object]]: + """Run ``sql`` and return every row as a dictionary of converted values.""" + body = { + "query": sql, + "useLegacySql": False, + "parameterMode": "NAMED", + "queryParameters": parameters, + "defaultDataset": { + "projectId": self.project_id, + "datasetId": LINKED_DATASET, + }, + "maximumBytesBilled": str(MAXIMUM_BYTES_BILLED), + "labels": {"purpose": "telemetry-error-issues", "query": label}, + "timeoutMs": QUERY_WAIT_MILLISECONDS, + "formatOptions": {"useInt64Timestamp": True}, + } + response = self._send("POST", self._queries_url, body) + job = response["jobReference"] + for _ in range(MAX_QUERY_POLLS): + if response.get("jobComplete"): + break + response = self._results(job, page_token=None) + else: + msg = "BigQuery query did not complete in time" + raise BigQueryError(msg) + rows = _convert_rows(response) + while response.get("pageToken"): + response = self._results(job, page_token=response["pageToken"]) + rows.extend(_convert_rows(response)) + return rows + + def _results( + self, job: Mapping[str, str], page_token: str | None + ) -> dict[str, Any]: + parameters = { + "location": job["location"], + "timeoutMs": str(QUERY_WAIT_MILLISECONDS), + "formatOptions.useInt64Timestamp": "true", + } + if page_token: + parameters["pageToken"] = page_token + job_id = quote(job["jobId"], safe="") + url = f"{self._queries_url}/{job_id}?{urlencode(parameters)}" + return self._send("GET", url, None) + + def _send( + self, method: str, url: str, body: Mapping[str, object] | None + ) -> dict[str, Any]: + headers = {"Authorization": f"Bearer {self._access_token}"} + if body is not None: + headers["Content-Type"] = "application/json" + encoded = None if body is None else json.dumps(body).encode() + response = self._transport(HttpRequest(method, url, headers, encoded)) + if response.status >= HTTPStatus.BAD_REQUEST: + raise BigQueryError.from_response(response) + return json.loads(response.body) + + +def _timestamp_parameter(name: str, moment: datetime) -> dict[str, Any]: + return { + "name": name, + "parameterType": {"type": "TIMESTAMP"}, + "parameterValue": {"value": moment.astimezone(UTC).isoformat(sep=" ")}, + } + + +def _query_parameters( + timestamps: Mapping[str, datetime], strings: Mapping[str, str] +) -> list[dict[str, Any]]: + parameters = [ + _timestamp_parameter(name, value) for name, value in timestamps.items() + ] + parameters.extend( + { + "name": name, + "parameterType": {"type": "STRING"}, + "parameterValue": {"value": value}, + } + for name, value in strings.items() + ) + parameters.append( + { + "name": "excluded_routes", + "parameterType": {"type": "ARRAY", "arrayType": {"type": "STRING"}}, + "parameterValue": { + "arrayValues": [{"value": route} for route in HEALTH_CHECK_ROUTES] + }, + } + ) + return parameters + + +def _text(value: object) -> str | None: + return None if value is None or value == "" else str(value) + + +def _integer(value: object) -> int | None: + if isinstance(value, int) and not isinstance(value, bool): + return value + if isinstance(value, str) and value.isascii() and value.isdigit(): + return int(value) + return None + + +def _boolean(value: object) -> bool | None: + if isinstance(value, bool): + return value + return {"true": True, "false": False}.get(str(value)) + + +def error_record_from_row(row: Mapping[str, object]) -> ErrorRecord: + """Build an ``ErrorRecord`` from one row of ``ERROR_RECORDS_QUERY``.""" + timestamp = row["timestamp"] + if not isinstance(timestamp, datetime): + msg = "an error record has no timestamp" + raise TypeError(msg) + frames = row["frames"] + return ErrorRecord( + timestamp=timestamp, + service_name=_text(row["service_name"]) or MISSING, + environment_name=_text(row["environment_name"]), + event_name=_text(row["event_name"]), + error_type=_text(row["error_type"]), + error_handled=_boolean(row["error_handled"]), + frames=tuple(map(str, frames)) if isinstance(frames, list) else (), + stack_truncated=row["stack_truncated"] is True, + message_template=_text(row["message_template"]), + attribute_error_type=_text(row["attribute_error_type"]), + failure_category=_text(row["failure_category"]), + route=_text(row["route"]), + status_code=_integer(row["status_code"]), + operation_name=_text(row["operation_name"]), + request_key=_text(row["request_key"]), + observability_key=_text(row["observability_key"]), + trace_key=_text(row["trace_key"]), + ) + + +def fetch_error_records( + client: BigQueryClient, window: ReportingWindow +) -> list[ErrorRecord]: + """Return the ERROR records of the window and the seven days before it.""" + parameters = _query_parameters( + {"query_start": window.lookback_start, "window_end": window.end}, + {"frame_pattern": FRAME_PATTERN}, + ) + rows = client.query(ERROR_RECORDS_QUERY, parameters, "error-records") + return [error_record_from_row(row) for row in rows] + + +def fetch_request_counts( + client: BigQueryClient, window: ReportingWindow +) -> list[RequestCount]: + """Return the requests each service completed per route in the window.""" + parameters = _query_parameters( + {"window_start": window.start, "window_end": window.end}, {} + ) + rows = client.query(REQUEST_COUNTS_QUERY, parameters, "request-counts") + return [ + RequestCount( + service_name=_text(row["service_name"]) or MISSING, + environment_name=_text(row["environment_name"]), + route=_text(row["route"]), + requests=_integer(row["requests"]) or 0, + ) + for row in rows + ] + + def _report_date_argument(value: str) -> date | None: if not value: return None diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index ab94545c4..4e79627be 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -4,11 +4,13 @@ import functools import importlib.util +import json import re import sys from datetime import UTC, date, datetime, timedelta from pathlib import Path from typing import TYPE_CHECKING +from urllib.parse import parse_qs, urlparse from zoneinfo import ZoneInfo import pytest @@ -1190,3 +1192,417 @@ def test_measures_count_production_occurrences_since_a_moment( assert group.production_since(_at(-1)) == (2, 2) assert group.production_since(_at(-48)) == (3, 3) + + +# BigQuery queries and client + +ACCESS_TOKEN = "fake-access-token" + + +class FakeTransport: + """Records each HTTP request and answers with queued responses.""" + + def __init__(self, sync_script: ModuleType, responses: list[object]) -> None: + self.sync_script = sync_script + self.responses = list(responses) + self.requests: list[object] = [] + + def __call__(self, request: object) -> object: + self.requests.append(request) + return self.responses.pop(0) + + def json_body(self, index: int) -> dict: + return json.loads(self.requests[index].body) + + +def _json_response( + sync_script: ModuleType, payload: object, status: int = 200, reason: str = "OK" +) -> object: + return sync_script.HttpResponse( + status=status, reason=reason, headers={}, body=json.dumps(payload).encode() + ) + + +def _cell(value: object) -> dict: + if isinstance(value, list): + return {"v": [{"v": item} for item in value]} + return {"v": value} + + +def _query_response( + fields: list[dict], + rows: list[list[object]], + *, + page_token: str | None = None, + job_complete: bool = True, +) -> dict: + response: dict[str, object] = { + "jobComplete": job_complete, + "jobReference": { + "projectId": "policyengine-observability", + "jobId": "job-1", + "location": "US", + }, + } + if job_complete: + response["schema"] = {"fields": fields} + response["rows"] = [{"f": [_cell(value) for value in row]} for row in rows] + if page_token: + response["pageToken"] = page_token + return response + + +STRING_FIELD = {"name": "name", "type": "STRING", "mode": "NULLABLE"} + + +def _client(sync_script: ModuleType, transport: FakeTransport) -> object: + return sync_script.BigQueryClient( + "policyengine-observability", ACCESS_TOKEN, transport + ) + + +def _sent_parameters(transport: FakeTransport, index: int = 0) -> dict: + return { + parameter["name"]: parameter + for parameter in transport.json_body(index)["queryParameters"] + } + + +def test_bigquery_error_query_request_limits_cost_and_names_its_parameters( + sync_script: ModuleType, +) -> None: + transport = FakeTransport( + sync_script, [_json_response(sync_script, _query_response([], []))] + ) + window = sync_script.reporting_window(REPORT_DATE) + + sync_script.fetch_error_records(_client(sync_script, transport), window) + + request = transport.requests[0] + body = transport.json_body(0) + parameters = _sent_parameters(transport) + assert request.method == "POST" + assert request.url == ( + "https://bigquery.googleapis.com/bigquery/v2/projects/" + "policyengine-observability/queries" + ) + assert request.headers["Authorization"] == f"Bearer {ACCESS_TOKEN}" + assert body["useLegacySql"] is False + assert body["parameterMode"] == "NAMED" + assert body["maximumBytesBilled"] == str(5 * 1024**3) + assert body["labels"] == { + "purpose": "telemetry-error-issues", + "query": "error-records", + } + assert body["defaultDataset"] == { + "projectId": "policyengine-observability", + "datasetId": "observability_logs", + } + assert body["formatOptions"] == {"useInt64Timestamp": True} + assert parameters["query_start"]["parameterType"] == {"type": "TIMESTAMP"} + assert parameters["query_start"]["parameterValue"]["value"] == ( + "2026-10-13 04:00:00+00:00" + ) + assert parameters["window_end"]["parameterValue"]["value"] == ( + "2026-10-21 04:00:00+00:00" + ) + assert parameters["frame_pattern"]["parameterValue"]["value"] == ( + sync_script.FRAME_PATTERN + ) + assert [ + item["value"] + for item in parameters["excluded_routes"]["parameterValue"]["arrayValues"] + ] == ["/health", "/simulation-gateway-check", "/liveness-check", "/readiness-check"] + + +def test_bigquery_error_query_filters_application_error_records( + sync_script: ModuleType, +) -> None: + sql = sync_script.ERROR_RECORDS_QUERY + + assert "FROM `_AllLogs`" in sql + assert "timestamp >= @query_start AND timestamp < @window_end" in sql + assert ( + "JSON_VALUE(json_payload, '$.schema_version') = " + "'policyengine.observability.v2'" in sql + ) + assert ( + "JSON_VALUE(json_payload, '$.\"service.namespace\"') = " + "'policyengine.api-v1'" in sql + ) + assert "severity_number >= 500" in sql + assert "NOT IN UNNEST(@excluded_routes)" in sql + assert "QUALIFY ROW_NUMBER() OVER" in sql + + +def test_bigquery_error_query_reads_stacks_and_messages_only_through_expressions( + sync_script: ModuleType, +) -> None: + sql = sync_script.ERROR_RECORDS_QUERY + stack = "JSON_VALUE(json_payload, '$.\"error.stack\"')" + + assert sql.count("error.stack") == 2 + assert f"REGEXP_EXTRACT_ALL({stack}, @frame_pattern)" in sql + assert f"LENGTH({stack}) >= 16384" in sql + assert sql.count("$.message") == 1 + assert "REGEXP_REPLACE(JSON_VALUE(json_payload, '$.message')" in sql + assert "error.message" not in sql + assert "SELECT *" not in sql + assert "json_payload AS" not in sql + + +@pytest.mark.parametrize( + "identifier", + [ + "JSON_VALUE(json_payload, '$.\"request.id\"')", + "JSON_VALUE(json_payload, '$.attributes.observability_id')", + ], +) +def test_bigquery_error_query_hashes_identifiers( + sync_script: ModuleType, identifier: str +) -> None: + sql = sync_script.ERROR_RECORDS_QUERY + + assert sql.count(identifier) == 1 + assert f"TO_HEX(SHA256({identifier}))" in sql + + +def test_bigquery_error_query_hashes_the_trace(sync_script: ModuleType) -> None: + sql = sync_script.ERROR_RECORDS_QUERY + + assert "TO_HEX(SHA256(COALESCE(" in sql + assert sql.count("$.trace_id") == 1 + + +def test_bigquery_request_count_query_counts_completed_requests_in_the_window( + sync_script: ModuleType, +) -> None: + sql = sync_script.REQUEST_COUNTS_QUERY + transport = FakeTransport( + sync_script, [_json_response(sync_script, _query_response([], []))] + ) + + sync_script.fetch_request_counts( + _client(sync_script, transport), sync_script.reporting_window(REPORT_DATE) + ) + + parameters = _sent_parameters(transport) + assert "JSON_VALUE(json_payload, '$.\"event.name\"') = 'request.completed'" in sql + assert "COUNT(*) AS requests" in sql + assert "GROUP BY service_name, environment_name, route" in sql + assert "timestamp >= @window_start AND timestamp < @window_end" in sql + assert parameters["window_start"]["parameterValue"]["value"] == ( + "2026-10-20 04:00:00+00:00" + ) + assert transport.json_body(0)["labels"]["query"] == "request-counts" + + +def test_bigquery_client_polls_until_the_job_completes(sync_script: ModuleType) -> None: + transport = FakeTransport( + sync_script, + [ + _json_response(sync_script, _query_response([], [], job_complete=False)), + _json_response(sync_script, _query_response([], [], job_complete=False)), + _json_response(sync_script, _query_response([STRING_FIELD], [["done"]])), + ], + ) + + rows = _client(sync_script, transport).query("SELECT 1", [], "test") + + assert rows == [{"name": "done"}] + poll = urlparse(transport.requests[1].url) + assert transport.requests[1].method == "GET" + assert poll.path.endswith("/projects/policyengine-observability/queries/job-1") + assert parse_qs(poll.query)["location"] == ["US"] + assert parse_qs(poll.query)["formatOptions.useInt64Timestamp"] == ["true"] + + +def test_bigquery_client_follows_page_tokens(sync_script: ModuleType) -> None: + transport = FakeTransport( + sync_script, + [ + _json_response( + sync_script, + _query_response([STRING_FIELD], [["first"]], page_token="page-2"), + ), + _json_response(sync_script, _query_response([STRING_FIELD], [["second"]])), + ], + ) + + rows = _client(sync_script, transport).query("SELECT 1", [], "test") + + assert rows == [{"name": "first"}, {"name": "second"}] + page = parse_qs(urlparse(transport.requests[1].url).query) + assert page["pageToken"] == ["page-2"] + assert page["location"] == ["US"] + + +def test_bigquery_client_gives_up_on_a_job_that_never_completes( + sync_script: ModuleType, +) -> None: + pending = _json_response(sync_script, _query_response([], [], job_complete=False)) + transport = FakeTransport(sync_script, [pending] * 200) + + with pytest.raises(sync_script.BigQueryError, match="did not complete"): + _client(sync_script, transport).query("SELECT 1", [], "test") + + +def test_bigquery_client_converts_row_values_from_strings( + sync_script: ModuleType, +) -> None: + fields = [ + {"name": "text", "type": "STRING", "mode": "NULLABLE"}, + {"name": "count", "type": "INTEGER", "mode": "NULLABLE"}, + {"name": "flag", "type": "BOOLEAN", "mode": "NULLABLE"}, + {"name": "moment", "type": "TIMESTAMP", "mode": "NULLABLE"}, + {"name": "items", "type": "STRING", "mode": "REPEATED"}, + {"name": "missing", "type": "INTEGER", "mode": "NULLABLE"}, + ] + row = ["value", "42", "true", "1792512000000001", ["a", "b"], None] + transport = FakeTransport( + sync_script, [_json_response(sync_script, _query_response(fields, [row]))] + ) + + rows = _client(sync_script, transport).query("SELECT 1", [], "test") + + assert rows == [ + { + "text": "value", + "count": 42, + "flag": True, + "moment": datetime(2026, 10, 20, 16, 0, 0, 1, tzinfo=UTC), + "items": ["a", "b"], + "missing": None, + } + ] + + +def test_bigquery_http_error_reports_only_the_status_and_reason( + sync_script: ModuleType, +) -> None: + error_body = { + "error": { + "code": 400, + "message": "SENTINEL value echoed from a row", + "errors": [{"reason": "invalidQuery", "message": "SENTINEL detail"}], + } + } + transport = FakeTransport( + sync_script, + [_json_response(sync_script, error_body, status=400, reason="Bad Request")], + ) + + with pytest.raises(sync_script.BigQueryError) as raised: + _client(sync_script, transport).query("SELECT 1", [], "test") + + assert str(raised.value) == "BigQuery request failed: HTTP 400 invalidQuery" + assert raised.value.status == 400 + assert raised.value.reason == "invalidQuery" + + +def test_bigquery_http_error_without_a_json_reason_uses_the_status_phrase( + sync_script: ModuleType, +) -> None: + response = sync_script.HttpResponse( + status=503, reason="Service Unavailable", headers={}, body=b"SENTINEL " + ) + transport = FakeTransport(sync_script, [response]) + + with pytest.raises(sync_script.BigQueryError) as raised: + _client(sync_script, transport).query("SELECT 1", [], "test") + + assert str(raised.value) == "BigQuery request failed: HTTP 503 Service Unavailable" + + +def test_bigquery_rows_become_error_records(sync_script: ModuleType) -> None: + row = { + "timestamp": WINDOW_NOON, + "service_name": "policyengine-api", + "environment_name": "production", + "event_name": "exception.recorded", + "error_type": "StaleDataError", + "error_handled": "false", + "frames": list(APPLICATION_FRAMES), + "stack_truncated": False, + "message_template": "", + "attribute_error_type": None, + "failure_category": None, + "route": "//report", + "status_code": "500", + "operation_name": None, + "request_key": "aa", + "observability_key": None, + "trace_key": "bb", + } + + record = sync_script.error_record_from_row(row) + + assert record == _exception_record(sync_script, request_key="aa", trace_key="bb") + + +def test_bigquery_rows_tolerate_missing_and_malformed_values( + sync_script: ModuleType, +) -> None: + row = { + "timestamp": WINDOW_NOON, + "service_name": None, + "environment_name": None, + "event_name": None, + "error_type": None, + "error_handled": "SENTINEL", + "frames": None, + "stack_truncated": None, + "message_template": None, + "attribute_error_type": None, + "failure_category": None, + "route": None, + "status_code": "SENTINEL", + "operation_name": None, + "request_key": None, + "observability_key": None, + "trace_key": None, + } + + record = sync_script.error_record_from_row(row) + + assert record.service_name == "-" + assert record.error_handled is None + assert record.frames == () + assert record.stack_truncated is False + assert record.status_code is None + + +def test_bigquery_rows_become_request_counts(sync_script: ModuleType) -> None: + rows = [ + { + "service_name": "policyengine-api", + "environment_name": "production", + "route": "//report", + "requests": 120, + } + ] + transport = FakeTransport( + sync_script, + [ + _json_response( + sync_script, + _query_response( + [ + {"name": "service_name", "type": "STRING"}, + {"name": "environment_name", "type": "STRING"}, + {"name": "route", "type": "STRING"}, + {"name": "requests", "type": "INTEGER"}, + ], + [list(map(str, rows[0].values()))], + ), + ) + ], + ) + + counts = sync_script.fetch_request_counts( + _client(sync_script, transport), sync_script.reporting_window(REPORT_DATE) + ) + + assert counts == [ + _request_count(sync_script, "//report", 120), + ] From 60f4df5a981b4e31fdf7ab955f7ac94182c10286 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:54:01 +0400 Subject: [PATCH 06/18] Render telemetry error groups as GitHub issue titles and bodies Titles name the service family and the exception type, "logged error", or "HTTP without recorded exception", then the most frequent route (or operation) and the fingerprint. The fingerprint lets the job find its issue even if someone removes the marked section. The route is dropped when the title would exceed GitHub's 256-character limit. The body has a short note, then a marked section that each run replaces: - the fingerprint, kind, service, type and route; - the window in Eastern time, with its length in hours; - a production and staging table of the measures; - related fingerprints; - a Logs Explorer link to the _AllLogs view, filtered by service, exception type (or route and status) and the window's timestamps; - a hidden state comment with the fingerprint, algorithm, kind and window end. Every value comes from the validated fields of the group, so a test with marker strings in every record field finds none of them in any title, body or link. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 223 +++++++++++- tests/unit/test_telemetry_error_issues.py | 342 +++++++++++++++++- 2 files changed, 562 insertions(+), 3 deletions(-) diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index bd5d136d5..28704e93a 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -101,6 +101,17 @@ QUERY_WAIT_MILLISECONDS = 30_000 MAX_QUERY_POLLS = 40 UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=UTC) +ISSUE_TITLE_PREFIX = "[telemetry]" +MAX_TITLE_LENGTH = 256 +SECTION_START = "" +SECTION_END = "" +STATE_PATTERN = re.compile(r"") +NOT_AVAILABLE = "n/a" +LOGS_EXPLORER_URL = "https://console.cloud.google.com/logs/query" +LOG_BUCKET = "policyengine-observability" +LOG_VIEW = "_AllLogs" +SCHEMA_VERSION = "policyengine.observability.v2" +SIMULATION_FAMILY = "policyengine-simulation-py" HEALTH_CHECK_ROUTES = ( "/health", "/simulation-gateway-check", @@ -639,8 +650,8 @@ def _most_frequent_route(occurrences: list[Occurrence], window: ReportingWindow) """Return the route most failures in the window used, preferring production.""" in_window = [item for item in occurrences if item.timestamp >= window.start] production = [item for item in in_window if item.environment == "production"] - routes = Counter(item.route or MISSING for item in production or in_window) - route, _ = min(routes.items(), key=lambda item: (-item[1], item[0])) + routes = Counter(item.route for item in production or in_window) + route, _ = min(routes.items(), key=lambda item: (-item[1], item[0] or "")) return public_route(route) @@ -1033,6 +1044,214 @@ def fetch_request_counts( ] +def _code(value: str) -> str: + return f"`{value}`" + + +def _place(group: ErrorGroup) -> str: + """Describe where the group fails: its route, else its operation, else nothing.""" + if group.route != MISSING: + return f" on {_code(group.route)}" + if group.kind == "message" and group.operation != MISSING: + return f" in {_code(group.operation)}" + return "" + + +def _status_text(group: ErrorGroup) -> str: + return "HTTP error" if group.status_code is None else f"HTTP {group.status_code}" + + +def _summary(group: ErrorGroup) -> str: + if group.kind == "exception": + return group.error_type + if group.kind == "message": + return "logged error" + return f"{_status_text(group)} without recorded exception" + + +def issue_title(group: ErrorGroup) -> str: + """Return the issue title, which names the fingerprint for matching. + + The route is left out when the title would exceed GitHub's length limit. + """ + lead = f"{ISSUE_TITLE_PREFIX} {group.family}: {_summary(group)}" + suffix = f" ({group.fingerprint})" + title = f"{lead}{_place(group)}{suffix}" + return title if len(title) <= MAX_TITLE_LENGTH else f"{lead}{suffix}" + + +def _logs_timestamp(moment: datetime) -> str: + return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _logs_filters(group: ErrorGroup) -> list[str]: + """Return logging query clauses that select the group's records.""" + filters = [f'jsonPayload.schema_version="{SCHEMA_VERSION}"'] + if group.family == SIMULATION_FAMILY: + filters.append( + f'jsonPayload."service.name"=~"^{SIMULATION_FAMILY}[0-9]+-[0-9]+-[0-9]+$"' + ) + elif group.family not in {MISSING, UNRECOGNIZED}: + filters.append(f'jsonPayload."service.name"="{group.family}"') + if group.kind == "failed_request": + filters.append('jsonPayload."event.name"="request.completed"') + else: + filters.append("severity>=ERROR") + if group.kind == "exception" and group.error_type not in {MISSING, UNRECOGNIZED}: + filters.append(f'jsonPayload."error.type"="{group.error_type}"') + if group.kind != "exception" and group.route not in {MISSING, UNRECOGNIZED}: + filters.append(f'jsonPayload."http.route"="{group.route}"') + elif group.kind == "message" and group.operation not in {MISSING, UNRECOGNIZED}: + filters.append(f'jsonPayload."operation.name"="{group.operation}"') + if group.kind == "failed_request" and group.status_code is not None: + filters.append(f'jsonPayload."http.response.status_code"={group.status_code}') + return filters + + +def logs_explorer_url( + group: ErrorGroup, window: ReportingWindow, project_id: str +) -> str: + """Return a Logs Explorer link to the group's records in the window.""" + query = "\n".join( + [ + *_logs_filters(group), + f'timestamp>="{_logs_timestamp(window.start)}"', + f'timestamp<"{_logs_timestamp(window.end)}"', + ] + ) + view = ( + f"projects/{project_id}/locations/global/buckets/{LOG_BUCKET}/views/{LOG_VIEW}" + ) + settings = {"query": query, "storageScope": f"storage,{view}"} + encoded = "".join( + f";{name}={quote(value, safe='')}" for name, value in settings.items() + ) + return f"{LOGS_EXPLORER_URL}{encoded}?{urlencode({'project': project_id})}" + + +def _eastern_time(moment: datetime | None) -> str: + if moment is None: + return NOT_AVAILABLE + return moment.astimezone(REPORTING_TIME_ZONE).strftime("%Y-%m-%d %H:%M %Z") + + +def _error_rate(measures: EnvironmentMeasures) -> str: + if measures.error_rate is None: + return NOT_AVAILABLE + return ( + f"{measures.error_rate:.2%} ({measures.affected_requests:,} of " + f"{measures.route_requests:,} requests)" + ) + + +def _measures_table(group: ErrorGroup) -> list[str]: + rows = [ + ("Occurrences", lambda measures: f"{measures.occurrences:,}"), + ("Affected requests", lambda measures: f"{measures.affected_requests:,}"), + ("Observability IDs", lambda measures: f"{measures.observability_ids:,}"), + ("Error rate on affected routes", _error_rate), + ("Occurrences per hour", lambda measures: f"{measures.hourly_rate:.2f}"), + ( + "Occurrences per hour, previous 7 days", + lambda measures: f"{measures.previous_hourly_rate:.2f}", + ), + ( + "First seen in the last 8 days", + lambda measures: _eastern_time(measures.first_seen), + ), + ("Last seen", lambda measures: _eastern_time(measures.last_seen)), + ] + return [ + "| Measure | Production | Staging |", + "| --- | --- | --- |", + *( + f"| {name} | {describe(group.production)} | {describe(group.staging)} |" + for name, describe in rows + ), + ] + + +def _group_details(group: ErrorGroup) -> list[str]: + details = [ + f"- Kind: {group.kind} (fingerprint algorithm {FINGERPRINT_ALGORITHM})", + f"- Service: {_code(group.family)}", + ] + if group.kind == "exception": + details.append(f"- Exception type: {_code(group.error_type)}") + if group.kind == "failed_request": + details.append(f"- Response: {_status_text(group)}") + if group.kind == "message" and group.operation != MISSING: + details.append(f"- Operation: {_code(group.operation)}") + details.append(f"- Most frequent route: {_code(group.route)}") + return details + + +def _state_comment(group: ErrorGroup, window: ReportingWindow) -> str: + state = { + "algorithm": FINGERPRINT_ALGORITHM, + "fingerprint": group.fingerprint, + "kind": group.kind, + "window_end": window.end.astimezone(UTC).isoformat(), + } + return f"" + + +def parse_state(body: str) -> dict[str, Any] | None: + """Return the state the job last wrote into an issue body, if it is there.""" + match = STATE_PATTERN.search(body) + if match is None: + return None + try: + state = json.loads(match.group(1)) + except ValueError: + return None + return state if isinstance(state, dict) else None + + +def issue_section(group: ErrorGroup, window: ReportingWindow, project_id: str) -> str: + """Return the marked section of the issue body, which each run replaces.""" + related = ", ".join(map(_code, group.related_fingerprints)) + window_text = ( + f"{_eastern_time(window.start)} to {_eastern_time(window.end)} " + f"({window.hours:.0f} hours)" + ) + lines = [ + SECTION_START, + f"### Error group {_code(group.fingerprint)}", + "", + *_group_details(group), + f"- Window: {window_text}", + "", + *_measures_table(group), + "", + ( + f"Related fingerprints (sharing traces): {related}" + if related + else "Related fingerprints: none" + ), + "", + ( + f"[Matching log records in Logs Explorer]" + f"({logs_explorer_url(group, window, project_id)}), for the window only. " + f"It needs read access to the `{project_id}` project." + ), + "", + _state_comment(group, window), + SECTION_END, + ] + return "\n".join(lines) + + +def issue_body(group: ErrorGroup, window: ReportingWindow, project_id: str) -> str: + """Return a new issue's body: a short note, then the marked section.""" + note = ( + "The daily telemetry error report opened this issue for a production " + "error group. Each run replaces the marked section below with the " + "latest day's measures; text outside it is kept." + ) + return f"{note}\n\n{issue_section(group, window, project_id)}\n" + + def _report_date_argument(value: str) -> date | None: if not value: return None diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index 4e79627be..f52d9f58d 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -10,7 +10,7 @@ from datetime import UTC, date, datetime, timedelta from pathlib import Path from typing import TYPE_CHECKING -from urllib.parse import parse_qs, urlparse +from urllib.parse import parse_qs, quote, unquote, urlparse, urlsplit from zoneinfo import ZoneInfo import pytest @@ -1606,3 +1606,343 @@ def test_bigquery_rows_become_request_counts(sync_script: ModuleType) -> None: assert counts == [ _request_count(sync_script, "//report", 120), ] + + +# Issue rendering + +PROJECT_ID = "policyengine-observability" + + +def _group(sync_script: ModuleType, records: list, request_counts=()) -> object: + groups = sync_script.build_groups( + records, + list(request_counts), + sync_script.reporting_window(REPORT_DATE), + FINGERPRINT_KEY, + ) + return groups[0] + + +def _render(sync_script: ModuleType, group: object) -> tuple[str, str]: + window = sync_script.reporting_window(REPORT_DATE) + return ( + sync_script.issue_title(group), + sync_script.issue_body(group, window, PROJECT_ID), + ) + + +def _section(sync_script: ModuleType, body: str) -> str: + start = body.index(sync_script.SECTION_START) + end = body.index(sync_script.SECTION_END) + len(sync_script.SECTION_END) + return body[start:end] + + +def _logs_query(url: str) -> tuple[str, dict[str, str]]: + """Split a Logs Explorer URL into its decoded query and other settings.""" + parsed = urlsplit(url) + settings = dict( + segment.split("=", 1) + for segment in parsed.path.split(";")[1:] + if "=" in segment + ) + return unquote(settings.pop("query")), settings + + +def test_render_title_names_the_exception_and_its_most_frequent_route( + sync_script: ModuleType, +) -> None: + group = _group(sync_script, [_exception_record(sync_script)]) + + title, _ = _render(sync_script, group) + + assert title == ( + "[telemetry] policyengine-api: StaleDataError on `//report` " + f"({group.fingerprint})" + ) + + +def test_render_title_for_a_logged_error(sync_script: ModuleType) -> None: + route_group = _group(sync_script, [_message_record(sync_script)]) + operation_group = _group( + sync_script, + [ + _message_record( + sync_script, route=None, operation_name="simulation.dispatch" + ) + ], + ) + + assert sync_script.issue_title(route_group).startswith( + "[telemetry] policyengine-api: logged error on `//report` (" + ) + assert sync_script.issue_title(operation_group).startswith( + "[telemetry] policyengine-api: logged error in `simulation.dispatch` (" + ) + + +def test_render_title_for_a_failed_request_without_an_exception( + sync_script: ModuleType, +) -> None: + group = _group(sync_script, [_failed_request_record(sync_script, status_code=502)]) + + assert sync_script.issue_title(group) == ( + "[telemetry] policyengine-api: HTTP 502 without recorded exception on " + f"`//report` ({group.fingerprint})" + ) + + +def test_render_title_without_a_route(sync_script: ModuleType) -> None: + group = _group( + sync_script, + [ + _exception_record( + sync_script, + service_name="policyengine-stage12-us-worker", + route=None, + ) + ], + ) + + assert sync_script.issue_title(group) == ( + "[telemetry] policyengine-stage12-us-worker: StaleDataError " + f"({group.fingerprint})" + ) + + +def test_render_body_marked_section_holds_the_measures_table( + sync_script: ModuleType, +) -> None: + records = [ + _exception_record(sync_script, request_key="r1", observability_key="o1"), + _exception_record(sync_script, request_key="r2", timestamp=_at(2)), + _exception_record( + sync_script, request_key="s1", environment_name="staging", timestamp=_at(1) + ), + ] + group = _group( + sync_script, + records, + [_request_count(sync_script, "//report", 400)], + ) + + _, body = _render(sync_script, group) + section = _section(sync_script, body) + + assert body.index(sync_script.SECTION_START) > 0 + assert f"`{group.fingerprint}`" in section + assert "2026-10-20 00:00 EDT to 2026-10-21 00:00 EDT (24 hours)" in section + assert "| Measure | Production | Staging |" in section + assert "| Occurrences | 2 | 1 |" in section + assert "| Affected requests | 2 | 1 |" in section + assert "| Observability IDs | 1 | 0 |" in section + assert ( + "| Error rate on affected routes | 0.50% (2 of 400 requests) | n/a |" in section + ) + assert "| Occurrences per hour | 0.08 | 0.04 |" in section + assert "| Occurrences per hour, previous 7 days | 0.00 | 0.00 |" in section + assert ( + "| First seen in the last 8 days " + "| 2026-10-20 12:00 EDT | 2026-10-20 13:00 EDT |" in section + ) + assert "| Last seen | 2026-10-20 14:00 EDT | 2026-10-20 13:00 EDT |" in section + assert "Related fingerprints: none" in section + assert "https://console.cloud.google.com/logs/query;" in section + + +def test_render_body_lists_related_fingerprints(sync_script: ModuleType) -> None: + records = [ + _exception_record(sync_script, trace_key="t1"), + _message_record( + sync_script, service_name="policyengine-simulation-entry", trace_key="t1" + ), + ] + groups = sync_script.build_groups( + records, [], sync_script.reporting_window(REPORT_DATE), FINGERPRINT_KEY + ) + exception_group = next(group for group in groups if group.kind == "exception") + other = exception_group.related_fingerprints[0] + + _, body = _render(sync_script, exception_group) + + assert f"Related fingerprints (sharing traces): `{other}`" in body + + +def test_render_body_state_comment_records_the_group_and_window( + sync_script: ModuleType, +) -> None: + group = _group(sync_script, [_exception_record(sync_script)]) + + _, body = _render(sync_script, group) + + assert sync_script.parse_state(body) == { + "algorithm": 1, + "fingerprint": group.fingerprint, + "kind": "exception", + "window_end": "2026-10-21T04:00:00+00:00", + } + + +def test_render_body_stays_under_the_github_limit(sync_script: ModuleType) -> None: + # The longest route that still passes validation (195 characters). + long_route = "/" + "/".join([""] * 13) + records = [ + _exception_record( + sync_script, + error_type="E" * 120, + route=long_route, + trace_key=f"t{index}", + request_key=f"r{index}", + ) + for index in range(4) + ] + records += [ + _message_record( + sync_script, + request_key=f"m{index}", + trace_key=f"t{index}", + message_template=f"template {index}", + service_name="policyengine-simulation-entry", + ) + for index in range(4) + ] + groups = sync_script.build_groups( + records, [], sync_script.reporting_window(REPORT_DATE), FINGERPRINT_KEY + ) + + for group in groups: + title, body = _render(sync_script, group) + assert len(body) < 65_536 + assert len(title) <= 256 + + +def test_render_logs_link_filters_exceptions_by_service_and_type( + sync_script: ModuleType, +) -> None: + group = _group(sync_script, [_exception_record(sync_script)]) + url = sync_script.logs_explorer_url( + group, sync_script.reporting_window(REPORT_DATE), PROJECT_ID + ) + + query, settings = _logs_query(url) + + assert urlparse(url).query == "project=policyengine-observability" + assert settings["storageScope"] == quote( + "storage,projects/policyengine-observability/locations/global/" + "buckets/policyengine-observability/views/_AllLogs", + safe="", + ) + assert query.splitlines() == [ + 'jsonPayload.schema_version="policyengine.observability.v2"', + 'jsonPayload."service.name"="policyengine-api"', + "severity>=ERROR", + 'jsonPayload."error.type"="StaleDataError"', + 'timestamp>="2026-10-20T04:00:00Z"', + 'timestamp<"2026-10-21T04:00:00Z"', + ] + for character in (" ", "\n", '"'): + assert character not in url + + +def test_render_logs_link_filters_failed_requests_by_route_and_status( + sync_script: ModuleType, +) -> None: + group = _group( + sync_script, + [ + _failed_request_record( + sync_script, + service_name="policyengine-simulation-py1-2-3", + route="/jobs/{job_id}", + status_code=504, + ) + ], + ) + + query, _ = _logs_query( + sync_script.logs_explorer_url( + group, sync_script.reporting_window(REPORT_DATE), PROJECT_ID + ) + ) + + assert query.splitlines()[1:5] == [ + 'jsonPayload."service.name"=~"^policyengine-simulation-py[0-9]+-[0-9]+-[0-9]+$"', + 'jsonPayload."event.name"="request.completed"', + 'jsonPayload."http.route"="/jobs/{job_id}"', + 'jsonPayload."http.response.status_code"=504', + ] + + +def test_render_logs_link_filters_logged_errors_by_route_or_operation( + sync_script: ModuleType, +) -> None: + window = sync_script.reporting_window(REPORT_DATE) + route_group = _group(sync_script, [_message_record(sync_script)]) + operation_group = _group( + sync_script, + [ + _message_record( + sync_script, route=None, operation_name="simulation.dispatch" + ) + ], + ) + + route_query, _ = _logs_query( + sync_script.logs_explorer_url(route_group, window, PROJECT_ID) + ) + operation_query, _ = _logs_query( + sync_script.logs_explorer_url(operation_group, window, PROJECT_ID) + ) + + assert 'jsonPayload."http.route"="//report"' in route_query + assert "severity>=ERROR" in route_query + assert 'jsonPayload."operation.name"="simulation.dispatch"' in operation_query + + +def _sentinel_row(sync_script: ModuleType, **overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "timestamp": WINDOW_NOON, + "service_name": "SENTINEL service name", + "environment_name": "production", + "event_name": "SENTINEL event", + "error_type": "SENTINEL error type", + "error_handled": "SENTINEL handled", + "frames": [ + '/app/policyengine_api/SENTINEL.py", line 1, in SENTINEL_function', + "SENTINEL frame text", + ], + "stack_truncated": "SENTINEL", + "message_template": "SENTINEL message words", + "attribute_error_type": "SENTINEL attribute type", + "failure_category": "SENTINEL category", + "route": "/SENTINEL/route?query=SENTINEL", + "status_code": "SENTINEL", + "operation_name": "SENTINEL operation", + "request_key": "SENTINEL request", + "observability_key": "SENTINEL observability", + "trace_key": "SENTINEL trace", + } + row.update(overrides) + return row + + +@pytest.mark.parametrize( + "overrides", + [ + {}, + {"error_type": None}, + {"error_type": None, "event_name": "request.completed"}, + {"route": None}, + ], +) +def test_render_never_includes_values_from_records( + sync_script: ModuleType, overrides: dict[str, object] +) -> None: + record = sync_script.error_record_from_row(_sentinel_row(sync_script, **overrides)) + group = _group(sync_script, [record]) + window = sync_script.reporting_window(REPORT_DATE) + + title, body = _render(sync_script, group) + url = sync_script.logs_explorer_url(group, window, PROJECT_ID) + + for output in (title, body, url, unquote(url)): + assert "SENTINEL" not in output From 5056cf14ff4e835c64f3df58fee68918db7e27be Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:58:42 +0400 Subject: [PATCH 07/18] Create and update one GitHub issue per production error group The job matches error groups to the open and closed issues that github-actions[bot] opened, by the fingerprint in the hidden state comment, or in the title if someone removed the marked section. Issues from other authors are ignored. - A group without an issue opens one only after 3 production occurrences in at least 2 distinct requests during the window. Staging-only groups never open issues. At most 5 issues open per run, largest production groups first; the rest wait for a later run. - An existing issue gets the new marked section only when its content changed. Text outside the section and the title are never changed, and a removed section is added back at the end. A section from a later window is kept rather than overwritten by a rerun of an earlier day. - An issue closed as completed, or without a reason, reopens with the regression label and a comment, but only when production failures starting 24 hours after it closed meet the same thresholds. Issues closed as not planned or duplicate stay closed. - Missing labels are created first, and a created issue whose labels GitHub didn't apply gets them added, or the run fails. - Writes go one at a time, at least a second apart. A 429, or a 403 with retry-after, is retried after the requested wait, up to 3 times. - A dry run reads issues but makes no write requests. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 414 ++++++++++++- tests/unit/test_telemetry_error_issues.py | 565 ++++++++++++++++++ 2 files changed, 976 insertions(+), 3 deletions(-) diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index 28704e93a..29db3d63c 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -17,9 +17,11 @@ import hmac import json import re +import time from collections import Counter from dataclasses import dataclass -from datetime import UTC, date, datetime, time, timedelta +from datetime import UTC, date, datetime, timedelta +from datetime import time as time_of_day from http import HTTPStatus from typing import TYPE_CHECKING, Any, NamedTuple from urllib.error import HTTPError @@ -31,6 +33,7 @@ from collections.abc import Callable, Iterable, Mapping REPORTING_TIME_ZONE = ZoneInfo("America/New_York") +MIDNIGHT = time_of_day(0, 0) # Shown in place of a value that is missing, or that fails validation and so # might not be code-defined. @@ -112,6 +115,36 @@ LOG_VIEW = "_AllLogs" SCHEMA_VERSION = "policyengine.observability.v2" SIMULATION_FAMILY = "policyengine-simulation-py" +GITHUB_API = "https://api.github.com" +GITHUB_API_VERSION = "2022-11-28" +GITHUB_PAGE_SIZE = 100 +BOT_LOGIN = "github-actions[bot]" +TITLE_FINGERPRINT = re.compile(r"\(([0-9a-f]{16})\)$") +BASE_LABEL = "telemetry-error" +REGRESSION_LABEL = "regression" +SERVICE_LABEL_PREFIX = "service: " +MAX_LABEL_LENGTH = 50 +LABEL_DETAILS = { + BASE_LABEL: ("b60205", "Opened by the daily telemetry error report"), + REGRESSION_LABEL: ("d93f0b", "An error that returned after its issue was closed"), +} +SERVICE_LABEL_DETAILS = ("1d76db", "Service that produced a telemetry error group") +# A production group opens an issue only when it fails this often, in this many +# distinct requests, during the window; the same thresholds reopen an issue. +MIN_PRODUCTION_OCCURRENCES = 3 +MIN_PRODUCTION_REQUESTS = 2 +MAX_NEW_ISSUES_PER_RUN = 5 +# Failures in the first day after an issue is closed may come from code that +# hasn't been deployed yet, so they never reopen it. +REOPEN_DELAY = timedelta(hours=24) +REOPENABLE_CLOSE_REASONS = frozenset({None, "completed"}) +# GitHub asks clients to send content-creating requests one at a time, at +# least a second apart, and to wait as its rate-limit responses say. +MIN_SECONDS_BETWEEN_WRITES = 1.0 +MAX_RATE_LIMIT_RETRIES = 3 +DEFAULT_RATE_LIMIT_WAIT_SECONDS = 60 +MAX_RATE_LIMIT_WAIT_SECONDS = 300 +RATE_LIMIT_STATUSES = frozenset({HTTPStatus.FORBIDDEN, HTTPStatus.TOO_MANY_REQUESTS}) HEALTH_CHECK_ROUTES = ( "/health", "/simulation-gateway-check", @@ -232,8 +265,8 @@ def reporting_window(report_date: date) -> ReportingWindow: """Return the window from midnight Eastern on ``report_date`` to the next.""" next_date = report_date + timedelta(days=1) return ReportingWindow( - start=datetime.combine(report_date, time(0), tzinfo=REPORTING_TIME_ZONE), - end=datetime.combine(next_date, time(0), tzinfo=REPORTING_TIME_ZONE), + start=datetime.combine(report_date, MIDNIGHT, tzinfo=REPORTING_TIME_ZONE), + end=datetime.combine(next_date, MIDNIGHT, tzinfo=REPORTING_TIME_ZONE), ) @@ -1252,6 +1285,381 @@ def issue_body(group: ErrorGroup, window: ReportingWindow, project_id: str) -> s return f"{note}\n\n{issue_section(group, window, project_id)}\n" +class GitHubError(ApiError): + """A GitHub request failed.""" + + service = "GitHub" + + +def _rate_limit_wait(response: HttpResponse) -> int | None: + """Return the seconds to wait before retrying a rate-limited request. + + A 403 without ``retry-after`` is a permission failure, so it isn't retried. + """ + if response.status not in RATE_LIMIT_STATUSES: + return None + retry_after = response.headers.get("retry-after", "") + if retry_after.isascii() and retry_after.isdigit(): + seconds = int(retry_after) + elif response.status == HTTPStatus.TOO_MANY_REQUESTS: + seconds = DEFAULT_RATE_LIMIT_WAIT_SECONDS + else: + return None + return seconds if seconds <= MAX_RATE_LIMIT_WAIT_SECONDS else None + + +def _json_object(response: HttpResponse) -> dict[str, Any]: + payload = json.loads(response.body) + if not isinstance(payload, dict): + msg = "GitHub returned an unexpected response" + raise GitHubError(msg) + return payload + + +def _json_list(response: HttpResponse) -> list[dict[str, Any]]: + payload = json.loads(response.body) + if not isinstance(payload, list): + msg = "GitHub returned an unexpected response" + raise GitHubError(msg) + return payload + + +class GitHubClient: + """Reads and writes one repository's issues and labels. + + Writes go one at a time, at least a second apart. A rate-limited request is + retried after the wait GitHub asks for, up to three times. + """ + + def __init__( + self, + repository: str, + credential: str, + transport: Callable[[HttpRequest], HttpResponse] = urllib_transport, + *, + sleep: Callable[[float], None] = time.sleep, + clock: Callable[[], float] = time.monotonic, + ) -> None: + """Send requests for ``repository`` (``owner/name``) with ``credential``.""" + self._repository_url = f"{GITHUB_API}/repos/{repository}" + self._headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {credential}", + "User-Agent": "policyengine-telemetry-error-issues", + "X-GitHub-Api-Version": GITHUB_API_VERSION, + } + self._transport = transport + self._sleep = sleep + self._clock = clock + self._last_write: float | None = None + + def list_issues(self, creator: str) -> list[dict[str, Any]]: + """Return the open and closed issues ``creator`` opened, without PRs.""" + issues = self._pages("/issues", {"state": "all", "creator": creator}) + return [issue for issue in issues if "pull_request" not in issue] + + def list_labels(self) -> set[str]: + """Return the names of the repository's labels.""" + return {label["name"] for label in self._pages("/labels", {})} + + def create_label(self, name: str, color: str, description: str) -> None: + """Create a label.""" + self._send( + "POST", + "/labels", + {"name": name, "color": color, "description": description}, + ) + + def create_issue(self, title: str, body: str, labels: list[str]) -> dict[str, Any]: + """Open an issue and return it as GitHub describes it.""" + payload = {"title": title, "body": body, "labels": labels} + return _json_object(self._send("POST", "/issues", payload)) + + def update_issue(self, number: int, fields: Mapping[str, object]) -> None: + """Change an issue's fields, such as its body or state.""" + self._send("PATCH", f"/issues/{number}", fields) + + def add_labels(self, number: int, labels: list[str]) -> set[str]: + """Add labels to an issue and return all the labels it now has.""" + response = self._send("POST", f"/issues/{number}/labels", {"labels": labels}) + return {label["name"] for label in _json_list(response)} + + def add_comment(self, number: int, body: str) -> None: + """Comment on an issue.""" + self._send("POST", f"/issues/{number}/comments", {"body": body}) + + def _pages(self, path: str, parameters: Mapping[str, str]) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + page = 1 + while True: + query = urlencode( + {**parameters, "per_page": GITHUB_PAGE_SIZE, "page": page} + ) + batch = _json_list(self._send("GET", f"{path}?{query}", None)) + items.extend(batch) + if len(batch) < GITHUB_PAGE_SIZE: + return items + page += 1 + + def _wait_between_writes(self) -> None: + if self._last_write is None: + return + elapsed = self._clock() - self._last_write + if elapsed < MIN_SECONDS_BETWEEN_WRITES: + self._sleep(MIN_SECONDS_BETWEEN_WRITES - elapsed) + + def _send( + self, method: str, path: str, body: Mapping[str, object] | None + ) -> HttpResponse: + headers = dict(self._headers) + encoded = None + if body is not None: + headers["Content-Type"] = "application/json" + encoded = json.dumps(body).encode() + request = HttpRequest(method, f"{self._repository_url}{path}", headers, encoded) + is_write = method != "GET" + for retry in range(MAX_RATE_LIMIT_RETRIES + 1): + if is_write: + self._wait_between_writes() + response = self._transport(request) + if is_write: + self._last_write = self._clock() + wait = _rate_limit_wait(response) + if wait is None or retry == MAX_RATE_LIMIT_RETRIES: + break + self._sleep(wait) + if response.status >= HTTPStatus.BAD_REQUEST: + raise GitHubError.from_response(response) + return response + + +class SyncAction(NamedTuple): + """What one run did, or in a dry run would do, for one error group.""" + + fingerprint: str + kind: str + family: str + production_occurrences: int + staging_occurrences: int + issue_number: int | None + action: str + + +class SyncResult(NamedTuple): + """Every group's action, and how many new issues waited for a later run.""" + + actions: list[SyncAction] + deferred: int + + +def replace_section(body: str, section: str) -> str: + """Replace the marked section of ``body``, or add it at the end if missing.""" + start = body.find(SECTION_START) + end = body.find(SECTION_END, start) if start >= 0 else -1 + if end >= 0: + return f"{body[:start]}{section}{body[end + len(SECTION_END) :]}" + kept = body.rstrip() + return f"{kept}\n\n{section}\n" if kept else f"{section}\n" + + +def _issue_fingerprint(issue: Mapping[str, Any]) -> str | None: + """Return the fingerprint from the state comment, or else from the title.""" + state = parse_state(issue.get("body") or "") + if state and state.get("algorithm") == FINGERPRINT_ALGORITHM: + fingerprint_value = state.get("fingerprint") + if isinstance(fingerprint_value, str): + return fingerprint_value + title = issue.get("title") or "" + match = TITLE_FINGERPRINT.search(title) + if match and title.startswith(ISSUE_TITLE_PREFIX): + return match.group(1) + return None + + +def _issues_by_fingerprint( + issues: Iterable[Mapping[str, Any]], +) -> dict[str, Mapping[str, Any]]: + """Index the bot's issues by fingerprint, preferring open, then newest.""" + index: dict[str, Mapping[str, Any]] = {} + for issue in issues: + if (issue.get("user") or {}).get("login") != BOT_LOGIN: + continue + group = _issue_fingerprint(issue) + if group is None: + continue + rank = (issue.get("state") == "open", issue.get("number", 0)) + current = index.get(group) + if current is None or rank > ( + current.get("state") == "open", + current["number"], + ): + index[group] = issue + return index + + +def _meets_thresholds(occurrences: int, affected_requests: int) -> bool: + return ( + occurrences >= MIN_PRODUCTION_OCCURRENCES + and affected_requests >= MIN_PRODUCTION_REQUESTS + ) + + +def _issue_labels(group: ErrorGroup) -> list[str]: + service_label = f"{SERVICE_LABEL_PREFIX}{group.family}" + if len(service_label) > MAX_LABEL_LENGTH: + return [BASE_LABEL] + return [BASE_LABEL, service_label] + + +def _parse_time(value: object) -> datetime | None: + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value) + except ValueError: + return None + return parsed if parsed.tzinfo is not None else None + + +class IssueSynchronizer: + """Creates and updates one issue per production error group.""" + + def __init__( + self, + github: GitHubClient, + window: ReportingWindow, + project_id: str, + *, + dry_run: bool, + ) -> None: + """Sync the groups of ``window``; a dry run reads but never writes.""" + self._github = github + self._window = window + self._project_id = project_id + self._dry_run = dry_run + self._known_labels: set[str] | None = None + + def run(self, groups: Iterable[ErrorGroup]) -> SyncResult: + """Sync each group, opening at most five issues, largest groups first.""" + issues = _issues_by_fingerprint(self._github.list_issues(BOT_LOGIN)) + actions: list[SyncAction] = [] + opened = 0 + for group in groups: + issue = issues.get(group.fingerprint) + number: int | None = None + if issue is not None: + number = issue["number"] + action = self._update(group, issue) + elif not _meets_thresholds( + group.production.occurrences, group.production.affected_requests + ): + action = "below-threshold" + elif opened >= MAX_NEW_ISSUES_PER_RUN: + action = "deferred" + else: + number = self._open(group) + opened += 1 + action = "created" + actions.append( + SyncAction( + fingerprint=group.fingerprint, + kind=group.kind, + family=group.family, + production_occurrences=group.production.occurrences, + staging_occurrences=group.staging.occurrences, + issue_number=number, + action=action, + ) + ) + deferred = sum(action.action == "deferred" for action in actions) + return SyncResult(actions, deferred) + + def _ensure_labels(self, names: Iterable[str]) -> None: + if self._known_labels is None: + self._known_labels = self._github.list_labels() + for name in names: + if name not in self._known_labels: + color, description = LABEL_DETAILS.get(name, SERVICE_LABEL_DETAILS) + self._github.create_label(name, color, description) + self._known_labels.add(name) + + def _open(self, group: ErrorGroup) -> int | None: + if self._dry_run: + return None + labels = _issue_labels(group) + self._ensure_labels(labels) + issue = self._github.create_issue( + issue_title(group), + issue_body(group, self._window, self._project_id), + labels, + ) + number = issue["number"] + applied = {label["name"] for label in issue.get("labels", [])} + missing = [label for label in labels if label not in applied] + if missing and not set(missing) <= self._github.add_labels(number, missing): + msg = f"GitHub did not apply the labels to issue #{number}" + raise GitHubError(msg) + return number + + def _reopen_since(self, issue: Mapping[str, Any]) -> datetime | None: + """Return when regressions count from, if the issue should reopen.""" + if issue.get("state") != "closed": + return None + if issue.get("state_reason") not in REOPENABLE_CLOSE_REASONS: + return None + closed_at = _parse_time(issue.get("closed_at")) + return None if closed_at is None else closed_at + REOPEN_DELAY + + def _update(self, group: ErrorGroup, issue: Mapping[str, Any]) -> str: + body = issue.get("body") or "" + state = parse_state(body) + reported_end = _parse_time(state.get("window_end")) if state else None + if reported_end is not None and reported_end > self._window.end: + return "kept-newer" + new_body = replace_section( + body, issue_section(group, self._window, self._project_id) + ) + since = self._reopen_since(issue) + if since is not None and _meets_thresholds(*group.production_since(since)): + if not self._dry_run: + self._reopen(issue["number"], new_body, group, since) + return "reopened" + if new_body == body: + return "unchanged" + if not self._dry_run: + self._github.update_issue(issue["number"], {"body": new_body}) + return "updated" + + def _reopen( + self, number: int, body: str, group: ErrorGroup, since: datetime + ) -> None: + occurrences, affected_requests = group.production_since(since) + self._ensure_labels([REGRESSION_LABEL]) + self._github.update_issue( + number, {"body": body, "state": "open", "state_reason": "reopened"} + ) + self._github.add_labels(number, [REGRESSION_LABEL]) + self._github.add_comment( + number, + "Production failures returned after this issue was closed: " + f"{occurrences:,} occurrences in {affected_requests:,} requests since " + f"{_eastern_time(since)}, 24 hours after it was closed.", + ) + + +def sync_issues( + groups: Iterable[ErrorGroup], + window: ReportingWindow, + project_id: str, + github: GitHubClient, + *, + dry_run: bool, +) -> SyncResult: + """Create or update the issue for each group; a dry run makes no changes.""" + synchronizer = IssueSynchronizer(github, window, project_id, dry_run=dry_run) + return synchronizer.run(groups) + + def _report_date_argument(value: str) -> date | None: if not value: return None diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index f52d9f58d..99a838ff3 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -4,6 +4,7 @@ import functools import importlib.util +import itertools import json import re import sys @@ -1946,3 +1947,567 @@ def test_render_never_includes_values_from_records( for output in (title, body, url, unquote(url)): assert "SENTINEL" not in output + + +# GitHub issue sync + +REPOSITORY = "PolicyEngine/policyengine-api" +BOT = "github-actions[bot]" + + +class FakeGitHub: + """An in-memory GitHub issues API with a clock that advances when it sleeps.""" + + def __init__( + self, + sync_script: ModuleType, + issues: list[dict] = (), + labels: tuple[str, ...] = (), + *, + drop_created_labels: bool = False, + ) -> None: + self.sync_script = sync_script + self.issues = {issue["number"]: dict(issue) for issue in issues} + self.labels = set(labels) + self.drop_created_labels = drop_created_labels + self.failures: list[object] = [] + self.requests: list[tuple[float, object]] = [] + self.comments: dict[int, list[str]] = {} + self.sleeps: list[float] = [] + self.now = 0.0 + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += seconds + + def clock(self) -> float: + return self.now + + def client(self) -> object: + return self.sync_script.GitHubClient( + REPOSITORY, "fake-github-token", self, sleep=self.sleep, clock=self.clock + ) + + @property + def writes(self) -> list[tuple[float, object]]: + return [entry for entry in self.requests if entry[1].method != "GET"] + + def _respond(self, payload: object, status: int = 200) -> object: + return self.sync_script.HttpResponse( + status=status, reason="OK", headers={}, body=json.dumps(payload).encode() + ) + + def __call__(self, request: object) -> object: + self.requests.append((self.now, request)) + if request.method != "GET" and self.failures: + return self.failures.pop(0) + split = urlsplit(request.url) + path = split.path.removeprefix(f"/repos/{REPOSITORY}") + query = {name: values[0] for name, values in parse_qs(split.query).items()} + body = json.loads(request.body) if request.body else None + return self._route(request.method, path, query, body) + + def _route(self, method: str, path: str, query: dict, body: dict | None) -> object: + parts = path.strip("/").split("/") + if len(parts) > 1: + return self._route_issue(method, int(parts[1]), parts[2:], body) + if method == "GET" and parts == ["issues"]: + issues = sorted(self.issues.values(), key=lambda issue: issue["number"]) + return self._respond(self._page(issues, query)) + if method == "GET": + labels = [{"name": name} for name in sorted(self.labels)] + return self._respond(self._page(labels, query)) + if parts == ["labels"]: + self.labels.add(body["name"]) + return self._respond({"name": body["name"]}, status=201) + return self._create(body) + + def _route_issue( + self, method: str, number: int, rest: list[str], body: dict + ) -> object: + if method == "PATCH": + self.issues[number].update(body) + return self._respond(self.issues[number]) + if rest == ["labels"]: + names = {label["name"] for label in self.issues[number]["labels"]} + if not self.drop_created_labels: + names |= set(body["labels"]) + self.issues[number]["labels"] = [{"name": name} for name in sorted(names)] + return self._respond(self.issues[number]["labels"]) + self.comments.setdefault(number, []).append(body["body"]) + return self._respond({"id": 1}, status=201) + + def _page(self, items: list, query: dict) -> list: + page = int(query.get("page", "1")) + size = int(query.get("per_page", "30")) + return items[(page - 1) * size : page * size] + + def _create(self, body: dict) -> object: + number = max(self.issues, default=0) + 1 + labels = [] if self.drop_created_labels else body["labels"] + self.issues[number] = { + "number": number, + "title": body["title"], + "body": body["body"], + "state": "open", + "state_reason": None, + "closed_at": None, + "labels": [{"name": name} for name in labels], + "user": {"login": BOT}, + } + return self._respond(self.issues[number], status=201) + + +def _bot_issue(number: int, **fields: object) -> dict: + issue = { + "number": number, + "title": f"Issue {number}", + "body": "", + "state": "open", + "state_reason": None, + "closed_at": None, + "labels": [], + "user": {"login": BOT}, + } + issue.update(fields) + return issue + + +def _failures( + sync_script: ModuleType, + count: int, + *, + request_key_prefix: str = "r", + **fields: object, +) -> list: + return [ + _exception_record( + sync_script, + request_key=f"{request_key_prefix}{index}", + timestamp=_at(index), + **fields, + ) + for index in range(count) + ] + + +def _groups(sync_script: ModuleType, records: list) -> list: + return sync_script.build_groups( + records, [], sync_script.reporting_window(REPORT_DATE), FINGERPRINT_KEY + ) + + +def _sync(sync_script: ModuleType, groups: list, github: FakeGitHub, *, dry_run=False): + return sync_script.sync_issues( + groups, + sync_script.reporting_window(REPORT_DATE), + PROJECT_ID, + github.client(), + dry_run=dry_run, + ) + + +def _existing_issue( + sync_script: ModuleType, group: object, number: int = 7, **fields: object +) -> dict: + """Return the issue the job opened for ``group`` on an earlier day.""" + earlier = sync_script.reporting_window(REPORT_DATE - timedelta(days=1)) + issue = _bot_issue( + number, + title=sync_script.issue_title(group), + body=sync_script.issue_body(group, earlier, PROJECT_ID), + ) + issue.update(fields) + return issue + + +def test_sync_lists_open_and_closed_issues_from_the_bot_page_by_page( + sync_script: ModuleType, +) -> None: + github = FakeGitHub(sync_script, [_bot_issue(number) for number in range(1, 151)]) + + _sync(sync_script, [], github) + + listing = [ + parse_qs(urlsplit(request.url).query) + for _, request in github.requests + if urlsplit(request.url).path.endswith("/issues") + ] + assert len(listing) == 2 + assert listing[0]["creator"] == [BOT] + assert listing[0]["state"] == ["all"] + assert listing[0]["per_page"] == ["100"] + assert [query["page"] for query in listing] == [["1"], ["2"]] + + +def test_sync_opens_an_issue_at_three_production_occurrences_in_two_requests( + sync_script: ModuleType, +) -> None: + eligible = _failures(sync_script, 3) + # Three failures that all affected the request with observability ID o1. + one_request = [ + _exception_record( + sync_script, + error_type="KeyError", + request_key=None, + trace_key=None, + observability_key="o1", + timestamp=_at(index), + ) + for index in range(3) + ] + too_few = _failures( + sync_script, 2, request_key_prefix="few", error_type="TypeError" + ) + groups = _groups(sync_script, eligible + one_request + too_few) + github = FakeGitHub(sync_script) + + result = _sync(sync_script, groups, github) + + created = list(github.issues.values()) + eligible_group = next( + group for group in groups if group.error_type == "StaleDataError" + ) + assert [issue["title"] for issue in created] == [ + sync_script.issue_title(eligible_group) + ] + assert created[0]["labels"] == [ + {"name": "telemetry-error"}, + {"name": "service: policyengine-api"}, + ] + actions = {action.fingerprint: action.action for action in result.actions} + assert sorted(actions.values()) == ["below-threshold", "below-threshold", "created"] + + +def test_sync_never_opens_an_issue_for_a_staging_only_group( + sync_script: ModuleType, +) -> None: + groups = _groups( + sync_script, _failures(sync_script, 10, environment_name="staging") + ) + github = FakeGitHub(sync_script) + + result = _sync(sync_script, groups, github) + + assert github.issues == {} + assert result.actions[0].action == "below-threshold" + + +def test_sync_opens_at_most_five_issues_in_order_of_production_volume( + sync_script: ModuleType, +) -> None: + records = [] + for index in range(7): + records += _failures( + sync_script, + 3 + index, + request_key_prefix=f"g{index}-", + error_type=f"Error{index}", + ) + groups = _groups(sync_script, records) + github = FakeGitHub(sync_script) + + result = _sync(sync_script, groups, github) + + titles = [issue["title"] for issue in github.issues.values()] + assert [title.split(": ")[1].split(" ")[0] for title in titles] == [ + "Error6", + "Error5", + "Error4", + "Error3", + "Error2", + ] + assert result.deferred == 2 + assert [action.action for action in result.actions].count("deferred") == 2 + + +def test_sync_replaces_only_the_marked_section_and_never_the_title( + sync_script: ModuleType, +) -> None: + group = _groups(sync_script, _failures(sync_script, 3))[0] + issue = _existing_issue(sync_script, group, title="Renamed by a person") + issue["body"] = f"Notes before.\n\n{issue['body']}\nNotes after.\n" + github = FakeGitHub(sync_script, [issue]) + + result = _sync(sync_script, [group], github) + + patch = json.loads(github.writes[0][1].body) + expected_section = sync_script.issue_section( + group, sync_script.reporting_window(REPORT_DATE), PROJECT_ID + ) + assert list(patch) == ["body"] + assert patch["body"].startswith("Notes before.\n\n") + assert patch["body"].endswith("\nNotes after.\n") + assert expected_section in patch["body"] + assert github.issues[7]["title"] == "Renamed by a person" + assert result.actions[0].action == "updated" + assert result.actions[0].issue_number == 7 + + +def test_sync_does_not_write_when_the_section_is_unchanged( + sync_script: ModuleType, +) -> None: + group = _groups(sync_script, _failures(sync_script, 3))[0] + github = FakeGitHub(sync_script, [_existing_issue(sync_script, group)]) + + _sync(sync_script, [group], github) + first_run_writes = len(github.writes) + second = _sync(sync_script, [group], github) + + assert first_run_writes == 1 + assert len(github.writes) == 1 + assert second.actions[0].action == "unchanged" + + +def test_sync_adds_a_deleted_section_back_at_the_end(sync_script: ModuleType) -> None: + group = _groups(sync_script, _failures(sync_script, 3))[0] + issue = _existing_issue(sync_script, group, body="Only human notes now.") + github = FakeGitHub(sync_script, [issue]) + + _sync(sync_script, [group], github) + + body = github.issues[7]["body"] + assert body.startswith("Only human notes now.\n\n") + assert body.rstrip().endswith(sync_script.SECTION_END) + assert len(github.issues) == 1 + + +def test_sync_ignores_issues_opened_by_other_authors(sync_script: ModuleType) -> None: + group = _groups(sync_script, _failures(sync_script, 3))[0] + person_issue = _existing_issue(sync_script, group, user={"login": "someone"}) + github = FakeGitHub(sync_script, [person_issue]) + + _sync(sync_script, [group], github) + + assert github.issues[7]["body"] == person_issue["body"] + assert len(github.issues) == 2 + + +def test_sync_reopens_a_completed_issue_when_production_failures_return( + sync_script: ModuleType, +) -> None: + group = _groups(sync_script, _failures(sync_script, 4))[0] + closed_at = (WINDOW_NOON - timedelta(days=2)).isoformat().replace("+00:00", "Z") + issue = _existing_issue( + sync_script, + group, + state="closed", + state_reason="completed", + closed_at=closed_at, + ) + github = FakeGitHub(sync_script, [issue], labels=("telemetry-error",)) + + result = _sync(sync_script, [group], github) + + reopened = github.issues[7] + assert reopened["state"] == "open" + assert reopened["state_reason"] == "reopened" + assert {"name": "regression"} in reopened["labels"] + assert "regression" in github.labels + assert github.comments[7] == [ + "Production failures returned after this issue was closed: 4 occurrences " + "in 4 requests since 2026-10-19 12:00 EDT, 24 hours after it was closed." + ] + assert result.actions[0].action == "reopened" + + +def test_sync_treats_an_issue_closed_without_a_reason_as_completed( + sync_script: ModuleType, +) -> None: + group = _groups(sync_script, _failures(sync_script, 3))[0] + closed_at = (WINDOW_NOON - timedelta(days=2)).isoformat() + issue = _existing_issue( + sync_script, group, state="closed", state_reason=None, closed_at=closed_at + ) + github = FakeGitHub(sync_script, [issue]) + + result = _sync(sync_script, [group], github) + + assert github.issues[7]["state"] == "open" + assert result.actions[0].action == "reopened" + + +def test_sync_keeps_a_completed_issue_closed_within_24_hours_of_closing( + sync_script: ModuleType, +) -> None: + group = _groups(sync_script, _failures(sync_script, 4))[0] + closed_at = (WINDOW_NOON - timedelta(hours=12)).isoformat() + issue = _existing_issue( + sync_script, + group, + state="closed", + state_reason="completed", + closed_at=closed_at, + ) + github = FakeGitHub(sync_script, [issue]) + + result = _sync(sync_script, [group], github) + + assert github.issues[7]["state"] == "closed" + assert 7 not in github.comments + assert ( + sync_script.issue_section( + group, sync_script.reporting_window(REPORT_DATE), PROJECT_ID + ) + in github.issues[7]["body"] + ) + assert result.actions[0].action == "updated" + + +@pytest.mark.parametrize("reason", ["not_planned", "duplicate"]) +def test_sync_updates_but_never_reopens_not_planned_or_duplicate_issues( + sync_script: ModuleType, reason: str +) -> None: + group = _groups(sync_script, _failures(sync_script, 10))[0] + closed_at = (WINDOW_NOON - timedelta(days=5)).isoformat() + issue = _existing_issue( + sync_script, group, state="closed", state_reason=reason, closed_at=closed_at + ) + github = FakeGitHub(sync_script, [issue]) + + result = _sync(sync_script, [group], github) + + assert github.issues[7]["state"] == "closed" + assert result.actions[0].action == "updated" + assert len(github.issues) == 1 + + +def test_sync_keeps_a_newer_report_instead_of_overwriting_it( + sync_script: ModuleType, +) -> None: + group = _groups(sync_script, _failures(sync_script, 3))[0] + later = sync_script.reporting_window(REPORT_DATE + timedelta(days=1)) + issue = _bot_issue( + 7, + title=sync_script.issue_title(group), + body=sync_script.issue_body(group, later, PROJECT_ID), + ) + github = FakeGitHub(sync_script, [issue]) + + result = _sync(sync_script, [group], github) + + assert github.writes == [] + assert result.actions[0].action == "kept-newer" + + +def test_sync_creates_missing_labels_before_opening_issues( + sync_script: ModuleType, +) -> None: + groups = _groups(sync_script, _failures(sync_script, 3)) + github = FakeGitHub(sync_script, labels=("telemetry-error",)) + + _sync(sync_script, groups, github) + + created_labels = [ + json.loads(request.body)["name"] + for _, request in github.writes + if urlsplit(request.url).path.endswith("/labels") + ] + assert created_labels == ["service: policyengine-api"] + + +def test_sync_adds_labels_a_create_response_is_missing( + sync_script: ModuleType, +) -> None: + groups = _groups(sync_script, _failures(sync_script, 3)) + github = FakeGitHub(sync_script, drop_created_labels=True) + + with pytest.raises(sync_script.GitHubError, match="labels"): + _sync(sync_script, groups, github) + + added = [ + request + for _, request in github.writes + if urlsplit(request.url).path.endswith("/issues/1/labels") + ] + assert len(added) == 1 + + +def test_sync_writes_one_at_a_time_at_least_a_second_apart( + sync_script: ModuleType, +) -> None: + records = [] + for index in range(3): + records += _failures( + sync_script, 3, request_key_prefix=f"g{index}-", error_type=f"Error{index}" + ) + github = FakeGitHub(sync_script) + + _sync(sync_script, _groups(sync_script, records), github) + + write_times = [moment for moment, _ in github.writes] + assert len(write_times) == 5 + assert all( + later - earlier >= 1.0 for earlier, later in itertools.pairwise(write_times) + ) + + +@pytest.mark.parametrize("status", [403, 429]) +def test_sync_waits_for_retry_after_on_rate_limits( + sync_script: ModuleType, status: int +) -> None: + groups = _groups(sync_script, _failures(sync_script, 3)) + github = FakeGitHub( + sync_script, labels=("telemetry-error", "service: policyengine-api") + ) + github.failures = [ + sync_script.HttpResponse( + status=status, reason="Forbidden", headers={"retry-after": "7"}, body=b"{}" + ) + ] + + _sync(sync_script, groups, github) + + assert 7 in github.sleeps + assert len(github.issues) == 1 + + +def test_sync_stops_after_three_rate_limit_retries(sync_script: ModuleType) -> None: + groups = _groups(sync_script, _failures(sync_script, 3)) + github = FakeGitHub( + sync_script, labels=("telemetry-error", "service: policyengine-api") + ) + limited = sync_script.HttpResponse( + status=429, reason="Too Many Requests", headers={"retry-after": "2"}, body=b"{}" + ) + github.failures = [limited] * 4 + + with pytest.raises(sync_script.GitHubError) as raised: + _sync(sync_script, groups, github) + + assert str(raised.value) == "GitHub request failed: HTTP 429 Too Many Requests" + assert github.sleeps.count(2) == 3 + + +def test_sync_reports_a_forbidden_write_without_retrying( + sync_script: ModuleType, +) -> None: + groups = _groups(sync_script, _failures(sync_script, 3)) + github = FakeGitHub( + sync_script, labels=("telemetry-error", "service: policyengine-api") + ) + github.failures = [ + sync_script.HttpResponse( + status=403, + reason="Forbidden", + headers={}, + body=b'{"message": "SENTINEL Resource not accessible"}', + ) + ] + + with pytest.raises(sync_script.GitHubError) as raised: + _sync(sync_script, groups, github) + + assert str(raised.value) == "GitHub request failed: HTTP 403 Forbidden" + assert len(github.writes) == 1 + + +def test_sync_dry_run_makes_no_write_calls(sync_script: ModuleType) -> None: + eligible = _groups(sync_script, _failures(sync_script, 3))[0] + existing = _groups(sync_script, _failures(sync_script, 3, error_type="KeyError"))[0] + github = FakeGitHub(sync_script, [_existing_issue(sync_script, existing)]) + + result = _sync(sync_script, [eligible, existing], github, dry_run=True) + + assert github.writes == [] + assert sorted(action.action for action in result.actions) == ["created", "updated"] From 511a3ae92337d1a2cb63b009ea81a689f7ebad5c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:01:39 +0400 Subject: [PATCH 08/18] Run the telemetry error report end to end with public-safe output main reads its settings from the environment, refuses a report date that hasn't ended in Eastern time, queries BigQuery, groups the errors, syncs the issues and prints a summary, which it also appends to GITHUB_STEP_SUMMARY. The summary lists only fingerprints, kinds, service families, counts, issue numbers and actions. Each stage runs inside a handler that replaces any exception with one line, printed to stderr: the stage, the exception class, and for an API error its HTTP status and reason. A configuration error also names the missing variables. The run then exits with 1. A test with marker strings in every record field finds none of them in stdout, stderr, the job summary or the issues created. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 225 ++++++++++- tests/unit/test_telemetry_error_issues.py | 372 ++++++++++++++++++ 2 files changed, 586 insertions(+), 11 deletions(-) diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index 29db3d63c..17e36baf2 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -16,14 +16,17 @@ import hashlib import hmac import json +import os import re +import sys import time from collections import Counter from dataclasses import dataclass from datetime import UTC, date, datetime, timedelta from datetime import time as time_of_day from http import HTTPStatus -from typing import TYPE_CHECKING, Any, NamedTuple +from pathlib import Path +from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar from urllib.error import HTTPError from urllib.parse import quote, urlencode from urllib.request import Request, urlopen @@ -31,6 +34,9 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterable, Mapping + from typing import TextIO + +StageResult = TypeVar("StageResult") REPORTING_TIME_ZONE = ZoneInfo("America/New_York") MIDNIGHT = time_of_day(0, 0) @@ -145,6 +151,13 @@ DEFAULT_RATE_LIMIT_WAIT_SECONDS = 60 MAX_RATE_LIMIT_WAIT_SECONDS = 300 RATE_LIMIT_STATUSES = frozenset({HTTPStatus.FORBIDDEN, HTTPStatus.TOO_MANY_REQUESTS}) +REQUIRED_ENVIRONMENT_VARIABLES = ( + "GOOGLE_OAUTH_ACCESS_TOKEN", + "GITHUB_TOKEN", + "GITHUB_REPOSITORY", + "TELEMETRY_ERROR_FINGERPRINT_KEY", +) +REPOSITORY_PATTERN = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") HEALTH_CHECK_ROUTES = ( "/health", "/simulation-gateway-check", @@ -834,6 +847,10 @@ class BigQueryError(ApiError): service = "BigQuery" +class BigQueryTimeoutError(BigQueryError): + """A BigQuery query job didn't complete within the polling limit.""" + + def _error_reason(response: HttpResponse) -> str: """Return a Google API error's reason code, else the HTTP reason phrase.""" try: @@ -932,7 +949,7 @@ def query( response = self._results(job, page_token=None) else: msg = "BigQuery query did not complete in time" - raise BigQueryError(msg) + raise BigQueryTimeoutError(msg) rows = _convert_rows(response) while response.get("pageToken"): response = self._results(job, page_token=response["pageToken"]) @@ -1168,6 +1185,13 @@ def _eastern_time(moment: datetime | None) -> str: return moment.astimezone(REPORTING_TIME_ZONE).strftime("%Y-%m-%d %H:%M %Z") +def _window_text(window: ReportingWindow) -> str: + return ( + f"{_eastern_time(window.start)} to {_eastern_time(window.end)} " + f"({window.hours:.0f} hours)" + ) + + def _error_rate(measures: EnvironmentMeasures) -> str: if measures.error_rate is None: return NOT_AVAILABLE @@ -1244,16 +1268,12 @@ def parse_state(body: str) -> dict[str, Any] | None: def issue_section(group: ErrorGroup, window: ReportingWindow, project_id: str) -> str: """Return the marked section of the issue body, which each run replaces.""" related = ", ".join(map(_code, group.related_fingerprints)) - window_text = ( - f"{_eastern_time(window.start)} to {_eastern_time(window.end)} " - f"({window.hours:.0f} hours)" - ) lines = [ SECTION_START, f"### Error group {_code(group.fingerprint)}", "", *_group_details(group), - f"- Window: {window_text}", + f"- Window: {_window_text(window)}", "", *_measures_table(group), "", @@ -1291,6 +1311,10 @@ class GitHubError(ApiError): service = "GitHub" +class GitHubLabelError(GitHubError): + """GitHub didn't apply the labels the job asked for.""" + + def _rate_limit_wait(response: HttpResponse) -> int | None: """Return the seconds to wait before retrying a rate-limited request. @@ -1598,7 +1622,7 @@ def _open(self, group: ErrorGroup) -> int | None: missing = [label for label in labels if label not in applied] if missing and not set(missing) <= self._github.add_labels(number, missing): msg = f"GitHub did not apply the labels to issue #{number}" - raise GitHubError(msg) + raise GitHubLabelError(msg) return number def _reopen_since(self, issue: Mapping[str, Any]) -> datetime | None: @@ -1692,9 +1716,188 @@ def parse_arguments(argv: list[str] | None = None) -> argparse.Namespace: return parser.parse_args(argv) -def main(argv: list[str] | None = None) -> int: - """Run one report and return the process exit code.""" - parse_arguments(argv) +class ConfigurationError(Exception): + """The run's settings are missing or invalid. The text names only settings.""" + + +class StageFailedError(Exception): + """A stage of the run failed. + + The text holds only the stage, the exception class, and for an API error + the HTTP status and reason, so it can be printed in a public log. + """ + + +def _current_time() -> datetime: + return datetime.now(UTC) + + +@dataclass(frozen=True) +class Runtime: + """What a run reads from outside the process, so tests can replace it.""" + + environ: Mapping[str, str] + stdout: TextIO + stderr: TextIO + bigquery_transport: Callable[[HttpRequest], HttpResponse] = urllib_transport + github_transport: Callable[[HttpRequest], HttpResponse] = urllib_transport + now: Callable[[], datetime] = _current_time + sleep: Callable[[float], None] = time.sleep + clock: Callable[[], float] = time.monotonic + + @classmethod + def from_process(cls) -> Runtime: + """Use the process environment, standard streams and the network.""" + return cls(environ=os.environ, stdout=sys.stdout, stderr=sys.stderr) + + +class Settings(NamedTuple): + """Settings read from the environment; none of them is ever printed.""" + + google_access_token: str + github_credential: str + repository: str + fingerprint_key: bytes + summary_path: str | None + + +def read_settings(environ: Mapping[str, str]) -> Settings: + """Read and check the environment variables the run needs.""" + missing = [name for name in REQUIRED_ENVIRONMENT_VARIABLES if not environ.get(name)] + if missing: + msg = f"missing environment variables: {', '.join(missing)}" + raise ConfigurationError(msg) + if not REPOSITORY_PATTERN.match(environ["GITHUB_REPOSITORY"]): + msg = "GITHUB_REPOSITORY must be owner/name" + raise ConfigurationError(msg) + try: + key = fingerprint_key(environ["TELEMETRY_ERROR_FINGERPRINT_KEY"]) + except ValueError as error: + raise ConfigurationError(str(error)) from None + return Settings( + google_access_token=environ["GOOGLE_OAUTH_ACCESS_TOKEN"], + github_credential=environ["GITHUB_TOKEN"], + repository=environ["GITHUB_REPOSITORY"], + fingerprint_key=key, + summary_path=environ.get("GITHUB_STEP_SUMMARY") or None, + ) + + +def _report_window(report_date: date | None, now: datetime) -> ReportingWindow: + window = reporting_window(report_date or default_report_date(now)) + if window.end > now: + msg = f"{window.start.date().isoformat()} has not ended in US Eastern time" + raise ConfigurationError(msg) + return window + + +def _failure_description(stage: str, error: Exception) -> str: + detail = "" + if isinstance(error, ConfigurationError): + detail = f" ({error})" + elif isinstance(error, ApiError) and error.status is not None: + detail = f" (HTTP {error.status} {error.reason})" + return f"{stage} failed: {type(error).__name__}{detail}" + + +def _stage(name: str, operation: Callable[[], StageResult]) -> StageResult: + """Run one stage, replacing any failure with a description safe to print.""" + try: + return operation() + except Exception as error: + raise StageFailedError(_failure_description(name, error)) from error + + +def render_summary( + result: SyncResult, window: ReportingWindow, *, dry_run: bool +) -> str: + """Return the job summary: fingerprints, counts, issue numbers and actions.""" + lines = [ + f"## Telemetry error issues for {window.start.date().isoformat()}", + "", + f"Window: {_window_text(window)}.", + ] + if dry_run: + lines.append("Dry run: no issues were changed.") + if not result.actions: + lines += ["", "No error groups had failures in the window."] + return "\n".join(lines) + totals = Counter(action.action for action in result.actions) + described = ", ".join(f"{count} {name}" for name, count in sorted(totals.items())) + lines += [ + "", + f"{len(result.actions)} error groups: {described}.", + "", + "| Fingerprint | Kind | Service | Production | Staging | Issue | Action |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] + for action in result.actions: + issue = MISSING if action.issue_number is None else f"#{action.issue_number}" + lines.append( + f"| {_code(action.fingerprint)} | {action.kind} | {_code(action.family)} " + f"| {action.production_occurrences:,} | {action.staging_occurrences:,} " + f"| {issue} | {action.action} |" + ) + return "\n".join(lines) + + +def _append_summary(path: str, summary: str) -> None: + with Path(path).open("a", encoding="utf-8") as summary_file: + summary_file.write(f"{summary}\n") + + +def _report(arguments: argparse.Namespace, runtime: Runtime) -> str: + settings = _stage("configuration", lambda: read_settings(runtime.environ)) + window = _stage( + "configuration", lambda: _report_window(arguments.report_date, runtime.now()) + ) + bigquery = BigQueryClient( + arguments.project_id, settings.google_access_token, runtime.bigquery_transport + ) + records = _stage( + "query error records", lambda: fetch_error_records(bigquery, window) + ) + request_counts = _stage( + "query request counts", lambda: fetch_request_counts(bigquery, window) + ) + groups = _stage( + "group errors", + lambda: build_groups(records, request_counts, window, settings.fingerprint_key), + ) + github = GitHubClient( + settings.repository, + settings.github_credential, + runtime.github_transport, + sleep=runtime.sleep, + clock=runtime.clock, + ) + result = _stage( + "sync issues", + lambda: sync_issues( + groups, window, arguments.project_id, github, dry_run=arguments.dry_run + ), + ) + summary = render_summary(result, window, dry_run=arguments.dry_run) + summary_path = settings.summary_path + if summary_path: + _stage("write job summary", lambda: _append_summary(summary_path, summary)) + return summary + + +def main(argv: list[str] | None = None, runtime: Runtime | None = None) -> int: + """Run one report and return the process exit code. + + A failed stage prints one line naming the stage and exception class, never + the exception's message, and returns 1. + """ + arguments = parse_arguments(argv) + runtime = runtime or Runtime.from_process() + try: + summary = _report(arguments, runtime) + except StageFailedError as failure: + print(failure, file=runtime.stderr) + return 1 + print(summary, file=runtime.stdout) return 0 diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index 99a838ff3..96acbe37f 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -4,6 +4,7 @@ import functools import importlib.util +import io import itertools import json import re @@ -2511,3 +2512,374 @@ def test_sync_dry_run_makes_no_write_calls(sync_script: ModuleType) -> None: assert github.writes == [] assert sorted(action.action for action in result.actions) == ["created", "updated"] + + +# End-to-end run and public output + +ERROR_RECORD_FIELDS = [ + {"name": "timestamp", "type": "TIMESTAMP"}, + {"name": "service_name", "type": "STRING"}, + {"name": "environment_name", "type": "STRING"}, + {"name": "event_name", "type": "STRING"}, + {"name": "error_type", "type": "STRING"}, + {"name": "error_handled", "type": "STRING"}, + {"name": "frames", "type": "STRING", "mode": "REPEATED"}, + {"name": "stack_truncated", "type": "BOOLEAN"}, + {"name": "message_template", "type": "STRING"}, + {"name": "attribute_error_type", "type": "STRING"}, + {"name": "failure_category", "type": "STRING"}, + {"name": "route", "type": "STRING"}, + {"name": "status_code", "type": "STRING"}, + {"name": "operation_name", "type": "STRING"}, + {"name": "request_key", "type": "STRING"}, + {"name": "observability_key", "type": "STRING"}, + {"name": "trace_key", "type": "STRING"}, +] +REQUEST_COUNT_FIELDS = [ + {"name": "service_name", "type": "STRING"}, + {"name": "environment_name", "type": "STRING"}, + {"name": "route", "type": "STRING"}, + {"name": "requests", "type": "INTEGER"}, +] +SCHEDULED_RUN_TIME = datetime(2026, 10, 21, 0, 45, tzinfo=EASTERN) + + +def _microseconds(moment: datetime) -> str: + return str((moment - datetime(1970, 1, 1, tzinfo=UTC)) // timedelta(microseconds=1)) + + +def _error_row(**overrides: object) -> list[object]: + values: dict[str, object] = { + "timestamp": _microseconds(WINDOW_NOON), + "service_name": "policyengine-api", + "environment_name": "production", + "event_name": "exception.recorded", + "error_type": "StaleDataError", + "error_handled": "false", + "frames": list(APPLICATION_FRAMES), + "stack_truncated": "false", + "message_template": None, + "attribute_error_type": None, + "failure_category": None, + "route": "//report", + "status_code": "500", + "operation_name": None, + "request_key": "r1", + "observability_key": None, + "trace_key": "t1", + } + values.update(overrides) + return [values[field["name"]] for field in ERROR_RECORD_FIELDS] + + +class FakeBigQuery: + """Answers the error-record and request-count queries with fixed rows.""" + + def __init__( + self, + sync_script: ModuleType, + error_rows: list[list[object]], + count_rows: list[list[object]] = (), + ) -> None: + self.sync_script = sync_script + self.error_rows = error_rows + self.count_rows = list(count_rows) + self.requests: list[object] = [] + + def __call__(self, request: object) -> object: + self.requests.append(request) + label = json.loads(request.body)["labels"]["query"] + if label == "error-records": + response = _query_response(ERROR_RECORD_FIELDS, self.error_rows) + else: + response = _query_response(REQUEST_COUNT_FIELDS, self.count_rows) + return _json_response(self.sync_script, response) + + def json_body(self, index: int) -> dict: + return json.loads(self.requests[index].body) + + +def _environment(tmp_path: Path, **overrides: str) -> dict[str, str]: + environment = { + "GOOGLE_OAUTH_ACCESS_TOKEN": ACCESS_TOKEN, + "GITHUB_TOKEN": "fake-github-token", + "GITHUB_REPOSITORY": REPOSITORY, + "TELEMETRY_ERROR_FINGERPRINT_KEY": "f" * 64, + "GITHUB_STEP_SUMMARY": str(tmp_path / "summary.md"), + } + environment.update(overrides) + return {name: value for name, value in environment.items() if value} + + +def _run_main( + sync_script: ModuleType, + tmp_path: Path, + bigquery: object, + github: FakeGitHub, + *arguments: str, + environment: dict[str, str] | None = None, +) -> tuple[int, str, str]: + stdout, stderr = io.StringIO(), io.StringIO() + runtime = sync_script.Runtime( + environ=_environment(tmp_path) if environment is None else environment, + bigquery_transport=bigquery, + github_transport=github, + now=lambda: SCHEDULED_RUN_TIME, + sleep=github.sleep, + clock=github.clock, + stdout=stdout, + stderr=stderr, + ) + exit_code = sync_script.main( + ["--project-id", PROJECT_ID, *arguments], runtime=runtime + ) + return exit_code, stdout.getvalue(), stderr.getvalue() + + +def _three_failures(**overrides: object) -> list[list[object]]: + return [ + _error_row( + request_key=f"r{index}", + timestamp=_microseconds(_at(index)), + **overrides, + ) + for index in range(3) + ] + + +def test_main_queries_groups_and_opens_issues_for_the_previous_day( + sync_script: ModuleType, tmp_path: Path +) -> None: + bigquery = FakeBigQuery( + sync_script, + _three_failures(), + [["policyengine-api", "production", "//report", "600"]], + ) + github = FakeGitHub(sync_script) + + exit_code, stdout, stderr = _run_main(sync_script, tmp_path, bigquery, github) + + assert exit_code == 0, stderr + issue = github.issues[1] + assert "StaleDataError" in issue["title"] + assert ( + "| Error rate on affected routes | 0.50% (3 of 600 requests) | n/a |" + in (issue["body"]) + ) + window_end = _sent_parameters(bigquery)["window_end"]["parameterValue"] + assert window_end["value"] == "2026-10-21 04:00:00+00:00" + summary = (tmp_path / "summary.md").read_text() + fingerprint = issue["title"].rsplit("(", 1)[1].rstrip(")") + assert ( + f"| `{fingerprint}` | exception | `policyengine-api` | 3 | 0 | #1 | created |" + in summary + ) + assert summary in stdout + + +def test_main_dry_run_reports_actions_without_writing( + sync_script: ModuleType, tmp_path: Path +) -> None: + github = FakeGitHub(sync_script) + + exit_code, stdout, _ = _run_main( + sync_script, + tmp_path, + FakeBigQuery(sync_script, _three_failures()), + github, + "--dry-run", + ) + + assert exit_code == 0 + assert github.writes == [] + assert "Dry run: no issues were changed." in stdout + assert "| - | created |" in stdout + + +def test_main_reports_the_requested_day( + sync_script: ModuleType, tmp_path: Path +) -> None: + bigquery = FakeBigQuery(sync_script, []) + + exit_code, stdout, _ = _run_main( + sync_script, + tmp_path, + bigquery, + FakeGitHub(sync_script), + "--report-date", + "2026-10-15", + ) + + assert exit_code == 0 + window_end = _sent_parameters(bigquery)["window_end"]["parameterValue"] + assert window_end["value"] == "2026-10-16 04:00:00+00:00" + assert "## Telemetry error issues for 2026-10-15" in stdout + assert "No error groups had failures in the window." in stdout + + +def test_main_rejects_a_day_that_has_not_ended( + sync_script: ModuleType, tmp_path: Path +) -> None: + exit_code, stdout, stderr = _run_main( + sync_script, + tmp_path, + FakeBigQuery(sync_script, []), + FakeGitHub(sync_script), + "--report-date", + "2026-10-21", + ) + + assert exit_code == 1 + assert stdout == "" + assert stderr == ( + "configuration failed: ConfigurationError " + "(2026-10-21 has not ended in US Eastern time)\n" + ) + + +def test_main_names_missing_environment_variables_without_values( + sync_script: ModuleType, tmp_path: Path +) -> None: + environment = _environment( + tmp_path, GITHUB_TOKEN="", TELEMETRY_ERROR_FINGERPRINT_KEY="" + ) + + exit_code, _, stderr = _run_main( + sync_script, + tmp_path, + FakeBigQuery(sync_script, []), + FakeGitHub(sync_script), + environment=environment, + ) + + assert exit_code == 1 + assert stderr == ( + "configuration failed: ConfigurationError (missing environment variables: " + "GITHUB_TOKEN, TELEMETRY_ERROR_FINGERPRINT_KEY)\n" + ) + + +def test_main_rejects_a_short_fingerprint_key_without_printing_it( + sync_script: ModuleType, tmp_path: Path +) -> None: + environment = _environment( + tmp_path, TELEMETRY_ERROR_FINGERPRINT_KEY="SENTINEL-short-key" + ) + + exit_code, stdout, stderr = _run_main( + sync_script, + tmp_path, + FakeBigQuery(sync_script, []), + FakeGitHub(sync_script), + environment=environment, + ) + + assert exit_code == 1 + assert "at least 32 characters" in stderr + assert "SENTINEL" not in stdout + stderr + + +def test_main_reports_an_api_failure_by_stage_class_and_status_only( + sync_script: ModuleType, tmp_path: Path +) -> None: + error_body = { + "error": { + "message": "SENTINEL echoed value", + "errors": [{"reason": "backendError", "message": "SENTINEL"}], + } + } + failing = FakeTransport( + sync_script, + [_json_response(sync_script, error_body, status=500, reason="SENTINEL")], + ) + + exit_code, stdout, stderr = _run_main( + sync_script, tmp_path, failing, FakeGitHub(sync_script) + ) + + assert exit_code == 1 + assert stdout == "" + assert stderr == ( + "query error records failed: BigQueryError (HTTP 500 backendError)\n" + ) + + +def test_main_reports_an_unexpected_exception_by_class_only( + sync_script: ModuleType, tmp_path: Path +) -> None: + def broken_transport(request: object) -> object: + message = "SENTINEL connection detail" + raise RuntimeError(message) + + exit_code, _, stderr = _run_main( + sync_script, tmp_path, broken_transport, FakeGitHub(sync_script) + ) + + assert exit_code == 1 + assert stderr == "query error records failed: RuntimeError\n" + + +def test_main_reports_a_github_failure_by_stage( + sync_script: ModuleType, tmp_path: Path +) -> None: + github = FakeGitHub(sync_script) + github.failures = [ + sync_script.HttpResponse( + status=403, reason="Forbidden", headers={}, body=b'{"message": "SENTINEL"}' + ) + ] + + exit_code, _, stderr = _run_main( + sync_script, tmp_path, FakeBigQuery(sync_script, _three_failures()), github + ) + + assert exit_code == 1 + assert stderr == "sync issues failed: GitHubError (HTTP 403 Forbidden)\n" + + +@pytest.mark.parametrize( + "overrides", + [ + {}, + {"error_type": None}, + {"error_type": None, "event_name": "request.completed"}, + ], +) +def test_main_never_publishes_values_from_records( + sync_script: ModuleType, tmp_path: Path, overrides: dict[str, object] +) -> None: + sentinel_values = { + name: value + for name, value in _sentinel_row(sync_script, **overrides).items() + if name not in {"timestamp", "environment_name", "frames", "stack_truncated"} + } + sentinel_values["stack_truncated"] = "true" + sentinel_values["frames"] = ['/app/SENTINEL.py", line 1, in SENTINEL'] + rows = [ + _error_row( + **{ + **sentinel_values, + "request_key": f"SENTINEL request {index}", + "timestamp": _microseconds(_at(index)), + } + ) + for index in range(3) + ] + count_rows = [["SENTINEL service", "production", "SENTINEL route", "10"]] + github = FakeGitHub(sync_script) + + exit_code, stdout, stderr = _run_main( + sync_script, tmp_path, FakeBigQuery(sync_script, rows, count_rows), github + ) + + assert exit_code == 0, stderr + assert len(github.issues) == 1 + published = [ + stdout, + stderr, + (tmp_path / "summary.md").read_text(), + *(issue["title"] for issue in github.issues.values()), + *(issue["body"] for issue in github.issues.values()), + ] + for text in published: + assert "SENTINEL" not in unquote(text) From efbaa064d262eda05c1f74aa0d96595ed7d6355b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:04:19 +0400 Subject: [PATCH 09/18] Add the daily telemetry error issues workflow and its wrapper The workflow runs at 00:45 America/New_York. That time is never skipped or repeated when the clocks change, and it avoids GitHub's delays for schedules at the start of an hour. Manual runs default to a dry run and can report a chosen Eastern day. The job: - runs only in PolicyEngine/policyengine-api, only from master, and in the telemetry-error-issues environment, whose deployments are limited to master; - has permission to read contents, write issues and request an OIDC token, and runs one at a time; - authenticates as the report's service account through Workload Identity Federation, without writing a credentials file; - calls sync_telemetry_error_issues.sh, which requires its variables, masks the access token and fingerprint key, and adds --dry-run and --report-date when set. A scheduled run is never a dry run. gcp/observability/README.md describes the job, how to run it by hand, the infrastructure it needs, why the fingerprint key must not be rotated, and the missing Cloud Run platform failures. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.sh | 26 +++ .github/workflows/telemetry-error-issues.yml | 65 ++++++ changelog.d/telemetry-error-issues.added.md | 1 + gcp/observability/README.md | 82 ++++++++ tests/unit/test_telemetry_error_issues.py | 186 ++++++++++++++++++ 5 files changed, 360 insertions(+) create mode 100644 .github/scripts/sync_telemetry_error_issues.sh create mode 100644 .github/workflows/telemetry-error-issues.yml create mode 100644 changelog.d/telemetry-error-issues.added.md diff --git a/.github/scripts/sync_telemetry_error_issues.sh b/.github/scripts/sync_telemetry_error_issues.sh new file mode 100644 index 000000000..00032e369 --- /dev/null +++ b/.github/scripts/sync_telemetry_error_issues.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Run the daily telemetry error report with the settings the workflow passes +# in the environment. DRY_RUN=true adds --dry-run; a non-empty REPORT_DATE +# (YYYY-MM-DD, an Eastern calendar day) replaces the default previous day. +set -euo pipefail + +: "${GOOGLE_OAUTH_ACCESS_TOKEN:?GOOGLE_OAUTH_ACCESS_TOKEN is required}" +: "${GITHUB_TOKEN:?GITHUB_TOKEN is required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" +: "${TELEMETRY_ERROR_FINGERPRINT_KEY:?TELEMETRY_ERROR_FINGERPRINT_KEY is required}" +: "${OBSERVABILITY_PROJECT_ID:?OBSERVABILITY_PROJECT_ID is required}" + +# Hide both values in the job log even if a later command prints them. +echo "::add-mask::${GOOGLE_OAUTH_ACCESS_TOKEN}" +echo "::add-mask::${TELEMETRY_ERROR_FINGERPRINT_KEY}" + +arguments=(--project-id "${OBSERVABILITY_PROJECT_ID}") +if [[ "${DRY_RUN:-false}" == "true" ]]; then + arguments+=(--dry-run) +fi +if [[ -n "${REPORT_DATE:-}" ]]; then + arguments+=(--report-date "${REPORT_DATE}") +fi + +script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec python3 "${script_directory}/sync_telemetry_error_issues.py" "${arguments[@]}" diff --git a/.github/workflows/telemetry-error-issues.yml b/.github/workflows/telemetry-error-issues.yml new file mode 100644 index 000000000..fc36e0020 --- /dev/null +++ b/.github/workflows/telemetry-error-issues.yml @@ -0,0 +1,65 @@ +name: Telemetry error issues + +# Each day at 00:45 US Eastern, report the previous Eastern calendar day's +# error groups from the central telemetry log bucket as GitHub issues. 00:45 +# is never skipped or repeated when the clocks change, and it avoids the +# delays GitHub has for schedules at the start of an hour. +on: + schedule: + - cron: "45 0 * * *" + timezone: "America/New_York" + workflow_dispatch: + inputs: + dry_run: + description: "Query and group errors without changing any issue" + type: boolean + default: true + report_date: + description: "Eastern calendar day to report, as YYYY-MM-DD (default: the previous day)" + type: string + default: "" + +concurrency: + group: telemetry-error-issues + cancel-in-progress: false + +jobs: + sync: + name: Sync telemetry error issues + if: github.repository == 'PolicyEngine/policyengine-api' + runs-on: ubuntu-latest + # Deployments to this environment are limited to master, which keeps other + # branches from authenticating as its service account. + environment: telemetry-error-issues + timeout-minutes: 15 + permissions: + contents: read + issues: write + id-token: write + steps: + - name: Require master branch + if: github.ref != 'refs/heads/master' + run: exit 1 + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.TELEMETRY_ERROR_ISSUES_SERVICE_ACCOUNT }} + token_format: access_token + create_credentials_file: false + - name: Sync telemetry error issues + env: + GOOGLE_OAUTH_ACCESS_TOKEN: ${{ steps.auth.outputs.access_token }} + GITHUB_TOKEN: ${{ github.token }} + TELEMETRY_ERROR_FINGERPRINT_KEY: ${{ secrets.TELEMETRY_ERROR_FINGERPRINT_KEY }} + OBSERVABILITY_PROJECT_ID: ${{ vars.OBSERVABILITY_PROJECT_ID }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} + REPORT_DATE: ${{ inputs.report_date }} + run: bash .github/scripts/sync_telemetry_error_issues.sh diff --git a/changelog.d/telemetry-error-issues.added.md b/changelog.d/telemetry-error-issues.added.md new file mode 100644 index 000000000..d712f6a11 --- /dev/null +++ b/changelog.d/telemetry-error-issues.added.md @@ -0,0 +1 @@ +A daily workflow reports the previous day's production error groups from the central telemetry logs as GitHub issues, with production and staging counts and no error messages or stack text. diff --git a/gcp/observability/README.md b/gcp/observability/README.md index 695a57ff7..04332c1f5 100644 --- a/gcp/observability/README.md +++ b/gcp/observability/README.md @@ -131,6 +131,88 @@ Compare the deployed interval with the recorded pre-fix baseline: 46 collector metric export errors comprising 863 rejected points. Trace export had no corresponding rejected points in that interval. +## Daily telemetry error issues + +The `Telemetry error issues` workflow +(`.github/workflows/telemetry-error-issues.yml`) runs at 00:45 US Eastern each +day. It reports the previous Eastern calendar day, from midnight to midnight, +which lasts 23 or 25 hours on the days the clocks change. The script it runs, +`.github/scripts/sync_telemetry_error_issues.py`: + +1. Queries the central bucket through its linked BigQuery dataset + `observability_logs`. It reads the `_AllLogs` view for ERROR records with + the `policyengine.api-v1` service namespace in the report day and the seven + days before it, excluding the health-check routes. It also counts + `request.completed` records per route in the report day, for error rates. + Each query is limited to 5 GiB billed. +2. Chooses one record per failed request and service: the exception record, + else the error message, else the failed `request.completed` record. +3. Groups failures by a 16-character HMAC-SHA256 fingerprint of the service + family and either the exception type and innermost application stack + frames, the route and message template, or the route and HTTP status. +4. Creates or updates one issue per group in this repository, labelled + `telemetry-error` and `service: `. A group opens an issue + only after 3 production failures in at least 2 requests in one day, and at + most 5 issues open per run. Staging counts appear in each issue but never + open one. An issue closed as completed reopens, with the `regression` + label, when production failures that start 24 hours after it closed meet + the same thresholds. + +This repository is public, so issues, workflow logs and job summaries hold +only fingerprints, counts, service names, exception class names, route +templates and a Logs Explorer link. Error messages, stack text and +identifiers stay in BigQuery: the queries return stacks only as file and +function names, messages only as their first three words, and identifiers +only as hashes. A failed run prints only the failed stage, the exception class +and the HTTP status. + +### Running it by hand + +Dispatch the workflow from `master`. `dry_run` defaults to `true`: the job +queries and groups errors and writes the actions it would take to the job +summary, without changing any issue. `report_date` (`YYYY-MM-DD`) reports a +different Eastern day, such as one a scheduled run missed. + +To run it locally with your own credentials, use a throwaway fingerprint key. +Its fingerprints won't match existing issues. + +```bash +GOOGLE_OAUTH_ACCESS_TOKEN="$(gcloud auth print-access-token)" \ +GITHUB_TOKEN="$(gh auth token)" \ +GITHUB_REPOSITORY=PolicyEngine/policyengine-api \ +TELEMETRY_ERROR_FINGERPRINT_KEY="$(openssl rand -hex 32)" \ +python3 .github/scripts/sync_telemetry_error_issues.py \ + --project-id policyengine-observability --dry-run +``` + +### Required infrastructure + +| Resource | Purpose | +| --- | --- | +| Linked BigQuery dataset `observability_logs` on the `policyengine-observability` bucket | SQL access to the bucket's log views | +| Service account `telemetry-error-issues@policyengine-observability.iam.gserviceaccount.com` | The identity the workflow queries BigQuery as | +| `roles/bigquery.jobUser` on the project, and `roles/bigquery.dataViewer` on the linked dataset | Run query jobs and read the log views | +| `roles/iam.workloadIdentityUser` on the service account for `principal://iam.googleapis.com/projects/389282473430/locations/global/workloadIdentityPools/github/subject/repo:PolicyEngine/policyengine-api:environment:telemetry-error-issues` | Lets only jobs in the `telemetry-error-issues` GitHub environment act as the service account | +| GitHub environment `telemetry-error-issues`, with deployments limited to `master` | Keeps other branches from acting as the service account | +| Environment variable `TELEMETRY_ERROR_ISSUES_SERVICE_ACCOUNT` | The service account's email address | +| Environment secret `TELEMETRY_ERROR_FINGERPRINT_KEY` | The 32-byte hexadecimal HMAC key for fingerprints | + +The workflow also uses the repository secret `GCP_WORKLOAD_IDENTITY_PROVIDER` +and the repository variable `OBSERVABILITY_PROJECT_ID`. + +Never rotate `TELEMETRY_ERROR_FINGERPRINT_KEY` on its own. Every fingerprint +would change, so the next run would open a second issue for each group that +already has one. A key change needs a plan for moving the existing issues to +their new fingerprints, as does any change to `FINGERPRINT_ALGORITHM` in the +script. + +### Known gap: Cloud Run platform failures + +The source sinks forward only records in the application log format +(`schema_version = policyengine.observability.v2`). Cloud Run request logs +are not forwarded, so requests that time out, run out of memory or crash the +container before the application logs anything never reach these reports. + ## Rollback 1. Remove the OTel endpoint from participating service configuration. diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index 96acbe37f..781f93f33 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -7,7 +7,9 @@ import io import itertools import json +import os import re +import subprocess import sys from datetime import UTC, date, datetime, timedelta from pathlib import Path @@ -16,6 +18,7 @@ from zoneinfo import ZoneInfo import pytest +import yaml if TYPE_CHECKING: from types import ModuleType @@ -2883,3 +2886,186 @@ def test_main_never_publishes_values_from_records( ] for text in published: assert "SENTINEL" not in unquote(text) + + +# Workflow and wrapper + +WORKFLOW_PATH = REPO / ".github" / "workflows" / "telemetry-error-issues.yml" +WRAPPER_PATH = REPO / ".github" / "scripts" / "sync_telemetry_error_issues.sh" +WRAPPER_ENVIRONMENT = { + "GOOGLE_OAUTH_ACCESS_TOKEN": "fake-access-token", + "GITHUB_TOKEN": "fake-github-token", + "GITHUB_REPOSITORY": REPOSITORY, + "TELEMETRY_ERROR_FINGERPRINT_KEY": "f" * 64, + "OBSERVABILITY_PROJECT_ID": PROJECT_ID, +} + + +def _workflow() -> dict: + return yaml.safe_load(WORKFLOW_PATH.read_text(encoding="utf-8")) + + +def _sync_job() -> dict: + return _workflow()["jobs"]["sync"] + + +def _step(name: str) -> dict: + return next(step for step in _sync_job()["steps"] if step["name"] == name) + + +def test_workflow_runs_at_00_45_eastern_and_on_request() -> None: + # PyYAML reads the bare key "on" as the boolean True. + triggers = _workflow()[True] + + assert triggers["schedule"] == [ + {"cron": "45 0 * * *", "timezone": "America/New_York"} + ] + inputs = triggers["workflow_dispatch"]["inputs"] + assert inputs["dry_run"]["type"] == "boolean" + assert inputs["dry_run"]["default"] is True + assert inputs["report_date"]["type"] == "string" + assert inputs["report_date"]["default"] == "" + + +def test_workflow_job_is_limited_to_master_in_its_environment() -> None: + workflow = _workflow() + job = _sync_job() + + assert workflow["concurrency"] == { + "group": "telemetry-error-issues", + "cancel-in-progress": False, + } + assert job["if"] == "github.repository == 'PolicyEngine/policyengine-api'" + assert job["environment"] == "telemetry-error-issues" + assert job["timeout-minutes"] == 15 + assert job["permissions"] == { + "contents": "read", + "issues": "write", + "id-token": "write", + } + assert job["steps"][0] == { + "name": "Require master branch", + "if": "github.ref != 'refs/heads/master'", + "run": "exit 1", + } + + +def test_workflow_authenticates_as_the_report_service_account() -> None: + auth = _step("Authenticate to Google Cloud") + + assert auth["id"] == "auth" + assert auth["uses"] == "google-github-actions/auth@v2" + assert auth["with"] == { + "workload_identity_provider": "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}", + "service_account": "${{ vars.TELEMETRY_ERROR_ISSUES_SERVICE_ACCOUNT }}", + "token_format": "access_token", + "create_credentials_file": False, + } + assert _step("Set up Python")["with"] == {"python-version": "3.12"} + + +def test_workflow_passes_settings_to_the_wrapper() -> None: + sync = _step("Sync telemetry error issues") + + assert sync["run"] == "bash .github/scripts/sync_telemetry_error_issues.sh" + assert sync["env"] == { + "GOOGLE_OAUTH_ACCESS_TOKEN": "${{ steps.auth.outputs.access_token }}", + "GITHUB_TOKEN": "${{ github.token }}", + "TELEMETRY_ERROR_FINGERPRINT_KEY": ( + "${{ secrets.TELEMETRY_ERROR_FINGERPRINT_KEY }}" + ), + "OBSERVABILITY_PROJECT_ID": "${{ vars.OBSERVABILITY_PROJECT_ID }}", + # A scheduled run has no inputs, so it is never a dry run. + "DRY_RUN": "${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}", + "REPORT_DATE": "${{ inputs.report_date }}", + } + + +def _run_wrapper( + tmp_path: Path, environment: dict[str, str] +) -> tuple[subprocess.CompletedProcess[str], list[str] | None]: + """Run the wrapper with a fake python3 that records its arguments.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + calls = tmp_path / "python-arguments.txt" + fake_python = bin_dir / "python3" + fake_python.write_text(f'#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "{calls}"\n') + fake_python.chmod(0o755) + result = subprocess.run( + ["bash", str(WRAPPER_PATH)], + cwd=REPO, + env={"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", **environment}, + text=True, + capture_output=True, + check=False, + ) + arguments = calls.read_text().splitlines() if calls.exists() else None + return result, arguments + + +def test_wrapper_passes_bash_syntax_checking() -> None: + result = subprocess.run( + ["bash", "-n", str(WRAPPER_PATH)], capture_output=True, text=True, check=False + ) + + assert result.returncode == 0, result.stderr + + +def test_wrapper_runs_the_report_for_the_project(tmp_path: Path) -> None: + result, arguments = _run_wrapper(tmp_path, WRAPPER_ENVIRONMENT) + + assert result.returncode == 0, result.stderr + assert arguments == [str(SCRIPT_PATH), "--project-id", PROJECT_ID] + + +def test_wrapper_masks_the_access_token_and_fingerprint_key(tmp_path: Path) -> None: + result, _ = _run_wrapper(tmp_path, WRAPPER_ENVIRONMENT) + + assert result.stdout.splitlines() == [ + "::add-mask::fake-access-token", + f"::add-mask::{'f' * 64}", + ] + + +def test_wrapper_adds_dry_run_and_report_date_when_set(tmp_path: Path) -> None: + environment = { + **WRAPPER_ENVIRONMENT, + "DRY_RUN": "true", + "REPORT_DATE": "2026-10-15", + } + + result, arguments = _run_wrapper(tmp_path, environment) + + assert result.returncode == 0, result.stderr + assert arguments == [ + str(SCRIPT_PATH), + "--project-id", + PROJECT_ID, + "--dry-run", + "--report-date", + "2026-10-15", + ] + + +def test_wrapper_runs_live_when_dry_run_is_false(tmp_path: Path) -> None: + environment = {**WRAPPER_ENVIRONMENT, "DRY_RUN": "false", "REPORT_DATE": ""} + + _, arguments = _run_wrapper(tmp_path, environment) + + assert arguments == [str(SCRIPT_PATH), "--project-id", PROJECT_ID] + + +@pytest.mark.parametrize("missing", sorted(WRAPPER_ENVIRONMENT)) +def test_wrapper_fails_before_running_when_a_variable_is_missing( + tmp_path: Path, missing: str +) -> None: + environment = { + name: value for name, value in WRAPPER_ENVIRONMENT.items() if name != missing + } + + result, arguments = _run_wrapper(tmp_path, environment) + + assert result.returncode != 0 + assert f"{missing} is required" in result.stderr + assert arguments is None + assert result.stdout == "" From 1538f61c9545934aa36372922e237a258a9cc506 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:13:14 +0400 Subject: [PATCH 10/18] Record the telemetry error issues infrastructure created on 2026-10-20 The linked dataset is in the BigQuery US location. A dataset-level dataViewer grant was enough, with no Cloud Logging role, and a dry run as the service account scanned about 0.2 GiB for one day's report. Co-Authored-By: Claude Opus 5.5 --- gcp/observability/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/gcp/observability/README.md b/gcp/observability/README.md index 04332c1f5..dd5cc0295 100644 --- a/gcp/observability/README.md +++ b/gcp/observability/README.md @@ -200,6 +200,19 @@ python3 .github/scripts/sync_telemetry_error_issues.py \ The workflow also uses the repository secret `GCP_WORKLOAD_IDENTITY_PROVIDER` and the repository variable `OBSERVABILITY_PROJECT_ID`. +These resources were created on 2026-10-20: + +- The linked dataset is in the BigQuery `US` location and holds one view, + `_AllLogs`. +- `roles/bigquery.dataViewer` was granted on the dataset alone with + `GRANT ... ON SCHEMA`; `bq update --source` would replace the dataset's + access list. The service account needs no Cloud Logging role. +- A dry run as the service account, for 2026-10-19, read 1,648 ERROR records + across eight days and found 15 error groups, 5 of them above the issue + thresholds. The queries scanned about 0.18 GiB of error records and + 0.02 GiB of request counts, so a daily run stays well inside BigQuery's free + monthly allowance. + Never rotate `TELEMETRY_ERROR_FINGERPRINT_KEY` on its own. Every fingerprint would change, so the next run would open a second issue for each group that already has one. A key change needs a plan for moving the existing issues to From 0a9f0c54dc5aaf99da0e0414e4d2972631609f1d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:47:15 +0400 Subject: [PATCH 11/18] Fingerprint exceptions by the first traceback of a chained stack The logging library keeps only the first 16,384 characters of error.stack, and Python prints a chained exception's cause first. A long cause traceback could therefore drop the final traceback. The same exception then got its frames from different tracebacks depending on how long its cause was, and so two fingerprints. Its counts split, often below the issue thresholds. The query now removes everything after the first traceback, matched by CHAINED_EXCEPTION_PATTERN as a query parameter, before extracting frames. For a chained exception that is its first cause, which is always kept. A test builds a chained stack with a short and a long cause, shortens the long one as the library does, and checks that both give the same fingerprint. Both queries pass a BigQuery dry run. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 31 +++++-- gcp/observability/README.md | 4 + tests/unit/test_telemetry_error_issues.py | 86 +++++++++++++++++-- 3 files changed, 109 insertions(+), 12 deletions(-) diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index 17e36baf2..5b768d3eb 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -55,6 +55,16 @@ # one capturing group because BigQuery's REGEXP_EXTRACT_ALL allows only one, # and the trailing newline skips a header that stack truncation cut short. FRAME_PATTERN = r'File "([^"\n]+", line [0-9]+, in [^\s]+)\n' +# Matches everything after the first traceback of a chained exception, so the +# query can remove it before extracting frames. The logging library keeps only +# the start of a long stack, which can drop the final traceback; the first +# traceback is always kept, so frames from it give an exception the same +# fingerprint however long its stack is. +CHAINED_EXCEPTION_PATTERN = ( + r"(?s)\n\n(?:The above exception was the direct cause of the following " + r"exception|During handling of the above exception, another exception " + r"occurred):\n.*" +) FRAME_PARTS = re.compile(r'^(?P[^"\n]+)", line [0-9]+, in (?P\S+)$') FINGERPRINT_FRAME_COUNT = 3 INSTALLED_PACKAGE_PREFIX = re.compile(r"^.*/(?:site|dist)-packages/") @@ -177,8 +187,12 @@ JSON_VALUE(json_payload, '$."event.name"') AS event_name, JSON_VALUE(json_payload, '$."error.type"') AS error_type, JSON_VALUE(json_payload, '$."error.handled"') AS error_handled, - REGEXP_EXTRACT_ALL(JSON_VALUE(json_payload, '$."error.stack"'), @frame_pattern) - AS frames, + REGEXP_EXTRACT_ALL( + REGEXP_REPLACE( + JSON_VALUE(json_payload, '$."error.stack"'), @chained_exception_pattern, '' + ), + @frame_pattern + ) AS frames, COALESCE(LENGTH(JSON_VALUE(json_payload, '$."error.stack"')) >= 16384, FALSE) AS stack_truncated, ARRAY_TO_STRING( @@ -330,9 +344,11 @@ def frame_key(frame: str) -> str | None: def fingerprint_frames(frames: Iterable[str] | None) -> tuple[str, ...]: """Choose the frames that identify where an exception was raised. - These are the three innermost frames in application packages. Without - application frames they are the three innermost frames of any kind, and - without a stack, a placeholder. + ``frames`` come from the first traceback of the stack only (see + ``CHAINED_EXCEPTION_PATTERN``); for a chained exception that is its first + cause. These are the three innermost frames in application packages. + Without application frames they are the three innermost frames of any + kind, and without a stack, a placeholder. """ keys = [key for key in map(frame_key, frames or ()) if key is not None] if not keys: @@ -1069,7 +1085,10 @@ def fetch_error_records( """Return the ERROR records of the window and the seven days before it.""" parameters = _query_parameters( {"query_start": window.lookback_start, "window_end": window.end}, - {"frame_pattern": FRAME_PATTERN}, + { + "frame_pattern": FRAME_PATTERN, + "chained_exception_pattern": CHAINED_EXCEPTION_PATTERN, + }, ) rows = client.query(ERROR_RECORDS_QUERY, parameters, "error-records") return [error_record_from_row(row) for row in rows] diff --git a/gcp/observability/README.md b/gcp/observability/README.md index dd5cc0295..cb12b2bfa 100644 --- a/gcp/observability/README.md +++ b/gcp/observability/README.md @@ -150,6 +150,10 @@ which lasts 23 or 25 hours on the days the clocks change. The script it runs, 3. Groups failures by a 16-character HMAC-SHA256 fingerprint of the service family and either the exception type and innermost application stack frames, the route and message template, or the route and HTTP status. + Frames come from the first traceback of the stack, which for a chained + exception is its first cause. The logging library keeps only the first + 16,384 characters of a stack, so a long cause can drop the final + traceback, but the first one is always kept. 4. Creates or updates one issue per group in this repository, labelled `telemetry-error` and `service: `. A group opens an issue only after 3 production failures in at least 2 requests in one day, and at diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index 781f93f33..31b728b09 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -199,8 +199,36 @@ def test_cli_rejects_missing_project_or_invalid_date_with_exit_code_2( def _extract_frames(sync_script: ModuleType, stack: str) -> list[str]: - """Apply the pattern the way BigQuery's REGEXP_EXTRACT_ALL does.""" - return re.findall(sync_script.FRAME_PATTERN, stack) + """Extract frames as the query does: REGEXP_REPLACE, then REGEXP_EXTRACT_ALL.""" + first_traceback = re.sub(sync_script.CHAINED_EXCEPTION_PATTERN, "", stack) + return re.findall(sync_script.FRAME_PATTERN, first_traceback) + + +def _chained_stack(recursion_depth: int) -> str: + """Return a chained traceback whose cause recursed ``recursion_depth`` times.""" + library_frame = ( + ' File "/venv/lib/python3.12/site-packages/policyengine_core/simulation.py", ' + "line 500, in calculate\n return self._calculate(variable_name, period)\n" + ) + cause = ( + "Traceback (most recent call last):\n" + ' File "/app/policyengine_api/services/household.py", line 40, in calculate\n' + " simulation.calculate(name)\n" + f"{library_frame * recursion_depth}" + "ValueError: SENTINEL cause\n" + ) + final = ( + "Traceback (most recent call last):\n" + ' File "/app/policyengine_api/routes/household_routes.py", line 12, in get\n' + " return service.calculate()\n" + ' File "/app/policyengine_api/services/household.py", line 44, in calculate\n' + " raise CalculationError(name) from error\n" + "CalculationError: SENTINEL final\n" + ) + separator = ( + "\nThe above exception was the direct cause of the following exception:\n\n" + ) + return f"{cause}{separator}{final}" @pytest.mark.parametrize( @@ -256,15 +284,25 @@ def test_frames_pattern_extracts_frame_headers_without_source_or_message( assert not any("SENTINEL" in frame or "\n" in frame for frame in frames) -def test_frames_pattern_reads_every_block_of_a_chained_traceback( +def test_frames_chained_traceback_contributes_only_its_first_traceback( sync_script: ModuleType, ) -> None: frames = _extract_frames(sync_script, CHAINED_STACK) assert [sync_script.frame_key(frame) for frame in frames] == [ "policyengine_api/services/cache.py:read", - "policyengine_api/routes/policy_routes.py:get_policy", - "policyengine_api/services/policy_service.py:get", + ] + + +def test_frames_explicitly_chained_traceback_contributes_only_its_cause( + sync_script: ModuleType, +) -> None: + frames = _extract_frames(sync_script, _chained_stack(recursion_depth=2)) + + assert [sync_script.frame_key(frame) for frame in frames] == [ + "policyengine_api/services/household.py:calculate", + "policyengine_core/simulation.py:calculate", + "policyengine_core/simulation.py:calculate", ] @@ -683,6 +721,32 @@ def test_fingerprint_exception_differs_by_type_frames_and_service( ) != sync_script.fingerprint(changed, FINGERPRINT_KEY) +def test_fingerprint_is_the_same_when_truncation_drops_the_final_traceback( + sync_script: ModuleType, +) -> None: + # The logging library keeps only the first 16,384 characters of a stack. + short_stack = _chained_stack(recursion_depth=20) + long_stack = _chained_stack(recursion_depth=120)[:16_384] + assert "CalculationError" in short_stack + assert "CalculationError" not in long_stack + + short = _exception_record( + sync_script, + error_type="CalculationError", + frames=tuple(_extract_frames(sync_script, short_stack)), + ) + long = _exception_record( + sync_script, + error_type="CalculationError", + frames=tuple(_extract_frames(sync_script, long_stack)), + stack_truncated=True, + ) + + assert sync_script.fingerprint(short, FINGERPRINT_KEY) == sync_script.fingerprint( + long, FINGERPRINT_KEY + ) + + def test_fingerprint_depends_on_the_key(sync_script: ModuleType) -> None: record = _exception_record(sync_script) @@ -1314,6 +1378,9 @@ def test_bigquery_error_query_request_limits_cost_and_names_its_parameters( assert parameters["frame_pattern"]["parameterValue"]["value"] == ( sync_script.FRAME_PATTERN ) + assert parameters["chained_exception_pattern"]["parameterValue"]["value"] == ( + sync_script.CHAINED_EXCEPTION_PATTERN + ) assert [ item["value"] for item in parameters["excluded_routes"]["parameterValue"]["arrayValues"] @@ -1347,7 +1414,14 @@ def test_bigquery_error_query_reads_stacks_and_messages_only_through_expressions stack = "JSON_VALUE(json_payload, '$.\"error.stack\"')" assert sql.count("error.stack") == 2 - assert f"REGEXP_EXTRACT_ALL({stack}, @frame_pattern)" in sql + assert ( + " REGEXP_EXTRACT_ALL(\n" + " REGEXP_REPLACE(\n" + f" {stack}, @chained_exception_pattern, ''\n" + " ),\n" + " @frame_pattern\n" + " ) AS frames" + ) in sql assert f"LENGTH({stack}) >= 16384" in sql assert sql.count("$.message") == 1 assert "REGEXP_REPLACE(JSON_VALUE(json_payload, '$.message')" in sql From 3820f0d8a4eb365c8b53753a8708757a559827d3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:49:17 +0400 Subject: [PATCH 12/18] Keep apostrophes from turning message values into the template The message template removed '[^']*' spans, so the apostrophe in a contraction or possessive paired with the next quote. In "Can't load policy 'alpha'" it removed the fixed words and kept the value, so each value formed its own group, usually below the issue thresholds. A single quote now opens a value only at the start of the message or after a character that isn't a letter or digit. The quote and word patterns are module constants, passed to the query as parameters, so the tests apply exactly what BigQuery runs. Running the patterns in BigQuery on the test strings gave the same templates and frames as Python. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 25 +++++++-- tests/unit/test_telemetry_error_issues.py | 55 ++++++++++++++++++- 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index 5b768d3eb..43e047ff3 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -67,6 +67,14 @@ ) FRAME_PARTS = re.compile(r'^(?P[^"\n]+)", line [0-9]+, in (?P\S+)$') FINGERPRINT_FRAME_COUNT = 3 +# A message's template is its first three whole words after removing quoted +# values, so values don't split one error into many groups. A single quote +# opens a value only after the start of the message or a character that is not +# a letter or digit; otherwise it is an apostrophe, as in "can't" or "users'". +# The query passes these to REGEXP_REPLACE and REGEXP_EXTRACT_ALL. +DOUBLE_QUOTED_PATTERN = r'"[^"]*"' +SINGLE_QUOTED_PATTERN = r"(^|[^A-Za-z0-9])'[^']*'" +TEMPLATE_WORD_PATTERN = r"\b[A-Za-z]+\b" INSTALLED_PACKAGE_PREFIX = re.compile(r"^.*/(?:site|dist)-packages/") STANDARD_LIBRARY_PREFIX = re.compile(r"^.*/lib/python3(?:\.\d+)?/") APPLICATION_PACKAGES = ( @@ -177,8 +185,8 @@ # Each query reads the linked dataset's _AllLogs view. Only code-defined or # normalized values leave BigQuery: stacks only as frame headers, messages -# only as their first three whole words outside quotes, and identifiers only -# as SHA-256 hashes. Duplicate deliveries of one log entry are removed. +# only as templates (see DOUBLE_QUOTED_PATTERN), and identifiers only as +# SHA-256 hashes. Duplicate deliveries of one log entry are removed. ERROR_RECORDS_QUERY = r""" SELECT timestamp, @@ -201,11 +209,13 @@ FROM UNNEST( REGEXP_EXTRACT_ALL( REGEXP_REPLACE( - REGEXP_REPLACE(JSON_VALUE(json_payload, '$.message'), r'"[^"]*"', ''), - r"'[^']*'", - '' + REGEXP_REPLACE( + JSON_VALUE(json_payload, '$.message'), @double_quoted_pattern, '' + ), + @single_quoted_pattern, + r'\1' ), - r'\b[A-Za-z]+\b' + @template_word_pattern ) ) AS word WITH OFFSET AS position WHERE position < 3 @@ -1088,6 +1098,9 @@ def fetch_error_records( { "frame_pattern": FRAME_PATTERN, "chained_exception_pattern": CHAINED_EXCEPTION_PATTERN, + "double_quoted_pattern": DOUBLE_QUOTED_PATTERN, + "single_quoted_pattern": SINGLE_QUOTED_PATTERN, + "template_word_pattern": TEMPLATE_WORD_PATTERN, }, ) rows = client.query(ERROR_RECORDS_QUERY, parameters, "error-records") diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index 31b728b09..cbe05cb90 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -1381,6 +1381,12 @@ def test_bigquery_error_query_request_limits_cost_and_names_its_parameters( assert parameters["chained_exception_pattern"]["parameterValue"]["value"] == ( sync_script.CHAINED_EXCEPTION_PATTERN ) + for name, pattern in ( + ("double_quoted_pattern", sync_script.DOUBLE_QUOTED_PATTERN), + ("single_quoted_pattern", sync_script.SINGLE_QUOTED_PATTERN), + ("template_word_pattern", sync_script.TEMPLATE_WORD_PATTERN), + ): + assert parameters[name]["parameterValue"]["value"] == pattern assert [ item["value"] for item in parameters["excluded_routes"]["parameterValue"]["arrayValues"] @@ -1424,7 +1430,17 @@ def test_bigquery_error_query_reads_stacks_and_messages_only_through_expressions ) in sql assert f"LENGTH({stack}) >= 16384" in sql assert sql.count("$.message") == 1 - assert "REGEXP_REPLACE(JSON_VALUE(json_payload, '$.message')" in sql + assert ( + " REGEXP_REPLACE(\n" + " REGEXP_REPLACE(\n" + " JSON_VALUE(json_payload, '$.message'), " + "@double_quoted_pattern, ''\n" + " ),\n" + " @single_quoted_pattern,\n" + " r'\\1'\n" + " ),\n" + " @template_word_pattern\n" + ) in sql assert "error.message" not in sql assert "SELECT *" not in sql assert "json_payload AS" not in sql @@ -1446,6 +1462,43 @@ def test_bigquery_error_query_hashes_identifiers( assert f"TO_HEX(SHA256({identifier}))" in sql +def _message_template(sync_script: ModuleType, message: str) -> str: + """Build the template the way the query does, with RE2's ASCII word rules.""" + without_double_quotes = re.sub(sync_script.DOUBLE_QUOTED_PATTERN, "", message) + without_quotes = re.sub( + sync_script.SINGLE_QUOTED_PATTERN, r"\1", without_double_quotes + ) + words = re.findall(sync_script.TEMPLATE_WORD_PATTERN, without_quotes, re.ASCII) + return " ".join(word.lower() for word in words[:3]) + + +def test_bigquery_template_keeps_words_around_an_apostrophe( + sync_script: ModuleType, +) -> None: + templates = { + _message_template(sync_script, f"Can't load policy '{value}'") + for value in ("alpha", "beta", "gamma") + } + + assert templates == {"can t load"} + + +@pytest.mark.parametrize( + ("message", "template"), + [ + ("Failed to save 'report 12' for user", "failed to save"), + ("'SENTINEL' could not be parsed", "could not be"), + ('Missing "SENTINEL" in household input', "missing in household"), + ("Timeout after 30s calling the simulation API", "timeout after calling"), + ("The users' policies 'SENTINEL' failed", "the users policies"), + ], +) +def test_bigquery_template_drops_quoted_values_and_partial_words( + sync_script: ModuleType, message: str, template: str +) -> None: + assert _message_template(sync_script, message) == template + + def test_bigquery_error_query_hashes_the_trace(sync_script: ModuleType) -> None: sql = sync_script.ERROR_RECORDS_QUERY From 72fb8550c482f1192956c1c024438943e98e0688 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:49:57 +0400 Subject: [PATCH 13/18] Compute the error rate only from failures on routes The numerator counted every affected request in the window, including failures without a route, while the denominator counted completed requests only on the routes the failures used. Exception fingerprints leave the route out, so one group can hold failed requests and background failures, and the rate could exceed 100%: one failed request on a route with one completed request, plus two route-less failures, showed "300.00% (3 of 1 requests)". The rate now divides the distinct affected requests that have a route (new route_affected_requests measure) by the completed requests on those routes, and the issue table shows that count. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 13 ++++-- tests/unit/test_telemetry_error_issues.py | 44 +++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index 43e047ff3..cbcd30395 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -595,6 +595,7 @@ class EnvironmentMeasures: occurrences: int affected_requests: int observability_ids: int + route_affected_requests: int route_requests: int error_rate: float | None hourly_rate: float @@ -700,7 +701,12 @@ def _environment_measures( in_window = [item for item in in_environment if item.timestamp >= window.start] before_window = len(in_environment) - len(in_window) affected_requests = len({item.affected_key for item in in_window}) - routes = {item.route for item in in_window if item.route} + # The error rate compares failed requests with completed requests on the + # same routes, so failures without a route, such as background + # operations, are left out of both. + on_routes = [item for item in in_window if item.route] + route_affected_requests = len({item.affected_key for item in on_routes}) + routes = {item.route for item in on_routes} requests = route_requests.total(family, environment, routes) timestamps = [item.timestamp for item in in_environment] return EnvironmentMeasures( @@ -709,8 +715,9 @@ def _environment_measures( observability_ids=len( {item.observability_key for item in in_window if item.observability_key} ), + route_affected_requests=route_affected_requests, route_requests=requests, - error_rate=affected_requests / requests if requests else None, + error_rate=route_affected_requests / requests if requests else None, hourly_rate=len(in_window) / window.hours, previous_hourly_rate=before_window / window.lookback_hours, first_seen=min(timestamps, default=None), @@ -1228,7 +1235,7 @@ def _error_rate(measures: EnvironmentMeasures) -> str: if measures.error_rate is None: return NOT_AVAILABLE return ( - f"{measures.error_rate:.2%} ({measures.affected_requests:,} of " + f"{measures.error_rate:.2%} ({measures.route_affected_requests:,} of " f"{measures.route_requests:,} requests)" ) diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index cbe05cb90..dca886edc 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -968,6 +968,36 @@ def test_measures_error_rate_uses_requests_to_the_affected_routes( assert group.staging.error_rate is None +def _failures_with_and_without_a_route(sync_script: ModuleType) -> list: + """Return one failed request on a route and two route-less failures.""" + return [ + _exception_record( + sync_script, request_key="r1", route="//calculate" + ), + _exception_record( + sync_script, request_key=None, trace_key="t2", route=None, timestamp=_at(1) + ), + _exception_record( + sync_script, request_key=None, trace_key="t3", route=None, timestamp=_at(2) + ), + ] + + +def test_measures_error_rate_counts_only_failures_on_routes( + sync_script: ModuleType, +) -> None: + group = _only_group( + sync_script, + _failures_with_and_without_a_route(sync_script), + [_request_count(sync_script, "//calculate", 1)], + ) + + assert group.production.affected_requests == 3 + assert group.production.route_affected_requests == 1 + assert group.production.route_requests == 1 + assert group.production.error_rate == pytest.approx(1.0) + + def test_measures_error_rate_counts_every_simulation_service_version( sync_script: ModuleType, ) -> None: @@ -1881,6 +1911,20 @@ def test_render_body_marked_section_holds_the_measures_table( assert "https://console.cloud.google.com/logs/query;" in section +def test_render_error_rate_names_only_failures_on_routes( + sync_script: ModuleType, +) -> None: + group = _group( + sync_script, + _failures_with_and_without_a_route(sync_script), + [_request_count(sync_script, "//calculate", 1)], + ) + + _, body = _render(sync_script, group) + + assert "| Error rate on affected routes | 100.00% (1 of 1 requests) | n/a |" in body + + def test_render_body_lists_related_fingerprints(sync_script: ModuleType) -> None: records = [ _exception_record(sync_script, trace_key="t1"), From a142e7a87bbecf2171e791138a132a70959a59e1 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:50:52 +0400 Subject: [PATCH 14/18] Match existing GitHub labels regardless of case GitHub label names are unique regardless of case, and applying a label by name matches it the same way. The job compared names exactly. If the repository had, say, "Regression", the job would try to create "regression", get HTTP 422 and fail the run. It would also report the labels on a new issue as missing when GitHub had applied the existing spelling. Known, applied and returned label names are now compared case-folded. The test fake now behaves like GitHub: creating a name that differs only in case returns 422, and labels are applied with their stored spelling. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 20 +++++-- tests/unit/test_telemetry_error_issues.py | 59 ++++++++++++++++++- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index cbcd30395..0cf6e18c2 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -1567,6 +1567,11 @@ def _meets_thresholds(occurrences: int, affected_requests: int) -> bool: ) +def _folded(names: Iterable[str]) -> set[str]: + """Return label names case-folded, as GitHub compares them.""" + return {name.casefold() for name in names} + + def _issue_labels(group: ErrorGroup) -> list[str]: service_label = f"{SERVICE_LABEL_PREFIX}{group.family}" if len(service_label) > MAX_LABEL_LENGTH: @@ -1600,6 +1605,7 @@ def __init__( self._window = window self._project_id = project_id self._dry_run = dry_run + # Case-folded names: GitHub label names are unique regardless of case. self._known_labels: set[str] | None = None def run(self, groups: Iterable[ErrorGroup]) -> SyncResult: @@ -1639,12 +1645,12 @@ def run(self, groups: Iterable[ErrorGroup]) -> SyncResult: def _ensure_labels(self, names: Iterable[str]) -> None: if self._known_labels is None: - self._known_labels = self._github.list_labels() + self._known_labels = _folded(self._github.list_labels()) for name in names: - if name not in self._known_labels: + if name.casefold() not in self._known_labels: color, description = LABEL_DETAILS.get(name, SERVICE_LABEL_DETAILS) self._github.create_label(name, color, description) - self._known_labels.add(name) + self._known_labels.add(name.casefold()) def _open(self, group: ErrorGroup) -> int | None: if self._dry_run: @@ -1657,9 +1663,11 @@ def _open(self, group: ErrorGroup) -> int | None: labels, ) number = issue["number"] - applied = {label["name"] for label in issue.get("labels", [])} - missing = [label for label in labels if label not in applied] - if missing and not set(missing) <= self._github.add_labels(number, missing): + applied = _folded(label["name"] for label in issue.get("labels", [])) + missing = [label for label in labels if label.casefold() not in applied] + if missing and not _folded(missing) <= _folded( + self._github.add_labels(number, missing) + ): msg = f"GitHub did not apply the labels to issue #{number}" raise GitHubLabelError(msg) return number diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index dca886edc..fd5177c2b 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -2193,10 +2193,18 @@ def _route(self, method: str, path: str, query: dict, body: dict | None) -> obje labels = [{"name": name} for name in sorted(self.labels)] return self._respond(self._page(labels, query)) if parts == ["labels"]: + # GitHub label names are unique regardless of case. + if body["name"].casefold() in {name.casefold() for name in self.labels}: + return self._respond({"message": "Validation Failed"}, status=422) self.labels.add(body["name"]) return self._respond({"name": body["name"]}, status=201) return self._create(body) + def _existing_label_names(self, requested: list[str]) -> list[str]: + """Return the stored spelling of each label, as GitHub applies them.""" + stored = {name.casefold(): name for name in self.labels} + return [stored.get(name.casefold(), name) for name in requested] + def _route_issue( self, method: str, number: int, rest: list[str], body: dict ) -> object: @@ -2206,7 +2214,7 @@ def _route_issue( if rest == ["labels"]: names = {label["name"] for label in self.issues[number]["labels"]} if not self.drop_created_labels: - names |= set(body["labels"]) + names |= set(self._existing_label_names(body["labels"])) self.issues[number]["labels"] = [{"name": name} for name in sorted(names)] return self._respond(self.issues[number]["labels"]) self.comments.setdefault(number, []).append(body["body"]) @@ -2219,7 +2227,11 @@ def _page(self, items: list, query: dict) -> list: def _create(self, body: dict) -> object: number = max(self.issues, default=0) + 1 - labels = [] if self.drop_created_labels else body["labels"] + labels = ( + [] + if self.drop_created_labels + else self._existing_label_names(body["labels"]) + ) self.issues[number] = { "number": number, "title": body["title"], @@ -2581,6 +2593,49 @@ def test_sync_creates_missing_labels_before_opening_issues( assert created_labels == ["service: policyengine-api"] +def test_sync_uses_existing_labels_whatever_their_case( + sync_script: ModuleType, +) -> None: + groups = _groups(sync_script, _failures(sync_script, 3)) + github = FakeGitHub( + sync_script, labels=("Telemetry-Error", "Service: Policyengine-Api") + ) + + _sync(sync_script, groups, github) + + created_labels = [ + request + for _, request in github.writes + if urlsplit(request.url).path.endswith("/labels") + ] + assert created_labels == [] + assert github.issues[1]["labels"] == [ + {"name": "Telemetry-Error"}, + {"name": "Service: Policyengine-Api"}, + ] + + +def test_sync_reopens_with_an_existing_regression_label_of_another_case( + sync_script: ModuleType, +) -> None: + group = _groups(sync_script, _failures(sync_script, 4))[0] + closed_at = (WINDOW_NOON - timedelta(days=2)).isoformat() + issue = _existing_issue( + sync_script, + group, + state="closed", + state_reason="completed", + closed_at=closed_at, + ) + github = FakeGitHub(sync_script, [issue], labels=("Regression",)) + + result = _sync(sync_script, [group], github) + + assert result.actions[0].action == "reopened" + assert {"name": "Regression"} in github.issues[7]["labels"] + assert github.labels == {"Regression"} + + def test_sync_adds_labels_a_create_response_is_missing( sync_script: ModuleType, ) -> None: From 594414785a489449b408f3ef2c1eff23b57f1259 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:51:55 +0400 Subject: [PATCH 15/18] Follow GitHub's rate-limit guidance for 403 responses without retry-after A 403 or 429 was retried only when it had retry-after, or for a 429. GitHub can send a secondary-rate-limit 403 without retry-after, which failed the run and left that day unreported until someone dispatched it again. The job now retries a rate-limited 403 or 429 as GitHub advises: after retry-after; when x-ratelimit-remaining is 0, until x-ratelimit-reset, measured from the response's date header; otherwise after a minute. A 403 whose headers and message don't mention a rate limit is still a permission failure and isn't retried. Waits over five minutes still fail the run. The message is only searched, never printed. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 29 +++++++- tests/unit/test_telemetry_error_issues.py | 70 +++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index 0cf6e18c2..b58859bbd 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -24,6 +24,7 @@ from dataclasses import dataclass from datetime import UTC, date, datetime, timedelta from datetime import time as time_of_day +from email.utils import parsedate_to_datetime from http import HTTPStatus from pathlib import Path from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar @@ -1354,17 +1355,41 @@ class GitHubLabelError(GitHubError): """GitHub didn't apply the labels the job asked for.""" +def _seconds_until_reset(response: HttpResponse) -> int | None: + """Return the seconds from the response's ``date`` to ``x-ratelimit-reset``.""" + reset = response.headers.get("x-ratelimit-reset", "") + sent = response.headers.get("date") + if not (reset.isascii() and reset.isdigit()) or not sent: + return None + try: + sent_at = parsedate_to_datetime(sent) + except (TypeError, ValueError): + return None + return max(int(reset) - int(sent_at.timestamp()), 0) + + def _rate_limit_wait(response: HttpResponse) -> int | None: """Return the seconds to wait before retrying a rate-limited request. - A 403 without ``retry-after`` is a permission failure, so it isn't retried. + Follows GitHub's guidance: wait for ``retry-after``; when no requests + remain, wait until ``x-ratelimit-reset``; otherwise wait a minute. A 403 + whose headers and message don't mention a rate limit is a permission + failure, so it isn't retried. The message is only searched, never printed. """ if response.status not in RATE_LIMIT_STATUSES: return None retry_after = response.headers.get("retry-after", "") if retry_after.isascii() and retry_after.isdigit(): seconds = int(retry_after) - elif response.status == HTTPStatus.TOO_MANY_REQUESTS: + elif response.headers.get("x-ratelimit-remaining") == "0": + until_reset = _seconds_until_reset(response) + seconds = ( + DEFAULT_RATE_LIMIT_WAIT_SECONDS if until_reset is None else until_reset + ) + elif ( + response.status == HTTPStatus.TOO_MANY_REQUESTS + or b"rate limit" in response.body.lower() + ): seconds = DEFAULT_RATE_LIMIT_WAIT_SECONDS else: return None diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py index fd5177c2b..17b1e3257 100644 --- a/tests/unit/test_telemetry_error_issues.py +++ b/tests/unit/test_telemetry_error_issues.py @@ -2692,6 +2692,76 @@ def test_sync_waits_for_retry_after_on_rate_limits( assert len(github.issues) == 1 +def _limited_github(sync_script: ModuleType, response: object) -> FakeGitHub: + """Return a fake whose first write gets ``response`` and later writes succeed.""" + github = FakeGitHub( + sync_script, labels=("telemetry-error", "service: policyengine-api") + ) + github.failures = [response] + return github + + +def test_sync_waits_a_minute_on_a_secondary_rate_limit_without_retry_after( + sync_script: ModuleType, +) -> None: + body = b'{"message": "You have exceeded a secondary rate limit. Please wait."}' + github = _limited_github( + sync_script, + sync_script.HttpResponse(status=403, reason="Forbidden", headers={}, body=body), + ) + + _sync(sync_script, _groups(sync_script, _failures(sync_script, 3)), github) + + assert 60 in github.sleeps + assert len(github.issues) == 1 + + +def test_sync_waits_until_an_exhausted_rate_limit_resets( + sync_script: ModuleType, +) -> None: + headers = { + "date": "Tue, 20 Oct 2026 23:00:00 GMT", + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": str( + int(datetime(2026, 10, 20, 23, 0, 42, tzinfo=UTC).timestamp()) + ), + } + github = _limited_github( + sync_script, + sync_script.HttpResponse( + status=403, reason="Forbidden", headers=headers, body=b"{}" + ), + ) + + _sync(sync_script, _groups(sync_script, _failures(sync_script, 3)), github) + + assert 42 in github.sleeps + assert len(github.issues) == 1 + + +def test_sync_fails_instead_of_waiting_hours_for_a_rate_limit_reset( + sync_script: ModuleType, +) -> None: + headers = { + "date": "Tue, 20 Oct 2026 23:00:00 GMT", + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": str( + int(datetime(2026, 10, 21, 0, 0, tzinfo=UTC).timestamp()) + ), + } + github = _limited_github( + sync_script, + sync_script.HttpResponse( + status=403, reason="Forbidden", headers=headers, body=b"{}" + ), + ) + + with pytest.raises(sync_script.GitHubError, match="HTTP 403"): + _sync(sync_script, _groups(sync_script, _failures(sync_script, 3)), github) + + assert github.sleeps == [] + + def test_sync_stops_after_three_rate_limit_retries(sync_script: ModuleType) -> None: groups = _groups(sync_script, _failures(sync_script, 3)) github = FakeGitHub( From 98d23091fae6a54353549ac478ac68835dcb6a97 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:15:03 +0400 Subject: [PATCH 16/18] Split the telemetry error issue script and its tests into modules The script was 1,977 lines in one module, and its test file 3,367 lines. The code now lives in a telemetry_error_issues package beside the script, with one module for each part of a run: window, normalization, records, groups, transport, bigquery, rendering, github_client, sync and cli. The largest is 395 lines. sync_telemetry_error_issues.py is now a short entry point that imports main from the package, so the wrapper and the workflow don't change. Python puts the directory of a script it runs by path first on sys.path, so the package imports from any working directory. The code moved without changing. Three private helpers in rendering that other modules now use lost their leading underscore and gained a docstring: _code, _window_text and _eastern_time. The tests moved to tests/unit/telemetry_error_issues, one module for each source module, plus one for whole runs of main and one for the workflow and its wrapper. They import the package directly instead of loading the script by its path. The records, fakes and helpers that several modules share are in three support modules, and the shared helpers lost their leading underscore. One test is new: it runs the script by path from another directory, to check that the entry point finds the package. This is one commit rather than two because the old tests loaded the script by path and read about 45 names from it. The entry point defines only main, so the old tests couldn't run against the split script without the rewrite of their imports. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.py | 1974 +--------- .../telemetry_error_issues/__init__.py | 13 + .../telemetry_error_issues/bigquery.py | 395 ++ .github/scripts/telemetry_error_issues/cli.py | 274 ++ .../telemetry_error_issues/github_client.py | 208 + .../scripts/telemetry_error_issues/groups.py | 271 ++ .../telemetry_error_issues/normalization.py | 168 + .../scripts/telemetry_error_issues/records.py | 166 + .../telemetry_error_issues/rendering.py | 242 ++ .../scripts/telemetry_error_issues/sync.py | 282 ++ .../telemetry_error_issues/transport.py | 96 + .../scripts/telemetry_error_issues/window.py | 57 + gcp/observability/README.md | 9 +- tests/unit/telemetry_error_issues/__init__.py | 1 + .../bigquery_support.py | 66 + tests/unit/telemetry_error_issues/conftest.py | 13 + .../telemetry_error_issues/github_support.py | 181 + .../telemetry_error_issues/report_support.py | 171 + .../telemetry_error_issues/test_bigquery.py | 407 ++ tests/unit/telemetry_error_issues/test_cli.py | 56 + .../telemetry_error_issues/test_end_to_end.py | 373 ++ .../test_github_client.py | 159 + .../telemetry_error_issues/test_groups.py | 378 ++ .../test_normalization.py | 328 ++ .../telemetry_error_issues/test_records.py | 246 ++ .../telemetry_error_issues/test_rendering.py | 309 ++ .../unit/telemetry_error_issues/test_sync.py | 338 ++ .../telemetry_error_issues/test_window.py | 58 + .../test_workflow_and_wrapper.py | 211 ++ tests/unit/test_telemetry_error_issues.py | 3367 ----------------- 30 files changed, 5478 insertions(+), 5339 deletions(-) create mode 100644 .github/scripts/telemetry_error_issues/__init__.py create mode 100644 .github/scripts/telemetry_error_issues/bigquery.py create mode 100644 .github/scripts/telemetry_error_issues/cli.py create mode 100644 .github/scripts/telemetry_error_issues/github_client.py create mode 100644 .github/scripts/telemetry_error_issues/groups.py create mode 100644 .github/scripts/telemetry_error_issues/normalization.py create mode 100644 .github/scripts/telemetry_error_issues/records.py create mode 100644 .github/scripts/telemetry_error_issues/rendering.py create mode 100644 .github/scripts/telemetry_error_issues/sync.py create mode 100644 .github/scripts/telemetry_error_issues/transport.py create mode 100644 .github/scripts/telemetry_error_issues/window.py create mode 100644 tests/unit/telemetry_error_issues/__init__.py create mode 100644 tests/unit/telemetry_error_issues/bigquery_support.py create mode 100644 tests/unit/telemetry_error_issues/conftest.py create mode 100644 tests/unit/telemetry_error_issues/github_support.py create mode 100644 tests/unit/telemetry_error_issues/report_support.py create mode 100644 tests/unit/telemetry_error_issues/test_bigquery.py create mode 100644 tests/unit/telemetry_error_issues/test_cli.py create mode 100644 tests/unit/telemetry_error_issues/test_end_to_end.py create mode 100644 tests/unit/telemetry_error_issues/test_github_client.py create mode 100644 tests/unit/telemetry_error_issues/test_groups.py create mode 100644 tests/unit/telemetry_error_issues/test_normalization.py create mode 100644 tests/unit/telemetry_error_issues/test_records.py create mode 100644 tests/unit/telemetry_error_issues/test_rendering.py create mode 100644 tests/unit/telemetry_error_issues/test_sync.py create mode 100644 tests/unit/telemetry_error_issues/test_window.py create mode 100644 tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py delete mode 100644 tests/unit/test_telemetry_error_issues.py diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py index b58859bbd..9b80188ae 100644 --- a/.github/scripts/sync_telemetry_error_issues.py +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -1,1977 +1,11 @@ """Report each day's telemetry error groups as GitHub issues. -Each run reads one US Eastern calendar day of error records from the central -Cloud Logging bucket through its linked BigQuery dataset. It groups the records -by a keyed fingerprint, measures each group in production and staging, and -creates or updates one GitHub issue per production error group. - -Workflow logs, job summaries and issues in this repository are public. The -script therefore publishes only fingerprints, counts, code-defined names and a -Logs Explorer link, never error messages, stack text or identifiers. +The code is in the ``telemetry_error_issues`` package beside this file. Python +puts the directory of a script it runs by path first on ``sys.path``, so the +package imports from any working directory. """ -from __future__ import annotations - -import argparse -import hashlib -import hmac -import json -import os -import re -import sys -import time -from collections import Counter -from dataclasses import dataclass -from datetime import UTC, date, datetime, timedelta -from datetime import time as time_of_day -from email.utils import parsedate_to_datetime -from http import HTTPStatus -from pathlib import Path -from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar -from urllib.error import HTTPError -from urllib.parse import quote, urlencode -from urllib.request import Request, urlopen -from zoneinfo import ZoneInfo - -if TYPE_CHECKING: - from collections.abc import Callable, Iterable, Mapping - from typing import TextIO - -StageResult = TypeVar("StageResult") - -REPORTING_TIME_ZONE = ZoneInfo("America/New_York") -MIDNIGHT = time_of_day(0, 0) - -# Shown in place of a value that is missing, or that fails validation and so -# might not be code-defined. -MISSING = "-" -UNRECOGNIZED = "" - -PRODUCTION_ENVIRONMENT_NAMES = frozenset({"production", "main"}) -STAGING_ENVIRONMENT_NAMES = frozenset({"staging"}) -VERSIONED_SIMULATION_SERVICE = re.compile(r"^(policyengine-simulation-py)\d+-\d+-\d+$") - -# Captures 'path", line N, in function' from each frame header of a Python -# traceback, including indented and "|"-prefixed exception group frames. It has -# one capturing group because BigQuery's REGEXP_EXTRACT_ALL allows only one, -# and the trailing newline skips a header that stack truncation cut short. -FRAME_PATTERN = r'File "([^"\n]+", line [0-9]+, in [^\s]+)\n' -# Matches everything after the first traceback of a chained exception, so the -# query can remove it before extracting frames. The logging library keeps only -# the start of a long stack, which can drop the final traceback; the first -# traceback is always kept, so frames from it give an exception the same -# fingerprint however long its stack is. -CHAINED_EXCEPTION_PATTERN = ( - r"(?s)\n\n(?:The above exception was the direct cause of the following " - r"exception|During handling of the above exception, another exception " - r"occurred):\n.*" -) -FRAME_PARTS = re.compile(r'^(?P[^"\n]+)", line [0-9]+, in (?P\S+)$') -FINGERPRINT_FRAME_COUNT = 3 -# A message's template is its first three whole words after removing quoted -# values, so values don't split one error into many groups. A single quote -# opens a value only after the start of the message or a character that is not -# a letter or digit; otherwise it is an apostrophe, as in "can't" or "users'". -# The query passes these to REGEXP_REPLACE and REGEXP_EXTRACT_ALL. -DOUBLE_QUOTED_PATTERN = r'"[^"]*"' -SINGLE_QUOTED_PATTERN = r"(^|[^A-Za-z0-9])'[^']*'" -TEMPLATE_WORD_PATTERN = r"\b[A-Za-z]+\b" -INSTALLED_PACKAGE_PREFIX = re.compile(r"^.*/(?:site|dist)-packages/") -STANDARD_LIBRARY_PREFIX = re.compile(r"^.*/lib/python3(?:\.\d+)?/") -APPLICATION_PACKAGES = ( - r"(?:policyengine_api|policyengine_fastapi" - r"|policyengine_simulation_[a-z0-9_]+|policyengine_stage12_[a-z0-9_]+)/" -) -APPLICATION_PATH = re.compile(rf"^{APPLICATION_PACKAGES}") -APPLICATION_PATH_IN_ABSOLUTE_PATH = re.compile(rf"(?:^|/)({APPLICATION_PACKAGES}.*)$") -FALLBACK_PATH_COMPONENTS = 2 - -ERROR_TYPE_PATTERN = re.compile(r"^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*$", re.ASCII) -MAX_ERROR_TYPE_LENGTH = 120 -UNMATCHED_ROUTE = "" -MAX_ROUTE_LENGTH = 200 -ROUTE_PARAMETER = re.compile( - r"^(?:<(?:[a-z]+:)?[A-Za-z_]\w*>|\{[A-Za-z_]\w*(?::[a-z]+)?\})$", re.ASCII -) -ROUTE_LITERAL = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,39}$") -# Literal route segments are words such as "v2" or "user-policies". A segment -# with more digits than this is more likely a record ID than code. -MAX_ROUTE_LITERAL_DIGITS = 2 -FAMILY_PATTERN = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") -MAX_FAMILY_LENGTH = 63 -OPERATION_PATTERN = re.compile(r"^[A-Za-z_][\w.:/-]*$", re.ASCII) -MAX_OPERATION_LENGTH = 100 -LOWEST_HTTP_STATUS = 100 -HIGHEST_HTTP_STATUS = 599 - -REQUEST_COMPLETED_EVENT = "request.completed" -# Changing how fingerprints are computed changes every fingerprint, and the -# job matches existing issues only by the current algorithm. A change to it -# must say how existing issues are carried over. -FINGERPRINT_ALGORITHM = 1 -FINGERPRINT_LENGTH = 16 -MIN_FINGERPRINT_KEY_LENGTH = 32 - -# Each run also reads the seven days before the window, for comparison rates, -# first-seen times and regressions. -LOOKBACK = timedelta(days=7) -REPORTED_ENVIRONMENTS = ("production", "staging") -MAX_RELATED_FINGERPRINTS = 3 - -HTTP_TIMEOUT_SECONDS = 60 -# A short code from an API error, such as "invalidQuery" or "Not Found". The -# error's message is never used, because it can repeat values from a record. -ERROR_REASON_PATTERN = re.compile(r"^[A-Za-z][A-Za-z ]{0,59}$") - -BIGQUERY_API = "https://bigquery.googleapis.com/bigquery/v2" -LINKED_DATASET = "observability_logs" -MAXIMUM_BYTES_BILLED = 5 * 1024**3 -QUERY_WAIT_MILLISECONDS = 30_000 -MAX_QUERY_POLLS = 40 -UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=UTC) -ISSUE_TITLE_PREFIX = "[telemetry]" -MAX_TITLE_LENGTH = 256 -SECTION_START = "" -SECTION_END = "" -STATE_PATTERN = re.compile(r"") -NOT_AVAILABLE = "n/a" -LOGS_EXPLORER_URL = "https://console.cloud.google.com/logs/query" -LOG_BUCKET = "policyengine-observability" -LOG_VIEW = "_AllLogs" -SCHEMA_VERSION = "policyengine.observability.v2" -SIMULATION_FAMILY = "policyengine-simulation-py" -GITHUB_API = "https://api.github.com" -GITHUB_API_VERSION = "2022-11-28" -GITHUB_PAGE_SIZE = 100 -BOT_LOGIN = "github-actions[bot]" -TITLE_FINGERPRINT = re.compile(r"\(([0-9a-f]{16})\)$") -BASE_LABEL = "telemetry-error" -REGRESSION_LABEL = "regression" -SERVICE_LABEL_PREFIX = "service: " -MAX_LABEL_LENGTH = 50 -LABEL_DETAILS = { - BASE_LABEL: ("b60205", "Opened by the daily telemetry error report"), - REGRESSION_LABEL: ("d93f0b", "An error that returned after its issue was closed"), -} -SERVICE_LABEL_DETAILS = ("1d76db", "Service that produced a telemetry error group") -# A production group opens an issue only when it fails this often, in this many -# distinct requests, during the window; the same thresholds reopen an issue. -MIN_PRODUCTION_OCCURRENCES = 3 -MIN_PRODUCTION_REQUESTS = 2 -MAX_NEW_ISSUES_PER_RUN = 5 -# Failures in the first day after an issue is closed may come from code that -# hasn't been deployed yet, so they never reopen it. -REOPEN_DELAY = timedelta(hours=24) -REOPENABLE_CLOSE_REASONS = frozenset({None, "completed"}) -# GitHub asks clients to send content-creating requests one at a time, at -# least a second apart, and to wait as its rate-limit responses say. -MIN_SECONDS_BETWEEN_WRITES = 1.0 -MAX_RATE_LIMIT_RETRIES = 3 -DEFAULT_RATE_LIMIT_WAIT_SECONDS = 60 -MAX_RATE_LIMIT_WAIT_SECONDS = 300 -RATE_LIMIT_STATUSES = frozenset({HTTPStatus.FORBIDDEN, HTTPStatus.TOO_MANY_REQUESTS}) -REQUIRED_ENVIRONMENT_VARIABLES = ( - "GOOGLE_OAUTH_ACCESS_TOKEN", - "GITHUB_TOKEN", - "GITHUB_REPOSITORY", - "TELEMETRY_ERROR_FINGERPRINT_KEY", -) -REPOSITORY_PATTERN = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") -HEALTH_CHECK_ROUTES = ( - "/health", - "/simulation-gateway-check", - "/liveness-check", - "/readiness-check", -) - -# Each query reads the linked dataset's _AllLogs view. Only code-defined or -# normalized values leave BigQuery: stacks only as frame headers, messages -# only as templates (see DOUBLE_QUOTED_PATTERN), and identifiers only as -# SHA-256 hashes. Duplicate deliveries of one log entry are removed. -ERROR_RECORDS_QUERY = r""" -SELECT - timestamp, - JSON_VALUE(json_payload, '$."service.name"') AS service_name, - JSON_VALUE(json_payload, '$."deployment.environment.name"') AS environment_name, - JSON_VALUE(json_payload, '$."event.name"') AS event_name, - JSON_VALUE(json_payload, '$."error.type"') AS error_type, - JSON_VALUE(json_payload, '$."error.handled"') AS error_handled, - REGEXP_EXTRACT_ALL( - REGEXP_REPLACE( - JSON_VALUE(json_payload, '$."error.stack"'), @chained_exception_pattern, '' - ), - @frame_pattern - ) AS frames, - COALESCE(LENGTH(JSON_VALUE(json_payload, '$."error.stack"')) >= 16384, FALSE) - AS stack_truncated, - ARRAY_TO_STRING( - ARRAY( - SELECT LOWER(word) - FROM UNNEST( - REGEXP_EXTRACT_ALL( - REGEXP_REPLACE( - REGEXP_REPLACE( - JSON_VALUE(json_payload, '$.message'), @double_quoted_pattern, '' - ), - @single_quoted_pattern, - r'\1' - ), - @template_word_pattern - ) - ) AS word WITH OFFSET AS position - WHERE position < 3 - ORDER BY position - ), - ' ' - ) AS message_template, - JSON_VALUE(json_payload, '$.attributes.error_type') AS attribute_error_type, - JSON_VALUE(json_payload, '$.attributes.failure_category') AS failure_category, - JSON_VALUE(json_payload, '$."http.route"') AS route, - JSON_VALUE(json_payload, '$."http.response.status_code"') AS status_code, - JSON_VALUE(json_payload, '$."operation.name"') AS operation_name, - TO_HEX(SHA256(JSON_VALUE(json_payload, '$."request.id"'))) AS request_key, - TO_HEX(SHA256(JSON_VALUE(json_payload, '$.attributes.observability_id'))) - AS observability_key, - TO_HEX(SHA256(COALESCE( - JSON_VALUE(json_payload, '$.trace_id'), - REGEXP_EXTRACT(trace, r'/traces/([0-9A-Fa-f]+)$') - ))) AS trace_key -FROM `_AllLogs` -WHERE timestamp >= @query_start AND timestamp < @window_end - AND JSON_VALUE(json_payload, '$.schema_version') = 'policyengine.observability.v2' - AND JSON_VALUE(json_payload, '$."service.namespace"') = 'policyengine.api-v1' - AND severity_number >= 500 - AND COALESCE(JSON_VALUE(json_payload, '$."http.route"'), '') - NOT IN UNNEST(@excluded_routes) -QUALIFY ROW_NUMBER() OVER ( - PARTITION BY insert_id, timestamp ORDER BY receive_timestamp -) = 1 -""" - -REQUEST_COUNTS_QUERY = r""" -WITH completed_requests AS ( - SELECT - JSON_VALUE(json_payload, '$."service.name"') AS service_name, - JSON_VALUE(json_payload, '$."deployment.environment.name"') AS environment_name, - JSON_VALUE(json_payload, '$."http.route"') AS route - FROM `_AllLogs` - WHERE timestamp >= @window_start AND timestamp < @window_end - AND JSON_VALUE(json_payload, '$.schema_version') = 'policyengine.observability.v2' - AND JSON_VALUE(json_payload, '$."service.namespace"') = 'policyengine.api-v1' - AND JSON_VALUE(json_payload, '$."event.name"') = 'request.completed' - AND COALESCE(JSON_VALUE(json_payload, '$."http.route"'), '') - NOT IN UNNEST(@excluded_routes) - QUALIFY ROW_NUMBER() OVER ( - PARTITION BY insert_id, timestamp ORDER BY receive_timestamp - ) = 1 -) -SELECT service_name, environment_name, route, COUNT(*) AS requests -FROM completed_requests -GROUP BY service_name, environment_name, route -""" - - -class ReportingWindow(NamedTuple): - """One Eastern calendar day, as the half-open interval [start, end).""" - - start: datetime - end: datetime - - @property - def hours(self) -> float: - """Return the elapsed hours: 23 or 25 on the days the clocks change.""" - return _hours_between(self.start, self.end) - - @property - def lookback_start(self) -> datetime: - """Return midnight Eastern seven days before the window starts.""" - return self.start - LOOKBACK - - @property - def lookback_hours(self) -> float: - """Return the elapsed hours of the seven days before the window.""" - return _hours_between(self.lookback_start, self.start) - - -def _hours_between(start: datetime, end: datetime) -> float: - # Subtracting two datetimes in the same zone ignores a clock change, so - # compare them in UTC. - return (end.astimezone(UTC) - start.astimezone(UTC)) / timedelta(hours=1) - - -def reporting_window(report_date: date) -> ReportingWindow: - """Return the window from midnight Eastern on ``report_date`` to the next.""" - next_date = report_date + timedelta(days=1) - return ReportingWindow( - start=datetime.combine(report_date, MIDNIGHT, tzinfo=REPORTING_TIME_ZONE), - end=datetime.combine(next_date, MIDNIGHT, tzinfo=REPORTING_TIME_ZONE), - ) - - -def default_report_date(now: datetime) -> date: - """Return the most recent Eastern calendar day that ended before ``now``.""" - return now.astimezone(REPORTING_TIME_ZONE).date() - timedelta(days=1) - - -def service_family(service_name: str) -> str: - """Return the service name, merging versioned simulation services into one.""" - return VERSIONED_SIMULATION_SERVICE.sub(r"\1", service_name) - - -def environment_class(environment_name: str | None) -> str: - """Classify a deployment environment name as production, staging or other. - - The Modal gateway reports its production environment as ``main``. - """ - if environment_name in PRODUCTION_ENVIRONMENT_NAMES: - return "production" - if environment_name in STAGING_ENVIRONMENT_NAMES: - return "staging" - return "other" - - -def normalize_path(path: str) -> str: - """Return a source path without its installation or checkout prefix.""" - installed = INSTALLED_PACKAGE_PREFIX.sub("", path, count=1) - if installed != path: - return installed - application = APPLICATION_PATH_IN_ABSOLUTE_PATH.search(path) - if application: - return application.group(1) - standard_library = STANDARD_LIBRARY_PREFIX.sub("", path, count=1) - if standard_library != path: - return standard_library - return "/".join(path.split("/")[-FALLBACK_PATH_COMPONENTS:]) - - -def frame_key(frame: str) -> str | None: - """Return ``path:function`` for one ``FRAME_PATTERN`` match, without its line.""" - parts = FRAME_PARTS.match(frame) - if parts is None: - return None - return f"{normalize_path(parts['path'])}:{parts['function']}" - - -def fingerprint_frames(frames: Iterable[str] | None) -> tuple[str, ...]: - """Choose the frames that identify where an exception was raised. - - ``frames`` come from the first traceback of the stack only (see - ``CHAINED_EXCEPTION_PATTERN``); for a chained exception that is its first - cause. These are the three innermost frames in application packages. - Without application frames they are the three innermost frames of any - kind, and without a stack, a placeholder. - """ - keys = [key for key in map(frame_key, frames or ()) if key is not None] - if not keys: - return (MISSING,) - application_keys = [key for key in keys if APPLICATION_PATH.match(key)] - return tuple((application_keys or keys)[-FINGERPRINT_FRAME_COUNT:]) - - -def _validated(value: str | None, pattern: re.Pattern[str], max_length: int) -> str: - if not value: - return MISSING - if len(value) <= max_length and pattern.fullmatch(value): - return value - return UNRECOGNIZED - - -def public_error_type(value: str | None) -> str: - """Return an exception class name, or a placeholder for anything else.""" - return _validated(value, ERROR_TYPE_PATTERN, MAX_ERROR_TYPE_LENGTH) - - -def public_family(value: str | None) -> str: - """Return a service name or family, or a placeholder for anything else.""" - return _validated(value, FAMILY_PATTERN, MAX_FAMILY_LENGTH) - - -def public_operation(value: str | None) -> str: - """Return a code-defined operation name, or a placeholder for anything else.""" - return _validated(value, OPERATION_PATTERN, MAX_OPERATION_LENGTH) - - -def _route_segment_is_code(segment: str) -> bool: - if ROUTE_PARAMETER.match(segment): - return True - digits = sum(character.isdigit() for character in segment) - return bool(ROUTE_LITERAL.match(segment)) and digits <= MAX_ROUTE_LITERAL_DIGITS - - -def public_route(value: str | None) -> str: - """Return a route template, or a placeholder for anything else. - - A path that holds a record ID, a query string or free text is not a - template, so it never reaches public output. - """ - if not value: - return MISSING - if value == UNMATCHED_ROUTE: - return value - if len(value) > MAX_ROUTE_LENGTH or not value.startswith("/"): - return UNRECOGNIZED - segments = [segment for segment in value.split("/") if segment] - if all(map(_route_segment_is_code, segments)): - return value - return UNRECOGNIZED - - -def public_status(value: object) -> int | None: - """Return an HTTP status code as an integer, or None for anything else.""" - if isinstance(value, bool): - return None - if isinstance(value, str) and value.isascii() and value.isdigit(): - value = int(value) - if isinstance(value, int) and LOWEST_HTTP_STATUS <= value <= HIGHEST_HTTP_STATUS: - return value - return None - - -@dataclass(frozen=True) -class ErrorRecord: - """One ERROR record from the central log bucket, as the query returns it. - - The query returns identifiers only as SHA-256 hashes, stacks only as frame - headers, and messages only as their first few words. - """ - - timestamp: datetime - service_name: str - environment_name: str | None - event_name: str | None - error_type: str | None - error_handled: bool | None - frames: tuple[str, ...] - stack_truncated: bool - message_template: str | None - attribute_error_type: str | None - failure_category: str | None - route: str | None - status_code: int | None - operation_name: str | None - request_key: str | None - observability_key: str | None - trace_key: str | None - - @property - def family(self) -> str: - """Return the service family that produced the record.""" - return service_family(self.service_name) - - @property - def environment(self) -> str: - """Return production, staging or other.""" - return environment_class(self.environment_name) - - @property - def kind(self) -> str: - """Return exception, message or failed_request. - - A record with an exception type is an exception, even a - ``request.completed`` record. Any other ``request.completed`` record is - a request that failed without a recorded exception. - """ - if self.error_type: - return "exception" - if self.event_name == REQUEST_COMPLETED_EVENT: - return "failed_request" - return "message" - - -def _primary_preference(record: ErrorRecord) -> tuple[int, int, datetime]: - """Sort key: exceptions before messages before request completions. - - Among exceptions, an unhandled one (the cause of an error response) comes - first; otherwise the earliest record does. - """ - if record.event_name == REQUEST_COMPLETED_EVENT: - source_rank = 2 - elif record.error_type: - source_rank = 0 - else: - source_rank = 1 - handled_rank = 0 if record.error_handled is False else 1 - return source_rank, handled_rank, record.timestamp - - -def primary_records(records: Iterable[ErrorRecord]) -> list[ErrorRecord]: - """Return one record per failure, in time order. - - Records of one service that share a request ID, or a trace when there is no - request ID, describe one failure. A record with neither is its own failure. - """ - failures: dict[tuple[str, str], list[ErrorRecord]] = {} - standalone: list[ErrorRecord] = [] - for record in records: - correlation = record.request_key or record.trace_key - if correlation is None: - standalone.append(record) - else: - failures.setdefault((record.family, correlation), []).append(record) - chosen = [min(group, key=_primary_preference) for group in failures.values()] - return sorted([*chosen, *standalone], key=lambda record: record.timestamp) - - -def fingerprint_parts(record: ErrorRecord) -> tuple[str, ...]: - """Return the kind and the values that identify the record's error group. - - An exception's route is left out, so one shared failure doesn't open an - issue for every route that reaches it. - """ - kind = record.kind - if kind == "exception": - return ( - kind, - record.family, - record.error_type or MISSING, - *fingerprint_frames(record.frames), - ) - if kind == "message": - return ( - kind, - record.family, - record.route or record.operation_name or MISSING, - record.message_template or MISSING, - record.attribute_error_type or MISSING, - record.failure_category or MISSING, - ) - status = MISSING if record.status_code is None else str(record.status_code) - return (kind, record.family, record.route or MISSING, status) - - -def fingerprint_key(secret: str) -> bytes: - """Return the HMAC key, rejecting a secret too short to be random.""" - if len(secret) < MIN_FINGERPRINT_KEY_LENGTH: - msg = ( - "the fingerprint key must be at least " - f"{MIN_FINGERPRINT_KEY_LENGTH} characters" - ) - raise ValueError(msg) - return secret.encode() - - -def fingerprint(record: ErrorRecord, key: bytes) -> str: - """Return the record's error group as 16 hex characters of HMAC-SHA256. - - The key keeps anyone from confirming a guessed message or stack frame by - hashing it and comparing the result with a public fingerprint. - """ - payload = json.dumps( - [FINGERPRINT_ALGORITHM, *fingerprint_parts(record)], - ensure_ascii=False, - separators=(",", ":"), - ) - digest = hmac.new(key, payload.encode(), hashlib.sha256).hexdigest() - return digest[:FINGERPRINT_LENGTH] - - -class RequestCount(NamedTuple): - """Requests that one service completed on one route during the window.""" - - service_name: str - environment_name: str | None - route: str | None - requests: int - - -class Occurrence(NamedTuple): - """One failure in an error group.""" - - timestamp: datetime - environment: str - route: str | None - affected_key: str - observability_key: str | None - trace_key: str | None - - -@dataclass(frozen=True) -class EnvironmentMeasures: - """An error group's measures in one environment. - - Counts and rates cover the window; first and last seen cover the window - and the seven days before it. - """ - - occurrences: int - affected_requests: int - observability_ids: int - route_affected_requests: int - route_requests: int - error_rate: float | None - hourly_rate: float - previous_hourly_rate: float - first_seen: datetime | None - last_seen: datetime | None - - -@dataclass(frozen=True) -class ErrorGroup: - """Failures that share a fingerprint, described only by validated values.""" - - fingerprint: str - kind: str - family: str - error_type: str - route: str - status_code: int | None - operation: str - production: EnvironmentMeasures - staging: EnvironmentMeasures - related_fingerprints: tuple[str, ...] - occurrences: tuple[Occurrence, ...] - - def production_since(self, moment: datetime) -> tuple[int, int]: - """Return production occurrences and affected requests from ``moment``.""" - recent = [ - occurrence - for occurrence in self.occurrences - if occurrence.environment == "production" and occurrence.timestamp >= moment - ] - return len(recent), len({occurrence.affected_key for occurrence in recent}) - - -def _occurrence(record: ErrorRecord, position: int) -> Occurrence: - """Describe a failure, identifying what it affected as precisely as possible. - - The affected request is the request ID, else the observability ID, else the - trace. A failure with none of them counts as its own request. - """ - if record.request_key: - affected_key = f"request:{record.request_key}" - elif record.observability_key: - affected_key = f"observability:{record.observability_key}" - elif record.trace_key: - affected_key = f"trace:{record.trace_key}" - else: - affected_key = f"record:{position}" - return Occurrence( - timestamp=record.timestamp, - environment=record.environment, - route=record.route, - affected_key=affected_key, - observability_key=record.observability_key, - trace_key=record.trace_key, - ) - - -class RouteRequests: - """Requests completed in the window, per service family, environment and route.""" - - def __init__(self, request_counts: Iterable[RequestCount]) -> None: - """Add up the request counts, merging versioned simulation services.""" - self._totals: Counter[tuple[str, str, str | None]] = Counter() - for count in request_counts: - environment = environment_class(count.environment_name) - family = service_family(count.service_name) - self._totals[family, environment, count.route] += count.requests - - def total(self, family: str, environment: str, routes: Iterable[str]) -> int: - """Return the requests one service family completed on the given routes.""" - return sum(self._totals[family, environment, route] for route in routes) - - -def _related_fingerprints( - occurrences_by_group: dict[str, list[Occurrence]], -) -> dict[str, tuple[str, ...]]: - """For each group, the groups that share the most traces with it.""" - groups_by_trace: dict[str, set[str]] = {} - for group, occurrences in occurrences_by_group.items(): - for occurrence in occurrences: - if occurrence.trace_key: - groups_by_trace.setdefault(occurrence.trace_key, set()).add(group) - related: dict[str, tuple[str, ...]] = {} - for group, occurrences in occurrences_by_group.items(): - shared_traces: Counter[str] = Counter() - traces = {occurrence.trace_key for occurrence in occurrences} - for trace in traces - {None}: - shared_traces.update(groups_by_trace[trace] - {group}) - ranked = sorted(shared_traces, key=lambda other: (-shared_traces[other], other)) - related[group] = tuple(ranked[:MAX_RELATED_FINGERPRINTS]) - return related - - -def _environment_measures( - occurrences: list[Occurrence], - environment: str, - family: str, - route_requests: RouteRequests, - window: ReportingWindow, -) -> EnvironmentMeasures: - in_environment = [item for item in occurrences if item.environment == environment] - in_window = [item for item in in_environment if item.timestamp >= window.start] - before_window = len(in_environment) - len(in_window) - affected_requests = len({item.affected_key for item in in_window}) - # The error rate compares failed requests with completed requests on the - # same routes, so failures without a route, such as background - # operations, are left out of both. - on_routes = [item for item in in_window if item.route] - route_affected_requests = len({item.affected_key for item in on_routes}) - routes = {item.route for item in on_routes} - requests = route_requests.total(family, environment, routes) - timestamps = [item.timestamp for item in in_environment] - return EnvironmentMeasures( - occurrences=len(in_window), - affected_requests=affected_requests, - observability_ids=len( - {item.observability_key for item in in_window if item.observability_key} - ), - route_affected_requests=route_affected_requests, - route_requests=requests, - error_rate=route_affected_requests / requests if requests else None, - hourly_rate=len(in_window) / window.hours, - previous_hourly_rate=before_window / window.lookback_hours, - first_seen=min(timestamps, default=None), - last_seen=max(timestamps, default=None), - ) - - -def _most_frequent_route(occurrences: list[Occurrence], window: ReportingWindow) -> str: - """Return the route most failures in the window used, preferring production.""" - in_window = [item for item in occurrences if item.timestamp >= window.start] - production = [item for item in in_window if item.environment == "production"] - routes = Counter(item.route for item in production or in_window) - route, _ = min(routes.items(), key=lambda item: (-item[1], item[0] or "")) - return public_route(route) - - -def _error_group( - group_fingerprint: str, - members: list[tuple[ErrorRecord, Occurrence]], - related_fingerprints: tuple[str, ...], - route_requests: RouteRequests, - window: ReportingWindow, -) -> ErrorGroup: - example = members[0][0] - occurrences = [occurrence for _, occurrence in members] - measures = { - environment: _environment_measures( - occurrences, environment, example.family, route_requests, window - ) - for environment in REPORTED_ENVIRONMENTS - } - return ErrorGroup( - fingerprint=group_fingerprint, - kind=example.kind, - family=public_family(example.family), - error_type=public_error_type(example.error_type), - route=_most_frequent_route(occurrences, window), - status_code=public_status(example.status_code), - operation=public_operation(example.operation_name), - production=measures["production"], - staging=measures["staging"], - related_fingerprints=related_fingerprints, - occurrences=tuple(occurrences), - ) - - -def build_groups( - records: Iterable[ErrorRecord], - request_counts: Iterable[RequestCount], - window: ReportingWindow, - key: bytes, -) -> list[ErrorGroup]: - """Group the failures in production and staging and measure each group. - - Only groups with a failure in the window are returned, most production - occurrences first. - """ - in_scope = [ - record - for record in records - if record.environment in REPORTED_ENVIRONMENTS - and window.lookback_start <= record.timestamp < window.end - ] - members_by_group: dict[str, list[tuple[ErrorRecord, Occurrence]]] = {} - for position, record in enumerate(primary_records(in_scope)): - members_by_group.setdefault(fingerprint(record, key), []).append( - (record, _occurrence(record, position)) - ) - related = _related_fingerprints( - { - group: [occurrence for _, occurrence in members] - for group, members in members_by_group.items() - } - ) - route_requests = RouteRequests(request_counts) - groups = [ - _error_group(group, members, related[group], route_requests, window) - for group, members in members_by_group.items() - if any(occurrence.timestamp >= window.start for _, occurrence in members) - ] - return sorted( - groups, - key=lambda group: ( - -group.production.occurrences, - -group.staging.occurrences, - group.fingerprint, - ), - ) - - -class HttpRequest(NamedTuple): - """An HTTP request for a transport to send.""" - - method: str - url: str - headers: dict[str, str] - body: bytes | None - - -class HttpResponse(NamedTuple): - """An HTTP response, including error responses. Header names are lowercase.""" - - status: int - reason: str - headers: Mapping[str, str] - body: bytes - - -def urllib_transport(request: HttpRequest) -> HttpResponse: - """Send a request with the standard library, returning HTTP errors too.""" - outgoing = Request( - request.url, - data=request.body, - headers=request.headers, - method=request.method, - ) - try: - response = urlopen(outgoing, timeout=HTTP_TIMEOUT_SECONDS) - except HTTPError as error: - with error: - return HttpResponse( - status=error.code, - reason=str(error.reason), - headers={name.lower(): value for name, value in error.headers.items()}, - body=error.read(), - ) - with response: - return HttpResponse( - status=response.status, - reason=response.reason, - headers={name.lower(): value for name, value in response.headers.items()}, - body=response.read(), - ) - - -class ApiError(Exception): - """An API request failed. The text holds only code-defined values.""" - - service = "API" - - def __init__( - self, message: str, *, status: int | None = None, reason: str | None = None - ) -> None: - """Keep the HTTP status and reason for the caller's summary.""" - super().__init__(message) - self.status = status - self.reason = reason - - @classmethod - def from_response(cls, response: HttpResponse) -> ApiError: - """Describe an error response by its status and reason, never its body.""" - reason = _error_reason(response) - message = f"{cls.service} request failed: HTTP {response.status} {reason}" - return cls(message, status=response.status, reason=reason) - - -class BigQueryError(ApiError): - """A BigQuery request failed.""" - - service = "BigQuery" - - -class BigQueryTimeoutError(BigQueryError): - """A BigQuery query job didn't complete within the polling limit.""" - - -def _error_reason(response: HttpResponse) -> str: - """Return a Google API error's reason code, else the HTTP reason phrase.""" - try: - payload = json.loads(response.body) - reason = payload["error"]["errors"][0]["reason"] - except (ValueError, KeyError, IndexError, TypeError): - reason = None - for candidate in (reason, response.reason): - if isinstance(candidate, str) and ERROR_REASON_PATTERN.match(candidate): - return candidate - return "unknown" - - -def _bigquery_boolean(text: str) -> bool: - return text == "true" - - -def _bigquery_timestamp(text: str) -> datetime: - """Convert microseconds since the epoch, as ``useInt64Timestamp`` returns.""" - return UNIX_EPOCH + timedelta(microseconds=int(text)) - - -CELL_CONVERTERS: dict[str, Callable[[str], object]] = { - "INTEGER": int, - "INT64": int, - "FLOAT": float, - "FLOAT64": float, - "BOOLEAN": _bigquery_boolean, - "BOOL": _bigquery_boolean, - "TIMESTAMP": _bigquery_timestamp, -} - - -def _convert_cell(field: Mapping[str, Any], value: object) -> object: - """Convert one BigQuery REST value, which arrives as a string, to Python.""" - if value is None: - return None - if field.get("mode") == "REPEATED" and isinstance(value, list): - item_field = {**field, "mode": "NULLABLE"} - return [_convert_cell(item_field, item["v"]) for item in value] - converter = CELL_CONVERTERS.get(str(field.get("type")), str) - return converter(str(value)) - - -def _convert_rows(response: Mapping[str, Any]) -> list[dict[str, object]]: - fields = response["schema"]["fields"] - return [ - { - field["name"]: _convert_cell(field, cell["v"]) - for field, cell in zip(fields, row["f"], strict=True) - } - for row in response.get("rows", []) - ] - - -class BigQueryClient: - """Runs parameterized standard SQL queries through the BigQuery REST API.""" - - def __init__( - self, - project_id: str, - access_token: str, - transport: Callable[[HttpRequest], HttpResponse] = urllib_transport, - ) -> None: - """Run jobs in ``project_id`` against its linked log dataset.""" - self.project_id = project_id - self._access_token = access_token - self._transport = transport - self._queries_url = ( - f"{BIGQUERY_API}/projects/{quote(project_id, safe='')}/queries" - ) - - def query( - self, sql: str, parameters: list[dict[str, Any]], label: str - ) -> list[dict[str, object]]: - """Run ``sql`` and return every row as a dictionary of converted values.""" - body = { - "query": sql, - "useLegacySql": False, - "parameterMode": "NAMED", - "queryParameters": parameters, - "defaultDataset": { - "projectId": self.project_id, - "datasetId": LINKED_DATASET, - }, - "maximumBytesBilled": str(MAXIMUM_BYTES_BILLED), - "labels": {"purpose": "telemetry-error-issues", "query": label}, - "timeoutMs": QUERY_WAIT_MILLISECONDS, - "formatOptions": {"useInt64Timestamp": True}, - } - response = self._send("POST", self._queries_url, body) - job = response["jobReference"] - for _ in range(MAX_QUERY_POLLS): - if response.get("jobComplete"): - break - response = self._results(job, page_token=None) - else: - msg = "BigQuery query did not complete in time" - raise BigQueryTimeoutError(msg) - rows = _convert_rows(response) - while response.get("pageToken"): - response = self._results(job, page_token=response["pageToken"]) - rows.extend(_convert_rows(response)) - return rows - - def _results( - self, job: Mapping[str, str], page_token: str | None - ) -> dict[str, Any]: - parameters = { - "location": job["location"], - "timeoutMs": str(QUERY_WAIT_MILLISECONDS), - "formatOptions.useInt64Timestamp": "true", - } - if page_token: - parameters["pageToken"] = page_token - job_id = quote(job["jobId"], safe="") - url = f"{self._queries_url}/{job_id}?{urlencode(parameters)}" - return self._send("GET", url, None) - - def _send( - self, method: str, url: str, body: Mapping[str, object] | None - ) -> dict[str, Any]: - headers = {"Authorization": f"Bearer {self._access_token}"} - if body is not None: - headers["Content-Type"] = "application/json" - encoded = None if body is None else json.dumps(body).encode() - response = self._transport(HttpRequest(method, url, headers, encoded)) - if response.status >= HTTPStatus.BAD_REQUEST: - raise BigQueryError.from_response(response) - return json.loads(response.body) - - -def _timestamp_parameter(name: str, moment: datetime) -> dict[str, Any]: - return { - "name": name, - "parameterType": {"type": "TIMESTAMP"}, - "parameterValue": {"value": moment.astimezone(UTC).isoformat(sep=" ")}, - } - - -def _query_parameters( - timestamps: Mapping[str, datetime], strings: Mapping[str, str] -) -> list[dict[str, Any]]: - parameters = [ - _timestamp_parameter(name, value) for name, value in timestamps.items() - ] - parameters.extend( - { - "name": name, - "parameterType": {"type": "STRING"}, - "parameterValue": {"value": value}, - } - for name, value in strings.items() - ) - parameters.append( - { - "name": "excluded_routes", - "parameterType": {"type": "ARRAY", "arrayType": {"type": "STRING"}}, - "parameterValue": { - "arrayValues": [{"value": route} for route in HEALTH_CHECK_ROUTES] - }, - } - ) - return parameters - - -def _text(value: object) -> str | None: - return None if value is None or value == "" else str(value) - - -def _integer(value: object) -> int | None: - if isinstance(value, int) and not isinstance(value, bool): - return value - if isinstance(value, str) and value.isascii() and value.isdigit(): - return int(value) - return None - - -def _boolean(value: object) -> bool | None: - if isinstance(value, bool): - return value - return {"true": True, "false": False}.get(str(value)) - - -def error_record_from_row(row: Mapping[str, object]) -> ErrorRecord: - """Build an ``ErrorRecord`` from one row of ``ERROR_RECORDS_QUERY``.""" - timestamp = row["timestamp"] - if not isinstance(timestamp, datetime): - msg = "an error record has no timestamp" - raise TypeError(msg) - frames = row["frames"] - return ErrorRecord( - timestamp=timestamp, - service_name=_text(row["service_name"]) or MISSING, - environment_name=_text(row["environment_name"]), - event_name=_text(row["event_name"]), - error_type=_text(row["error_type"]), - error_handled=_boolean(row["error_handled"]), - frames=tuple(map(str, frames)) if isinstance(frames, list) else (), - stack_truncated=row["stack_truncated"] is True, - message_template=_text(row["message_template"]), - attribute_error_type=_text(row["attribute_error_type"]), - failure_category=_text(row["failure_category"]), - route=_text(row["route"]), - status_code=_integer(row["status_code"]), - operation_name=_text(row["operation_name"]), - request_key=_text(row["request_key"]), - observability_key=_text(row["observability_key"]), - trace_key=_text(row["trace_key"]), - ) - - -def fetch_error_records( - client: BigQueryClient, window: ReportingWindow -) -> list[ErrorRecord]: - """Return the ERROR records of the window and the seven days before it.""" - parameters = _query_parameters( - {"query_start": window.lookback_start, "window_end": window.end}, - { - "frame_pattern": FRAME_PATTERN, - "chained_exception_pattern": CHAINED_EXCEPTION_PATTERN, - "double_quoted_pattern": DOUBLE_QUOTED_PATTERN, - "single_quoted_pattern": SINGLE_QUOTED_PATTERN, - "template_word_pattern": TEMPLATE_WORD_PATTERN, - }, - ) - rows = client.query(ERROR_RECORDS_QUERY, parameters, "error-records") - return [error_record_from_row(row) for row in rows] - - -def fetch_request_counts( - client: BigQueryClient, window: ReportingWindow -) -> list[RequestCount]: - """Return the requests each service completed per route in the window.""" - parameters = _query_parameters( - {"window_start": window.start, "window_end": window.end}, {} - ) - rows = client.query(REQUEST_COUNTS_QUERY, parameters, "request-counts") - return [ - RequestCount( - service_name=_text(row["service_name"]) or MISSING, - environment_name=_text(row["environment_name"]), - route=_text(row["route"]), - requests=_integer(row["requests"]) or 0, - ) - for row in rows - ] - - -def _code(value: str) -> str: - return f"`{value}`" - - -def _place(group: ErrorGroup) -> str: - """Describe where the group fails: its route, else its operation, else nothing.""" - if group.route != MISSING: - return f" on {_code(group.route)}" - if group.kind == "message" and group.operation != MISSING: - return f" in {_code(group.operation)}" - return "" - - -def _status_text(group: ErrorGroup) -> str: - return "HTTP error" if group.status_code is None else f"HTTP {group.status_code}" - - -def _summary(group: ErrorGroup) -> str: - if group.kind == "exception": - return group.error_type - if group.kind == "message": - return "logged error" - return f"{_status_text(group)} without recorded exception" - - -def issue_title(group: ErrorGroup) -> str: - """Return the issue title, which names the fingerprint for matching. - - The route is left out when the title would exceed GitHub's length limit. - """ - lead = f"{ISSUE_TITLE_PREFIX} {group.family}: {_summary(group)}" - suffix = f" ({group.fingerprint})" - title = f"{lead}{_place(group)}{suffix}" - return title if len(title) <= MAX_TITLE_LENGTH else f"{lead}{suffix}" - - -def _logs_timestamp(moment: datetime) -> str: - return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def _logs_filters(group: ErrorGroup) -> list[str]: - """Return logging query clauses that select the group's records.""" - filters = [f'jsonPayload.schema_version="{SCHEMA_VERSION}"'] - if group.family == SIMULATION_FAMILY: - filters.append( - f'jsonPayload."service.name"=~"^{SIMULATION_FAMILY}[0-9]+-[0-9]+-[0-9]+$"' - ) - elif group.family not in {MISSING, UNRECOGNIZED}: - filters.append(f'jsonPayload."service.name"="{group.family}"') - if group.kind == "failed_request": - filters.append('jsonPayload."event.name"="request.completed"') - else: - filters.append("severity>=ERROR") - if group.kind == "exception" and group.error_type not in {MISSING, UNRECOGNIZED}: - filters.append(f'jsonPayload."error.type"="{group.error_type}"') - if group.kind != "exception" and group.route not in {MISSING, UNRECOGNIZED}: - filters.append(f'jsonPayload."http.route"="{group.route}"') - elif group.kind == "message" and group.operation not in {MISSING, UNRECOGNIZED}: - filters.append(f'jsonPayload."operation.name"="{group.operation}"') - if group.kind == "failed_request" and group.status_code is not None: - filters.append(f'jsonPayload."http.response.status_code"={group.status_code}') - return filters - - -def logs_explorer_url( - group: ErrorGroup, window: ReportingWindow, project_id: str -) -> str: - """Return a Logs Explorer link to the group's records in the window.""" - query = "\n".join( - [ - *_logs_filters(group), - f'timestamp>="{_logs_timestamp(window.start)}"', - f'timestamp<"{_logs_timestamp(window.end)}"', - ] - ) - view = ( - f"projects/{project_id}/locations/global/buckets/{LOG_BUCKET}/views/{LOG_VIEW}" - ) - settings = {"query": query, "storageScope": f"storage,{view}"} - encoded = "".join( - f";{name}={quote(value, safe='')}" for name, value in settings.items() - ) - return f"{LOGS_EXPLORER_URL}{encoded}?{urlencode({'project': project_id})}" - - -def _eastern_time(moment: datetime | None) -> str: - if moment is None: - return NOT_AVAILABLE - return moment.astimezone(REPORTING_TIME_ZONE).strftime("%Y-%m-%d %H:%M %Z") - - -def _window_text(window: ReportingWindow) -> str: - return ( - f"{_eastern_time(window.start)} to {_eastern_time(window.end)} " - f"({window.hours:.0f} hours)" - ) - - -def _error_rate(measures: EnvironmentMeasures) -> str: - if measures.error_rate is None: - return NOT_AVAILABLE - return ( - f"{measures.error_rate:.2%} ({measures.route_affected_requests:,} of " - f"{measures.route_requests:,} requests)" - ) - - -def _measures_table(group: ErrorGroup) -> list[str]: - rows = [ - ("Occurrences", lambda measures: f"{measures.occurrences:,}"), - ("Affected requests", lambda measures: f"{measures.affected_requests:,}"), - ("Observability IDs", lambda measures: f"{measures.observability_ids:,}"), - ("Error rate on affected routes", _error_rate), - ("Occurrences per hour", lambda measures: f"{measures.hourly_rate:.2f}"), - ( - "Occurrences per hour, previous 7 days", - lambda measures: f"{measures.previous_hourly_rate:.2f}", - ), - ( - "First seen in the last 8 days", - lambda measures: _eastern_time(measures.first_seen), - ), - ("Last seen", lambda measures: _eastern_time(measures.last_seen)), - ] - return [ - "| Measure | Production | Staging |", - "| --- | --- | --- |", - *( - f"| {name} | {describe(group.production)} | {describe(group.staging)} |" - for name, describe in rows - ), - ] - - -def _group_details(group: ErrorGroup) -> list[str]: - details = [ - f"- Kind: {group.kind} (fingerprint algorithm {FINGERPRINT_ALGORITHM})", - f"- Service: {_code(group.family)}", - ] - if group.kind == "exception": - details.append(f"- Exception type: {_code(group.error_type)}") - if group.kind == "failed_request": - details.append(f"- Response: {_status_text(group)}") - if group.kind == "message" and group.operation != MISSING: - details.append(f"- Operation: {_code(group.operation)}") - details.append(f"- Most frequent route: {_code(group.route)}") - return details - - -def _state_comment(group: ErrorGroup, window: ReportingWindow) -> str: - state = { - "algorithm": FINGERPRINT_ALGORITHM, - "fingerprint": group.fingerprint, - "kind": group.kind, - "window_end": window.end.astimezone(UTC).isoformat(), - } - return f"" - - -def parse_state(body: str) -> dict[str, Any] | None: - """Return the state the job last wrote into an issue body, if it is there.""" - match = STATE_PATTERN.search(body) - if match is None: - return None - try: - state = json.loads(match.group(1)) - except ValueError: - return None - return state if isinstance(state, dict) else None - - -def issue_section(group: ErrorGroup, window: ReportingWindow, project_id: str) -> str: - """Return the marked section of the issue body, which each run replaces.""" - related = ", ".join(map(_code, group.related_fingerprints)) - lines = [ - SECTION_START, - f"### Error group {_code(group.fingerprint)}", - "", - *_group_details(group), - f"- Window: {_window_text(window)}", - "", - *_measures_table(group), - "", - ( - f"Related fingerprints (sharing traces): {related}" - if related - else "Related fingerprints: none" - ), - "", - ( - f"[Matching log records in Logs Explorer]" - f"({logs_explorer_url(group, window, project_id)}), for the window only. " - f"It needs read access to the `{project_id}` project." - ), - "", - _state_comment(group, window), - SECTION_END, - ] - return "\n".join(lines) - - -def issue_body(group: ErrorGroup, window: ReportingWindow, project_id: str) -> str: - """Return a new issue's body: a short note, then the marked section.""" - note = ( - "The daily telemetry error report opened this issue for a production " - "error group. Each run replaces the marked section below with the " - "latest day's measures; text outside it is kept." - ) - return f"{note}\n\n{issue_section(group, window, project_id)}\n" - - -class GitHubError(ApiError): - """A GitHub request failed.""" - - service = "GitHub" - - -class GitHubLabelError(GitHubError): - """GitHub didn't apply the labels the job asked for.""" - - -def _seconds_until_reset(response: HttpResponse) -> int | None: - """Return the seconds from the response's ``date`` to ``x-ratelimit-reset``.""" - reset = response.headers.get("x-ratelimit-reset", "") - sent = response.headers.get("date") - if not (reset.isascii() and reset.isdigit()) or not sent: - return None - try: - sent_at = parsedate_to_datetime(sent) - except (TypeError, ValueError): - return None - return max(int(reset) - int(sent_at.timestamp()), 0) - - -def _rate_limit_wait(response: HttpResponse) -> int | None: - """Return the seconds to wait before retrying a rate-limited request. - - Follows GitHub's guidance: wait for ``retry-after``; when no requests - remain, wait until ``x-ratelimit-reset``; otherwise wait a minute. A 403 - whose headers and message don't mention a rate limit is a permission - failure, so it isn't retried. The message is only searched, never printed. - """ - if response.status not in RATE_LIMIT_STATUSES: - return None - retry_after = response.headers.get("retry-after", "") - if retry_after.isascii() and retry_after.isdigit(): - seconds = int(retry_after) - elif response.headers.get("x-ratelimit-remaining") == "0": - until_reset = _seconds_until_reset(response) - seconds = ( - DEFAULT_RATE_LIMIT_WAIT_SECONDS if until_reset is None else until_reset - ) - elif ( - response.status == HTTPStatus.TOO_MANY_REQUESTS - or b"rate limit" in response.body.lower() - ): - seconds = DEFAULT_RATE_LIMIT_WAIT_SECONDS - else: - return None - return seconds if seconds <= MAX_RATE_LIMIT_WAIT_SECONDS else None - - -def _json_object(response: HttpResponse) -> dict[str, Any]: - payload = json.loads(response.body) - if not isinstance(payload, dict): - msg = "GitHub returned an unexpected response" - raise GitHubError(msg) - return payload - - -def _json_list(response: HttpResponse) -> list[dict[str, Any]]: - payload = json.loads(response.body) - if not isinstance(payload, list): - msg = "GitHub returned an unexpected response" - raise GitHubError(msg) - return payload - - -class GitHubClient: - """Reads and writes one repository's issues and labels. - - Writes go one at a time, at least a second apart. A rate-limited request is - retried after the wait GitHub asks for, up to three times. - """ - - def __init__( - self, - repository: str, - credential: str, - transport: Callable[[HttpRequest], HttpResponse] = urllib_transport, - *, - sleep: Callable[[float], None] = time.sleep, - clock: Callable[[], float] = time.monotonic, - ) -> None: - """Send requests for ``repository`` (``owner/name``) with ``credential``.""" - self._repository_url = f"{GITHUB_API}/repos/{repository}" - self._headers = { - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {credential}", - "User-Agent": "policyengine-telemetry-error-issues", - "X-GitHub-Api-Version": GITHUB_API_VERSION, - } - self._transport = transport - self._sleep = sleep - self._clock = clock - self._last_write: float | None = None - - def list_issues(self, creator: str) -> list[dict[str, Any]]: - """Return the open and closed issues ``creator`` opened, without PRs.""" - issues = self._pages("/issues", {"state": "all", "creator": creator}) - return [issue for issue in issues if "pull_request" not in issue] - - def list_labels(self) -> set[str]: - """Return the names of the repository's labels.""" - return {label["name"] for label in self._pages("/labels", {})} - - def create_label(self, name: str, color: str, description: str) -> None: - """Create a label.""" - self._send( - "POST", - "/labels", - {"name": name, "color": color, "description": description}, - ) - - def create_issue(self, title: str, body: str, labels: list[str]) -> dict[str, Any]: - """Open an issue and return it as GitHub describes it.""" - payload = {"title": title, "body": body, "labels": labels} - return _json_object(self._send("POST", "/issues", payload)) - - def update_issue(self, number: int, fields: Mapping[str, object]) -> None: - """Change an issue's fields, such as its body or state.""" - self._send("PATCH", f"/issues/{number}", fields) - - def add_labels(self, number: int, labels: list[str]) -> set[str]: - """Add labels to an issue and return all the labels it now has.""" - response = self._send("POST", f"/issues/{number}/labels", {"labels": labels}) - return {label["name"] for label in _json_list(response)} - - def add_comment(self, number: int, body: str) -> None: - """Comment on an issue.""" - self._send("POST", f"/issues/{number}/comments", {"body": body}) - - def _pages(self, path: str, parameters: Mapping[str, str]) -> list[dict[str, Any]]: - items: list[dict[str, Any]] = [] - page = 1 - while True: - query = urlencode( - {**parameters, "per_page": GITHUB_PAGE_SIZE, "page": page} - ) - batch = _json_list(self._send("GET", f"{path}?{query}", None)) - items.extend(batch) - if len(batch) < GITHUB_PAGE_SIZE: - return items - page += 1 - - def _wait_between_writes(self) -> None: - if self._last_write is None: - return - elapsed = self._clock() - self._last_write - if elapsed < MIN_SECONDS_BETWEEN_WRITES: - self._sleep(MIN_SECONDS_BETWEEN_WRITES - elapsed) - - def _send( - self, method: str, path: str, body: Mapping[str, object] | None - ) -> HttpResponse: - headers = dict(self._headers) - encoded = None - if body is not None: - headers["Content-Type"] = "application/json" - encoded = json.dumps(body).encode() - request = HttpRequest(method, f"{self._repository_url}{path}", headers, encoded) - is_write = method != "GET" - for retry in range(MAX_RATE_LIMIT_RETRIES + 1): - if is_write: - self._wait_between_writes() - response = self._transport(request) - if is_write: - self._last_write = self._clock() - wait = _rate_limit_wait(response) - if wait is None or retry == MAX_RATE_LIMIT_RETRIES: - break - self._sleep(wait) - if response.status >= HTTPStatus.BAD_REQUEST: - raise GitHubError.from_response(response) - return response - - -class SyncAction(NamedTuple): - """What one run did, or in a dry run would do, for one error group.""" - - fingerprint: str - kind: str - family: str - production_occurrences: int - staging_occurrences: int - issue_number: int | None - action: str - - -class SyncResult(NamedTuple): - """Every group's action, and how many new issues waited for a later run.""" - - actions: list[SyncAction] - deferred: int - - -def replace_section(body: str, section: str) -> str: - """Replace the marked section of ``body``, or add it at the end if missing.""" - start = body.find(SECTION_START) - end = body.find(SECTION_END, start) if start >= 0 else -1 - if end >= 0: - return f"{body[:start]}{section}{body[end + len(SECTION_END) :]}" - kept = body.rstrip() - return f"{kept}\n\n{section}\n" if kept else f"{section}\n" - - -def _issue_fingerprint(issue: Mapping[str, Any]) -> str | None: - """Return the fingerprint from the state comment, or else from the title.""" - state = parse_state(issue.get("body") or "") - if state and state.get("algorithm") == FINGERPRINT_ALGORITHM: - fingerprint_value = state.get("fingerprint") - if isinstance(fingerprint_value, str): - return fingerprint_value - title = issue.get("title") or "" - match = TITLE_FINGERPRINT.search(title) - if match and title.startswith(ISSUE_TITLE_PREFIX): - return match.group(1) - return None - - -def _issues_by_fingerprint( - issues: Iterable[Mapping[str, Any]], -) -> dict[str, Mapping[str, Any]]: - """Index the bot's issues by fingerprint, preferring open, then newest.""" - index: dict[str, Mapping[str, Any]] = {} - for issue in issues: - if (issue.get("user") or {}).get("login") != BOT_LOGIN: - continue - group = _issue_fingerprint(issue) - if group is None: - continue - rank = (issue.get("state") == "open", issue.get("number", 0)) - current = index.get(group) - if current is None or rank > ( - current.get("state") == "open", - current["number"], - ): - index[group] = issue - return index - - -def _meets_thresholds(occurrences: int, affected_requests: int) -> bool: - return ( - occurrences >= MIN_PRODUCTION_OCCURRENCES - and affected_requests >= MIN_PRODUCTION_REQUESTS - ) - - -def _folded(names: Iterable[str]) -> set[str]: - """Return label names case-folded, as GitHub compares them.""" - return {name.casefold() for name in names} - - -def _issue_labels(group: ErrorGroup) -> list[str]: - service_label = f"{SERVICE_LABEL_PREFIX}{group.family}" - if len(service_label) > MAX_LABEL_LENGTH: - return [BASE_LABEL] - return [BASE_LABEL, service_label] - - -def _parse_time(value: object) -> datetime | None: - if not isinstance(value, str): - return None - try: - parsed = datetime.fromisoformat(value) - except ValueError: - return None - return parsed if parsed.tzinfo is not None else None - - -class IssueSynchronizer: - """Creates and updates one issue per production error group.""" - - def __init__( - self, - github: GitHubClient, - window: ReportingWindow, - project_id: str, - *, - dry_run: bool, - ) -> None: - """Sync the groups of ``window``; a dry run reads but never writes.""" - self._github = github - self._window = window - self._project_id = project_id - self._dry_run = dry_run - # Case-folded names: GitHub label names are unique regardless of case. - self._known_labels: set[str] | None = None - - def run(self, groups: Iterable[ErrorGroup]) -> SyncResult: - """Sync each group, opening at most five issues, largest groups first.""" - issues = _issues_by_fingerprint(self._github.list_issues(BOT_LOGIN)) - actions: list[SyncAction] = [] - opened = 0 - for group in groups: - issue = issues.get(group.fingerprint) - number: int | None = None - if issue is not None: - number = issue["number"] - action = self._update(group, issue) - elif not _meets_thresholds( - group.production.occurrences, group.production.affected_requests - ): - action = "below-threshold" - elif opened >= MAX_NEW_ISSUES_PER_RUN: - action = "deferred" - else: - number = self._open(group) - opened += 1 - action = "created" - actions.append( - SyncAction( - fingerprint=group.fingerprint, - kind=group.kind, - family=group.family, - production_occurrences=group.production.occurrences, - staging_occurrences=group.staging.occurrences, - issue_number=number, - action=action, - ) - ) - deferred = sum(action.action == "deferred" for action in actions) - return SyncResult(actions, deferred) - - def _ensure_labels(self, names: Iterable[str]) -> None: - if self._known_labels is None: - self._known_labels = _folded(self._github.list_labels()) - for name in names: - if name.casefold() not in self._known_labels: - color, description = LABEL_DETAILS.get(name, SERVICE_LABEL_DETAILS) - self._github.create_label(name, color, description) - self._known_labels.add(name.casefold()) - - def _open(self, group: ErrorGroup) -> int | None: - if self._dry_run: - return None - labels = _issue_labels(group) - self._ensure_labels(labels) - issue = self._github.create_issue( - issue_title(group), - issue_body(group, self._window, self._project_id), - labels, - ) - number = issue["number"] - applied = _folded(label["name"] for label in issue.get("labels", [])) - missing = [label for label in labels if label.casefold() not in applied] - if missing and not _folded(missing) <= _folded( - self._github.add_labels(number, missing) - ): - msg = f"GitHub did not apply the labels to issue #{number}" - raise GitHubLabelError(msg) - return number - - def _reopen_since(self, issue: Mapping[str, Any]) -> datetime | None: - """Return when regressions count from, if the issue should reopen.""" - if issue.get("state") != "closed": - return None - if issue.get("state_reason") not in REOPENABLE_CLOSE_REASONS: - return None - closed_at = _parse_time(issue.get("closed_at")) - return None if closed_at is None else closed_at + REOPEN_DELAY - - def _update(self, group: ErrorGroup, issue: Mapping[str, Any]) -> str: - body = issue.get("body") or "" - state = parse_state(body) - reported_end = _parse_time(state.get("window_end")) if state else None - if reported_end is not None and reported_end > self._window.end: - return "kept-newer" - new_body = replace_section( - body, issue_section(group, self._window, self._project_id) - ) - since = self._reopen_since(issue) - if since is not None and _meets_thresholds(*group.production_since(since)): - if not self._dry_run: - self._reopen(issue["number"], new_body, group, since) - return "reopened" - if new_body == body: - return "unchanged" - if not self._dry_run: - self._github.update_issue(issue["number"], {"body": new_body}) - return "updated" - - def _reopen( - self, number: int, body: str, group: ErrorGroup, since: datetime - ) -> None: - occurrences, affected_requests = group.production_since(since) - self._ensure_labels([REGRESSION_LABEL]) - self._github.update_issue( - number, {"body": body, "state": "open", "state_reason": "reopened"} - ) - self._github.add_labels(number, [REGRESSION_LABEL]) - self._github.add_comment( - number, - "Production failures returned after this issue was closed: " - f"{occurrences:,} occurrences in {affected_requests:,} requests since " - f"{_eastern_time(since)}, 24 hours after it was closed.", - ) - - -def sync_issues( - groups: Iterable[ErrorGroup], - window: ReportingWindow, - project_id: str, - github: GitHubClient, - *, - dry_run: bool, -) -> SyncResult: - """Create or update the issue for each group; a dry run makes no changes.""" - synchronizer = IssueSynchronizer(github, window, project_id, dry_run=dry_run) - return synchronizer.run(groups) - - -def _report_date_argument(value: str) -> date | None: - if not value: - return None - try: - return date.fromisoformat(value) - except ValueError as error: - msg = "expected an Eastern calendar date as YYYY-MM-DD" - raise argparse.ArgumentTypeError(msg) from error - - -def parse_arguments(argv: list[str] | None = None) -> argparse.Namespace: - """Parse the command line; argparse exits with code 2 on invalid input.""" - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument( - "--project-id", - required=True, - help="Google Cloud project that holds the linked BigQuery dataset", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="query and group errors, but make no changes to GitHub issues", - ) - parser.add_argument( - "--report-date", - type=_report_date_argument, - default=None, - help="Eastern calendar day to report (default: the previous day)", - ) - return parser.parse_args(argv) - - -class ConfigurationError(Exception): - """The run's settings are missing or invalid. The text names only settings.""" - - -class StageFailedError(Exception): - """A stage of the run failed. - - The text holds only the stage, the exception class, and for an API error - the HTTP status and reason, so it can be printed in a public log. - """ - - -def _current_time() -> datetime: - return datetime.now(UTC) - - -@dataclass(frozen=True) -class Runtime: - """What a run reads from outside the process, so tests can replace it.""" - - environ: Mapping[str, str] - stdout: TextIO - stderr: TextIO - bigquery_transport: Callable[[HttpRequest], HttpResponse] = urllib_transport - github_transport: Callable[[HttpRequest], HttpResponse] = urllib_transport - now: Callable[[], datetime] = _current_time - sleep: Callable[[float], None] = time.sleep - clock: Callable[[], float] = time.monotonic - - @classmethod - def from_process(cls) -> Runtime: - """Use the process environment, standard streams and the network.""" - return cls(environ=os.environ, stdout=sys.stdout, stderr=sys.stderr) - - -class Settings(NamedTuple): - """Settings read from the environment; none of them is ever printed.""" - - google_access_token: str - github_credential: str - repository: str - fingerprint_key: bytes - summary_path: str | None - - -def read_settings(environ: Mapping[str, str]) -> Settings: - """Read and check the environment variables the run needs.""" - missing = [name for name in REQUIRED_ENVIRONMENT_VARIABLES if not environ.get(name)] - if missing: - msg = f"missing environment variables: {', '.join(missing)}" - raise ConfigurationError(msg) - if not REPOSITORY_PATTERN.match(environ["GITHUB_REPOSITORY"]): - msg = "GITHUB_REPOSITORY must be owner/name" - raise ConfigurationError(msg) - try: - key = fingerprint_key(environ["TELEMETRY_ERROR_FINGERPRINT_KEY"]) - except ValueError as error: - raise ConfigurationError(str(error)) from None - return Settings( - google_access_token=environ["GOOGLE_OAUTH_ACCESS_TOKEN"], - github_credential=environ["GITHUB_TOKEN"], - repository=environ["GITHUB_REPOSITORY"], - fingerprint_key=key, - summary_path=environ.get("GITHUB_STEP_SUMMARY") or None, - ) - - -def _report_window(report_date: date | None, now: datetime) -> ReportingWindow: - window = reporting_window(report_date or default_report_date(now)) - if window.end > now: - msg = f"{window.start.date().isoformat()} has not ended in US Eastern time" - raise ConfigurationError(msg) - return window - - -def _failure_description(stage: str, error: Exception) -> str: - detail = "" - if isinstance(error, ConfigurationError): - detail = f" ({error})" - elif isinstance(error, ApiError) and error.status is not None: - detail = f" (HTTP {error.status} {error.reason})" - return f"{stage} failed: {type(error).__name__}{detail}" - - -def _stage(name: str, operation: Callable[[], StageResult]) -> StageResult: - """Run one stage, replacing any failure with a description safe to print.""" - try: - return operation() - except Exception as error: - raise StageFailedError(_failure_description(name, error)) from error - - -def render_summary( - result: SyncResult, window: ReportingWindow, *, dry_run: bool -) -> str: - """Return the job summary: fingerprints, counts, issue numbers and actions.""" - lines = [ - f"## Telemetry error issues for {window.start.date().isoformat()}", - "", - f"Window: {_window_text(window)}.", - ] - if dry_run: - lines.append("Dry run: no issues were changed.") - if not result.actions: - lines += ["", "No error groups had failures in the window."] - return "\n".join(lines) - totals = Counter(action.action for action in result.actions) - described = ", ".join(f"{count} {name}" for name, count in sorted(totals.items())) - lines += [ - "", - f"{len(result.actions)} error groups: {described}.", - "", - "| Fingerprint | Kind | Service | Production | Staging | Issue | Action |", - "| --- | --- | --- | --- | --- | --- | --- |", - ] - for action in result.actions: - issue = MISSING if action.issue_number is None else f"#{action.issue_number}" - lines.append( - f"| {_code(action.fingerprint)} | {action.kind} | {_code(action.family)} " - f"| {action.production_occurrences:,} | {action.staging_occurrences:,} " - f"| {issue} | {action.action} |" - ) - return "\n".join(lines) - - -def _append_summary(path: str, summary: str) -> None: - with Path(path).open("a", encoding="utf-8") as summary_file: - summary_file.write(f"{summary}\n") - - -def _report(arguments: argparse.Namespace, runtime: Runtime) -> str: - settings = _stage("configuration", lambda: read_settings(runtime.environ)) - window = _stage( - "configuration", lambda: _report_window(arguments.report_date, runtime.now()) - ) - bigquery = BigQueryClient( - arguments.project_id, settings.google_access_token, runtime.bigquery_transport - ) - records = _stage( - "query error records", lambda: fetch_error_records(bigquery, window) - ) - request_counts = _stage( - "query request counts", lambda: fetch_request_counts(bigquery, window) - ) - groups = _stage( - "group errors", - lambda: build_groups(records, request_counts, window, settings.fingerprint_key), - ) - github = GitHubClient( - settings.repository, - settings.github_credential, - runtime.github_transport, - sleep=runtime.sleep, - clock=runtime.clock, - ) - result = _stage( - "sync issues", - lambda: sync_issues( - groups, window, arguments.project_id, github, dry_run=arguments.dry_run - ), - ) - summary = render_summary(result, window, dry_run=arguments.dry_run) - summary_path = settings.summary_path - if summary_path: - _stage("write job summary", lambda: _append_summary(summary_path, summary)) - return summary - - -def main(argv: list[str] | None = None, runtime: Runtime | None = None) -> int: - """Run one report and return the process exit code. - - A failed stage prints one line naming the stage and exception class, never - the exception's message, and returns 1. - """ - arguments = parse_arguments(argv) - runtime = runtime or Runtime.from_process() - try: - summary = _report(arguments, runtime) - except StageFailedError as failure: - print(failure, file=runtime.stderr) - return 1 - print(summary, file=runtime.stdout) - return 0 - +from telemetry_error_issues.cli import main if __name__ == "__main__": raise SystemExit(main()) diff --git a/.github/scripts/telemetry_error_issues/__init__.py b/.github/scripts/telemetry_error_issues/__init__.py new file mode 100644 index 000000000..0f7cb75ee --- /dev/null +++ b/.github/scripts/telemetry_error_issues/__init__.py @@ -0,0 +1,13 @@ +"""Report each day's telemetry error groups as GitHub issues. + +Each run reads one US Eastern calendar day of error records from the central +Cloud Logging bucket through its linked BigQuery dataset. It groups the records +by a keyed fingerprint, measures each group in production and staging, and +creates or updates one GitHub issue per production error group. + +Workflow logs, job summaries and issues in this repository are public. The +script therefore publishes only fingerprints, counts, code-defined names and a +Logs Explorer link, never error messages, stack text or identifiers. + +``sync_telemetry_error_issues.py`` beside this package runs ``cli.main``. +""" diff --git a/.github/scripts/telemetry_error_issues/bigquery.py b/.github/scripts/telemetry_error_issues/bigquery.py new file mode 100644 index 000000000..b2a9d614a --- /dev/null +++ b/.github/scripts/telemetry_error_issues/bigquery.py @@ -0,0 +1,395 @@ +"""Query error records and request counts through the BigQuery REST API.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta +from http import HTTPStatus +from typing import TYPE_CHECKING, Any +from urllib.parse import quote, urlencode + +from telemetry_error_issues.groups import RequestCount +from telemetry_error_issues.normalization import MISSING +from telemetry_error_issues.records import ErrorRecord +from telemetry_error_issues.transport import ( + ApiError, + HttpRequest, + HttpResponse, + urllib_transport, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + from telemetry_error_issues.window import ReportingWindow + +# Captures 'path", line N, in function' from each frame header of a Python +# traceback, including indented and "|"-prefixed exception group frames. It has +# one capturing group because BigQuery's REGEXP_EXTRACT_ALL allows only one, +# and the trailing newline skips a header that stack truncation cut short. +FRAME_PATTERN = r'File "([^"\n]+", line [0-9]+, in [^\s]+)\n' +# Matches everything after the first traceback of a chained exception, so the +# query can remove it before extracting frames. The logging library keeps only +# the start of a long stack, which can drop the final traceback; the first +# traceback is always kept, so frames from it give an exception the same +# fingerprint however long its stack is. +CHAINED_EXCEPTION_PATTERN = ( + r"(?s)\n\n(?:The above exception was the direct cause of the following " + r"exception|During handling of the above exception, another exception " + r"occurred):\n.*" +) + +# A message's template is its first three whole words after removing quoted +# values, so values don't split one error into many groups. A single quote +# opens a value only after the start of the message or a character that is not +# a letter or digit; otherwise it is an apostrophe, as in "can't" or "users'". +# The query passes these to REGEXP_REPLACE and REGEXP_EXTRACT_ALL. +DOUBLE_QUOTED_PATTERN = r'"[^"]*"' +SINGLE_QUOTED_PATTERN = r"(^|[^A-Za-z0-9])'[^']*'" +TEMPLATE_WORD_PATTERN = r"\b[A-Za-z]+\b" + +BIGQUERY_API = "https://bigquery.googleapis.com/bigquery/v2" +LINKED_DATASET = "observability_logs" +MAXIMUM_BYTES_BILLED = 5 * 1024**3 +QUERY_WAIT_MILLISECONDS = 30_000 +MAX_QUERY_POLLS = 40 +UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=UTC) + +HEALTH_CHECK_ROUTES = ( + "/health", + "/simulation-gateway-check", + "/liveness-check", + "/readiness-check", +) + +# Each query reads the linked dataset's _AllLogs view. Only code-defined or +# normalized values leave BigQuery: stacks only as frame headers, messages +# only as templates (see DOUBLE_QUOTED_PATTERN), and identifiers only as +# SHA-256 hashes. Duplicate deliveries of one log entry are removed. +ERROR_RECORDS_QUERY = r""" +SELECT + timestamp, + JSON_VALUE(json_payload, '$."service.name"') AS service_name, + JSON_VALUE(json_payload, '$."deployment.environment.name"') AS environment_name, + JSON_VALUE(json_payload, '$."event.name"') AS event_name, + JSON_VALUE(json_payload, '$."error.type"') AS error_type, + JSON_VALUE(json_payload, '$."error.handled"') AS error_handled, + REGEXP_EXTRACT_ALL( + REGEXP_REPLACE( + JSON_VALUE(json_payload, '$."error.stack"'), @chained_exception_pattern, '' + ), + @frame_pattern + ) AS frames, + COALESCE(LENGTH(JSON_VALUE(json_payload, '$."error.stack"')) >= 16384, FALSE) + AS stack_truncated, + ARRAY_TO_STRING( + ARRAY( + SELECT LOWER(word) + FROM UNNEST( + REGEXP_EXTRACT_ALL( + REGEXP_REPLACE( + REGEXP_REPLACE( + JSON_VALUE(json_payload, '$.message'), @double_quoted_pattern, '' + ), + @single_quoted_pattern, + r'\1' + ), + @template_word_pattern + ) + ) AS word WITH OFFSET AS position + WHERE position < 3 + ORDER BY position + ), + ' ' + ) AS message_template, + JSON_VALUE(json_payload, '$.attributes.error_type') AS attribute_error_type, + JSON_VALUE(json_payload, '$.attributes.failure_category') AS failure_category, + JSON_VALUE(json_payload, '$."http.route"') AS route, + JSON_VALUE(json_payload, '$."http.response.status_code"') AS status_code, + JSON_VALUE(json_payload, '$."operation.name"') AS operation_name, + TO_HEX(SHA256(JSON_VALUE(json_payload, '$."request.id"'))) AS request_key, + TO_HEX(SHA256(JSON_VALUE(json_payload, '$.attributes.observability_id'))) + AS observability_key, + TO_HEX(SHA256(COALESCE( + JSON_VALUE(json_payload, '$.trace_id'), + REGEXP_EXTRACT(trace, r'/traces/([0-9A-Fa-f]+)$') + ))) AS trace_key +FROM `_AllLogs` +WHERE timestamp >= @query_start AND timestamp < @window_end + AND JSON_VALUE(json_payload, '$.schema_version') = 'policyengine.observability.v2' + AND JSON_VALUE(json_payload, '$."service.namespace"') = 'policyengine.api-v1' + AND severity_number >= 500 + AND COALESCE(JSON_VALUE(json_payload, '$."http.route"'), '') + NOT IN UNNEST(@excluded_routes) +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY insert_id, timestamp ORDER BY receive_timestamp +) = 1 +""" + +REQUEST_COUNTS_QUERY = r""" +WITH completed_requests AS ( + SELECT + JSON_VALUE(json_payload, '$."service.name"') AS service_name, + JSON_VALUE(json_payload, '$."deployment.environment.name"') AS environment_name, + JSON_VALUE(json_payload, '$."http.route"') AS route + FROM `_AllLogs` + WHERE timestamp >= @window_start AND timestamp < @window_end + AND JSON_VALUE(json_payload, '$.schema_version') = 'policyengine.observability.v2' + AND JSON_VALUE(json_payload, '$."service.namespace"') = 'policyengine.api-v1' + AND JSON_VALUE(json_payload, '$."event.name"') = 'request.completed' + AND COALESCE(JSON_VALUE(json_payload, '$."http.route"'), '') + NOT IN UNNEST(@excluded_routes) + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY insert_id, timestamp ORDER BY receive_timestamp + ) = 1 +) +SELECT service_name, environment_name, route, COUNT(*) AS requests +FROM completed_requests +GROUP BY service_name, environment_name, route +""" + + +class BigQueryError(ApiError): + """A BigQuery request failed.""" + + service = "BigQuery" + + +class BigQueryTimeoutError(BigQueryError): + """A BigQuery query job didn't complete within the polling limit.""" + + +def _bigquery_boolean(text: str) -> bool: + return text == "true" + + +def _bigquery_timestamp(text: str) -> datetime: + """Convert microseconds since the epoch, as ``useInt64Timestamp`` returns.""" + return UNIX_EPOCH + timedelta(microseconds=int(text)) + + +CELL_CONVERTERS: dict[str, Callable[[str], object]] = { + "INTEGER": int, + "INT64": int, + "FLOAT": float, + "FLOAT64": float, + "BOOLEAN": _bigquery_boolean, + "BOOL": _bigquery_boolean, + "TIMESTAMP": _bigquery_timestamp, +} + + +def _convert_cell(field: Mapping[str, Any], value: object) -> object: + """Convert one BigQuery REST value, which arrives as a string, to Python.""" + if value is None: + return None + if field.get("mode") == "REPEATED" and isinstance(value, list): + item_field = {**field, "mode": "NULLABLE"} + return [_convert_cell(item_field, item["v"]) for item in value] + converter = CELL_CONVERTERS.get(str(field.get("type")), str) + return converter(str(value)) + + +def _convert_rows(response: Mapping[str, Any]) -> list[dict[str, object]]: + fields = response["schema"]["fields"] + return [ + { + field["name"]: _convert_cell(field, cell["v"]) + for field, cell in zip(fields, row["f"], strict=True) + } + for row in response.get("rows", []) + ] + + +class BigQueryClient: + """Runs parameterized standard SQL queries through the BigQuery REST API.""" + + def __init__( + self, + project_id: str, + access_token: str, + transport: Callable[[HttpRequest], HttpResponse] = urllib_transport, + ) -> None: + """Run jobs in ``project_id`` against its linked log dataset.""" + self.project_id = project_id + self._access_token = access_token + self._transport = transport + self._queries_url = ( + f"{BIGQUERY_API}/projects/{quote(project_id, safe='')}/queries" + ) + + def query( + self, sql: str, parameters: list[dict[str, Any]], label: str + ) -> list[dict[str, object]]: + """Run ``sql`` and return every row as a dictionary of converted values.""" + body = { + "query": sql, + "useLegacySql": False, + "parameterMode": "NAMED", + "queryParameters": parameters, + "defaultDataset": { + "projectId": self.project_id, + "datasetId": LINKED_DATASET, + }, + "maximumBytesBilled": str(MAXIMUM_BYTES_BILLED), + "labels": {"purpose": "telemetry-error-issues", "query": label}, + "timeoutMs": QUERY_WAIT_MILLISECONDS, + "formatOptions": {"useInt64Timestamp": True}, + } + response = self._send("POST", self._queries_url, body) + job = response["jobReference"] + for _ in range(MAX_QUERY_POLLS): + if response.get("jobComplete"): + break + response = self._results(job, page_token=None) + else: + msg = "BigQuery query did not complete in time" + raise BigQueryTimeoutError(msg) + rows = _convert_rows(response) + while response.get("pageToken"): + response = self._results(job, page_token=response["pageToken"]) + rows.extend(_convert_rows(response)) + return rows + + def _results( + self, job: Mapping[str, str], page_token: str | None + ) -> dict[str, Any]: + parameters = { + "location": job["location"], + "timeoutMs": str(QUERY_WAIT_MILLISECONDS), + "formatOptions.useInt64Timestamp": "true", + } + if page_token: + parameters["pageToken"] = page_token + job_id = quote(job["jobId"], safe="") + url = f"{self._queries_url}/{job_id}?{urlencode(parameters)}" + return self._send("GET", url, None) + + def _send( + self, method: str, url: str, body: Mapping[str, object] | None + ) -> dict[str, Any]: + headers = {"Authorization": f"Bearer {self._access_token}"} + if body is not None: + headers["Content-Type"] = "application/json" + encoded = None if body is None else json.dumps(body).encode() + response = self._transport(HttpRequest(method, url, headers, encoded)) + if response.status >= HTTPStatus.BAD_REQUEST: + raise BigQueryError.from_response(response) + return json.loads(response.body) + + +def _timestamp_parameter(name: str, moment: datetime) -> dict[str, Any]: + return { + "name": name, + "parameterType": {"type": "TIMESTAMP"}, + "parameterValue": {"value": moment.astimezone(UTC).isoformat(sep=" ")}, + } + + +def _query_parameters( + timestamps: Mapping[str, datetime], strings: Mapping[str, str] +) -> list[dict[str, Any]]: + parameters = [ + _timestamp_parameter(name, value) for name, value in timestamps.items() + ] + parameters.extend( + { + "name": name, + "parameterType": {"type": "STRING"}, + "parameterValue": {"value": value}, + } + for name, value in strings.items() + ) + parameters.append( + { + "name": "excluded_routes", + "parameterType": {"type": "ARRAY", "arrayType": {"type": "STRING"}}, + "parameterValue": { + "arrayValues": [{"value": route} for route in HEALTH_CHECK_ROUTES] + }, + } + ) + return parameters + + +def _text(value: object) -> str | None: + return None if value is None or value == "" else str(value) + + +def _integer(value: object) -> int | None: + if isinstance(value, int) and not isinstance(value, bool): + return value + if isinstance(value, str) and value.isascii() and value.isdigit(): + return int(value) + return None + + +def _boolean(value: object) -> bool | None: + if isinstance(value, bool): + return value + return {"true": True, "false": False}.get(str(value)) + + +def error_record_from_row(row: Mapping[str, object]) -> ErrorRecord: + """Build an ``ErrorRecord`` from one row of ``ERROR_RECORDS_QUERY``.""" + timestamp = row["timestamp"] + if not isinstance(timestamp, datetime): + msg = "an error record has no timestamp" + raise TypeError(msg) + frames = row["frames"] + return ErrorRecord( + timestamp=timestamp, + service_name=_text(row["service_name"]) or MISSING, + environment_name=_text(row["environment_name"]), + event_name=_text(row["event_name"]), + error_type=_text(row["error_type"]), + error_handled=_boolean(row["error_handled"]), + frames=tuple(map(str, frames)) if isinstance(frames, list) else (), + stack_truncated=row["stack_truncated"] is True, + message_template=_text(row["message_template"]), + attribute_error_type=_text(row["attribute_error_type"]), + failure_category=_text(row["failure_category"]), + route=_text(row["route"]), + status_code=_integer(row["status_code"]), + operation_name=_text(row["operation_name"]), + request_key=_text(row["request_key"]), + observability_key=_text(row["observability_key"]), + trace_key=_text(row["trace_key"]), + ) + + +def fetch_error_records( + client: BigQueryClient, window: ReportingWindow +) -> list[ErrorRecord]: + """Return the ERROR records of the window and the seven days before it.""" + parameters = _query_parameters( + {"query_start": window.lookback_start, "window_end": window.end}, + { + "frame_pattern": FRAME_PATTERN, + "chained_exception_pattern": CHAINED_EXCEPTION_PATTERN, + "double_quoted_pattern": DOUBLE_QUOTED_PATTERN, + "single_quoted_pattern": SINGLE_QUOTED_PATTERN, + "template_word_pattern": TEMPLATE_WORD_PATTERN, + }, + ) + rows = client.query(ERROR_RECORDS_QUERY, parameters, "error-records") + return [error_record_from_row(row) for row in rows] + + +def fetch_request_counts( + client: BigQueryClient, window: ReportingWindow +) -> list[RequestCount]: + """Return the requests each service completed per route in the window.""" + parameters = _query_parameters( + {"window_start": window.start, "window_end": window.end}, {} + ) + rows = client.query(REQUEST_COUNTS_QUERY, parameters, "request-counts") + return [ + RequestCount( + service_name=_text(row["service_name"]) or MISSING, + environment_name=_text(row["environment_name"]), + route=_text(row["route"]), + requests=_integer(row["requests"]) or 0, + ) + for row in rows + ] diff --git a/.github/scripts/telemetry_error_issues/cli.py b/.github/scripts/telemetry_error_issues/cli.py new file mode 100644 index 000000000..95e10d042 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/cli.py @@ -0,0 +1,274 @@ +"""Report each day's telemetry error groups as GitHub issues. + +The command line: it parses the arguments, reads the settings from the +environment, runs each stage of the report and writes the job summary. +``parse_arguments`` uses the first line of this docstring as the description +that ``--help`` prints. +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +import time +from collections import Counter +from dataclasses import dataclass +from datetime import UTC, date, datetime +from pathlib import Path +from typing import TYPE_CHECKING, NamedTuple, TypeVar + +from telemetry_error_issues.bigquery import ( + BigQueryClient, + fetch_error_records, + fetch_request_counts, +) +from telemetry_error_issues.github_client import GitHubClient +from telemetry_error_issues.groups import build_groups +from telemetry_error_issues.normalization import MISSING +from telemetry_error_issues.records import fingerprint_key +from telemetry_error_issues.rendering import code, window_text +from telemetry_error_issues.sync import SyncResult, sync_issues +from telemetry_error_issues.transport import ( + ApiError, + HttpRequest, + HttpResponse, + urllib_transport, +) +from telemetry_error_issues.window import ( + ReportingWindow, + default_report_date, + reporting_window, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + from typing import TextIO + +StageResult = TypeVar("StageResult") + +REQUIRED_ENVIRONMENT_VARIABLES = ( + "GOOGLE_OAUTH_ACCESS_TOKEN", + "GITHUB_TOKEN", + "GITHUB_REPOSITORY", + "TELEMETRY_ERROR_FINGERPRINT_KEY", +) +REPOSITORY_PATTERN = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") + + +def _report_date_argument(value: str) -> date | None: + if not value: + return None + try: + return date.fromisoformat(value) + except ValueError as error: + msg = "expected an Eastern calendar date as YYYY-MM-DD" + raise argparse.ArgumentTypeError(msg) from error + + +def parse_arguments(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the command line; argparse exits with code 2 on invalid input.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--project-id", + required=True, + help="Google Cloud project that holds the linked BigQuery dataset", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="query and group errors, but make no changes to GitHub issues", + ) + parser.add_argument( + "--report-date", + type=_report_date_argument, + default=None, + help="Eastern calendar day to report (default: the previous day)", + ) + return parser.parse_args(argv) + + +class ConfigurationError(Exception): + """The run's settings are missing or invalid. The text names only settings.""" + + +class StageFailedError(Exception): + """A stage of the run failed. + + The text holds only the stage, the exception class, and for an API error + the HTTP status and reason, so it can be printed in a public log. + """ + + +def _current_time() -> datetime: + return datetime.now(UTC) + + +@dataclass(frozen=True) +class Runtime: + """What a run reads from outside the process, so tests can replace it.""" + + environ: Mapping[str, str] + stdout: TextIO + stderr: TextIO + bigquery_transport: Callable[[HttpRequest], HttpResponse] = urllib_transport + github_transport: Callable[[HttpRequest], HttpResponse] = urllib_transport + now: Callable[[], datetime] = _current_time + sleep: Callable[[float], None] = time.sleep + clock: Callable[[], float] = time.monotonic + + @classmethod + def from_process(cls) -> Runtime: + """Use the process environment, standard streams and the network.""" + return cls(environ=os.environ, stdout=sys.stdout, stderr=sys.stderr) + + +class Settings(NamedTuple): + """Settings read from the environment; none of them is ever printed.""" + + google_access_token: str + github_credential: str + repository: str + fingerprint_key: bytes + summary_path: str | None + + +def read_settings(environ: Mapping[str, str]) -> Settings: + """Read and check the environment variables the run needs.""" + missing = [name for name in REQUIRED_ENVIRONMENT_VARIABLES if not environ.get(name)] + if missing: + msg = f"missing environment variables: {', '.join(missing)}" + raise ConfigurationError(msg) + if not REPOSITORY_PATTERN.match(environ["GITHUB_REPOSITORY"]): + msg = "GITHUB_REPOSITORY must be owner/name" + raise ConfigurationError(msg) + try: + key = fingerprint_key(environ["TELEMETRY_ERROR_FINGERPRINT_KEY"]) + except ValueError as error: + raise ConfigurationError(str(error)) from None + return Settings( + google_access_token=environ["GOOGLE_OAUTH_ACCESS_TOKEN"], + github_credential=environ["GITHUB_TOKEN"], + repository=environ["GITHUB_REPOSITORY"], + fingerprint_key=key, + summary_path=environ.get("GITHUB_STEP_SUMMARY") or None, + ) + + +def _report_window(report_date: date | None, now: datetime) -> ReportingWindow: + window = reporting_window(report_date or default_report_date(now)) + if window.end > now: + msg = f"{window.start.date().isoformat()} has not ended in US Eastern time" + raise ConfigurationError(msg) + return window + + +def _failure_description(stage: str, error: Exception) -> str: + detail = "" + if isinstance(error, ConfigurationError): + detail = f" ({error})" + elif isinstance(error, ApiError) and error.status is not None: + detail = f" (HTTP {error.status} {error.reason})" + return f"{stage} failed: {type(error).__name__}{detail}" + + +def _stage(name: str, operation: Callable[[], StageResult]) -> StageResult: + """Run one stage, replacing any failure with a description safe to print.""" + try: + return operation() + except Exception as error: + raise StageFailedError(_failure_description(name, error)) from error + + +def render_summary( + result: SyncResult, window: ReportingWindow, *, dry_run: bool +) -> str: + """Return the job summary: fingerprints, counts, issue numbers and actions.""" + lines = [ + f"## Telemetry error issues for {window.start.date().isoformat()}", + "", + f"Window: {window_text(window)}.", + ] + if dry_run: + lines.append("Dry run: no issues were changed.") + if not result.actions: + lines += ["", "No error groups had failures in the window."] + return "\n".join(lines) + totals = Counter(action.action for action in result.actions) + described = ", ".join(f"{count} {name}" for name, count in sorted(totals.items())) + lines += [ + "", + f"{len(result.actions)} error groups: {described}.", + "", + "| Fingerprint | Kind | Service | Production | Staging | Issue | Action |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] + for action in result.actions: + issue = MISSING if action.issue_number is None else f"#{action.issue_number}" + lines.append( + f"| {code(action.fingerprint)} | {action.kind} | {code(action.family)} " + f"| {action.production_occurrences:,} | {action.staging_occurrences:,} " + f"| {issue} | {action.action} |" + ) + return "\n".join(lines) + + +def _append_summary(path: str, summary: str) -> None: + with Path(path).open("a", encoding="utf-8") as summary_file: + summary_file.write(f"{summary}\n") + + +def _report(arguments: argparse.Namespace, runtime: Runtime) -> str: + settings = _stage("configuration", lambda: read_settings(runtime.environ)) + window = _stage( + "configuration", lambda: _report_window(arguments.report_date, runtime.now()) + ) + bigquery = BigQueryClient( + arguments.project_id, settings.google_access_token, runtime.bigquery_transport + ) + records = _stage( + "query error records", lambda: fetch_error_records(bigquery, window) + ) + request_counts = _stage( + "query request counts", lambda: fetch_request_counts(bigquery, window) + ) + groups = _stage( + "group errors", + lambda: build_groups(records, request_counts, window, settings.fingerprint_key), + ) + github = GitHubClient( + settings.repository, + settings.github_credential, + runtime.github_transport, + sleep=runtime.sleep, + clock=runtime.clock, + ) + result = _stage( + "sync issues", + lambda: sync_issues( + groups, window, arguments.project_id, github, dry_run=arguments.dry_run + ), + ) + summary = render_summary(result, window, dry_run=arguments.dry_run) + summary_path = settings.summary_path + if summary_path: + _stage("write job summary", lambda: _append_summary(summary_path, summary)) + return summary + + +def main(argv: list[str] | None = None, runtime: Runtime | None = None) -> int: + """Run one report and return the process exit code. + + A failed stage prints one line naming the stage and exception class, never + the exception's message, and returns 1. + """ + arguments = parse_arguments(argv) + runtime = runtime or Runtime.from_process() + try: + summary = _report(arguments, runtime) + except StageFailedError as failure: + print(failure, file=runtime.stderr) + return 1 + print(summary, file=runtime.stdout) + return 0 diff --git a/.github/scripts/telemetry_error_issues/github_client.py b/.github/scripts/telemetry_error_issues/github_client.py new file mode 100644 index 000000000..ee1642847 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/github_client.py @@ -0,0 +1,208 @@ +"""A client for one GitHub repository's issues and labels, with rate limiting.""" + +from __future__ import annotations + +import json +import time +from email.utils import parsedate_to_datetime +from http import HTTPStatus +from typing import TYPE_CHECKING, Any +from urllib.parse import urlencode + +from telemetry_error_issues.transport import ( + ApiError, + HttpRequest, + HttpResponse, + urllib_transport, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + +GITHUB_API = "https://api.github.com" +GITHUB_API_VERSION = "2022-11-28" +GITHUB_PAGE_SIZE = 100 + +# GitHub asks clients to send content-creating requests one at a time, at +# least a second apart, and to wait as its rate-limit responses say. +MIN_SECONDS_BETWEEN_WRITES = 1.0 +MAX_RATE_LIMIT_RETRIES = 3 +DEFAULT_RATE_LIMIT_WAIT_SECONDS = 60 +MAX_RATE_LIMIT_WAIT_SECONDS = 300 +RATE_LIMIT_STATUSES = frozenset({HTTPStatus.FORBIDDEN, HTTPStatus.TOO_MANY_REQUESTS}) + + +class GitHubError(ApiError): + """A GitHub request failed.""" + + service = "GitHub" + + +class GitHubLabelError(GitHubError): + """GitHub didn't apply the labels the job asked for.""" + + +def _seconds_until_reset(response: HttpResponse) -> int | None: + """Return the seconds from the response's ``date`` to ``x-ratelimit-reset``.""" + reset = response.headers.get("x-ratelimit-reset", "") + sent = response.headers.get("date") + if not (reset.isascii() and reset.isdigit()) or not sent: + return None + try: + sent_at = parsedate_to_datetime(sent) + except (TypeError, ValueError): + return None + return max(int(reset) - int(sent_at.timestamp()), 0) + + +def _rate_limit_wait(response: HttpResponse) -> int | None: + """Return the seconds to wait before retrying a rate-limited request. + + Follows GitHub's guidance: wait for ``retry-after``; when no requests + remain, wait until ``x-ratelimit-reset``; otherwise wait a minute. A 403 + whose headers and message don't mention a rate limit is a permission + failure, so it isn't retried. The message is only searched, never printed. + """ + if response.status not in RATE_LIMIT_STATUSES: + return None + retry_after = response.headers.get("retry-after", "") + if retry_after.isascii() and retry_after.isdigit(): + seconds = int(retry_after) + elif response.headers.get("x-ratelimit-remaining") == "0": + until_reset = _seconds_until_reset(response) + seconds = ( + DEFAULT_RATE_LIMIT_WAIT_SECONDS if until_reset is None else until_reset + ) + elif ( + response.status == HTTPStatus.TOO_MANY_REQUESTS + or b"rate limit" in response.body.lower() + ): + seconds = DEFAULT_RATE_LIMIT_WAIT_SECONDS + else: + return None + return seconds if seconds <= MAX_RATE_LIMIT_WAIT_SECONDS else None + + +def _json_object(response: HttpResponse) -> dict[str, Any]: + payload = json.loads(response.body) + if not isinstance(payload, dict): + msg = "GitHub returned an unexpected response" + raise GitHubError(msg) + return payload + + +def _json_list(response: HttpResponse) -> list[dict[str, Any]]: + payload = json.loads(response.body) + if not isinstance(payload, list): + msg = "GitHub returned an unexpected response" + raise GitHubError(msg) + return payload + + +class GitHubClient: + """Reads and writes one repository's issues and labels. + + Writes go one at a time, at least a second apart. A rate-limited request is + retried after the wait GitHub asks for, up to three times. + """ + + def __init__( + self, + repository: str, + credential: str, + transport: Callable[[HttpRequest], HttpResponse] = urllib_transport, + *, + sleep: Callable[[float], None] = time.sleep, + clock: Callable[[], float] = time.monotonic, + ) -> None: + """Send requests for ``repository`` (``owner/name``) with ``credential``.""" + self._repository_url = f"{GITHUB_API}/repos/{repository}" + self._headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {credential}", + "User-Agent": "policyengine-telemetry-error-issues", + "X-GitHub-Api-Version": GITHUB_API_VERSION, + } + self._transport = transport + self._sleep = sleep + self._clock = clock + self._last_write: float | None = None + + def list_issues(self, creator: str) -> list[dict[str, Any]]: + """Return the open and closed issues ``creator`` opened, without PRs.""" + issues = self._pages("/issues", {"state": "all", "creator": creator}) + return [issue for issue in issues if "pull_request" not in issue] + + def list_labels(self) -> set[str]: + """Return the names of the repository's labels.""" + return {label["name"] for label in self._pages("/labels", {})} + + def create_label(self, name: str, color: str, description: str) -> None: + """Create a label.""" + self._send( + "POST", + "/labels", + {"name": name, "color": color, "description": description}, + ) + + def create_issue(self, title: str, body: str, labels: list[str]) -> dict[str, Any]: + """Open an issue and return it as GitHub describes it.""" + payload = {"title": title, "body": body, "labels": labels} + return _json_object(self._send("POST", "/issues", payload)) + + def update_issue(self, number: int, fields: Mapping[str, object]) -> None: + """Change an issue's fields, such as its body or state.""" + self._send("PATCH", f"/issues/{number}", fields) + + def add_labels(self, number: int, labels: list[str]) -> set[str]: + """Add labels to an issue and return all the labels it now has.""" + response = self._send("POST", f"/issues/{number}/labels", {"labels": labels}) + return {label["name"] for label in _json_list(response)} + + def add_comment(self, number: int, body: str) -> None: + """Comment on an issue.""" + self._send("POST", f"/issues/{number}/comments", {"body": body}) + + def _pages(self, path: str, parameters: Mapping[str, str]) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + page = 1 + while True: + query = urlencode( + {**parameters, "per_page": GITHUB_PAGE_SIZE, "page": page} + ) + batch = _json_list(self._send("GET", f"{path}?{query}", None)) + items.extend(batch) + if len(batch) < GITHUB_PAGE_SIZE: + return items + page += 1 + + def _wait_between_writes(self) -> None: + if self._last_write is None: + return + elapsed = self._clock() - self._last_write + if elapsed < MIN_SECONDS_BETWEEN_WRITES: + self._sleep(MIN_SECONDS_BETWEEN_WRITES - elapsed) + + def _send( + self, method: str, path: str, body: Mapping[str, object] | None + ) -> HttpResponse: + headers = dict(self._headers) + encoded = None + if body is not None: + headers["Content-Type"] = "application/json" + encoded = json.dumps(body).encode() + request = HttpRequest(method, f"{self._repository_url}{path}", headers, encoded) + is_write = method != "GET" + for retry in range(MAX_RATE_LIMIT_RETRIES + 1): + if is_write: + self._wait_between_writes() + response = self._transport(request) + if is_write: + self._last_write = self._clock() + wait = _rate_limit_wait(response) + if wait is None or retry == MAX_RATE_LIMIT_RETRIES: + break + self._sleep(wait) + if response.status >= HTTPStatus.BAD_REQUEST: + raise GitHubError.from_response(response) + return response diff --git a/.github/scripts/telemetry_error_issues/groups.py b/.github/scripts/telemetry_error_issues/groups.py new file mode 100644 index 000000000..2d04f4080 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/groups.py @@ -0,0 +1,271 @@ +"""Group failures by fingerprint, and measure each group in each environment.""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass +from typing import TYPE_CHECKING, NamedTuple + +from telemetry_error_issues.normalization import ( + environment_class, + public_error_type, + public_family, + public_operation, + public_route, + public_status, + service_family, +) +from telemetry_error_issues.records import ErrorRecord, fingerprint, primary_records + +if TYPE_CHECKING: + from collections.abc import Iterable + from datetime import datetime + + from telemetry_error_issues.window import ReportingWindow + +REPORTED_ENVIRONMENTS = ("production", "staging") +MAX_RELATED_FINGERPRINTS = 3 + + +class RequestCount(NamedTuple): + """Requests that one service completed on one route during the window.""" + + service_name: str + environment_name: str | None + route: str | None + requests: int + + +class Occurrence(NamedTuple): + """One failure in an error group.""" + + timestamp: datetime + environment: str + route: str | None + affected_key: str + observability_key: str | None + trace_key: str | None + + +@dataclass(frozen=True) +class EnvironmentMeasures: + """An error group's measures in one environment. + + Counts and rates cover the window; first and last seen cover the window + and the seven days before it. + """ + + occurrences: int + affected_requests: int + observability_ids: int + route_affected_requests: int + route_requests: int + error_rate: float | None + hourly_rate: float + previous_hourly_rate: float + first_seen: datetime | None + last_seen: datetime | None + + +@dataclass(frozen=True) +class ErrorGroup: + """Failures that share a fingerprint, described only by validated values.""" + + fingerprint: str + kind: str + family: str + error_type: str + route: str + status_code: int | None + operation: str + production: EnvironmentMeasures + staging: EnvironmentMeasures + related_fingerprints: tuple[str, ...] + occurrences: tuple[Occurrence, ...] + + def production_since(self, moment: datetime) -> tuple[int, int]: + """Return production occurrences and affected requests from ``moment``.""" + recent = [ + occurrence + for occurrence in self.occurrences + if occurrence.environment == "production" and occurrence.timestamp >= moment + ] + return len(recent), len({occurrence.affected_key for occurrence in recent}) + + +def _occurrence(record: ErrorRecord, position: int) -> Occurrence: + """Describe a failure, identifying what it affected as precisely as possible. + + The affected request is the request ID, else the observability ID, else the + trace. A failure with none of them counts as its own request. + """ + if record.request_key: + affected_key = f"request:{record.request_key}" + elif record.observability_key: + affected_key = f"observability:{record.observability_key}" + elif record.trace_key: + affected_key = f"trace:{record.trace_key}" + else: + affected_key = f"record:{position}" + return Occurrence( + timestamp=record.timestamp, + environment=record.environment, + route=record.route, + affected_key=affected_key, + observability_key=record.observability_key, + trace_key=record.trace_key, + ) + + +class RouteRequests: + """Requests completed in the window, per service family, environment and route.""" + + def __init__(self, request_counts: Iterable[RequestCount]) -> None: + """Add up the request counts, merging versioned simulation services.""" + self._totals: Counter[tuple[str, str, str | None]] = Counter() + for count in request_counts: + environment = environment_class(count.environment_name) + family = service_family(count.service_name) + self._totals[family, environment, count.route] += count.requests + + def total(self, family: str, environment: str, routes: Iterable[str]) -> int: + """Return the requests one service family completed on the given routes.""" + return sum(self._totals[family, environment, route] for route in routes) + + +def _related_fingerprints( + occurrences_by_group: dict[str, list[Occurrence]], +) -> dict[str, tuple[str, ...]]: + """For each group, the groups that share the most traces with it.""" + groups_by_trace: dict[str, set[str]] = {} + for group, occurrences in occurrences_by_group.items(): + for occurrence in occurrences: + if occurrence.trace_key: + groups_by_trace.setdefault(occurrence.trace_key, set()).add(group) + related: dict[str, tuple[str, ...]] = {} + for group, occurrences in occurrences_by_group.items(): + shared_traces: Counter[str] = Counter() + traces = {occurrence.trace_key for occurrence in occurrences} + for trace in traces - {None}: + shared_traces.update(groups_by_trace[trace] - {group}) + ranked = sorted(shared_traces, key=lambda other: (-shared_traces[other], other)) + related[group] = tuple(ranked[:MAX_RELATED_FINGERPRINTS]) + return related + + +def _environment_measures( + occurrences: list[Occurrence], + environment: str, + family: str, + route_requests: RouteRequests, + window: ReportingWindow, +) -> EnvironmentMeasures: + in_environment = [item for item in occurrences if item.environment == environment] + in_window = [item for item in in_environment if item.timestamp >= window.start] + before_window = len(in_environment) - len(in_window) + affected_requests = len({item.affected_key for item in in_window}) + # The error rate compares failed requests with completed requests on the + # same routes, so failures without a route, such as background + # operations, are left out of both. + on_routes = [item for item in in_window if item.route] + route_affected_requests = len({item.affected_key for item in on_routes}) + routes = {item.route for item in on_routes} + requests = route_requests.total(family, environment, routes) + timestamps = [item.timestamp for item in in_environment] + return EnvironmentMeasures( + occurrences=len(in_window), + affected_requests=affected_requests, + observability_ids=len( + {item.observability_key for item in in_window if item.observability_key} + ), + route_affected_requests=route_affected_requests, + route_requests=requests, + error_rate=route_affected_requests / requests if requests else None, + hourly_rate=len(in_window) / window.hours, + previous_hourly_rate=before_window / window.lookback_hours, + first_seen=min(timestamps, default=None), + last_seen=max(timestamps, default=None), + ) + + +def _most_frequent_route(occurrences: list[Occurrence], window: ReportingWindow) -> str: + """Return the route most failures in the window used, preferring production.""" + in_window = [item for item in occurrences if item.timestamp >= window.start] + production = [item for item in in_window if item.environment == "production"] + routes = Counter(item.route for item in production or in_window) + route, _ = min(routes.items(), key=lambda item: (-item[1], item[0] or "")) + return public_route(route) + + +def _error_group( + group_fingerprint: str, + members: list[tuple[ErrorRecord, Occurrence]], + related_fingerprints: tuple[str, ...], + route_requests: RouteRequests, + window: ReportingWindow, +) -> ErrorGroup: + example = members[0][0] + occurrences = [occurrence for _, occurrence in members] + measures = { + environment: _environment_measures( + occurrences, environment, example.family, route_requests, window + ) + for environment in REPORTED_ENVIRONMENTS + } + return ErrorGroup( + fingerprint=group_fingerprint, + kind=example.kind, + family=public_family(example.family), + error_type=public_error_type(example.error_type), + route=_most_frequent_route(occurrences, window), + status_code=public_status(example.status_code), + operation=public_operation(example.operation_name), + production=measures["production"], + staging=measures["staging"], + related_fingerprints=related_fingerprints, + occurrences=tuple(occurrences), + ) + + +def build_groups( + records: Iterable[ErrorRecord], + request_counts: Iterable[RequestCount], + window: ReportingWindow, + key: bytes, +) -> list[ErrorGroup]: + """Group the failures in production and staging and measure each group. + + Only groups with a failure in the window are returned, most production + occurrences first. + """ + in_scope = [ + record + for record in records + if record.environment in REPORTED_ENVIRONMENTS + and window.lookback_start <= record.timestamp < window.end + ] + members_by_group: dict[str, list[tuple[ErrorRecord, Occurrence]]] = {} + for position, record in enumerate(primary_records(in_scope)): + members_by_group.setdefault(fingerprint(record, key), []).append( + (record, _occurrence(record, position)) + ) + related = _related_fingerprints( + { + group: [occurrence for _, occurrence in members] + for group, members in members_by_group.items() + } + ) + route_requests = RouteRequests(request_counts) + groups = [ + _error_group(group, members, related[group], route_requests, window) + for group, members in members_by_group.items() + if any(occurrence.timestamp >= window.start for _, occurrence in members) + ] + return sorted( + groups, + key=lambda group: ( + -group.production.occurrences, + -group.staging.occurrences, + group.fingerprint, + ), + ) diff --git a/.github/scripts/telemetry_error_issues/normalization.py b/.github/scripts/telemetry_error_issues/normalization.py new file mode 100644 index 000000000..3bc2737c3 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/normalization.py @@ -0,0 +1,168 @@ +"""Normalize service, environment and source names, and validate public values. + +Each ``public_*`` function returns a value only if it matches the pattern of a +code-defined name, and a placeholder otherwise, so that free text from a record +never reaches an issue, a log or a job summary. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + +# Shown in place of a value that is missing, or that fails validation and so +# might not be code-defined. +MISSING = "-" +UNRECOGNIZED = "" + +PRODUCTION_ENVIRONMENT_NAMES = frozenset({"production", "main"}) +STAGING_ENVIRONMENT_NAMES = frozenset({"staging"}) +VERSIONED_SIMULATION_SERVICE = re.compile(r"^(policyengine-simulation-py)\d+-\d+-\d+$") + +FRAME_PARTS = re.compile(r'^(?P[^"\n]+)", line [0-9]+, in (?P\S+)$') +FINGERPRINT_FRAME_COUNT = 3 + +INSTALLED_PACKAGE_PREFIX = re.compile(r"^.*/(?:site|dist)-packages/") +STANDARD_LIBRARY_PREFIX = re.compile(r"^.*/lib/python3(?:\.\d+)?/") +APPLICATION_PACKAGES = ( + r"(?:policyengine_api|policyengine_fastapi" + r"|policyengine_simulation_[a-z0-9_]+|policyengine_stage12_[a-z0-9_]+)/" +) +APPLICATION_PATH = re.compile(rf"^{APPLICATION_PACKAGES}") +APPLICATION_PATH_IN_ABSOLUTE_PATH = re.compile(rf"(?:^|/)({APPLICATION_PACKAGES}.*)$") +FALLBACK_PATH_COMPONENTS = 2 + +ERROR_TYPE_PATTERN = re.compile(r"^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*$", re.ASCII) +MAX_ERROR_TYPE_LENGTH = 120 +UNMATCHED_ROUTE = "" +MAX_ROUTE_LENGTH = 200 +ROUTE_PARAMETER = re.compile( + r"^(?:<(?:[a-z]+:)?[A-Za-z_]\w*>|\{[A-Za-z_]\w*(?::[a-z]+)?\})$", re.ASCII +) +ROUTE_LITERAL = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,39}$") +# Literal route segments are words such as "v2" or "user-policies". A segment +# with more digits than this is more likely a record ID than code. +MAX_ROUTE_LITERAL_DIGITS = 2 +FAMILY_PATTERN = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") +MAX_FAMILY_LENGTH = 63 +OPERATION_PATTERN = re.compile(r"^[A-Za-z_][\w.:/-]*$", re.ASCII) +MAX_OPERATION_LENGTH = 100 +LOWEST_HTTP_STATUS = 100 +HIGHEST_HTTP_STATUS = 599 + + +def service_family(service_name: str) -> str: + """Return the service name, merging versioned simulation services into one.""" + return VERSIONED_SIMULATION_SERVICE.sub(r"\1", service_name) + + +def environment_class(environment_name: str | None) -> str: + """Classify a deployment environment name as production, staging or other. + + The Modal gateway reports its production environment as ``main``. + """ + if environment_name in PRODUCTION_ENVIRONMENT_NAMES: + return "production" + if environment_name in STAGING_ENVIRONMENT_NAMES: + return "staging" + return "other" + + +def normalize_path(path: str) -> str: + """Return a source path without its installation or checkout prefix.""" + installed = INSTALLED_PACKAGE_PREFIX.sub("", path, count=1) + if installed != path: + return installed + application = APPLICATION_PATH_IN_ABSOLUTE_PATH.search(path) + if application: + return application.group(1) + standard_library = STANDARD_LIBRARY_PREFIX.sub("", path, count=1) + if standard_library != path: + return standard_library + return "/".join(path.split("/")[-FALLBACK_PATH_COMPONENTS:]) + + +def frame_key(frame: str) -> str | None: + """Return ``path:function`` for one ``FRAME_PATTERN`` match, without its line.""" + parts = FRAME_PARTS.match(frame) + if parts is None: + return None + return f"{normalize_path(parts['path'])}:{parts['function']}" + + +def fingerprint_frames(frames: Iterable[str] | None) -> tuple[str, ...]: + """Choose the frames that identify where an exception was raised. + + ``frames`` come from the first traceback of the stack only (see + ``CHAINED_EXCEPTION_PATTERN``); for a chained exception that is its first + cause. These are the three innermost frames in application packages. + Without application frames they are the three innermost frames of any + kind, and without a stack, a placeholder. + """ + keys = [key for key in map(frame_key, frames or ()) if key is not None] + if not keys: + return (MISSING,) + application_keys = [key for key in keys if APPLICATION_PATH.match(key)] + return tuple((application_keys or keys)[-FINGERPRINT_FRAME_COUNT:]) + + +def _validated(value: str | None, pattern: re.Pattern[str], max_length: int) -> str: + if not value: + return MISSING + if len(value) <= max_length and pattern.fullmatch(value): + return value + return UNRECOGNIZED + + +def public_error_type(value: str | None) -> str: + """Return an exception class name, or a placeholder for anything else.""" + return _validated(value, ERROR_TYPE_PATTERN, MAX_ERROR_TYPE_LENGTH) + + +def public_family(value: str | None) -> str: + """Return a service name or family, or a placeholder for anything else.""" + return _validated(value, FAMILY_PATTERN, MAX_FAMILY_LENGTH) + + +def public_operation(value: str | None) -> str: + """Return a code-defined operation name, or a placeholder for anything else.""" + return _validated(value, OPERATION_PATTERN, MAX_OPERATION_LENGTH) + + +def _route_segment_is_code(segment: str) -> bool: + if ROUTE_PARAMETER.match(segment): + return True + digits = sum(character.isdigit() for character in segment) + return bool(ROUTE_LITERAL.match(segment)) and digits <= MAX_ROUTE_LITERAL_DIGITS + + +def public_route(value: str | None) -> str: + """Return a route template, or a placeholder for anything else. + + A path that holds a record ID, a query string or free text is not a + template, so it never reaches public output. + """ + if not value: + return MISSING + if value == UNMATCHED_ROUTE: + return value + if len(value) > MAX_ROUTE_LENGTH or not value.startswith("/"): + return UNRECOGNIZED + segments = [segment for segment in value.split("/") if segment] + if all(map(_route_segment_is_code, segments)): + return value + return UNRECOGNIZED + + +def public_status(value: object) -> int | None: + """Return an HTTP status code as an integer, or None for anything else.""" + if isinstance(value, bool): + return None + if isinstance(value, str) and value.isascii() and value.isdigit(): + value = int(value) + if isinstance(value, int) and LOWEST_HTTP_STATUS <= value <= HIGHEST_HTTP_STATUS: + return value + return None diff --git a/.github/scripts/telemetry_error_issues/records.py b/.github/scripts/telemetry_error_issues/records.py new file mode 100644 index 000000000..d685968f6 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/records.py @@ -0,0 +1,166 @@ +"""Error records, the one record chosen per failure, and their fingerprints.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from telemetry_error_issues.normalization import ( + MISSING, + environment_class, + fingerprint_frames, + service_family, +) + +if TYPE_CHECKING: + from collections.abc import Iterable + from datetime import datetime + +REQUEST_COMPLETED_EVENT = "request.completed" +# Changing how fingerprints are computed changes every fingerprint, and the +# job matches existing issues only by the current algorithm. A change to it +# must say how existing issues are carried over. +FINGERPRINT_ALGORITHM = 1 +FINGERPRINT_LENGTH = 16 +MIN_FINGERPRINT_KEY_LENGTH = 32 + + +@dataclass(frozen=True) +class ErrorRecord: + """One ERROR record from the central log bucket, as the query returns it. + + The query returns identifiers only as SHA-256 hashes, stacks only as frame + headers, and messages only as their first few words. + """ + + timestamp: datetime + service_name: str + environment_name: str | None + event_name: str | None + error_type: str | None + error_handled: bool | None + frames: tuple[str, ...] + stack_truncated: bool + message_template: str | None + attribute_error_type: str | None + failure_category: str | None + route: str | None + status_code: int | None + operation_name: str | None + request_key: str | None + observability_key: str | None + trace_key: str | None + + @property + def family(self) -> str: + """Return the service family that produced the record.""" + return service_family(self.service_name) + + @property + def environment(self) -> str: + """Return production, staging or other.""" + return environment_class(self.environment_name) + + @property + def kind(self) -> str: + """Return exception, message or failed_request. + + A record with an exception type is an exception, even a + ``request.completed`` record. Any other ``request.completed`` record is + a request that failed without a recorded exception. + """ + if self.error_type: + return "exception" + if self.event_name == REQUEST_COMPLETED_EVENT: + return "failed_request" + return "message" + + +def _primary_preference(record: ErrorRecord) -> tuple[int, int, datetime]: + """Sort key: exceptions before messages before request completions. + + Among exceptions, an unhandled one (the cause of an error response) comes + first; otherwise the earliest record does. + """ + if record.event_name == REQUEST_COMPLETED_EVENT: + source_rank = 2 + elif record.error_type: + source_rank = 0 + else: + source_rank = 1 + handled_rank = 0 if record.error_handled is False else 1 + return source_rank, handled_rank, record.timestamp + + +def primary_records(records: Iterable[ErrorRecord]) -> list[ErrorRecord]: + """Return one record per failure, in time order. + + Records of one service that share a request ID, or a trace when there is no + request ID, describe one failure. A record with neither is its own failure. + """ + failures: dict[tuple[str, str], list[ErrorRecord]] = {} + standalone: list[ErrorRecord] = [] + for record in records: + correlation = record.request_key or record.trace_key + if correlation is None: + standalone.append(record) + else: + failures.setdefault((record.family, correlation), []).append(record) + chosen = [min(group, key=_primary_preference) for group in failures.values()] + return sorted([*chosen, *standalone], key=lambda record: record.timestamp) + + +def fingerprint_parts(record: ErrorRecord) -> tuple[str, ...]: + """Return the kind and the values that identify the record's error group. + + An exception's route is left out, so one shared failure doesn't open an + issue for every route that reaches it. + """ + kind = record.kind + if kind == "exception": + return ( + kind, + record.family, + record.error_type or MISSING, + *fingerprint_frames(record.frames), + ) + if kind == "message": + return ( + kind, + record.family, + record.route or record.operation_name or MISSING, + record.message_template or MISSING, + record.attribute_error_type or MISSING, + record.failure_category or MISSING, + ) + status = MISSING if record.status_code is None else str(record.status_code) + return (kind, record.family, record.route or MISSING, status) + + +def fingerprint_key(secret: str) -> bytes: + """Return the HMAC key, rejecting a secret too short to be random.""" + if len(secret) < MIN_FINGERPRINT_KEY_LENGTH: + msg = ( + "the fingerprint key must be at least " + f"{MIN_FINGERPRINT_KEY_LENGTH} characters" + ) + raise ValueError(msg) + return secret.encode() + + +def fingerprint(record: ErrorRecord, key: bytes) -> str: + """Return the record's error group as 16 hex characters of HMAC-SHA256. + + The key keeps anyone from confirming a guessed message or stack frame by + hashing it and comparing the result with a public fingerprint. + """ + payload = json.dumps( + [FINGERPRINT_ALGORITHM, *fingerprint_parts(record)], + ensure_ascii=False, + separators=(",", ":"), + ) + digest = hmac.new(key, payload.encode(), hashlib.sha256).hexdigest() + return digest[:FINGERPRINT_LENGTH] diff --git a/.github/scripts/telemetry_error_issues/rendering.py b/.github/scripts/telemetry_error_issues/rendering.py new file mode 100644 index 000000000..286fddea3 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/rendering.py @@ -0,0 +1,242 @@ +"""Issue titles and bodies: the measures table, the state and a Logs Explorer link.""" + +from __future__ import annotations + +import json +import re +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from urllib.parse import quote, urlencode + +from telemetry_error_issues.normalization import MISSING, UNRECOGNIZED +from telemetry_error_issues.records import FINGERPRINT_ALGORITHM +from telemetry_error_issues.window import REPORTING_TIME_ZONE, ReportingWindow + +if TYPE_CHECKING: + from telemetry_error_issues.groups import EnvironmentMeasures, ErrorGroup + +ISSUE_TITLE_PREFIX = "[telemetry]" +MAX_TITLE_LENGTH = 256 +SECTION_START = "" +SECTION_END = "" +STATE_PATTERN = re.compile(r"") +NOT_AVAILABLE = "n/a" +LOGS_EXPLORER_URL = "https://console.cloud.google.com/logs/query" +LOG_BUCKET = "policyengine-observability" +LOG_VIEW = "_AllLogs" +SCHEMA_VERSION = "policyengine.observability.v2" +SIMULATION_FAMILY = "policyengine-simulation-py" + + +def code(value: str) -> str: + """Return ``value`` as Markdown inline code.""" + return f"`{value}`" + + +def _place(group: ErrorGroup) -> str: + """Describe where the group fails: its route, else its operation, else nothing.""" + if group.route != MISSING: + return f" on {code(group.route)}" + if group.kind == "message" and group.operation != MISSING: + return f" in {code(group.operation)}" + return "" + + +def _status_text(group: ErrorGroup) -> str: + return "HTTP error" if group.status_code is None else f"HTTP {group.status_code}" + + +def _summary(group: ErrorGroup) -> str: + if group.kind == "exception": + return group.error_type + if group.kind == "message": + return "logged error" + return f"{_status_text(group)} without recorded exception" + + +def issue_title(group: ErrorGroup) -> str: + """Return the issue title, which names the fingerprint for matching. + + The route is left out when the title would exceed GitHub's length limit. + """ + lead = f"{ISSUE_TITLE_PREFIX} {group.family}: {_summary(group)}" + suffix = f" ({group.fingerprint})" + title = f"{lead}{_place(group)}{suffix}" + return title if len(title) <= MAX_TITLE_LENGTH else f"{lead}{suffix}" + + +def _logs_timestamp(moment: datetime) -> str: + return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _logs_filters(group: ErrorGroup) -> list[str]: + """Return logging query clauses that select the group's records.""" + filters = [f'jsonPayload.schema_version="{SCHEMA_VERSION}"'] + if group.family == SIMULATION_FAMILY: + filters.append( + f'jsonPayload."service.name"=~"^{SIMULATION_FAMILY}[0-9]+-[0-9]+-[0-9]+$"' + ) + elif group.family not in {MISSING, UNRECOGNIZED}: + filters.append(f'jsonPayload."service.name"="{group.family}"') + if group.kind == "failed_request": + filters.append('jsonPayload."event.name"="request.completed"') + else: + filters.append("severity>=ERROR") + if group.kind == "exception" and group.error_type not in {MISSING, UNRECOGNIZED}: + filters.append(f'jsonPayload."error.type"="{group.error_type}"') + if group.kind != "exception" and group.route not in {MISSING, UNRECOGNIZED}: + filters.append(f'jsonPayload."http.route"="{group.route}"') + elif group.kind == "message" and group.operation not in {MISSING, UNRECOGNIZED}: + filters.append(f'jsonPayload."operation.name"="{group.operation}"') + if group.kind == "failed_request" and group.status_code is not None: + filters.append(f'jsonPayload."http.response.status_code"={group.status_code}') + return filters + + +def logs_explorer_url( + group: ErrorGroup, window: ReportingWindow, project_id: str +) -> str: + """Return a Logs Explorer link to the group's records in the window.""" + query = "\n".join( + [ + *_logs_filters(group), + f'timestamp>="{_logs_timestamp(window.start)}"', + f'timestamp<"{_logs_timestamp(window.end)}"', + ] + ) + view = ( + f"projects/{project_id}/locations/global/buckets/{LOG_BUCKET}/views/{LOG_VIEW}" + ) + settings = {"query": query, "storageScope": f"storage,{view}"} + encoded = "".join( + f";{name}={quote(value, safe='')}" for name, value in settings.items() + ) + return f"{LOGS_EXPLORER_URL}{encoded}?{urlencode({'project': project_id})}" + + +def eastern_time(moment: datetime | None) -> str: + """Return ``moment`` in US Eastern time, such as 2026-10-20 12:00 EDT, or n/a.""" + if moment is None: + return NOT_AVAILABLE + return moment.astimezone(REPORTING_TIME_ZONE).strftime("%Y-%m-%d %H:%M %Z") + + +def window_text(window: ReportingWindow) -> str: + """Describe the window by its Eastern start and end, and its length in hours.""" + return ( + f"{eastern_time(window.start)} to {eastern_time(window.end)} " + f"({window.hours:.0f} hours)" + ) + + +def _error_rate(measures: EnvironmentMeasures) -> str: + if measures.error_rate is None: + return NOT_AVAILABLE + return ( + f"{measures.error_rate:.2%} ({measures.route_affected_requests:,} of " + f"{measures.route_requests:,} requests)" + ) + + +def _measures_table(group: ErrorGroup) -> list[str]: + rows = [ + ("Occurrences", lambda measures: f"{measures.occurrences:,}"), + ("Affected requests", lambda measures: f"{measures.affected_requests:,}"), + ("Observability IDs", lambda measures: f"{measures.observability_ids:,}"), + ("Error rate on affected routes", _error_rate), + ("Occurrences per hour", lambda measures: f"{measures.hourly_rate:.2f}"), + ( + "Occurrences per hour, previous 7 days", + lambda measures: f"{measures.previous_hourly_rate:.2f}", + ), + ( + "First seen in the last 8 days", + lambda measures: eastern_time(measures.first_seen), + ), + ("Last seen", lambda measures: eastern_time(measures.last_seen)), + ] + return [ + "| Measure | Production | Staging |", + "| --- | --- | --- |", + *( + f"| {name} | {describe(group.production)} | {describe(group.staging)} |" + for name, describe in rows + ), + ] + + +def _group_details(group: ErrorGroup) -> list[str]: + details = [ + f"- Kind: {group.kind} (fingerprint algorithm {FINGERPRINT_ALGORITHM})", + f"- Service: {code(group.family)}", + ] + if group.kind == "exception": + details.append(f"- Exception type: {code(group.error_type)}") + if group.kind == "failed_request": + details.append(f"- Response: {_status_text(group)}") + if group.kind == "message" and group.operation != MISSING: + details.append(f"- Operation: {code(group.operation)}") + details.append(f"- Most frequent route: {code(group.route)}") + return details + + +def _state_comment(group: ErrorGroup, window: ReportingWindow) -> str: + state = { + "algorithm": FINGERPRINT_ALGORITHM, + "fingerprint": group.fingerprint, + "kind": group.kind, + "window_end": window.end.astimezone(UTC).isoformat(), + } + return f"" + + +def parse_state(body: str) -> dict[str, Any] | None: + """Return the state the job last wrote into an issue body, if it is there.""" + match = STATE_PATTERN.search(body) + if match is None: + return None + try: + state = json.loads(match.group(1)) + except ValueError: + return None + return state if isinstance(state, dict) else None + + +def issue_section(group: ErrorGroup, window: ReportingWindow, project_id: str) -> str: + """Return the marked section of the issue body, which each run replaces.""" + related = ", ".join(map(code, group.related_fingerprints)) + lines = [ + SECTION_START, + f"### Error group {code(group.fingerprint)}", + "", + *_group_details(group), + f"- Window: {window_text(window)}", + "", + *_measures_table(group), + "", + ( + f"Related fingerprints (sharing traces): {related}" + if related + else "Related fingerprints: none" + ), + "", + ( + f"[Matching log records in Logs Explorer]" + f"({logs_explorer_url(group, window, project_id)}), for the window only. " + f"It needs read access to the `{project_id}` project." + ), + "", + _state_comment(group, window), + SECTION_END, + ] + return "\n".join(lines) + + +def issue_body(group: ErrorGroup, window: ReportingWindow, project_id: str) -> str: + """Return a new issue's body: a short note, then the marked section.""" + note = ( + "The daily telemetry error report opened this issue for a production " + "error group. Each run replaces the marked section below with the " + "latest day's measures; text outside it is kept." + ) + return f"{note}\n\n{issue_section(group, window, project_id)}\n" diff --git a/.github/scripts/telemetry_error_issues/sync.py b/.github/scripts/telemetry_error_issues/sync.py new file mode 100644 index 000000000..2f1ac46eb --- /dev/null +++ b/.github/scripts/telemetry_error_issues/sync.py @@ -0,0 +1,282 @@ +"""Create, update and reopen one GitHub issue per production error group.""" + +from __future__ import annotations + +import re +from datetime import datetime, timedelta +from typing import TYPE_CHECKING, Any, NamedTuple + +from telemetry_error_issues.github_client import GitHubClient, GitHubLabelError +from telemetry_error_issues.records import FINGERPRINT_ALGORITHM +from telemetry_error_issues.rendering import ( + ISSUE_TITLE_PREFIX, + SECTION_END, + SECTION_START, + eastern_time, + issue_body, + issue_section, + issue_title, + parse_state, +) + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + + from telemetry_error_issues.groups import ErrorGroup + from telemetry_error_issues.window import ReportingWindow + +BOT_LOGIN = "github-actions[bot]" +TITLE_FINGERPRINT = re.compile(r"\(([0-9a-f]{16})\)$") +BASE_LABEL = "telemetry-error" +REGRESSION_LABEL = "regression" +SERVICE_LABEL_PREFIX = "service: " +MAX_LABEL_LENGTH = 50 +LABEL_DETAILS = { + BASE_LABEL: ("b60205", "Opened by the daily telemetry error report"), + REGRESSION_LABEL: ("d93f0b", "An error that returned after its issue was closed"), +} +SERVICE_LABEL_DETAILS = ("1d76db", "Service that produced a telemetry error group") +# A production group opens an issue only when it fails this often, in this many +# distinct requests, during the window; the same thresholds reopen an issue. +MIN_PRODUCTION_OCCURRENCES = 3 +MIN_PRODUCTION_REQUESTS = 2 +MAX_NEW_ISSUES_PER_RUN = 5 +# Failures in the first day after an issue is closed may come from code that +# hasn't been deployed yet, so they never reopen it. +REOPEN_DELAY = timedelta(hours=24) +REOPENABLE_CLOSE_REASONS = frozenset({None, "completed"}) + + +class SyncAction(NamedTuple): + """What one run did, or in a dry run would do, for one error group.""" + + fingerprint: str + kind: str + family: str + production_occurrences: int + staging_occurrences: int + issue_number: int | None + action: str + + +class SyncResult(NamedTuple): + """Every group's action, and how many new issues waited for a later run.""" + + actions: list[SyncAction] + deferred: int + + +def replace_section(body: str, section: str) -> str: + """Replace the marked section of ``body``, or add it at the end if missing.""" + start = body.find(SECTION_START) + end = body.find(SECTION_END, start) if start >= 0 else -1 + if end >= 0: + return f"{body[:start]}{section}{body[end + len(SECTION_END) :]}" + kept = body.rstrip() + return f"{kept}\n\n{section}\n" if kept else f"{section}\n" + + +def _issue_fingerprint(issue: Mapping[str, Any]) -> str | None: + """Return the fingerprint from the state comment, or else from the title.""" + state = parse_state(issue.get("body") or "") + if state and state.get("algorithm") == FINGERPRINT_ALGORITHM: + fingerprint_value = state.get("fingerprint") + if isinstance(fingerprint_value, str): + return fingerprint_value + title = issue.get("title") or "" + match = TITLE_FINGERPRINT.search(title) + if match and title.startswith(ISSUE_TITLE_PREFIX): + return match.group(1) + return None + + +def _issues_by_fingerprint( + issues: Iterable[Mapping[str, Any]], +) -> dict[str, Mapping[str, Any]]: + """Index the bot's issues by fingerprint, preferring open, then newest.""" + index: dict[str, Mapping[str, Any]] = {} + for issue in issues: + if (issue.get("user") or {}).get("login") != BOT_LOGIN: + continue + group = _issue_fingerprint(issue) + if group is None: + continue + rank = (issue.get("state") == "open", issue.get("number", 0)) + current = index.get(group) + if current is None or rank > ( + current.get("state") == "open", + current["number"], + ): + index[group] = issue + return index + + +def _meets_thresholds(occurrences: int, affected_requests: int) -> bool: + return ( + occurrences >= MIN_PRODUCTION_OCCURRENCES + and affected_requests >= MIN_PRODUCTION_REQUESTS + ) + + +def _folded(names: Iterable[str]) -> set[str]: + """Return label names case-folded, as GitHub compares them.""" + return {name.casefold() for name in names} + + +def _issue_labels(group: ErrorGroup) -> list[str]: + service_label = f"{SERVICE_LABEL_PREFIX}{group.family}" + if len(service_label) > MAX_LABEL_LENGTH: + return [BASE_LABEL] + return [BASE_LABEL, service_label] + + +def _parse_time(value: object) -> datetime | None: + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value) + except ValueError: + return None + return parsed if parsed.tzinfo is not None else None + + +class IssueSynchronizer: + """Creates and updates one issue per production error group.""" + + def __init__( + self, + github: GitHubClient, + window: ReportingWindow, + project_id: str, + *, + dry_run: bool, + ) -> None: + """Sync the groups of ``window``; a dry run reads but never writes.""" + self._github = github + self._window = window + self._project_id = project_id + self._dry_run = dry_run + # Case-folded names: GitHub label names are unique regardless of case. + self._known_labels: set[str] | None = None + + def run(self, groups: Iterable[ErrorGroup]) -> SyncResult: + """Sync each group, opening at most five issues, largest groups first.""" + issues = _issues_by_fingerprint(self._github.list_issues(BOT_LOGIN)) + actions: list[SyncAction] = [] + opened = 0 + for group in groups: + issue = issues.get(group.fingerprint) + number: int | None = None + if issue is not None: + number = issue["number"] + action = self._update(group, issue) + elif not _meets_thresholds( + group.production.occurrences, group.production.affected_requests + ): + action = "below-threshold" + elif opened >= MAX_NEW_ISSUES_PER_RUN: + action = "deferred" + else: + number = self._open(group) + opened += 1 + action = "created" + actions.append( + SyncAction( + fingerprint=group.fingerprint, + kind=group.kind, + family=group.family, + production_occurrences=group.production.occurrences, + staging_occurrences=group.staging.occurrences, + issue_number=number, + action=action, + ) + ) + deferred = sum(action.action == "deferred" for action in actions) + return SyncResult(actions, deferred) + + def _ensure_labels(self, names: Iterable[str]) -> None: + if self._known_labels is None: + self._known_labels = _folded(self._github.list_labels()) + for name in names: + if name.casefold() not in self._known_labels: + color, description = LABEL_DETAILS.get(name, SERVICE_LABEL_DETAILS) + self._github.create_label(name, color, description) + self._known_labels.add(name.casefold()) + + def _open(self, group: ErrorGroup) -> int | None: + if self._dry_run: + return None + labels = _issue_labels(group) + self._ensure_labels(labels) + issue = self._github.create_issue( + issue_title(group), + issue_body(group, self._window, self._project_id), + labels, + ) + number = issue["number"] + applied = _folded(label["name"] for label in issue.get("labels", [])) + missing = [label for label in labels if label.casefold() not in applied] + if missing and not _folded(missing) <= _folded( + self._github.add_labels(number, missing) + ): + msg = f"GitHub did not apply the labels to issue #{number}" + raise GitHubLabelError(msg) + return number + + def _reopen_since(self, issue: Mapping[str, Any]) -> datetime | None: + """Return when regressions count from, if the issue should reopen.""" + if issue.get("state") != "closed": + return None + if issue.get("state_reason") not in REOPENABLE_CLOSE_REASONS: + return None + closed_at = _parse_time(issue.get("closed_at")) + return None if closed_at is None else closed_at + REOPEN_DELAY + + def _update(self, group: ErrorGroup, issue: Mapping[str, Any]) -> str: + body = issue.get("body") or "" + state = parse_state(body) + reported_end = _parse_time(state.get("window_end")) if state else None + if reported_end is not None and reported_end > self._window.end: + return "kept-newer" + new_body = replace_section( + body, issue_section(group, self._window, self._project_id) + ) + since = self._reopen_since(issue) + if since is not None and _meets_thresholds(*group.production_since(since)): + if not self._dry_run: + self._reopen(issue["number"], new_body, group, since) + return "reopened" + if new_body == body: + return "unchanged" + if not self._dry_run: + self._github.update_issue(issue["number"], {"body": new_body}) + return "updated" + + def _reopen( + self, number: int, body: str, group: ErrorGroup, since: datetime + ) -> None: + occurrences, affected_requests = group.production_since(since) + self._ensure_labels([REGRESSION_LABEL]) + self._github.update_issue( + number, {"body": body, "state": "open", "state_reason": "reopened"} + ) + self._github.add_labels(number, [REGRESSION_LABEL]) + self._github.add_comment( + number, + "Production failures returned after this issue was closed: " + f"{occurrences:,} occurrences in {affected_requests:,} requests since " + f"{eastern_time(since)}, 24 hours after it was closed.", + ) + + +def sync_issues( + groups: Iterable[ErrorGroup], + window: ReportingWindow, + project_id: str, + github: GitHubClient, + *, + dry_run: bool, +) -> SyncResult: + """Create or update the issue for each group; a dry run makes no changes.""" + synchronizer = IssueSynchronizer(github, window, project_id, dry_run=dry_run) + return synchronizer.run(groups) diff --git a/.github/scripts/telemetry_error_issues/transport.py b/.github/scripts/telemetry_error_issues/transport.py new file mode 100644 index 000000000..14c69497a --- /dev/null +++ b/.github/scripts/telemetry_error_issues/transport.py @@ -0,0 +1,96 @@ +"""HTTP requests and responses, the standard library transport, and API errors.""" + +from __future__ import annotations + +import json +import re +from typing import TYPE_CHECKING, NamedTuple +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +if TYPE_CHECKING: + from collections.abc import Mapping + +HTTP_TIMEOUT_SECONDS = 60 +# A short code from an API error, such as "invalidQuery" or "Not Found". The +# error's message is never used, because it can repeat values from a record. +ERROR_REASON_PATTERN = re.compile(r"^[A-Za-z][A-Za-z ]{0,59}$") + + +class HttpRequest(NamedTuple): + """An HTTP request for a transport to send.""" + + method: str + url: str + headers: dict[str, str] + body: bytes | None + + +class HttpResponse(NamedTuple): + """An HTTP response, including error responses. Header names are lowercase.""" + + status: int + reason: str + headers: Mapping[str, str] + body: bytes + + +def urllib_transport(request: HttpRequest) -> HttpResponse: + """Send a request with the standard library, returning HTTP errors too.""" + outgoing = Request( + request.url, + data=request.body, + headers=request.headers, + method=request.method, + ) + try: + response = urlopen(outgoing, timeout=HTTP_TIMEOUT_SECONDS) + except HTTPError as error: + with error: + return HttpResponse( + status=error.code, + reason=str(error.reason), + headers={name.lower(): value for name, value in error.headers.items()}, + body=error.read(), + ) + with response: + return HttpResponse( + status=response.status, + reason=response.reason, + headers={name.lower(): value for name, value in response.headers.items()}, + body=response.read(), + ) + + +class ApiError(Exception): + """An API request failed. The text holds only code-defined values.""" + + service = "API" + + def __init__( + self, message: str, *, status: int | None = None, reason: str | None = None + ) -> None: + """Keep the HTTP status and reason for the caller's summary.""" + super().__init__(message) + self.status = status + self.reason = reason + + @classmethod + def from_response(cls, response: HttpResponse) -> ApiError: + """Describe an error response by its status and reason, never its body.""" + reason = _error_reason(response) + message = f"{cls.service} request failed: HTTP {response.status} {reason}" + return cls(message, status=response.status, reason=reason) + + +def _error_reason(response: HttpResponse) -> str: + """Return a Google API error's reason code, else the HTTP reason phrase.""" + try: + payload = json.loads(response.body) + reason = payload["error"]["errors"][0]["reason"] + except (ValueError, KeyError, IndexError, TypeError): + reason = None + for candidate in (reason, response.reason): + if isinstance(candidate, str) and ERROR_REASON_PATTERN.match(candidate): + return candidate + return "unknown" diff --git a/.github/scripts/telemetry_error_issues/window.py b/.github/scripts/telemetry_error_issues/window.py new file mode 100644 index 000000000..5145fc1a0 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/window.py @@ -0,0 +1,57 @@ +"""The reporting window: one US Eastern calendar day, and the week before it.""" + +from __future__ import annotations + +from datetime import UTC, date, datetime, timedelta +from datetime import time as time_of_day +from typing import NamedTuple +from zoneinfo import ZoneInfo + +REPORTING_TIME_ZONE = ZoneInfo("America/New_York") +MIDNIGHT = time_of_day(0, 0) + +# Each run also reads the seven days before the window, for comparison rates, +# first-seen times and regressions. +LOOKBACK = timedelta(days=7) + + +class ReportingWindow(NamedTuple): + """One Eastern calendar day, as the half-open interval [start, end).""" + + start: datetime + end: datetime + + @property + def hours(self) -> float: + """Return the elapsed hours: 23 or 25 on the days the clocks change.""" + return _hours_between(self.start, self.end) + + @property + def lookback_start(self) -> datetime: + """Return midnight Eastern seven days before the window starts.""" + return self.start - LOOKBACK + + @property + def lookback_hours(self) -> float: + """Return the elapsed hours of the seven days before the window.""" + return _hours_between(self.lookback_start, self.start) + + +def _hours_between(start: datetime, end: datetime) -> float: + # Subtracting two datetimes in the same zone ignores a clock change, so + # compare them in UTC. + return (end.astimezone(UTC) - start.astimezone(UTC)) / timedelta(hours=1) + + +def reporting_window(report_date: date) -> ReportingWindow: + """Return the window from midnight Eastern on ``report_date`` to the next.""" + next_date = report_date + timedelta(days=1) + return ReportingWindow( + start=datetime.combine(report_date, MIDNIGHT, tzinfo=REPORTING_TIME_ZONE), + end=datetime.combine(next_date, MIDNIGHT, tzinfo=REPORTING_TIME_ZONE), + ) + + +def default_report_date(now: datetime) -> date: + """Return the most recent Eastern calendar day that ended before ``now``.""" + return now.astimezone(REPORTING_TIME_ZONE).date() - timedelta(days=1) diff --git a/gcp/observability/README.md b/gcp/observability/README.md index cb12b2bfa..bc6943414 100644 --- a/gcp/observability/README.md +++ b/gcp/observability/README.md @@ -136,8 +136,13 @@ corresponding rejected points in that interval. The `Telemetry error issues` workflow (`.github/workflows/telemetry-error-issues.yml`) runs at 00:45 US Eastern each day. It reports the previous Eastern calendar day, from midnight to midnight, -which lasts 23 or 25 hours on the days the clocks change. The script it runs, -`.github/scripts/sync_telemetry_error_issues.py`: +which lasts 23 or 25 hours on the days the clocks change. It runs +`.github/scripts/sync_telemetry_error_issues.py`, a short entry point that calls +the `telemetry_error_issues` package beside it in `.github/scripts/`. The package's +`cli` module reads the arguments and settings and runs each step; `bigquery` +runs the queries, `records` and `groups` choose, fingerprint and measure the +failures, `rendering` writes the issue text, and `sync` updates the issues +through `github_client`. Each run: 1. Queries the central bucket through its linked BigQuery dataset `observability_logs`. It reads the `_AllLogs` view for ERROR records with diff --git a/tests/unit/telemetry_error_issues/__init__.py b/tests/unit/telemetry_error_issues/__init__.py new file mode 100644 index 000000000..72527b225 --- /dev/null +++ b/tests/unit/telemetry_error_issues/__init__.py @@ -0,0 +1 @@ +"""Tests for the telemetry error issues package under .github/scripts.""" diff --git a/tests/unit/telemetry_error_issues/bigquery_support.py b/tests/unit/telemetry_error_issues/bigquery_support.py new file mode 100644 index 000000000..beb1a784b --- /dev/null +++ b/tests/unit/telemetry_error_issues/bigquery_support.py @@ -0,0 +1,66 @@ +"""A fake HTTP transport and BigQuery REST API responses.""" + +from __future__ import annotations + +import json + +from telemetry_error_issues.transport import HttpResponse + +ACCESS_TOKEN = "fake-access-token" + + +class FakeTransport: + """Records each HTTP request and answers with queued responses.""" + + def __init__(self, responses: list[object]) -> None: + self.responses = list(responses) + self.requests: list[object] = [] + + def __call__(self, request: object) -> object: + self.requests.append(request) + return self.responses.pop(0) + + def json_body(self, index: int) -> dict: + return json.loads(self.requests[index].body) + + +def json_response(payload: object, status: int = 200, reason: str = "OK") -> object: + return HttpResponse( + status=status, reason=reason, headers={}, body=json.dumps(payload).encode() + ) + + +def _cell(value: object) -> dict: + if isinstance(value, list): + return {"v": [{"v": item} for item in value]} + return {"v": value} + + +def query_response( + fields: list[dict], + rows: list[list[object]], + *, + page_token: str | None = None, + job_complete: bool = True, +) -> dict: + response: dict[str, object] = { + "jobComplete": job_complete, + "jobReference": { + "projectId": "policyengine-observability", + "jobId": "job-1", + "location": "US", + }, + } + if job_complete: + response["schema"] = {"fields": fields} + response["rows"] = [{"f": [_cell(value) for value in row]} for row in rows] + if page_token: + response["pageToken"] = page_token + return response + + +def sent_parameters(transport: FakeTransport, index: int = 0) -> dict: + return { + parameter["name"]: parameter + for parameter in transport.json_body(index)["queryParameters"] + } diff --git a/tests/unit/telemetry_error_issues/conftest.py b/tests/unit/telemetry_error_issues/conftest.py new file mode 100644 index 000000000..c0a2b5304 --- /dev/null +++ b/tests/unit/telemetry_error_issues/conftest.py @@ -0,0 +1,13 @@ +"""Make the telemetry_error_issues package importable, as running the script does. + +Python puts the directory of a script it runs by path first on ``sys.path``, +which is how ``sync_telemetry_error_issues.py`` imports the package beside it. +""" + +import sys +from pathlib import Path + +SCRIPTS_DIRECTORY = Path(__file__).resolve().parents[3] / ".github" / "scripts" + +if str(SCRIPTS_DIRECTORY) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIRECTORY)) diff --git a/tests/unit/telemetry_error_issues/github_support.py b/tests/unit/telemetry_error_issues/github_support.py new file mode 100644 index 000000000..614a99e24 --- /dev/null +++ b/tests/unit/telemetry_error_issues/github_support.py @@ -0,0 +1,181 @@ +"""An in-memory GitHub issues API, and helpers that sync issues against it.""" + +from __future__ import annotations + +import json +from urllib.parse import parse_qs, urlsplit + +from telemetry_error_issues.github_client import GitHubClient +from telemetry_error_issues.groups import build_groups +from telemetry_error_issues.sync import sync_issues +from telemetry_error_issues.transport import HttpResponse +from telemetry_error_issues.window import reporting_window + +from tests.unit.telemetry_error_issues.report_support import ( + FINGERPRINT_KEY, + PROJECT_ID, + REPORT_DATE, + exception_record, + hours_after_window_noon, +) + +REPOSITORY = "PolicyEngine/policyengine-api" +BOT = "github-actions[bot]" + + +class FakeGitHub: + """An in-memory GitHub issues API with a clock that advances when it sleeps.""" + + def __init__( + self, + issues: list[dict] = (), + labels: tuple[str, ...] = (), + *, + drop_created_labels: bool = False, + ) -> None: + self.issues = {issue["number"]: dict(issue) for issue in issues} + self.labels = set(labels) + self.drop_created_labels = drop_created_labels + self.failures: list[object] = [] + self.requests: list[tuple[float, object]] = [] + self.comments: dict[int, list[str]] = {} + self.sleeps: list[float] = [] + self.now = 0.0 + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += seconds + + def clock(self) -> float: + return self.now + + def client(self) -> object: + return GitHubClient( + REPOSITORY, "fake-github-token", self, sleep=self.sleep, clock=self.clock + ) + + @property + def writes(self) -> list[tuple[float, object]]: + return [entry for entry in self.requests if entry[1].method != "GET"] + + def _respond(self, payload: object, status: int = 200) -> object: + return HttpResponse( + status=status, reason="OK", headers={}, body=json.dumps(payload).encode() + ) + + def __call__(self, request: object) -> object: + self.requests.append((self.now, request)) + if request.method != "GET" and self.failures: + return self.failures.pop(0) + split = urlsplit(request.url) + path = split.path.removeprefix(f"/repos/{REPOSITORY}") + query = {name: values[0] for name, values in parse_qs(split.query).items()} + body = json.loads(request.body) if request.body else None + return self._route(request.method, path, query, body) + + def _route(self, method: str, path: str, query: dict, body: dict | None) -> object: + parts = path.strip("/").split("/") + if len(parts) > 1: + return self._route_issue(method, int(parts[1]), parts[2:], body) + if method == "GET" and parts == ["issues"]: + issues = sorted(self.issues.values(), key=lambda issue: issue["number"]) + return self._respond(self._page(issues, query)) + if method == "GET": + labels = [{"name": name} for name in sorted(self.labels)] + return self._respond(self._page(labels, query)) + if parts == ["labels"]: + # GitHub label names are unique regardless of case. + if body["name"].casefold() in {name.casefold() for name in self.labels}: + return self._respond({"message": "Validation Failed"}, status=422) + self.labels.add(body["name"]) + return self._respond({"name": body["name"]}, status=201) + return self._create(body) + + def _existing_label_names(self, requested: list[str]) -> list[str]: + """Return the stored spelling of each label, as GitHub applies them.""" + stored = {name.casefold(): name for name in self.labels} + return [stored.get(name.casefold(), name) for name in requested] + + def _route_issue( + self, method: str, number: int, rest: list[str], body: dict + ) -> object: + if method == "PATCH": + self.issues[number].update(body) + return self._respond(self.issues[number]) + if rest == ["labels"]: + names = {label["name"] for label in self.issues[number]["labels"]} + if not self.drop_created_labels: + names |= set(self._existing_label_names(body["labels"])) + self.issues[number]["labels"] = [{"name": name} for name in sorted(names)] + return self._respond(self.issues[number]["labels"]) + self.comments.setdefault(number, []).append(body["body"]) + return self._respond({"id": 1}, status=201) + + def _page(self, items: list, query: dict) -> list: + page = int(query.get("page", "1")) + size = int(query.get("per_page", "30")) + return items[(page - 1) * size : page * size] + + def _create(self, body: dict) -> object: + number = max(self.issues, default=0) + 1 + labels = ( + [] + if self.drop_created_labels + else self._existing_label_names(body["labels"]) + ) + self.issues[number] = { + "number": number, + "title": body["title"], + "body": body["body"], + "state": "open", + "state_reason": None, + "closed_at": None, + "labels": [{"name": name} for name in labels], + "user": {"login": BOT}, + } + return self._respond(self.issues[number], status=201) + + +def bot_issue(number: int, **fields: object) -> dict: + issue = { + "number": number, + "title": f"Issue {number}", + "body": "", + "state": "open", + "state_reason": None, + "closed_at": None, + "labels": [], + "user": {"login": BOT}, + } + issue.update(fields) + return issue + + +def failures( + count: int, + *, + request_key_prefix: str = "r", + **fields: object, +) -> list: + return [ + exception_record( + request_key=f"{request_key_prefix}{index}", + timestamp=hours_after_window_noon(index), + **fields, + ) + for index in range(count) + ] + + +def groups_for_report_date(records: list) -> list: + return build_groups(records, [], reporting_window(REPORT_DATE), FINGERPRINT_KEY) + + +def sync(groups: list, github: FakeGitHub, *, dry_run=False): + return sync_issues( + groups, + reporting_window(REPORT_DATE), + PROJECT_ID, + github.client(), + dry_run=dry_run, + ) diff --git a/tests/unit/telemetry_error_issues/report_support.py b/tests/unit/telemetry_error_issues/report_support.py new file mode 100644 index 000000000..a597af8e4 --- /dev/null +++ b/tests/unit/telemetry_error_issues/report_support.py @@ -0,0 +1,171 @@ +"""Records, stacks, times and settings shared by several test modules.""" + +from __future__ import annotations + +import re +from datetime import UTC, date, datetime, timedelta +from zoneinfo import ZoneInfo + +from telemetry_error_issues.bigquery import CHAINED_EXCEPTION_PATTERN, FRAME_PATTERN +from telemetry_error_issues.groups import RequestCount +from telemetry_error_issues.records import ErrorRecord + +EASTERN = ZoneInfo("America/New_York") + + +def extract_frames(stack: str) -> list[str]: + """Extract frames as the query does: REGEXP_REPLACE, then REGEXP_EXTRACT_ALL.""" + first_traceback = re.sub(CHAINED_EXCEPTION_PATTERN, "", stack) + return re.findall(FRAME_PATTERN, first_traceback) + + +def chained_stack(recursion_depth: int) -> str: + """Return a chained traceback whose cause recursed ``recursion_depth`` times.""" + library_frame = ( + ' File "/venv/lib/python3.12/site-packages/policyengine_core/simulation.py", ' + "line 500, in calculate\n return self._calculate(variable_name, period)\n" + ) + cause = ( + "Traceback (most recent call last):\n" + ' File "/app/policyengine_api/services/household.py", line 40, in calculate\n' + " simulation.calculate(name)\n" + f"{library_frame * recursion_depth}" + "ValueError: SENTINEL cause\n" + ) + final = ( + "Traceback (most recent call last):\n" + ' File "/app/policyengine_api/routes/household_routes.py", line 12, in get\n' + " return service.calculate()\n" + ' File "/app/policyengine_api/services/household.py", line 44, in calculate\n' + " raise CalculationError(name) from error\n" + "CalculationError: SENTINEL final\n" + ) + separator = ( + "\nThe above exception was the direct cause of the following exception:\n\n" + ) + return f"{cause}{separator}{final}" + + +FINGERPRINT_KEY = b"k" * 32 +APPLICATION_FRAMES = ( + '/app/policyengine_api/routes/report_routes.py", line 41, in create_report', + '/app/policyengine_api/services/report_service.py", line 88, in create', + '/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 43, in flush', +) + + +def _record(**overrides: object) -> object: + fields: dict[str, object] = { + "timestamp": datetime(2026, 10, 20, 16, 0, tzinfo=UTC), + "service_name": "policyengine-api", + "environment_name": "production", + "event_name": None, + "error_type": None, + "error_handled": None, + "frames": (), + "stack_truncated": False, + "message_template": None, + "attribute_error_type": None, + "failure_category": None, + "route": "//report", + "status_code": None, + "operation_name": None, + "request_key": "request-1", + "observability_key": None, + "trace_key": "trace-1", + } + fields.update(overrides) + return ErrorRecord(**fields) + + +def exception_record(**overrides: object) -> object: + fields: dict[str, object] = { + "event_name": "exception.recorded", + "error_type": "StaleDataError", + "error_handled": False, + "frames": APPLICATION_FRAMES, + "status_code": 500, + } + fields.update(overrides) + return _record(**fields) + + +def message_record(**overrides: object) -> object: + fields: dict[str, object] = {"message_template": "failed to save"} + fields.update(overrides) + return _record(**fields) + + +def failed_request_record(**overrides: object) -> object: + fields: dict[str, object] = {"event_name": "request.completed", "status_code": 500} + fields.update(overrides) + return _record(**fields) + + +REPORT_DATE = date(2026, 10, 20) +# Noon Eastern on the report date: 12 hours after the window starts, and 180 +# hours after the seven days before it start. +WINDOW_NOON = datetime(2026, 10, 20, 16, 0, tzinfo=UTC) + + +def hours_after_window_noon(hours: float) -> datetime: + return WINDOW_NOON + timedelta(hours=hours) + + +def request_count( + route: str, + requests: int, + *, + service_name: str = "policyengine-api", + environment_name: str = "production", +) -> object: + return RequestCount(service_name, environment_name, route, requests) + + +def failures_with_and_without_a_route() -> list: + """Return one failed request on a route and two route-less failures.""" + return [ + exception_record(request_key="r1", route="//calculate"), + exception_record( + request_key=None, + trace_key="t2", + route=None, + timestamp=hours_after_window_noon(1), + ), + exception_record( + request_key=None, + trace_key="t3", + route=None, + timestamp=hours_after_window_noon(2), + ), + ] + + +PROJECT_ID = "policyengine-observability" + + +def sentinel_row(**overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "timestamp": WINDOW_NOON, + "service_name": "SENTINEL service name", + "environment_name": "production", + "event_name": "SENTINEL event", + "error_type": "SENTINEL error type", + "error_handled": "SENTINEL handled", + "frames": [ + '/app/policyengine_api/SENTINEL.py", line 1, in SENTINEL_function', + "SENTINEL frame text", + ], + "stack_truncated": "SENTINEL", + "message_template": "SENTINEL message words", + "attribute_error_type": "SENTINEL attribute type", + "failure_category": "SENTINEL category", + "route": "/SENTINEL/route?query=SENTINEL", + "status_code": "SENTINEL", + "operation_name": "SENTINEL operation", + "request_key": "SENTINEL request", + "observability_key": "SENTINEL observability", + "trace_key": "SENTINEL trace", + } + row.update(overrides) + return row diff --git a/tests/unit/telemetry_error_issues/test_bigquery.py b/tests/unit/telemetry_error_issues/test_bigquery.py new file mode 100644 index 000000000..166776e91 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_bigquery.py @@ -0,0 +1,407 @@ +"""Tests for telemetry_error_issues.bigquery: the queries, client and rows.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime +from urllib.parse import parse_qs, urlparse + +import pytest +from telemetry_error_issues.bigquery import ( + CHAINED_EXCEPTION_PATTERN, + DOUBLE_QUOTED_PATTERN, + ERROR_RECORDS_QUERY, + FRAME_PATTERN, + REQUEST_COUNTS_QUERY, + SINGLE_QUOTED_PATTERN, + TEMPLATE_WORD_PATTERN, + BigQueryClient, + BigQueryError, + error_record_from_row, + fetch_error_records, + fetch_request_counts, +) +from telemetry_error_issues.transport import HttpResponse +from telemetry_error_issues.window import reporting_window + +from tests.unit.telemetry_error_issues.bigquery_support import ( + ACCESS_TOKEN, + FakeTransport, + json_response, + query_response, + sent_parameters, +) +from tests.unit.telemetry_error_issues.report_support import ( + APPLICATION_FRAMES, + REPORT_DATE, + WINDOW_NOON, + exception_record, + request_count, +) + +STRING_FIELD = {"name": "name", "type": "STRING", "mode": "NULLABLE"} + + +def _client(transport: FakeTransport) -> object: + return BigQueryClient("policyengine-observability", ACCESS_TOKEN, transport) + + +def test_bigquery_error_query_request_limits_cost_and_names_its_parameters() -> None: + transport = FakeTransport([json_response(query_response([], []))]) + window = reporting_window(REPORT_DATE) + + fetch_error_records(_client(transport), window) + + request = transport.requests[0] + body = transport.json_body(0) + parameters = sent_parameters(transport) + assert request.method == "POST" + assert request.url == ( + "https://bigquery.googleapis.com/bigquery/v2/projects/" + "policyengine-observability/queries" + ) + assert request.headers["Authorization"] == f"Bearer {ACCESS_TOKEN}" + assert body["useLegacySql"] is False + assert body["parameterMode"] == "NAMED" + assert body["maximumBytesBilled"] == str(5 * 1024**3) + assert body["labels"] == { + "purpose": "telemetry-error-issues", + "query": "error-records", + } + assert body["defaultDataset"] == { + "projectId": "policyengine-observability", + "datasetId": "observability_logs", + } + assert body["formatOptions"] == {"useInt64Timestamp": True} + assert parameters["query_start"]["parameterType"] == {"type": "TIMESTAMP"} + assert parameters["query_start"]["parameterValue"]["value"] == ( + "2026-10-13 04:00:00+00:00" + ) + assert parameters["window_end"]["parameterValue"]["value"] == ( + "2026-10-21 04:00:00+00:00" + ) + assert parameters["frame_pattern"]["parameterValue"]["value"] == (FRAME_PATTERN) + assert parameters["chained_exception_pattern"]["parameterValue"]["value"] == ( + CHAINED_EXCEPTION_PATTERN + ) + for name, pattern in ( + ("double_quoted_pattern", DOUBLE_QUOTED_PATTERN), + ("single_quoted_pattern", SINGLE_QUOTED_PATTERN), + ("template_word_pattern", TEMPLATE_WORD_PATTERN), + ): + assert parameters[name]["parameterValue"]["value"] == pattern + assert [ + item["value"] + for item in parameters["excluded_routes"]["parameterValue"]["arrayValues"] + ] == ["/health", "/simulation-gateway-check", "/liveness-check", "/readiness-check"] + + +def test_bigquery_error_query_filters_application_error_records() -> None: + sql = ERROR_RECORDS_QUERY + + assert "FROM `_AllLogs`" in sql + assert "timestamp >= @query_start AND timestamp < @window_end" in sql + assert ( + "JSON_VALUE(json_payload, '$.schema_version') = " + "'policyengine.observability.v2'" in sql + ) + assert ( + "JSON_VALUE(json_payload, '$.\"service.namespace\"') = " + "'policyengine.api-v1'" in sql + ) + assert "severity_number >= 500" in sql + assert "NOT IN UNNEST(@excluded_routes)" in sql + assert "QUALIFY ROW_NUMBER() OVER" in sql + + +def test_bigquery_error_query_reads_stacks_and_messages_only_through_expressions(): + sql = ERROR_RECORDS_QUERY + stack = "JSON_VALUE(json_payload, '$.\"error.stack\"')" + + assert sql.count("error.stack") == 2 + assert ( + " REGEXP_EXTRACT_ALL(\n" + " REGEXP_REPLACE(\n" + f" {stack}, @chained_exception_pattern, ''\n" + " ),\n" + " @frame_pattern\n" + " ) AS frames" + ) in sql + assert f"LENGTH({stack}) >= 16384" in sql + assert sql.count("$.message") == 1 + assert ( + " REGEXP_REPLACE(\n" + " REGEXP_REPLACE(\n" + " JSON_VALUE(json_payload, '$.message'), " + "@double_quoted_pattern, ''\n" + " ),\n" + " @single_quoted_pattern,\n" + " r'\\1'\n" + " ),\n" + " @template_word_pattern\n" + ) in sql + assert "error.message" not in sql + assert "SELECT *" not in sql + assert "json_payload AS" not in sql + + +@pytest.mark.parametrize( + "identifier", + [ + "JSON_VALUE(json_payload, '$.\"request.id\"')", + "JSON_VALUE(json_payload, '$.attributes.observability_id')", + ], +) +def test_bigquery_error_query_hashes_identifiers(identifier: str) -> None: + sql = ERROR_RECORDS_QUERY + + assert sql.count(identifier) == 1 + assert f"TO_HEX(SHA256({identifier}))" in sql + + +def _message_template(message: str) -> str: + """Build the template the way the query does, with RE2's ASCII word rules.""" + without_double_quotes = re.sub(DOUBLE_QUOTED_PATTERN, "", message) + without_quotes = re.sub(SINGLE_QUOTED_PATTERN, r"\1", without_double_quotes) + words = re.findall(TEMPLATE_WORD_PATTERN, without_quotes, re.ASCII) + return " ".join(word.lower() for word in words[:3]) + + +def test_bigquery_template_keeps_words_around_an_apostrophe() -> None: + templates = { + _message_template(f"Can't load policy '{value}'") + for value in ("alpha", "beta", "gamma") + } + + assert templates == {"can t load"} + + +@pytest.mark.parametrize( + ("message", "template"), + [ + ("Failed to save 'report 12' for user", "failed to save"), + ("'SENTINEL' could not be parsed", "could not be"), + ('Missing "SENTINEL" in household input', "missing in household"), + ("Timeout after 30s calling the simulation API", "timeout after calling"), + ("The users' policies 'SENTINEL' failed", "the users policies"), + ], +) +def test_bigquery_template_drops_quoted_values_and_partial_words( + message: str, template: str +) -> None: + assert _message_template(message) == template + + +def test_bigquery_error_query_hashes_the_trace() -> None: + sql = ERROR_RECORDS_QUERY + + assert "TO_HEX(SHA256(COALESCE(" in sql + assert sql.count("$.trace_id") == 1 + + +def test_bigquery_request_count_query_counts_completed_requests_in_the_window() -> None: + sql = REQUEST_COUNTS_QUERY + transport = FakeTransport([json_response(query_response([], []))]) + + fetch_request_counts(_client(transport), reporting_window(REPORT_DATE)) + + parameters = sent_parameters(transport) + assert "JSON_VALUE(json_payload, '$.\"event.name\"') = 'request.completed'" in sql + assert "COUNT(*) AS requests" in sql + assert "GROUP BY service_name, environment_name, route" in sql + assert "timestamp >= @window_start AND timestamp < @window_end" in sql + assert parameters["window_start"]["parameterValue"]["value"] == ( + "2026-10-20 04:00:00+00:00" + ) + assert transport.json_body(0)["labels"]["query"] == "request-counts" + + +def test_bigquery_client_polls_until_the_job_completes() -> None: + transport = FakeTransport( + [ + json_response(query_response([], [], job_complete=False)), + json_response(query_response([], [], job_complete=False)), + json_response(query_response([STRING_FIELD], [["done"]])), + ], + ) + + rows = _client(transport).query("SELECT 1", [], "test") + + assert rows == [{"name": "done"}] + poll = urlparse(transport.requests[1].url) + assert transport.requests[1].method == "GET" + assert poll.path.endswith("/projects/policyengine-observability/queries/job-1") + assert parse_qs(poll.query)["location"] == ["US"] + assert parse_qs(poll.query)["formatOptions.useInt64Timestamp"] == ["true"] + + +def test_bigquery_client_follows_page_tokens() -> None: + transport = FakeTransport( + [ + json_response( + query_response([STRING_FIELD], [["first"]], page_token="page-2"), + ), + json_response(query_response([STRING_FIELD], [["second"]])), + ], + ) + + rows = _client(transport).query("SELECT 1", [], "test") + + assert rows == [{"name": "first"}, {"name": "second"}] + page = parse_qs(urlparse(transport.requests[1].url).query) + assert page["pageToken"] == ["page-2"] + assert page["location"] == ["US"] + + +def test_bigquery_client_gives_up_on_a_job_that_never_completes() -> None: + pending = json_response(query_response([], [], job_complete=False)) + transport = FakeTransport([pending] * 200) + + with pytest.raises(BigQueryError, match="did not complete"): + _client(transport).query("SELECT 1", [], "test") + + +def test_bigquery_client_converts_row_values_from_strings() -> None: + fields = [ + {"name": "text", "type": "STRING", "mode": "NULLABLE"}, + {"name": "count", "type": "INTEGER", "mode": "NULLABLE"}, + {"name": "flag", "type": "BOOLEAN", "mode": "NULLABLE"}, + {"name": "moment", "type": "TIMESTAMP", "mode": "NULLABLE"}, + {"name": "items", "type": "STRING", "mode": "REPEATED"}, + {"name": "missing", "type": "INTEGER", "mode": "NULLABLE"}, + ] + row = ["value", "42", "true", "1792512000000001", ["a", "b"], None] + transport = FakeTransport([json_response(query_response(fields, [row]))]) + + rows = _client(transport).query("SELECT 1", [], "test") + + assert rows == [ + { + "text": "value", + "count": 42, + "flag": True, + "moment": datetime(2026, 10, 20, 16, 0, 0, 1, tzinfo=UTC), + "items": ["a", "b"], + "missing": None, + } + ] + + +def test_bigquery_http_error_reports_only_the_status_and_reason() -> None: + error_body = { + "error": { + "code": 400, + "message": "SENTINEL value echoed from a row", + "errors": [{"reason": "invalidQuery", "message": "SENTINEL detail"}], + } + } + transport = FakeTransport( + [json_response(error_body, status=400, reason="Bad Request")], + ) + + with pytest.raises(BigQueryError) as raised: + _client(transport).query("SELECT 1", [], "test") + + assert str(raised.value) == "BigQuery request failed: HTTP 400 invalidQuery" + assert raised.value.status == 400 + assert raised.value.reason == "invalidQuery" + + +def test_bigquery_http_error_without_a_json_reason_uses_the_status_phrase() -> None: + response = HttpResponse( + status=503, reason="Service Unavailable", headers={}, body=b"SENTINEL " + ) + transport = FakeTransport([response]) + + with pytest.raises(BigQueryError) as raised: + _client(transport).query("SELECT 1", [], "test") + + assert str(raised.value) == "BigQuery request failed: HTTP 503 Service Unavailable" + + +def test_bigquery_rows_become_error_records() -> None: + row = { + "timestamp": WINDOW_NOON, + "service_name": "policyengine-api", + "environment_name": "production", + "event_name": "exception.recorded", + "error_type": "StaleDataError", + "error_handled": "false", + "frames": list(APPLICATION_FRAMES), + "stack_truncated": False, + "message_template": "", + "attribute_error_type": None, + "failure_category": None, + "route": "//report", + "status_code": "500", + "operation_name": None, + "request_key": "aa", + "observability_key": None, + "trace_key": "bb", + } + + record = error_record_from_row(row) + + assert record == exception_record(request_key="aa", trace_key="bb") + + +def test_bigquery_rows_tolerate_missing_and_malformed_values() -> None: + row = { + "timestamp": WINDOW_NOON, + "service_name": None, + "environment_name": None, + "event_name": None, + "error_type": None, + "error_handled": "SENTINEL", + "frames": None, + "stack_truncated": None, + "message_template": None, + "attribute_error_type": None, + "failure_category": None, + "route": None, + "status_code": "SENTINEL", + "operation_name": None, + "request_key": None, + "observability_key": None, + "trace_key": None, + } + + record = error_record_from_row(row) + + assert record.service_name == "-" + assert record.error_handled is None + assert record.frames == () + assert record.stack_truncated is False + assert record.status_code is None + + +def test_bigquery_rows_become_request_counts() -> None: + rows = [ + { + "service_name": "policyengine-api", + "environment_name": "production", + "route": "//report", + "requests": 120, + } + ] + transport = FakeTransport( + [ + json_response( + query_response( + [ + {"name": "service_name", "type": "STRING"}, + {"name": "environment_name", "type": "STRING"}, + {"name": "route", "type": "STRING"}, + {"name": "requests", "type": "INTEGER"}, + ], + [list(map(str, rows[0].values()))], + ), + ) + ], + ) + + counts = fetch_request_counts(_client(transport), reporting_window(REPORT_DATE)) + + assert counts == [ + request_count("//report", 120), + ] diff --git a/tests/unit/telemetry_error_issues/test_cli.py b/tests/unit/telemetry_error_issues/test_cli.py new file mode 100644 index 000000000..d3a955331 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_cli.py @@ -0,0 +1,56 @@ +"""Tests for telemetry_error_issues.cli: the command-line arguments.""" + +from __future__ import annotations + +from datetime import date + +import pytest +from telemetry_error_issues.cli import main, parse_arguments + + +def test_cli_parses_project_dry_run_and_report_date() -> None: + arguments = parse_arguments( + [ + "--project-id", + "policyengine-observability", + "--dry-run", + "--report-date", + "2026-10-19", + ] + ) + + assert arguments.project_id == "policyengine-observability" + assert arguments.dry_run is True + assert arguments.report_date == date(2026, 10, 19) + + +def test_cli_defaults_to_a_live_run_of_the_previous_day() -> None: + arguments = parse_arguments(["--project-id", "policyengine-observability"]) + + assert arguments.dry_run is False + assert arguments.report_date is None + + +def test_cli_treats_an_empty_report_date_as_unset() -> None: + arguments = parse_arguments( + ["--project-id", "policyengine-observability", "--report-date", ""] + ) + + assert arguments.report_date is None + + +@pytest.mark.parametrize( + "argv", + [ + [], + ["--dry-run"], + ["--project-id", "policyengine-observability", "--report-date", "20 Oct"], + ], +) +def test_cli_rejects_missing_project_or_invalid_date_with_exit_code_2( + argv: list[str], +) -> None: + with pytest.raises(SystemExit) as raised: + main(argv) + + assert raised.value.code == 2 diff --git a/tests/unit/telemetry_error_issues/test_end_to_end.py b/tests/unit/telemetry_error_issues/test_end_to_end.py new file mode 100644 index 000000000..090d7c9c3 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_end_to_end.py @@ -0,0 +1,373 @@ +"""End-to-end tests of telemetry_error_issues.cli.main and its public output.""" + +from __future__ import annotations + +import io +import json +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING +from urllib.parse import unquote + +import pytest +from telemetry_error_issues.cli import Runtime, main +from telemetry_error_issues.transport import HttpResponse + +from tests.unit.telemetry_error_issues.bigquery_support import ( + ACCESS_TOKEN, + FakeTransport, + json_response, + query_response, + sent_parameters, +) +from tests.unit.telemetry_error_issues.github_support import REPOSITORY, FakeGitHub +from tests.unit.telemetry_error_issues.report_support import ( + APPLICATION_FRAMES, + EASTERN, + PROJECT_ID, + WINDOW_NOON, + hours_after_window_noon, + sentinel_row, +) + +if TYPE_CHECKING: + from pathlib import Path + +ERROR_RECORD_FIELDS = [ + {"name": "timestamp", "type": "TIMESTAMP"}, + {"name": "service_name", "type": "STRING"}, + {"name": "environment_name", "type": "STRING"}, + {"name": "event_name", "type": "STRING"}, + {"name": "error_type", "type": "STRING"}, + {"name": "error_handled", "type": "STRING"}, + {"name": "frames", "type": "STRING", "mode": "REPEATED"}, + {"name": "stack_truncated", "type": "BOOLEAN"}, + {"name": "message_template", "type": "STRING"}, + {"name": "attribute_error_type", "type": "STRING"}, + {"name": "failure_category", "type": "STRING"}, + {"name": "route", "type": "STRING"}, + {"name": "status_code", "type": "STRING"}, + {"name": "operation_name", "type": "STRING"}, + {"name": "request_key", "type": "STRING"}, + {"name": "observability_key", "type": "STRING"}, + {"name": "trace_key", "type": "STRING"}, +] +REQUEST_COUNT_FIELDS = [ + {"name": "service_name", "type": "STRING"}, + {"name": "environment_name", "type": "STRING"}, + {"name": "route", "type": "STRING"}, + {"name": "requests", "type": "INTEGER"}, +] +SCHEDULED_RUN_TIME = datetime(2026, 10, 21, 0, 45, tzinfo=EASTERN) + + +def _microseconds(moment: datetime) -> str: + return str((moment - datetime(1970, 1, 1, tzinfo=UTC)) // timedelta(microseconds=1)) + + +def _error_row(**overrides: object) -> list[object]: + values: dict[str, object] = { + "timestamp": _microseconds(WINDOW_NOON), + "service_name": "policyengine-api", + "environment_name": "production", + "event_name": "exception.recorded", + "error_type": "StaleDataError", + "error_handled": "false", + "frames": list(APPLICATION_FRAMES), + "stack_truncated": "false", + "message_template": None, + "attribute_error_type": None, + "failure_category": None, + "route": "//report", + "status_code": "500", + "operation_name": None, + "request_key": "r1", + "observability_key": None, + "trace_key": "t1", + } + values.update(overrides) + return [values[field["name"]] for field in ERROR_RECORD_FIELDS] + + +class FakeBigQuery: + """Answers the error-record and request-count queries with fixed rows.""" + + def __init__( + self, + error_rows: list[list[object]], + count_rows: list[list[object]] = (), + ) -> None: + self.error_rows = error_rows + self.count_rows = list(count_rows) + self.requests: list[object] = [] + + def __call__(self, request: object) -> object: + self.requests.append(request) + label = json.loads(request.body)["labels"]["query"] + if label == "error-records": + response = query_response(ERROR_RECORD_FIELDS, self.error_rows) + else: + response = query_response(REQUEST_COUNT_FIELDS, self.count_rows) + return json_response(response) + + def json_body(self, index: int) -> dict: + return json.loads(self.requests[index].body) + + +def _environment(tmp_path: Path, **overrides: str) -> dict[str, str]: + environment = { + "GOOGLE_OAUTH_ACCESS_TOKEN": ACCESS_TOKEN, + "GITHUB_TOKEN": "fake-github-token", + "GITHUB_REPOSITORY": REPOSITORY, + "TELEMETRY_ERROR_FINGERPRINT_KEY": "f" * 64, + "GITHUB_STEP_SUMMARY": str(tmp_path / "summary.md"), + } + environment.update(overrides) + return {name: value for name, value in environment.items() if value} + + +def _run_main( + tmp_path: Path, + bigquery: object, + github: FakeGitHub, + *arguments: str, + environment: dict[str, str] | None = None, +) -> tuple[int, str, str]: + stdout, stderr = io.StringIO(), io.StringIO() + runtime = Runtime( + environ=_environment(tmp_path) if environment is None else environment, + bigquery_transport=bigquery, + github_transport=github, + now=lambda: SCHEDULED_RUN_TIME, + sleep=github.sleep, + clock=github.clock, + stdout=stdout, + stderr=stderr, + ) + exit_code = main(["--project-id", PROJECT_ID, *arguments], runtime=runtime) + return exit_code, stdout.getvalue(), stderr.getvalue() + + +def _three_failures(**overrides: object) -> list[list[object]]: + return [ + _error_row( + request_key=f"r{index}", + timestamp=_microseconds(hours_after_window_noon(index)), + **overrides, + ) + for index in range(3) + ] + + +def test_main_queries_groups_and_opens_issues_for_the_previous_day( + tmp_path: Path, +) -> None: + bigquery = FakeBigQuery( + _three_failures(), + [["policyengine-api", "production", "//report", "600"]], + ) + github = FakeGitHub() + + exit_code, stdout, stderr = _run_main(tmp_path, bigquery, github) + + assert exit_code == 0, stderr + issue = github.issues[1] + assert "StaleDataError" in issue["title"] + assert ( + "| Error rate on affected routes | 0.50% (3 of 600 requests) | n/a |" + in (issue["body"]) + ) + window_end = sent_parameters(bigquery)["window_end"]["parameterValue"] + assert window_end["value"] == "2026-10-21 04:00:00+00:00" + summary = (tmp_path / "summary.md").read_text() + fingerprint = issue["title"].rsplit("(", 1)[1].rstrip(")") + assert ( + f"| `{fingerprint}` | exception | `policyengine-api` | 3 | 0 | #1 | created |" + in summary + ) + assert summary in stdout + + +def test_main_dry_run_reports_actions_without_writing(tmp_path: Path) -> None: + github = FakeGitHub() + + exit_code, stdout, _ = _run_main( + tmp_path, + FakeBigQuery(_three_failures()), + github, + "--dry-run", + ) + + assert exit_code == 0 + assert github.writes == [] + assert "Dry run: no issues were changed." in stdout + assert "| - | created |" in stdout + + +def test_main_reports_the_requested_day(tmp_path: Path) -> None: + bigquery = FakeBigQuery([]) + + exit_code, stdout, _ = _run_main( + tmp_path, + bigquery, + FakeGitHub(), + "--report-date", + "2026-10-15", + ) + + assert exit_code == 0 + window_end = sent_parameters(bigquery)["window_end"]["parameterValue"] + assert window_end["value"] == "2026-10-16 04:00:00+00:00" + assert "## Telemetry error issues for 2026-10-15" in stdout + assert "No error groups had failures in the window." in stdout + + +def test_main_rejects_a_day_that_has_not_ended(tmp_path: Path) -> None: + exit_code, stdout, stderr = _run_main( + tmp_path, + FakeBigQuery([]), + FakeGitHub(), + "--report-date", + "2026-10-21", + ) + + assert exit_code == 1 + assert stdout == "" + assert stderr == ( + "configuration failed: ConfigurationError " + "(2026-10-21 has not ended in US Eastern time)\n" + ) + + +def test_main_names_missing_environment_variables_without_values( + tmp_path: Path, +) -> None: + environment = _environment( + tmp_path, GITHUB_TOKEN="", TELEMETRY_ERROR_FINGERPRINT_KEY="" + ) + + exit_code, _, stderr = _run_main( + tmp_path, + FakeBigQuery([]), + FakeGitHub(), + environment=environment, + ) + + assert exit_code == 1 + assert stderr == ( + "configuration failed: ConfigurationError (missing environment variables: " + "GITHUB_TOKEN, TELEMETRY_ERROR_FINGERPRINT_KEY)\n" + ) + + +def test_main_rejects_a_short_fingerprint_key_without_printing_it( + tmp_path: Path, +) -> None: + environment = _environment( + tmp_path, TELEMETRY_ERROR_FINGERPRINT_KEY="SENTINEL-short-key" + ) + + exit_code, stdout, stderr = _run_main( + tmp_path, + FakeBigQuery([]), + FakeGitHub(), + environment=environment, + ) + + assert exit_code == 1 + assert "at least 32 characters" in stderr + assert "SENTINEL" not in stdout + stderr + + +def test_main_reports_an_api_failure_by_stage_class_and_status_only( + tmp_path: Path, +) -> None: + error_body = { + "error": { + "message": "SENTINEL echoed value", + "errors": [{"reason": "backendError", "message": "SENTINEL"}], + } + } + failing = FakeTransport( + [json_response(error_body, status=500, reason="SENTINEL")], + ) + + exit_code, stdout, stderr = _run_main(tmp_path, failing, FakeGitHub()) + + assert exit_code == 1 + assert stdout == "" + assert stderr == ( + "query error records failed: BigQueryError (HTTP 500 backendError)\n" + ) + + +def test_main_reports_an_unexpected_exception_by_class_only(tmp_path: Path) -> None: + def broken_transport(request: object) -> object: + message = "SENTINEL connection detail" + raise RuntimeError(message) + + exit_code, _, stderr = _run_main(tmp_path, broken_transport, FakeGitHub()) + + assert exit_code == 1 + assert stderr == "query error records failed: RuntimeError\n" + + +def test_main_reports_a_github_failure_by_stage(tmp_path: Path) -> None: + github = FakeGitHub() + github.failures = [ + HttpResponse( + status=403, reason="Forbidden", headers={}, body=b'{"message": "SENTINEL"}' + ) + ] + + exit_code, _, stderr = _run_main(tmp_path, FakeBigQuery(_three_failures()), github) + + assert exit_code == 1 + assert stderr == "sync issues failed: GitHubError (HTTP 403 Forbidden)\n" + + +@pytest.mark.parametrize( + "overrides", + [ + {}, + {"error_type": None}, + {"error_type": None, "event_name": "request.completed"}, + ], +) +def test_main_never_publishes_values_from_records( + tmp_path: Path, overrides: dict[str, object] +) -> None: + sentinel_values = { + name: value + for name, value in sentinel_row(**overrides).items() + if name not in {"timestamp", "environment_name", "frames", "stack_truncated"} + } + sentinel_values["stack_truncated"] = "true" + sentinel_values["frames"] = ['/app/SENTINEL.py", line 1, in SENTINEL'] + rows = [ + _error_row( + **{ + **sentinel_values, + "request_key": f"SENTINEL request {index}", + "timestamp": _microseconds(hours_after_window_noon(index)), + } + ) + for index in range(3) + ] + count_rows = [["SENTINEL service", "production", "SENTINEL route", "10"]] + github = FakeGitHub() + + exit_code, stdout, stderr = _run_main( + tmp_path, FakeBigQuery(rows, count_rows), github + ) + + assert exit_code == 0, stderr + assert len(github.issues) == 1 + published = [ + stdout, + stderr, + (tmp_path / "summary.md").read_text(), + *(issue["title"] for issue in github.issues.values()), + *(issue["body"] for issue in github.issues.values()), + ] + for text in published: + assert "SENTINEL" not in unquote(text) diff --git a/tests/unit/telemetry_error_issues/test_github_client.py b/tests/unit/telemetry_error_issues/test_github_client.py new file mode 100644 index 000000000..318a00ffe --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_github_client.py @@ -0,0 +1,159 @@ +"""Tests for telemetry_error_issues.github_client: paging, write spacing, retries.""" + +from __future__ import annotations + +import itertools +from datetime import UTC, datetime +from urllib.parse import parse_qs, urlsplit + +import pytest +from telemetry_error_issues.github_client import GitHubError +from telemetry_error_issues.transport import HttpResponse + +from tests.unit.telemetry_error_issues.github_support import ( + BOT, + FakeGitHub, + bot_issue, + failures, + groups_for_report_date, + sync, +) + + +def test_sync_lists_open_and_closed_issues_from_the_bot_page_by_page() -> None: + github = FakeGitHub([bot_issue(number) for number in range(1, 151)]) + + sync([], github) + + listing = [ + parse_qs(urlsplit(request.url).query) + for _, request in github.requests + if urlsplit(request.url).path.endswith("/issues") + ] + assert len(listing) == 2 + assert listing[0]["creator"] == [BOT] + assert listing[0]["state"] == ["all"] + assert listing[0]["per_page"] == ["100"] + assert [query["page"] for query in listing] == [["1"], ["2"]] + + +def test_sync_writes_one_at_a_time_at_least_a_second_apart() -> None: + records = [] + for index in range(3): + records += failures( + 3, request_key_prefix=f"g{index}-", error_type=f"Error{index}" + ) + github = FakeGitHub() + + sync(groups_for_report_date(records), github) + + write_times = [moment for moment, _ in github.writes] + assert len(write_times) == 5 + assert all( + later - earlier >= 1.0 for earlier, later in itertools.pairwise(write_times) + ) + + +@pytest.mark.parametrize("status", [403, 429]) +def test_sync_waits_for_retry_after_on_rate_limits(status: int) -> None: + groups = groups_for_report_date(failures(3)) + github = FakeGitHub(labels=("telemetry-error", "service: policyengine-api")) + github.failures = [ + HttpResponse( + status=status, reason="Forbidden", headers={"retry-after": "7"}, body=b"{}" + ) + ] + + sync(groups, github) + + assert 7 in github.sleeps + assert len(github.issues) == 1 + + +def _limited_github(response: object) -> FakeGitHub: + """Return a fake whose first write gets ``response`` and later writes succeed.""" + github = FakeGitHub(labels=("telemetry-error", "service: policyengine-api")) + github.failures = [response] + return github + + +def test_sync_waits_a_minute_on_a_secondary_rate_limit_without_retry_after() -> None: + body = b'{"message": "You have exceeded a secondary rate limit. Please wait."}' + github = _limited_github( + HttpResponse(status=403, reason="Forbidden", headers={}, body=body), + ) + + sync(groups_for_report_date(failures(3)), github) + + assert 60 in github.sleeps + assert len(github.issues) == 1 + + +def test_sync_waits_until_an_exhausted_rate_limit_resets() -> None: + headers = { + "date": "Tue, 20 Oct 2026 23:00:00 GMT", + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": str( + int(datetime(2026, 10, 20, 23, 0, 42, tzinfo=UTC).timestamp()) + ), + } + github = _limited_github( + HttpResponse(status=403, reason="Forbidden", headers=headers, body=b"{}"), + ) + + sync(groups_for_report_date(failures(3)), github) + + assert 42 in github.sleeps + assert len(github.issues) == 1 + + +def test_sync_fails_instead_of_waiting_hours_for_a_rate_limit_reset() -> None: + headers = { + "date": "Tue, 20 Oct 2026 23:00:00 GMT", + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": str( + int(datetime(2026, 10, 21, 0, 0, tzinfo=UTC).timestamp()) + ), + } + github = _limited_github( + HttpResponse(status=403, reason="Forbidden", headers=headers, body=b"{}"), + ) + + with pytest.raises(GitHubError, match="HTTP 403"): + sync(groups_for_report_date(failures(3)), github) + + assert github.sleeps == [] + + +def test_sync_stops_after_three_rate_limit_retries() -> None: + groups = groups_for_report_date(failures(3)) + github = FakeGitHub(labels=("telemetry-error", "service: policyengine-api")) + limited = HttpResponse( + status=429, reason="Too Many Requests", headers={"retry-after": "2"}, body=b"{}" + ) + github.failures = [limited] * 4 + + with pytest.raises(GitHubError) as raised: + sync(groups, github) + + assert str(raised.value) == "GitHub request failed: HTTP 429 Too Many Requests" + assert github.sleeps.count(2) == 3 + + +def test_sync_reports_a_forbidden_write_without_retrying() -> None: + groups = groups_for_report_date(failures(3)) + github = FakeGitHub(labels=("telemetry-error", "service: policyengine-api")) + github.failures = [ + HttpResponse( + status=403, + reason="Forbidden", + headers={}, + body=b'{"message": "SENTINEL Resource not accessible"}', + ) + ] + + with pytest.raises(GitHubError) as raised: + sync(groups, github) + + assert str(raised.value) == "GitHub request failed: HTTP 403 Forbidden" + assert len(github.writes) == 1 diff --git a/tests/unit/telemetry_error_issues/test_groups.py b/tests/unit/telemetry_error_issues/test_groups.py new file mode 100644 index 000000000..41807ac43 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_groups.py @@ -0,0 +1,378 @@ +"""Tests for telemetry_error_issues.groups: grouping failures and their measures.""" + +from __future__ import annotations + +from datetime import UTC, date, datetime, timedelta + +import pytest +from telemetry_error_issues.groups import build_groups +from telemetry_error_issues.records import fingerprint +from telemetry_error_issues.window import reporting_window + +from tests.unit.telemetry_error_issues.report_support import ( + FINGERPRINT_KEY, + REPORT_DATE, + WINDOW_NOON, + exception_record, + failed_request_record, + failures_with_and_without_a_route, + hours_after_window_noon, + message_record, + request_count, +) + + +def _only_group(records: list, request_counts=()) -> object: + groups = build_groups( + records, + list(request_counts), + reporting_window(REPORT_DATE), + FINGERPRINT_KEY, + ) + assert len(groups) == 1 + return groups[0] + + +def test_measures_count_occurrences_requests_and_observability_ids() -> None: + records = [ + exception_record(request_key="r1", observability_key="o1"), + exception_record( + request_key="r2", + observability_key="o1", + timestamp=hours_after_window_noon(1), + ), + exception_record( + request_key="r3", + observability_key="o2", + timestamp=hours_after_window_noon(2), + ), + exception_record( + request_key="s1", + environment_name="staging", + timestamp=hours_after_window_noon(3), + ), + ] + + group = _only_group(records) + + assert group.production.occurrences == 3 + assert group.production.affected_requests == 3 + assert group.production.observability_ids == 2 + assert group.staging.occurrences == 1 + assert group.staging.affected_requests == 1 + assert group.staging.observability_ids == 0 + + +def test_measures_affected_requests_fall_back_to_observability_id_then_trace() -> None: + records = [ + exception_record(request_key=None, trace_key=None, observability_key="o1"), + exception_record( + request_key=None, + trace_key=None, + observability_key="o1", + timestamp=hours_after_window_noon(1), + ), + exception_record(request_key=None, trace_key="t1"), + exception_record(request_key=None, trace_key=None), + exception_record( + request_key=None, trace_key=None, timestamp=hours_after_window_noon(2) + ), + ] + + group = _only_group(records) + + assert group.production.occurrences == 5 + # o1 once, t1 once, and each record without any identifier on its own. + assert group.production.affected_requests == 4 + + +def test_measures_error_rate_uses_requests_to_the_affected_routes() -> None: + records = [ + exception_record(request_key="r1", route="//report"), + exception_record( + request_key="r2", + route="//calculate", + timestamp=hours_after_window_noon(1), + ), + exception_record( + request_key="r3", + route="//calculate", + timestamp=hours_after_window_noon(2), + ), + ] + request_counts = [ + request_count("//report", 200), + request_count("//calculate", 100), + request_count("//metadata", 5000), + request_count("//report", 10, environment_name="staging"), + request_count( + "//report", + 7000, + service_name="policyengine-simulation-entry", + ), + ] + + group = _only_group(records, request_counts) + + assert group.production.route_requests == 300 + assert group.production.error_rate == pytest.approx(0.01) + assert group.staging.error_rate is None + + +def test_measures_error_rate_counts_only_failures_on_routes() -> None: + group = _only_group( + failures_with_and_without_a_route(), + [request_count("//calculate", 1)], + ) + + assert group.production.affected_requests == 3 + assert group.production.route_affected_requests == 1 + assert group.production.route_requests == 1 + assert group.production.error_rate == pytest.approx(1.0) + + +def test_measures_error_rate_counts_every_simulation_service_version() -> None: + records = [ + failed_request_record( + service_name="policyengine-simulation-py1-2-3", + route="/jobs/{job_id}", + ) + ] + request_counts = [ + request_count( + "/jobs/{job_id}", + 30, + service_name="policyengine-simulation-py1-2-3", + ), + request_count( + "/jobs/{job_id}", + 70, + service_name="policyengine-simulation-py1-2-4", + ), + ] + + group = _only_group(records, request_counts) + + assert group.production.route_requests == 100 + assert group.production.error_rate == pytest.approx(0.01) + + +def test_measures_error_rate_is_unknown_without_route_requests() -> None: + group = _only_group([message_record(route=None, operation_name="sync")]) + + assert group.production.route_requests == 0 + assert group.production.error_rate is None + + +def test_measures_hourly_rates_for_the_window_and_the_previous_seven_days() -> None: + window_records = [ + message_record( + request_key=f"w{index}", timestamp=hours_after_window_noon(index) + ) + for index in range(6) + ] + previous_records = [ + message_record( + request_key=f"p{index}", timestamp=hours_after_window_noon(-13 - 4 * index) + ) + for index in range(42) + ] + + group = _only_group(window_records + previous_records) + + assert group.production.occurrences == 6 + assert group.production.hourly_rate == pytest.approx(6 / 24) + assert group.production.previous_hourly_rate == pytest.approx(42 / 168) + + +def test_measures_hourly_rate_uses_the_25_hour_fall_back_day() -> None: + fall_back_noon = datetime(2026, 11, 1, 17, 0, tzinfo=UTC) + records = [ + message_record( + request_key=f"w{index}", + timestamp=fall_back_noon + timedelta(hours=index), + ) + for index in range(5) + ] + + groups = build_groups( + records, [], reporting_window(date(2026, 11, 1)), FINGERPRINT_KEY + ) + + assert groups[0].production.hourly_rate == pytest.approx(5 / 25) + + +def test_measures_first_and_last_seen_cover_the_queried_eight_days() -> None: + records = [ + message_record(request_key="early", timestamp=hours_after_window_noon(-24 * 6)), + message_record(request_key="latest", timestamp=hours_after_window_noon(5)), + message_record(request_key="middle", timestamp=hours_after_window_noon(1)), + ] + + group = _only_group(records) + + assert group.production.first_seen == hours_after_window_noon(-24 * 6) + assert group.production.last_seen == hours_after_window_noon(5) + assert group.staging.first_seen is None + assert group.staging.last_seen is None + + +def test_measures_ignore_records_outside_the_queried_days() -> None: + records = [ + message_record(request_key="in-window"), + message_record( + request_key="too-old", timestamp=hours_after_window_noon(-24 * 9) + ), + message_record(request_key="too-new", timestamp=hours_after_window_noon(24)), + ] + + group = _only_group(records) + + assert group.production.occurrences == 1 + assert group.production.first_seen == WINDOW_NOON + + +def test_measures_report_only_groups_seen_in_the_window() -> None: + records = [ + message_record(request_key="today"), + message_record( + request_key="last-week", + message_template="something else", + timestamp=hours_after_window_noon(-48), + ), + ] + + groups = build_groups(records, [], reporting_window(REPORT_DATE), FINGERPRINT_KEY) + + assert [group.fingerprint for group in groups] == [ + fingerprint(records[0], FINGERPRINT_KEY) + ] + + +def test_measures_ignore_environments_other_than_production_and_staging() -> None: + records = [ + message_record(environment_name="local"), + message_record(environment_name=None, request_key="r2"), + ] + + groups = build_groups(records, [], reporting_window(REPORT_DATE), FINGERPRINT_KEY) + + assert groups == [] + + +def test_measures_describe_the_group_with_its_most_frequent_production_route() -> None: + records = [ + exception_record(request_key="r1", route="//report"), + exception_record(request_key="r2", route="//calculate"), + exception_record(request_key="r3", route="//calculate"), + exception_record( + request_key="s1", + route="//report", + environment_name="staging", + ), + exception_record( + request_key="s2", + route="//report", + environment_name="staging", + ), + ] + + group = _only_group(records) + + assert group.kind == "exception" + assert group.family == "policyengine-api" + assert group.error_type == "StaleDataError" + assert group.route == "//calculate" + + +def test_measures_describe_groups_only_with_validated_values() -> None: + group = _only_group( + [ + exception_record( + error_type="SENTINEL not a class", + route="/us/policy/12345", + ) + ], + ) + + assert group.error_type == "" + assert group.route == "" + + +def test_measures_list_up_to_three_related_fingerprints_by_shared_traces() -> None: + base = exception_record(request_key="r1", trace_key="t1") + base_second = exception_record( + request_key="r2", trace_key="t2", timestamp=hours_after_window_noon(1) + ) + related_twice = [ + message_record( + service_name="policyengine-simulation-entry", + request_key=f"x{index}", + trace_key=trace, + ) + for index, trace in enumerate(("t1", "t2")) + ] + related_once = [ + message_record( + service_name="policyengine-simulation-gateway", + request_key=f"y{index}", + trace_key="t1", + message_template=template, + ) + for index, template in enumerate(("first error", "second error", "third")) + ] + + groups = build_groups( + [base, base_second, *related_twice, *related_once], + [], + reporting_window(REPORT_DATE), + FINGERPRINT_KEY, + ) + by_fingerprint = {group.fingerprint: group for group in groups} + base_group = by_fingerprint[fingerprint(base, FINGERPRINT_KEY)] + twice = fingerprint(related_twice[0], FINGERPRINT_KEY) + once = sorted(fingerprint(record, FINGERPRINT_KEY) for record in related_once) + + assert base_group.related_fingerprints == (twice, once[0], once[1]) + + +def test_measures_sort_groups_by_production_occurrences() -> None: + frequent = [ + message_record(request_key=f"f{index}", message_template="often") + for index in range(3) + ] + staging_only = [ + message_record( + request_key=f"s{index}", + message_template="staging", + environment_name="staging", + ) + for index in range(5) + ] + rare = [message_record(request_key="rare", message_template="rare")] + + groups = build_groups( + staging_only + rare + frequent, + [], + reporting_window(REPORT_DATE), + FINGERPRINT_KEY, + ) + + assert [group.production.occurrences for group in groups] == [3, 1, 0] + + +def test_measures_count_production_occurrences_since_a_moment() -> None: + records = [ + exception_record(request_key="before", timestamp=hours_after_window_noon(-30)), + exception_record(request_key="r1", timestamp=hours_after_window_noon(1)), + exception_record(request_key="r2", timestamp=hours_after_window_noon(2)), + exception_record( + request_key="r2", trace_key="t9", timestamp=hours_after_window_noon(3) + ), + exception_record(request_key="staging", environment_name="staging"), + ] + + group = _only_group(records) + + assert group.production_since(hours_after_window_noon(-1)) == (2, 2) + assert group.production_since(hours_after_window_noon(-48)) == (3, 3) diff --git a/tests/unit/telemetry_error_issues/test_normalization.py b/tests/unit/telemetry_error_issues/test_normalization.py new file mode 100644 index 000000000..f035b2b7c --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_normalization.py @@ -0,0 +1,328 @@ +"""Tests for telemetry_error_issues.normalization: names, frames and public values.""" + +from __future__ import annotations + +import re + +import pytest +from telemetry_error_issues import normalization +from telemetry_error_issues.bigquery import FRAME_PATTERN +from telemetry_error_issues.normalization import ( + fingerprint_frames, + frame_key, + normalize_path, + public_error_type, + public_family, + public_operation, + public_route, + public_status, + service_family, +) + +from tests.unit.telemetry_error_issues.report_support import ( + chained_stack, + extract_frames, +) + +STANDARD_STACK = """\ +Traceback (most recent call last): + File "/venv/lib/python3.12/site-packages/flask/app.py", line 880, in dispatch_request + rv = self.dispatch_request() + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/policyengine_api/routes/report_routes.py", line 41, in create_report + return service.create(payload) + File "/app/policyengine_api/services/report_service.py", line 88, in create + session.flush() + File "/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 43, in flush + self._flush(objects) +sqlalchemy.orm.exc.StaleDataError: SENTINEL message about 'reports' +""" + +CHAINED_STACK = """\ +Traceback (most recent call last): + File "/app/policyengine_api/services/cache.py", line 12, in read + return store[key] +KeyError: 'SENTINEL key' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/app/policyengine_api/routes/policy_routes.py", line 30, in get_policy + return service.get(policy_id) + File "/app/policyengine_api/services/policy_service.py", line 55, in get + raise LookupError(policy_id) +LookupError: SENTINEL policy +""" + +EXCEPTION_GROUP_STACK = """\ + + Exception Group Traceback (most recent call last): + | File "/root/policyengine_simulation_gateway/app.py", line 10, in run + | raise ExceptionGroup("SENTINEL group", errors) + | ExceptionGroup: SENTINEL group (1 sub-exception) + +-+---------------- 1 ---------------- + | Traceback (most recent call last): + | File "/root/policyengine_simulation_gateway/worker.py", line 22, in step + | raise ValueError("SENTINEL value") + | ValueError: SENTINEL value + +------------------------------------ +""" + + +@pytest.mark.parametrize( + ("service_name", "family"), + [ + ("policyengine-simulation-py1-2-3", "policyengine-simulation-py"), + ("policyengine-simulation-py0-412-16", "policyengine-simulation-py"), + ("policyengine-api", "policyengine-api"), + ("policyengine-simulation-gateway", "policyengine-simulation-gateway"), + ("policyengine-stage12-us-worker", "policyengine-stage12-us-worker"), + ], +) +def test_normalize_service_family_merges_simulation_versions( + service_name: str, family: str +) -> None: + assert service_family(service_name) == family + + +@pytest.mark.parametrize( + ("environment_name", "environment_class"), + [ + ("production", "production"), + ("main", "production"), + ("staging", "staging"), + ("local", "other"), + ("", "other"), + (None, "other"), + ], +) +def test_normalize_environment_maps_modal_main_to_production( + environment_name: str | None, environment_class: str +) -> None: + assert normalization.environment_class(environment_name) == environment_class + + +def test_frames_pattern_has_one_capturing_group_for_bigquery() -> None: + assert re.compile(FRAME_PATTERN).groups == 1 + + +def test_frames_pattern_extracts_frame_headers_without_source_or_message() -> None: + frames = extract_frames(STANDARD_STACK) + + assert [frame_key(frame) for frame in frames] == [ + "flask/app.py:dispatch_request", + "policyengine_api/routes/report_routes.py:create_report", + "policyengine_api/services/report_service.py:create", + "sqlalchemy/orm/session.py:flush", + ] + assert not any("SENTINEL" in frame or "\n" in frame for frame in frames) + + +def test_frames_chained_traceback_contributes_only_its_first_traceback() -> None: + frames = extract_frames(CHAINED_STACK) + + assert [frame_key(frame) for frame in frames] == [ + "policyengine_api/services/cache.py:read", + ] + + +def test_frames_explicitly_chained_traceback_contributes_only_its_cause() -> None: + frames = extract_frames(chained_stack(recursion_depth=2)) + + assert [frame_key(frame) for frame in frames] == [ + "policyengine_api/services/household.py:calculate", + "policyengine_core/simulation.py:calculate", + "policyengine_core/simulation.py:calculate", + ] + + +def test_frames_pattern_reads_exception_group_tracebacks() -> None: + frames = extract_frames(EXCEPTION_GROUP_STACK) + + assert [frame_key(frame) for frame in frames] == [ + "policyengine_simulation_gateway/app.py:run", + "policyengine_simulation_gateway/worker.py:step", + ] + + +def test_frames_pattern_skips_a_frame_header_cut_off_by_truncation() -> None: + cut_at = STANDARD_STACK.index("create_report") + len("create_re") + frames = extract_frames(STANDARD_STACK[:cut_at]) + + assert [frame_key(frame) for frame in frames] == [ + "flask/app.py:dispatch_request", + ] + + +@pytest.mark.parametrize( + ("path", "normalized"), + [ + ( + "/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", + "sqlalchemy/orm/session.py", + ), + ("/usr/lib/python3/dist-packages/yaml/loader.py", "yaml/loader.py"), + ("/usr/local/lib/python3.13/asyncio/tasks.py", "asyncio/tasks.py"), + ("/app/policyengine_api/services/x.py", "policyengine_api/services/x.py"), + ( + "/home/runner/work/api/api/policyengine_api/services/x.py", + "policyengine_api/services/x.py", + ), + ( + "/root/policyengine_simulation_gateway/app.py", + "policyengine_simulation_gateway/app.py", + ), + ("/pkg/synchronicity/combined_types.py", "synchronicity/combined_types.py"), + ("", ""), + ], +) +def test_frames_paths_drop_installation_prefixes(path: str, normalized: str) -> None: + assert normalize_path(path) == normalized + + +def test_frames_keys_ignore_line_numbers() -> None: + first = frame_key('/app/policyengine_api/x.py", line 10, in handler') + second = frame_key('/srv/policyengine_api/x.py", line 99, in handler') + + assert first == second == "policyengine_api/x.py:handler" + + +def test_frames_choose_the_three_innermost_application_frames() -> None: + frames = [ + '/app/policyengine_api/routes/a.py", line 1, in outer', + '/app/policyengine_api/services/b.py", line 2, in middle', + '/app/policyengine_api/services/c.py", line 3, in inner', + '/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/d.py", line 4, in lib', + '/app/policyengine_api/services/e.py", line 5, in callback', + '/venv/lib/python3.12/site-packages/sqlalchemy/orm/f.py", line 6, in raises', + ] + + assert fingerprint_frames(frames) == ( + "policyengine_api/services/b.py:middle", + "policyengine_api/services/c.py:inner", + "policyengine_api/services/e.py:callback", + ) + + +def test_frames_count_simulation_and_stage12_packages_as_application_code() -> None: + frames = [ + '/pkg/modal/_runtime.py", line 1, in run', + '/root/policyengine_simulation_executor/run.py", line 2, in execute', + '/root/policyengine_stage12_persistence/store.py", line 3, in save', + '/pkg/synchronicity/combined_types.py", line 4, in __call__', + ] + + assert fingerprint_frames(frames) == ( + "policyengine_simulation_executor/run.py:execute", + "policyengine_stage12_persistence/store.py:save", + ) + + +def test_frames_without_application_frames_use_the_innermost_frames() -> None: + frames = [ + '/usr/local/lib/python3.13/asyncio/a.py", line 1, in first', + '/pkg/modal/b.py", line 2, in second', + '/pkg/modal/c.py", line 3, in third', + '/pkg/synchronicity/d.py", line 4, in fourth', + ] + + assert fingerprint_frames(frames) == ( + "modal/b.py:second", + "modal/c.py:third", + "synchronicity/d.py:fourth", + ) + + +@pytest.mark.parametrize("frames", [[], None, ["not a frame header"]]) +def test_frames_without_a_stack_use_a_placeholder(frames: list[str] | None) -> None: + assert fingerprint_frames(frames) == ("-",) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("StaleDataError", "StaleDataError"), + ("sqlalchemy.orm.exc.StaleDataError", "sqlalchemy.orm.exc.StaleDataError"), + ("SENTINEL message with spaces", ""), + ("Error: 'quoted'", ""), + ("E" * 200, ""), + (None, "-"), + ("", "-"), + ], +) +def test_public_error_type_allows_only_class_names( + value: str | None, expected: str +) -> None: + assert public_error_type(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ( + "//economy//over/", + "//economy//over/", + ), + ("/v2/user-policies", "/v2/user-policies"), + ("/jobs/{job_id}", "/jobs/{job_id}"), + ("", ""), + ("/us/policy/12345", ""), + ("/v2/households/3f2b1c9e-8d7a-4c1b-9e2f-0a1b2c3d4e5f", ""), + ("/us/report?token=SENTINEL", ""), + ("SENTINEL route", ""), + ("/" + "a" * 300, ""), + (None, "-"), + ("", "-"), + ], +) +def test_public_route_allows_only_route_templates( + value: str | None, expected: str +) -> None: + assert public_route(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("policyengine-api", "policyengine-api"), + ("policyengine-simulation-py", "policyengine-simulation-py"), + ("SENTINEL family", ""), + ("Policyengine_API", ""), + (None, "-"), + ], +) +def test_public_family_allows_only_service_names( + value: str | None, expected: str +) -> None: + assert public_family(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("simulation.dispatch", "simulation.dispatch"), + ("stage12/run_comparison", "stage12/run_comparison"), + ("SENTINEL operation", ""), + (None, "-"), + ], +) +def test_public_operation_allows_only_operation_names( + value: str | None, expected: str +) -> None: + assert public_operation(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (500, 500), + ("503", 503), + (99, None), + (600, None), + ("SENTINEL", None), + (None, None), + ], +) +def test_public_status_allows_only_http_status_codes( + value: object, expected: int | None +) -> None: + assert public_status(value) == expected diff --git a/tests/unit/telemetry_error_issues/test_records.py b/tests/unit/telemetry_error_issues/test_records.py new file mode 100644 index 000000000..cc5105b2a --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_records.py @@ -0,0 +1,246 @@ +"""Tests for telemetry_error_issues.records: primary records and fingerprints.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime + +import pytest +from telemetry_error_issues.records import ( + fingerprint, + fingerprint_key, + fingerprint_parts, + primary_records, +) + +from tests.unit.telemetry_error_issues.report_support import ( + APPLICATION_FRAMES, + FINGERPRINT_KEY, + chained_stack, + exception_record, + extract_frames, + failed_request_record, + message_record, +) + + +def test_primary_prefers_the_exception_record_of_a_failed_request() -> None: + exception = exception_record() + records = [ + message_record(), + exception, + failed_request_record(), + ] + + assert primary_records(records) == [exception] + + +def test_primary_falls_back_to_the_message_record() -> None: + message = message_record() + + primary = primary_records([failed_request_record(), message]) + + assert primary == [message] + assert primary[0].kind == "message" + + +def test_primary_falls_back_to_the_failed_request_record() -> None: + failed_request = failed_request_record() + + primary = primary_records([failed_request]) + + assert primary == [failed_request] + assert primary[0].kind == "failed_request" + + +def test_primary_prefers_an_unhandled_exception_over_a_handled_one() -> None: + handled = exception_record(error_type="KeyError", error_handled=True) + unhandled = exception_record( + timestamp=datetime(2026, 10, 20, 16, 0, 1, tzinfo=UTC), + error_type="LookupError", + ) + + assert primary_records([handled, unhandled]) == [unhandled] + + +def test_primary_treats_a_request_completed_record_with_error_fields_as_an_exception(): + completed = failed_request_record( + error_type="StaleDataError", frames=APPLICATION_FRAMES + ) + + assert completed.kind == "exception" + assert primary_records([completed]) == [completed] + + +def test_primary_correlates_by_trace_when_the_request_id_is_missing() -> None: + exception = exception_record(request_key=None) + message = message_record(request_key=None) + + assert primary_records([message, exception]) == [exception] + + +def test_primary_records_without_request_or_trace_stand_alone() -> None: + first = message_record(request_key=None, trace_key=None) + second = message_record( + request_key=None, + trace_key=None, + timestamp=datetime(2026, 10, 20, 17, 0, tzinfo=UTC), + ) + + assert primary_records([second, first]) == [first, second] + + +def test_primary_keeps_failures_of_different_services_separate() -> None: + api_failure = failed_request_record() + simulation_failure = exception_record( + service_name="policyengine-simulation-entry", + frames=(), + ) + + primary = primary_records([api_failure, simulation_failure]) + + assert primary == [api_failure, simulation_failure] + + +def test_fingerprint_exception_ignores_lines_path_prefixes_and_routes() -> None: + moved_frames = tuple( + frame.replace("/app/", "/srv/checkout/").replace("line 4", "line 9") + for frame in APPLICATION_FRAMES + ) + first = exception_record() + second = exception_record( + frames=moved_frames, + route="//economy//over/", + ) + + assert fingerprint(first, FINGERPRINT_KEY) == fingerprint(second, FINGERPRINT_KEY) + + +def test_fingerprint_exception_merges_simulation_service_versions() -> None: + first = exception_record(service_name="policyengine-simulation-py1-2-3") + second = exception_record(service_name="policyengine-simulation-py1-2-4") + + assert fingerprint(first, FINGERPRINT_KEY) == fingerprint(second, FINGERPRINT_KEY) + + +@pytest.mark.parametrize( + "change", + [ + {"error_type": "ValueError"}, + {"frames": APPLICATION_FRAMES[:1]}, + {"service_name": "policyengine-simulation-entry"}, + ], +) +def test_fingerprint_exception_differs_by_type_frames_and_service( + change: dict[str, object], +) -> None: + original = exception_record() + changed = exception_record(**change) + + assert fingerprint(original, FINGERPRINT_KEY) != fingerprint( + changed, FINGERPRINT_KEY + ) + + +def test_fingerprint_is_the_same_when_truncation_drops_the_final_traceback() -> None: + # The logging library keeps only the first 16,384 characters of a stack. + short_stack = chained_stack(recursion_depth=20) + long_stack = chained_stack(recursion_depth=120)[:16_384] + assert "CalculationError" in short_stack + assert "CalculationError" not in long_stack + + short = exception_record( + error_type="CalculationError", + frames=tuple(extract_frames(short_stack)), + ) + long = exception_record( + error_type="CalculationError", + frames=tuple(extract_frames(long_stack)), + stack_truncated=True, + ) + + assert fingerprint(short, FINGERPRINT_KEY) == fingerprint(long, FINGERPRINT_KEY) + + +def test_fingerprint_depends_on_the_key() -> None: + record = exception_record() + + assert fingerprint(record, FINGERPRINT_KEY) != fingerprint(record, b"j" * 32) + + +def test_fingerprint_is_16_hex_characters() -> None: + value = fingerprint(exception_record(), FINGERPRINT_KEY) + + assert re.fullmatch(r"[0-9a-f]{16}", value) + + +def test_fingerprint_key_parts_for_each_kind() -> None: + exception = exception_record() + message = message_record( + attribute_error_type="TimeoutError", + failure_category="upstream", + ) + operation_message = message_record(route=None, operation_name="simulation.dispatch") + failed_request = failed_request_record(status_code=502) + + assert fingerprint_parts(exception) == ( + "exception", + "policyengine-api", + "StaleDataError", + "policyengine_api/routes/report_routes.py:create_report", + "policyengine_api/services/report_service.py:create", + ) + assert fingerprint_parts(message) == ( + "message", + "policyengine-api", + "//report", + "failed to save", + "TimeoutError", + "upstream", + ) + assert fingerprint_parts(operation_message)[2] == "simulation.dispatch" + assert fingerprint_parts(failed_request) == ( + "failed_request", + "policyengine-api", + "//report", + "502", + ) + + +@pytest.mark.parametrize( + "change", + [ + {"message_template": "failed to load"}, + {"route": "//calculate"}, + {"attribute_error_type": "ValueError"}, + {"failure_category": "validation"}, + ], +) +def test_fingerprint_message_differs_by_each_key_part( + change: dict[str, object], +) -> None: + original = message_record() + changed = message_record(**change) + + assert fingerprint(original, FINGERPRINT_KEY) != fingerprint( + changed, FINGERPRINT_KEY + ) + + +def test_fingerprint_failed_request_differs_by_status() -> None: + first = failed_request_record(status_code=500) + second = failed_request_record(status_code=504) + + assert fingerprint(first, FINGERPRINT_KEY) != fingerprint(second, FINGERPRINT_KEY) + + +@pytest.mark.parametrize("value", ["", "too-short", "x" * 31]) +def test_fingerprint_key_rejects_a_secret_shorter_than_32_characters( + value: str, +) -> None: + with pytest.raises(ValueError, match="at least 32 characters"): + fingerprint_key(value) + + +def test_fingerprint_key_accepts_a_64_character_hex_secret() -> None: + assert fingerprint_key("ab" * 32) == ("ab" * 32).encode() diff --git a/tests/unit/telemetry_error_issues/test_rendering.py b/tests/unit/telemetry_error_issues/test_rendering.py new file mode 100644 index 000000000..d1d442a06 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_rendering.py @@ -0,0 +1,309 @@ +"""Tests for telemetry_error_issues.rendering: issue titles and bodies.""" + +from __future__ import annotations + +from urllib.parse import quote, unquote, urlparse, urlsplit + +import pytest +from telemetry_error_issues.bigquery import error_record_from_row +from telemetry_error_issues.groups import build_groups +from telemetry_error_issues.rendering import ( + SECTION_END, + SECTION_START, + issue_body, + issue_title, + logs_explorer_url, + parse_state, +) +from telemetry_error_issues.window import reporting_window + +from tests.unit.telemetry_error_issues.report_support import ( + FINGERPRINT_KEY, + PROJECT_ID, + REPORT_DATE, + exception_record, + failed_request_record, + failures_with_and_without_a_route, + hours_after_window_noon, + message_record, + request_count, + sentinel_row, +) + + +def _group(records: list, request_counts=()) -> object: + groups = build_groups( + records, + list(request_counts), + reporting_window(REPORT_DATE), + FINGERPRINT_KEY, + ) + return groups[0] + + +def _render(group: object) -> tuple[str, str]: + window = reporting_window(REPORT_DATE) + return ( + issue_title(group), + issue_body(group, window, PROJECT_ID), + ) + + +def _section(body: str) -> str: + start = body.index(SECTION_START) + end = body.index(SECTION_END) + len(SECTION_END) + return body[start:end] + + +def _logs_query(url: str) -> tuple[str, dict[str, str]]: + """Split a Logs Explorer URL into its decoded query and other settings.""" + parsed = urlsplit(url) + settings = dict( + segment.split("=", 1) + for segment in parsed.path.split(";")[1:] + if "=" in segment + ) + return unquote(settings.pop("query")), settings + + +def test_render_title_names_the_exception_and_its_most_frequent_route() -> None: + group = _group([exception_record()]) + + title, _ = _render(group) + + assert title == ( + "[telemetry] policyengine-api: StaleDataError on `//report` " + f"({group.fingerprint})" + ) + + +def test_render_title_for_a_logged_error() -> None: + route_group = _group([message_record()]) + operation_group = _group( + [message_record(route=None, operation_name="simulation.dispatch")], + ) + + assert issue_title(route_group).startswith( + "[telemetry] policyengine-api: logged error on `//report` (" + ) + assert issue_title(operation_group).startswith( + "[telemetry] policyengine-api: logged error in `simulation.dispatch` (" + ) + + +def test_render_title_for_a_failed_request_without_an_exception() -> None: + group = _group([failed_request_record(status_code=502)]) + + assert issue_title(group) == ( + "[telemetry] policyengine-api: HTTP 502 without recorded exception on " + f"`//report` ({group.fingerprint})" + ) + + +def test_render_title_without_a_route() -> None: + group = _group( + [ + exception_record( + service_name="policyengine-stage12-us-worker", + route=None, + ) + ], + ) + + assert issue_title(group) == ( + "[telemetry] policyengine-stage12-us-worker: StaleDataError " + f"({group.fingerprint})" + ) + + +def test_render_body_marked_section_holds_the_measures_table() -> None: + records = [ + exception_record(request_key="r1", observability_key="o1"), + exception_record(request_key="r2", timestamp=hours_after_window_noon(2)), + exception_record( + request_key="s1", + environment_name="staging", + timestamp=hours_after_window_noon(1), + ), + ] + group = _group( + records, + [request_count("//report", 400)], + ) + + _, body = _render(group) + section = _section(body) + + assert body.index(SECTION_START) > 0 + assert f"`{group.fingerprint}`" in section + assert "2026-10-20 00:00 EDT to 2026-10-21 00:00 EDT (24 hours)" in section + assert "| Measure | Production | Staging |" in section + assert "| Occurrences | 2 | 1 |" in section + assert "| Affected requests | 2 | 1 |" in section + assert "| Observability IDs | 1 | 0 |" in section + assert ( + "| Error rate on affected routes | 0.50% (2 of 400 requests) | n/a |" in section + ) + assert "| Occurrences per hour | 0.08 | 0.04 |" in section + assert "| Occurrences per hour, previous 7 days | 0.00 | 0.00 |" in section + assert ( + "| First seen in the last 8 days " + "| 2026-10-20 12:00 EDT | 2026-10-20 13:00 EDT |" in section + ) + assert "| Last seen | 2026-10-20 14:00 EDT | 2026-10-20 13:00 EDT |" in section + assert "Related fingerprints: none" in section + assert "https://console.cloud.google.com/logs/query;" in section + + +def test_render_error_rate_names_only_failures_on_routes() -> None: + group = _group( + failures_with_and_without_a_route(), + [request_count("//calculate", 1)], + ) + + _, body = _render(group) + + assert "| Error rate on affected routes | 100.00% (1 of 1 requests) | n/a |" in body + + +def test_render_body_lists_related_fingerprints() -> None: + records = [ + exception_record(trace_key="t1"), + message_record(service_name="policyengine-simulation-entry", trace_key="t1"), + ] + groups = build_groups(records, [], reporting_window(REPORT_DATE), FINGERPRINT_KEY) + exception_group = next(group for group in groups if group.kind == "exception") + other = exception_group.related_fingerprints[0] + + _, body = _render(exception_group) + + assert f"Related fingerprints (sharing traces): `{other}`" in body + + +def test_render_body_state_comment_records_the_group_and_window() -> None: + group = _group([exception_record()]) + + _, body = _render(group) + + assert parse_state(body) == { + "algorithm": 1, + "fingerprint": group.fingerprint, + "kind": "exception", + "window_end": "2026-10-21T04:00:00+00:00", + } + + +def test_render_body_stays_under_the_github_limit() -> None: + # The longest route that still passes validation (195 characters). + long_route = "/" + "/".join([""] * 13) + records = [ + exception_record( + error_type="E" * 120, + route=long_route, + trace_key=f"t{index}", + request_key=f"r{index}", + ) + for index in range(4) + ] + records += [ + message_record( + request_key=f"m{index}", + trace_key=f"t{index}", + message_template=f"template {index}", + service_name="policyengine-simulation-entry", + ) + for index in range(4) + ] + groups = build_groups(records, [], reporting_window(REPORT_DATE), FINGERPRINT_KEY) + + for group in groups: + title, body = _render(group) + assert len(body) < 65_536 + assert len(title) <= 256 + + +def test_render_logs_link_filters_exceptions_by_service_and_type() -> None: + group = _group([exception_record()]) + url = logs_explorer_url(group, reporting_window(REPORT_DATE), PROJECT_ID) + + query, settings = _logs_query(url) + + assert urlparse(url).query == "project=policyengine-observability" + assert settings["storageScope"] == quote( + "storage,projects/policyengine-observability/locations/global/" + "buckets/policyengine-observability/views/_AllLogs", + safe="", + ) + assert query.splitlines() == [ + 'jsonPayload.schema_version="policyengine.observability.v2"', + 'jsonPayload."service.name"="policyengine-api"', + "severity>=ERROR", + 'jsonPayload."error.type"="StaleDataError"', + 'timestamp>="2026-10-20T04:00:00Z"', + 'timestamp<"2026-10-21T04:00:00Z"', + ] + for character in (" ", "\n", '"'): + assert character not in url + + +def test_render_logs_link_filters_failed_requests_by_route_and_status() -> None: + group = _group( + [ + failed_request_record( + service_name="policyengine-simulation-py1-2-3", + route="/jobs/{job_id}", + status_code=504, + ) + ], + ) + + query, _ = _logs_query( + logs_explorer_url(group, reporting_window(REPORT_DATE), PROJECT_ID) + ) + + assert query.splitlines()[1:5] == [ + 'jsonPayload."service.name"=~"^policyengine-simulation-py[0-9]+-[0-9]+-[0-9]+$"', + 'jsonPayload."event.name"="request.completed"', + 'jsonPayload."http.route"="/jobs/{job_id}"', + 'jsonPayload."http.response.status_code"=504', + ] + + +def test_render_logs_link_filters_logged_errors_by_route_or_operation() -> None: + window = reporting_window(REPORT_DATE) + route_group = _group([message_record()]) + operation_group = _group( + [message_record(route=None, operation_name="simulation.dispatch")], + ) + + route_query, _ = _logs_query(logs_explorer_url(route_group, window, PROJECT_ID)) + operation_query, _ = _logs_query( + logs_explorer_url(operation_group, window, PROJECT_ID) + ) + + assert 'jsonPayload."http.route"="//report"' in route_query + assert "severity>=ERROR" in route_query + assert 'jsonPayload."operation.name"="simulation.dispatch"' in operation_query + + +@pytest.mark.parametrize( + "overrides", + [ + {}, + {"error_type": None}, + {"error_type": None, "event_name": "request.completed"}, + {"route": None}, + ], +) +def test_render_never_includes_values_from_records( + overrides: dict[str, object], +) -> None: + record = error_record_from_row(sentinel_row(**overrides)) + group = _group([record]) + window = reporting_window(REPORT_DATE) + + title, body = _render(group) + url = logs_explorer_url(group, window, PROJECT_ID) + + for output in (title, body, url, unquote(url)): + assert "SENTINEL" not in output diff --git a/tests/unit/telemetry_error_issues/test_sync.py b/tests/unit/telemetry_error_issues/test_sync.py new file mode 100644 index 000000000..5115d65f1 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_sync.py @@ -0,0 +1,338 @@ +"""Tests for telemetry_error_issues.sync: opening, updating and reopening issues.""" + +from __future__ import annotations + +import json +from datetime import timedelta +from urllib.parse import urlsplit + +import pytest +from telemetry_error_issues.github_client import GitHubError +from telemetry_error_issues.rendering import ( + SECTION_END, + issue_body, + issue_section, + issue_title, +) +from telemetry_error_issues.window import reporting_window + +from tests.unit.telemetry_error_issues.github_support import ( + FakeGitHub, + bot_issue, + failures, + groups_for_report_date, + sync, +) +from tests.unit.telemetry_error_issues.report_support import ( + PROJECT_ID, + REPORT_DATE, + WINDOW_NOON, + exception_record, + hours_after_window_noon, +) + + +def _existing_issue(group: object, number: int = 7, **fields: object) -> dict: + """Return the issue the job opened for ``group`` on an earlier day.""" + earlier = reporting_window(REPORT_DATE - timedelta(days=1)) + issue = bot_issue( + number, + title=issue_title(group), + body=issue_body(group, earlier, PROJECT_ID), + ) + issue.update(fields) + return issue + + +def test_sync_opens_an_issue_at_three_production_occurrences_in_two_requests() -> None: + eligible = failures(3) + # Three failures that all affected the request with observability ID o1. + one_request = [ + exception_record( + error_type="KeyError", + request_key=None, + trace_key=None, + observability_key="o1", + timestamp=hours_after_window_noon(index), + ) + for index in range(3) + ] + too_few = failures(2, request_key_prefix="few", error_type="TypeError") + groups = groups_for_report_date(eligible + one_request + too_few) + github = FakeGitHub() + + result = sync(groups, github) + + created = list(github.issues.values()) + eligible_group = next( + group for group in groups if group.error_type == "StaleDataError" + ) + assert [issue["title"] for issue in created] == [issue_title(eligible_group)] + assert created[0]["labels"] == [ + {"name": "telemetry-error"}, + {"name": "service: policyengine-api"}, + ] + actions = {action.fingerprint: action.action for action in result.actions} + assert sorted(actions.values()) == ["below-threshold", "below-threshold", "created"] + + +def test_sync_never_opens_an_issue_for_a_staging_only_group() -> None: + groups = groups_for_report_date(failures(10, environment_name="staging")) + github = FakeGitHub() + + result = sync(groups, github) + + assert github.issues == {} + assert result.actions[0].action == "below-threshold" + + +def test_sync_opens_at_most_five_issues_in_order_of_production_volume() -> None: + records = [] + for index in range(7): + records += failures( + 3 + index, + request_key_prefix=f"g{index}-", + error_type=f"Error{index}", + ) + groups = groups_for_report_date(records) + github = FakeGitHub() + + result = sync(groups, github) + + titles = [issue["title"] for issue in github.issues.values()] + assert [title.split(": ")[1].split(" ")[0] for title in titles] == [ + "Error6", + "Error5", + "Error4", + "Error3", + "Error2", + ] + assert result.deferred == 2 + assert [action.action for action in result.actions].count("deferred") == 2 + + +def test_sync_replaces_only_the_marked_section_and_never_the_title() -> None: + group = groups_for_report_date(failures(3))[0] + issue = _existing_issue(group, title="Renamed by a person") + issue["body"] = f"Notes before.\n\n{issue['body']}\nNotes after.\n" + github = FakeGitHub([issue]) + + result = sync([group], github) + + patch = json.loads(github.writes[0][1].body) + expected_section = issue_section(group, reporting_window(REPORT_DATE), PROJECT_ID) + assert list(patch) == ["body"] + assert patch["body"].startswith("Notes before.\n\n") + assert patch["body"].endswith("\nNotes after.\n") + assert expected_section in patch["body"] + assert github.issues[7]["title"] == "Renamed by a person" + assert result.actions[0].action == "updated" + assert result.actions[0].issue_number == 7 + + +def test_sync_does_not_write_when_the_section_is_unchanged() -> None: + group = groups_for_report_date(failures(3))[0] + github = FakeGitHub([_existing_issue(group)]) + + sync([group], github) + first_run_writes = len(github.writes) + second = sync([group], github) + + assert first_run_writes == 1 + assert len(github.writes) == 1 + assert second.actions[0].action == "unchanged" + + +def test_sync_adds_a_deleted_section_back_at_the_end() -> None: + group = groups_for_report_date(failures(3))[0] + issue = _existing_issue(group, body="Only human notes now.") + github = FakeGitHub([issue]) + + sync([group], github) + + body = github.issues[7]["body"] + assert body.startswith("Only human notes now.\n\n") + assert body.rstrip().endswith(SECTION_END) + assert len(github.issues) == 1 + + +def test_sync_ignores_issues_opened_by_other_authors() -> None: + group = groups_for_report_date(failures(3))[0] + person_issue = _existing_issue(group, user={"login": "someone"}) + github = FakeGitHub([person_issue]) + + sync([group], github) + + assert github.issues[7]["body"] == person_issue["body"] + assert len(github.issues) == 2 + + +def test_sync_reopens_a_completed_issue_when_production_failures_return() -> None: + group = groups_for_report_date(failures(4))[0] + closed_at = (WINDOW_NOON - timedelta(days=2)).isoformat().replace("+00:00", "Z") + issue = _existing_issue( + group, + state="closed", + state_reason="completed", + closed_at=closed_at, + ) + github = FakeGitHub([issue], labels=("telemetry-error",)) + + result = sync([group], github) + + reopened = github.issues[7] + assert reopened["state"] == "open" + assert reopened["state_reason"] == "reopened" + assert {"name": "regression"} in reopened["labels"] + assert "regression" in github.labels + assert github.comments[7] == [ + "Production failures returned after this issue was closed: 4 occurrences " + "in 4 requests since 2026-10-19 12:00 EDT, 24 hours after it was closed." + ] + assert result.actions[0].action == "reopened" + + +def test_sync_treats_an_issue_closed_without_a_reason_as_completed() -> None: + group = groups_for_report_date(failures(3))[0] + closed_at = (WINDOW_NOON - timedelta(days=2)).isoformat() + issue = _existing_issue( + group, state="closed", state_reason=None, closed_at=closed_at + ) + github = FakeGitHub([issue]) + + result = sync([group], github) + + assert github.issues[7]["state"] == "open" + assert result.actions[0].action == "reopened" + + +def test_sync_keeps_a_completed_issue_closed_within_24_hours_of_closing() -> None: + group = groups_for_report_date(failures(4))[0] + closed_at = (WINDOW_NOON - timedelta(hours=12)).isoformat() + issue = _existing_issue( + group, + state="closed", + state_reason="completed", + closed_at=closed_at, + ) + github = FakeGitHub([issue]) + + result = sync([group], github) + + assert github.issues[7]["state"] == "closed" + assert 7 not in github.comments + assert ( + issue_section(group, reporting_window(REPORT_DATE), PROJECT_ID) + in github.issues[7]["body"] + ) + assert result.actions[0].action == "updated" + + +@pytest.mark.parametrize("reason", ["not_planned", "duplicate"]) +def test_sync_updates_but_never_reopens_not_planned_or_duplicate_issues( + reason: str, +) -> None: + group = groups_for_report_date(failures(10))[0] + closed_at = (WINDOW_NOON - timedelta(days=5)).isoformat() + issue = _existing_issue( + group, state="closed", state_reason=reason, closed_at=closed_at + ) + github = FakeGitHub([issue]) + + result = sync([group], github) + + assert github.issues[7]["state"] == "closed" + assert result.actions[0].action == "updated" + assert len(github.issues) == 1 + + +def test_sync_keeps_a_newer_report_instead_of_overwriting_it() -> None: + group = groups_for_report_date(failures(3))[0] + later = reporting_window(REPORT_DATE + timedelta(days=1)) + issue = bot_issue( + 7, + title=issue_title(group), + body=issue_body(group, later, PROJECT_ID), + ) + github = FakeGitHub([issue]) + + result = sync([group], github) + + assert github.writes == [] + assert result.actions[0].action == "kept-newer" + + +def test_sync_creates_missing_labels_before_opening_issues() -> None: + groups = groups_for_report_date(failures(3)) + github = FakeGitHub(labels=("telemetry-error",)) + + sync(groups, github) + + created_labels = [ + json.loads(request.body)["name"] + for _, request in github.writes + if urlsplit(request.url).path.endswith("/labels") + ] + assert created_labels == ["service: policyengine-api"] + + +def test_sync_uses_existing_labels_whatever_their_case() -> None: + groups = groups_for_report_date(failures(3)) + github = FakeGitHub(labels=("Telemetry-Error", "Service: Policyengine-Api")) + + sync(groups, github) + + created_labels = [ + request + for _, request in github.writes + if urlsplit(request.url).path.endswith("/labels") + ] + assert created_labels == [] + assert github.issues[1]["labels"] == [ + {"name": "Telemetry-Error"}, + {"name": "Service: Policyengine-Api"}, + ] + + +def test_sync_reopens_with_an_existing_regression_label_of_another_case() -> None: + group = groups_for_report_date(failures(4))[0] + closed_at = (WINDOW_NOON - timedelta(days=2)).isoformat() + issue = _existing_issue( + group, + state="closed", + state_reason="completed", + closed_at=closed_at, + ) + github = FakeGitHub([issue], labels=("Regression",)) + + result = sync([group], github) + + assert result.actions[0].action == "reopened" + assert {"name": "Regression"} in github.issues[7]["labels"] + assert github.labels == {"Regression"} + + +def test_sync_adds_labels_a_create_response_is_missing() -> None: + groups = groups_for_report_date(failures(3)) + github = FakeGitHub(drop_created_labels=True) + + with pytest.raises(GitHubError, match="labels"): + sync(groups, github) + + added = [ + request + for _, request in github.writes + if urlsplit(request.url).path.endswith("/issues/1/labels") + ] + assert len(added) == 1 + + +def test_sync_dry_run_makes_no_write_calls() -> None: + eligible = groups_for_report_date(failures(3))[0] + existing = groups_for_report_date(failures(3, error_type="KeyError"))[0] + github = FakeGitHub([_existing_issue(existing)]) + + result = sync([eligible, existing], github, dry_run=True) + + assert github.writes == [] + assert sorted(action.action for action in result.actions) == ["created", "updated"] diff --git a/tests/unit/telemetry_error_issues/test_window.py b/tests/unit/telemetry_error_issues/test_window.py new file mode 100644 index 000000000..3deea8625 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_window.py @@ -0,0 +1,58 @@ +"""Tests for telemetry_error_issues.window: the reporting window.""" + +from __future__ import annotations + +from datetime import UTC, date, datetime, timedelta + +import pytest +from telemetry_error_issues.window import default_report_date, reporting_window + +from tests.unit.telemetry_error_issues.report_support import EASTERN + + +@pytest.mark.parametrize( + "run_time", + [ + datetime(2026, 10, 21, 0, 45, tzinfo=EASTERN), + datetime(2026, 10, 21, 3, 30, tzinfo=EASTERN), + datetime(2026, 10, 21, 4, 45, tzinfo=UTC), + ], +) +def test_window_scheduled_run_reports_the_previous_eastern_day( + run_time: datetime, +) -> None: + window = reporting_window(default_report_date(run_time)) + + assert window.start == datetime(2026, 10, 20, 0, 0, tzinfo=EASTERN) + assert window.end == datetime(2026, 10, 21, 0, 0, tzinfo=EASTERN) + assert window.hours == 24 + + +def test_window_default_date_follows_eastern_time_not_utc() -> None: + # 02:00 UTC on 21 October is still 22:00 on 20 October in New York. + run_time = datetime(2026, 10, 21, 2, 0, tzinfo=UTC) + + assert default_report_date(run_time) == date(2026, 10, 19) + + +def test_window_spring_forward_day_lasts_23_hours() -> None: + window = reporting_window(date(2026, 3, 8)) + + assert window.start.utcoffset() == timedelta(hours=-5) + assert window.end.utcoffset() == timedelta(hours=-4) + assert window.hours == 23 + + +def test_window_fall_back_day_lasts_25_hours() -> None: + window = reporting_window(date(2026, 11, 1)) + + assert window.start.utcoffset() == timedelta(hours=-4) + assert window.end.utcoffset() == timedelta(hours=-5) + assert window.hours == 25 + + +def test_window_bounds_convert_to_utc_for_queries() -> None: + window = reporting_window(date(2026, 11, 1)) + + assert window.start.astimezone(UTC) == datetime(2026, 11, 1, 4, 0, tzinfo=UTC) + assert window.end.astimezone(UTC) == datetime(2026, 11, 2, 5, 0, tzinfo=UTC) diff --git a/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py b/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py new file mode 100644 index 000000000..e7a1c778f --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py @@ -0,0 +1,211 @@ +"""Tests for the workflow, its shell wrapper and the script they run.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from tests.unit.telemetry_error_issues.github_support import REPOSITORY +from tests.unit.telemetry_error_issues.report_support import PROJECT_ID + +REPO = Path(__file__).resolve().parents[3] +SCRIPT_PATH = REPO / ".github" / "scripts" / "sync_telemetry_error_issues.py" +WORKFLOW_PATH = REPO / ".github" / "workflows" / "telemetry-error-issues.yml" +WRAPPER_PATH = REPO / ".github" / "scripts" / "sync_telemetry_error_issues.sh" +WRAPPER_ENVIRONMENT = { + "GOOGLE_OAUTH_ACCESS_TOKEN": "fake-access-token", + "GITHUB_TOKEN": "fake-github-token", + "GITHUB_REPOSITORY": REPOSITORY, + "TELEMETRY_ERROR_FINGERPRINT_KEY": "f" * 64, + "OBSERVABILITY_PROJECT_ID": PROJECT_ID, +} + + +def _workflow() -> dict: + return yaml.safe_load(WORKFLOW_PATH.read_text(encoding="utf-8")) + + +def _sync_job() -> dict: + return _workflow()["jobs"]["sync"] + + +def _step(name: str) -> dict: + return next(step for step in _sync_job()["steps"] if step["name"] == name) + + +def test_workflow_runs_at_00_45_eastern_and_on_request() -> None: + # PyYAML reads the bare key "on" as the boolean True. + triggers = _workflow()[True] + + assert triggers["schedule"] == [ + {"cron": "45 0 * * *", "timezone": "America/New_York"} + ] + inputs = triggers["workflow_dispatch"]["inputs"] + assert inputs["dry_run"]["type"] == "boolean" + assert inputs["dry_run"]["default"] is True + assert inputs["report_date"]["type"] == "string" + assert inputs["report_date"]["default"] == "" + + +def test_workflow_job_is_limited_to_master_in_its_environment() -> None: + workflow = _workflow() + job = _sync_job() + + assert workflow["concurrency"] == { + "group": "telemetry-error-issues", + "cancel-in-progress": False, + } + assert job["if"] == "github.repository == 'PolicyEngine/policyengine-api'" + assert job["environment"] == "telemetry-error-issues" + assert job["timeout-minutes"] == 15 + assert job["permissions"] == { + "contents": "read", + "issues": "write", + "id-token": "write", + } + assert job["steps"][0] == { + "name": "Require master branch", + "if": "github.ref != 'refs/heads/master'", + "run": "exit 1", + } + + +def test_workflow_authenticates_as_the_report_service_account() -> None: + auth = _step("Authenticate to Google Cloud") + + assert auth["id"] == "auth" + assert auth["uses"] == "google-github-actions/auth@v2" + assert auth["with"] == { + "workload_identity_provider": "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}", + "service_account": "${{ vars.TELEMETRY_ERROR_ISSUES_SERVICE_ACCOUNT }}", + "token_format": "access_token", + "create_credentials_file": False, + } + assert _step("Set up Python")["with"] == {"python-version": "3.12"} + + +def test_workflow_passes_settings_to_the_wrapper() -> None: + sync = _step("Sync telemetry error issues") + + assert sync["run"] == "bash .github/scripts/sync_telemetry_error_issues.sh" + assert sync["env"] == { + "GOOGLE_OAUTH_ACCESS_TOKEN": "${{ steps.auth.outputs.access_token }}", + "GITHUB_TOKEN": "${{ github.token }}", + "TELEMETRY_ERROR_FINGERPRINT_KEY": ( + "${{ secrets.TELEMETRY_ERROR_FINGERPRINT_KEY }}" + ), + "OBSERVABILITY_PROJECT_ID": "${{ vars.OBSERVABILITY_PROJECT_ID }}", + # A scheduled run has no inputs, so it is never a dry run. + "DRY_RUN": "${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}", + "REPORT_DATE": "${{ inputs.report_date }}", + } + + +def _run_wrapper( + tmp_path: Path, environment: dict[str, str] +) -> tuple[subprocess.CompletedProcess[str], list[str] | None]: + """Run the wrapper with a fake python3 that records its arguments.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + calls = tmp_path / "python-arguments.txt" + fake_python = bin_dir / "python3" + fake_python.write_text(f'#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "{calls}"\n') + fake_python.chmod(0o755) + result = subprocess.run( + ["bash", str(WRAPPER_PATH)], + cwd=REPO, + env={"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", **environment}, + text=True, + capture_output=True, + check=False, + ) + arguments = calls.read_text().splitlines() if calls.exists() else None + return result, arguments + + +def test_wrapper_passes_bash_syntax_checking() -> None: + result = subprocess.run( + ["bash", "-n", str(WRAPPER_PATH)], capture_output=True, text=True, check=False + ) + + assert result.returncode == 0, result.stderr + + +def test_wrapper_runs_the_report_for_the_project(tmp_path: Path) -> None: + result, arguments = _run_wrapper(tmp_path, WRAPPER_ENVIRONMENT) + + assert result.returncode == 0, result.stderr + assert arguments == [str(SCRIPT_PATH), "--project-id", PROJECT_ID] + + +def test_wrapper_masks_the_access_token_and_fingerprint_key(tmp_path: Path) -> None: + result, _ = _run_wrapper(tmp_path, WRAPPER_ENVIRONMENT) + + assert result.stdout.splitlines() == [ + "::add-mask::fake-access-token", + f"::add-mask::{'f' * 64}", + ] + + +def test_wrapper_adds_dry_run_and_report_date_when_set(tmp_path: Path) -> None: + environment = { + **WRAPPER_ENVIRONMENT, + "DRY_RUN": "true", + "REPORT_DATE": "2026-10-15", + } + + result, arguments = _run_wrapper(tmp_path, environment) + + assert result.returncode == 0, result.stderr + assert arguments == [ + str(SCRIPT_PATH), + "--project-id", + PROJECT_ID, + "--dry-run", + "--report-date", + "2026-10-15", + ] + + +def test_wrapper_runs_live_when_dry_run_is_false(tmp_path: Path) -> None: + environment = {**WRAPPER_ENVIRONMENT, "DRY_RUN": "false", "REPORT_DATE": ""} + + _, arguments = _run_wrapper(tmp_path, environment) + + assert arguments == [str(SCRIPT_PATH), "--project-id", PROJECT_ID] + + +@pytest.mark.parametrize("missing", sorted(WRAPPER_ENVIRONMENT)) +def test_wrapper_fails_before_running_when_a_variable_is_missing( + tmp_path: Path, missing: str +) -> None: + environment = { + name: value for name, value in WRAPPER_ENVIRONMENT.items() if name != missing + } + + result, arguments = _run_wrapper(tmp_path, environment) + + assert result.returncode != 0 + assert f"{missing} is required" in result.stderr + assert arguments is None + assert result.stdout == "" + + +def test_script_imports_its_package_from_another_working_directory( + tmp_path: Path, +) -> None: + result = subprocess.run( + [sys.executable, str(SCRIPT_PATH), "--help"], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "Report each day's telemetry error groups as GitHub issues." in result.stdout diff --git a/tests/unit/test_telemetry_error_issues.py b/tests/unit/test_telemetry_error_issues.py deleted file mode 100644 index 17b1e3257..000000000 --- a/tests/unit/test_telemetry_error_issues.py +++ /dev/null @@ -1,3367 +0,0 @@ -"""Tests for .github/scripts/sync_telemetry_error_issues.py.""" - -from __future__ import annotations - -import functools -import importlib.util -import io -import itertools -import json -import os -import re -import subprocess -import sys -from datetime import UTC, date, datetime, timedelta -from pathlib import Path -from typing import TYPE_CHECKING -from urllib.parse import parse_qs, quote, unquote, urlparse, urlsplit -from zoneinfo import ZoneInfo - -import pytest -import yaml - -if TYPE_CHECKING: - from types import ModuleType - -REPO = Path(__file__).resolve().parents[2] -SCRIPT_PATH = REPO / ".github" / "scripts" / "sync_telemetry_error_issues.py" -EASTERN = ZoneInfo("America/New_York") - - -@functools.cache -def _load_script() -> ModuleType: - spec = importlib.util.spec_from_file_location( - "sync_telemetry_error_issues", SCRIPT_PATH - ) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - # Dataclasses resolve their annotations through sys.modules. - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -@pytest.fixture -def sync_script() -> ModuleType: - return _load_script() - - -# Reporting window and command-line interface - - -@pytest.mark.parametrize( - "run_time", - [ - datetime(2026, 10, 21, 0, 45, tzinfo=EASTERN), - datetime(2026, 10, 21, 3, 30, tzinfo=EASTERN), - datetime(2026, 10, 21, 4, 45, tzinfo=UTC), - ], -) -def test_window_scheduled_run_reports_the_previous_eastern_day( - sync_script: ModuleType, run_time: datetime -) -> None: - window = sync_script.reporting_window(sync_script.default_report_date(run_time)) - - assert window.start == datetime(2026, 10, 20, 0, 0, tzinfo=EASTERN) - assert window.end == datetime(2026, 10, 21, 0, 0, tzinfo=EASTERN) - assert window.hours == 24 - - -def test_window_default_date_follows_eastern_time_not_utc( - sync_script: ModuleType, -) -> None: - # 02:00 UTC on 21 October is still 22:00 on 20 October in New York. - run_time = datetime(2026, 10, 21, 2, 0, tzinfo=UTC) - - assert sync_script.default_report_date(run_time) == date(2026, 10, 19) - - -def test_window_spring_forward_day_lasts_23_hours(sync_script: ModuleType) -> None: - window = sync_script.reporting_window(date(2026, 3, 8)) - - assert window.start.utcoffset() == timedelta(hours=-5) - assert window.end.utcoffset() == timedelta(hours=-4) - assert window.hours == 23 - - -def test_window_fall_back_day_lasts_25_hours(sync_script: ModuleType) -> None: - window = sync_script.reporting_window(date(2026, 11, 1)) - - assert window.start.utcoffset() == timedelta(hours=-4) - assert window.end.utcoffset() == timedelta(hours=-5) - assert window.hours == 25 - - -def test_window_bounds_convert_to_utc_for_queries(sync_script: ModuleType) -> None: - window = sync_script.reporting_window(date(2026, 11, 1)) - - assert window.start.astimezone(UTC) == datetime(2026, 11, 1, 4, 0, tzinfo=UTC) - assert window.end.astimezone(UTC) == datetime(2026, 11, 2, 5, 0, tzinfo=UTC) - - -def test_cli_parses_project_dry_run_and_report_date(sync_script: ModuleType) -> None: - arguments = sync_script.parse_arguments( - [ - "--project-id", - "policyengine-observability", - "--dry-run", - "--report-date", - "2026-10-19", - ] - ) - - assert arguments.project_id == "policyengine-observability" - assert arguments.dry_run is True - assert arguments.report_date == date(2026, 10, 19) - - -def test_cli_defaults_to_a_live_run_of_the_previous_day( - sync_script: ModuleType, -) -> None: - arguments = sync_script.parse_arguments( - ["--project-id", "policyengine-observability"] - ) - - assert arguments.dry_run is False - assert arguments.report_date is None - - -def test_cli_treats_an_empty_report_date_as_unset(sync_script: ModuleType) -> None: - arguments = sync_script.parse_arguments( - ["--project-id", "policyengine-observability", "--report-date", ""] - ) - - assert arguments.report_date is None - - -@pytest.mark.parametrize( - "argv", - [ - [], - ["--dry-run"], - ["--project-id", "policyengine-observability", "--report-date", "20 Oct"], - ], -) -def test_cli_rejects_missing_project_or_invalid_date_with_exit_code_2( - sync_script: ModuleType, argv: list[str] -) -> None: - with pytest.raises(SystemExit) as raised: - sync_script.main(argv) - - assert raised.value.code == 2 - - -# Normalization, stack frames and public value validation - -STANDARD_STACK = """\ -Traceback (most recent call last): - File "/venv/lib/python3.12/site-packages/flask/app.py", line 880, in dispatch_request - rv = self.dispatch_request() - ^^^^^^^^^^^^^^^^^^^^^^^ - File "/app/policyengine_api/routes/report_routes.py", line 41, in create_report - return service.create(payload) - File "/app/policyengine_api/services/report_service.py", line 88, in create - session.flush() - File "/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 43, in flush - self._flush(objects) -sqlalchemy.orm.exc.StaleDataError: SENTINEL message about 'reports' -""" - -CHAINED_STACK = """\ -Traceback (most recent call last): - File "/app/policyengine_api/services/cache.py", line 12, in read - return store[key] -KeyError: 'SENTINEL key' - -During handling of the above exception, another exception occurred: - -Traceback (most recent call last): - File "/app/policyengine_api/routes/policy_routes.py", line 30, in get_policy - return service.get(policy_id) - File "/app/policyengine_api/services/policy_service.py", line 55, in get - raise LookupError(policy_id) -LookupError: SENTINEL policy -""" - -EXCEPTION_GROUP_STACK = """\ - + Exception Group Traceback (most recent call last): - | File "/root/policyengine_simulation_gateway/app.py", line 10, in run - | raise ExceptionGroup("SENTINEL group", errors) - | ExceptionGroup: SENTINEL group (1 sub-exception) - +-+---------------- 1 ---------------- - | Traceback (most recent call last): - | File "/root/policyengine_simulation_gateway/worker.py", line 22, in step - | raise ValueError("SENTINEL value") - | ValueError: SENTINEL value - +------------------------------------ -""" - - -def _extract_frames(sync_script: ModuleType, stack: str) -> list[str]: - """Extract frames as the query does: REGEXP_REPLACE, then REGEXP_EXTRACT_ALL.""" - first_traceback = re.sub(sync_script.CHAINED_EXCEPTION_PATTERN, "", stack) - return re.findall(sync_script.FRAME_PATTERN, first_traceback) - - -def _chained_stack(recursion_depth: int) -> str: - """Return a chained traceback whose cause recursed ``recursion_depth`` times.""" - library_frame = ( - ' File "/venv/lib/python3.12/site-packages/policyengine_core/simulation.py", ' - "line 500, in calculate\n return self._calculate(variable_name, period)\n" - ) - cause = ( - "Traceback (most recent call last):\n" - ' File "/app/policyengine_api/services/household.py", line 40, in calculate\n' - " simulation.calculate(name)\n" - f"{library_frame * recursion_depth}" - "ValueError: SENTINEL cause\n" - ) - final = ( - "Traceback (most recent call last):\n" - ' File "/app/policyengine_api/routes/household_routes.py", line 12, in get\n' - " return service.calculate()\n" - ' File "/app/policyengine_api/services/household.py", line 44, in calculate\n' - " raise CalculationError(name) from error\n" - "CalculationError: SENTINEL final\n" - ) - separator = ( - "\nThe above exception was the direct cause of the following exception:\n\n" - ) - return f"{cause}{separator}{final}" - - -@pytest.mark.parametrize( - ("service_name", "family"), - [ - ("policyengine-simulation-py1-2-3", "policyengine-simulation-py"), - ("policyengine-simulation-py0-412-16", "policyengine-simulation-py"), - ("policyengine-api", "policyengine-api"), - ("policyengine-simulation-gateway", "policyengine-simulation-gateway"), - ("policyengine-stage12-us-worker", "policyengine-stage12-us-worker"), - ], -) -def test_normalize_service_family_merges_simulation_versions( - sync_script: ModuleType, service_name: str, family: str -) -> None: - assert sync_script.service_family(service_name) == family - - -@pytest.mark.parametrize( - ("environment_name", "environment_class"), - [ - ("production", "production"), - ("main", "production"), - ("staging", "staging"), - ("local", "other"), - ("", "other"), - (None, "other"), - ], -) -def test_normalize_environment_maps_modal_main_to_production( - sync_script: ModuleType, environment_name: str | None, environment_class: str -) -> None: - assert sync_script.environment_class(environment_name) == environment_class - - -def test_frames_pattern_has_one_capturing_group_for_bigquery( - sync_script: ModuleType, -) -> None: - assert re.compile(sync_script.FRAME_PATTERN).groups == 1 - - -def test_frames_pattern_extracts_frame_headers_without_source_or_message( - sync_script: ModuleType, -) -> None: - frames = _extract_frames(sync_script, STANDARD_STACK) - - assert [sync_script.frame_key(frame) for frame in frames] == [ - "flask/app.py:dispatch_request", - "policyengine_api/routes/report_routes.py:create_report", - "policyengine_api/services/report_service.py:create", - "sqlalchemy/orm/session.py:flush", - ] - assert not any("SENTINEL" in frame or "\n" in frame for frame in frames) - - -def test_frames_chained_traceback_contributes_only_its_first_traceback( - sync_script: ModuleType, -) -> None: - frames = _extract_frames(sync_script, CHAINED_STACK) - - assert [sync_script.frame_key(frame) for frame in frames] == [ - "policyengine_api/services/cache.py:read", - ] - - -def test_frames_explicitly_chained_traceback_contributes_only_its_cause( - sync_script: ModuleType, -) -> None: - frames = _extract_frames(sync_script, _chained_stack(recursion_depth=2)) - - assert [sync_script.frame_key(frame) for frame in frames] == [ - "policyengine_api/services/household.py:calculate", - "policyengine_core/simulation.py:calculate", - "policyengine_core/simulation.py:calculate", - ] - - -def test_frames_pattern_reads_exception_group_tracebacks( - sync_script: ModuleType, -) -> None: - frames = _extract_frames(sync_script, EXCEPTION_GROUP_STACK) - - assert [sync_script.frame_key(frame) for frame in frames] == [ - "policyengine_simulation_gateway/app.py:run", - "policyengine_simulation_gateway/worker.py:step", - ] - - -def test_frames_pattern_skips_a_frame_header_cut_off_by_truncation( - sync_script: ModuleType, -) -> None: - cut_at = STANDARD_STACK.index("create_report") + len("create_re") - frames = _extract_frames(sync_script, STANDARD_STACK[:cut_at]) - - assert [sync_script.frame_key(frame) for frame in frames] == [ - "flask/app.py:dispatch_request", - ] - - -@pytest.mark.parametrize( - ("path", "normalized"), - [ - ( - "/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", - "sqlalchemy/orm/session.py", - ), - ("/usr/lib/python3/dist-packages/yaml/loader.py", "yaml/loader.py"), - ("/usr/local/lib/python3.13/asyncio/tasks.py", "asyncio/tasks.py"), - ("/app/policyengine_api/services/x.py", "policyengine_api/services/x.py"), - ( - "/home/runner/work/api/api/policyengine_api/services/x.py", - "policyengine_api/services/x.py", - ), - ( - "/root/policyengine_simulation_gateway/app.py", - "policyengine_simulation_gateway/app.py", - ), - ("/pkg/synchronicity/combined_types.py", "synchronicity/combined_types.py"), - ("", ""), - ], -) -def test_frames_paths_drop_installation_prefixes( - sync_script: ModuleType, path: str, normalized: str -) -> None: - assert sync_script.normalize_path(path) == normalized - - -def test_frames_keys_ignore_line_numbers(sync_script: ModuleType) -> None: - first = sync_script.frame_key('/app/policyengine_api/x.py", line 10, in handler') - second = sync_script.frame_key('/srv/policyengine_api/x.py", line 99, in handler') - - assert first == second == "policyengine_api/x.py:handler" - - -def test_frames_choose_the_three_innermost_application_frames( - sync_script: ModuleType, -) -> None: - frames = [ - '/app/policyengine_api/routes/a.py", line 1, in outer', - '/app/policyengine_api/services/b.py", line 2, in middle', - '/app/policyengine_api/services/c.py", line 3, in inner', - '/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/d.py", line 4, in lib', - '/app/policyengine_api/services/e.py", line 5, in callback', - '/venv/lib/python3.12/site-packages/sqlalchemy/orm/f.py", line 6, in raises', - ] - - assert sync_script.fingerprint_frames(frames) == ( - "policyengine_api/services/b.py:middle", - "policyengine_api/services/c.py:inner", - "policyengine_api/services/e.py:callback", - ) - - -def test_frames_count_simulation_and_stage12_packages_as_application_code( - sync_script: ModuleType, -) -> None: - frames = [ - '/pkg/modal/_runtime.py", line 1, in run', - '/root/policyengine_simulation_executor/run.py", line 2, in execute', - '/root/policyengine_stage12_persistence/store.py", line 3, in save', - '/pkg/synchronicity/combined_types.py", line 4, in __call__', - ] - - assert sync_script.fingerprint_frames(frames) == ( - "policyengine_simulation_executor/run.py:execute", - "policyengine_stage12_persistence/store.py:save", - ) - - -def test_frames_without_application_frames_use_the_innermost_frames( - sync_script: ModuleType, -) -> None: - frames = [ - '/usr/local/lib/python3.13/asyncio/a.py", line 1, in first', - '/pkg/modal/b.py", line 2, in second', - '/pkg/modal/c.py", line 3, in third', - '/pkg/synchronicity/d.py", line 4, in fourth', - ] - - assert sync_script.fingerprint_frames(frames) == ( - "modal/b.py:second", - "modal/c.py:third", - "synchronicity/d.py:fourth", - ) - - -@pytest.mark.parametrize("frames", [[], None, ["not a frame header"]]) -def test_frames_without_a_stack_use_a_placeholder( - sync_script: ModuleType, frames: list[str] | None -) -> None: - assert sync_script.fingerprint_frames(frames) == ("-",) - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - ("StaleDataError", "StaleDataError"), - ("sqlalchemy.orm.exc.StaleDataError", "sqlalchemy.orm.exc.StaleDataError"), - ("SENTINEL message with spaces", ""), - ("Error: 'quoted'", ""), - ("E" * 200, ""), - (None, "-"), - ("", "-"), - ], -) -def test_public_error_type_allows_only_class_names( - sync_script: ModuleType, value: str | None, expected: str -) -> None: - assert sync_script.public_error_type(value) == expected - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - ( - "//economy//over/", - "//economy//over/", - ), - ("/v2/user-policies", "/v2/user-policies"), - ("/jobs/{job_id}", "/jobs/{job_id}"), - ("", ""), - ("/us/policy/12345", ""), - ("/v2/households/3f2b1c9e-8d7a-4c1b-9e2f-0a1b2c3d4e5f", ""), - ("/us/report?token=SENTINEL", ""), - ("SENTINEL route", ""), - ("/" + "a" * 300, ""), - (None, "-"), - ("", "-"), - ], -) -def test_public_route_allows_only_route_templates( - sync_script: ModuleType, value: str | None, expected: str -) -> None: - assert sync_script.public_route(value) == expected - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - ("policyengine-api", "policyengine-api"), - ("policyengine-simulation-py", "policyengine-simulation-py"), - ("SENTINEL family", ""), - ("Policyengine_API", ""), - (None, "-"), - ], -) -def test_public_family_allows_only_service_names( - sync_script: ModuleType, value: str | None, expected: str -) -> None: - assert sync_script.public_family(value) == expected - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - ("simulation.dispatch", "simulation.dispatch"), - ("stage12/run_comparison", "stage12/run_comparison"), - ("SENTINEL operation", ""), - (None, "-"), - ], -) -def test_public_operation_allows_only_operation_names( - sync_script: ModuleType, value: str | None, expected: str -) -> None: - assert sync_script.public_operation(value) == expected - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - (500, 500), - ("503", 503), - (99, None), - (600, None), - ("SENTINEL", None), - (None, None), - ], -) -def test_public_status_allows_only_http_status_codes( - sync_script: ModuleType, value: object, expected: int | None -) -> None: - assert sync_script.public_status(value) == expected - - -# Primary record per failure and fingerprints - -FINGERPRINT_KEY = b"k" * 32 -APPLICATION_FRAMES = ( - '/app/policyengine_api/routes/report_routes.py", line 41, in create_report', - '/app/policyengine_api/services/report_service.py", line 88, in create', - '/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 43, in flush', -) - - -def _record(sync_script: ModuleType, **overrides: object) -> object: - fields: dict[str, object] = { - "timestamp": datetime(2026, 10, 20, 16, 0, tzinfo=UTC), - "service_name": "policyengine-api", - "environment_name": "production", - "event_name": None, - "error_type": None, - "error_handled": None, - "frames": (), - "stack_truncated": False, - "message_template": None, - "attribute_error_type": None, - "failure_category": None, - "route": "//report", - "status_code": None, - "operation_name": None, - "request_key": "request-1", - "observability_key": None, - "trace_key": "trace-1", - } - fields.update(overrides) - return sync_script.ErrorRecord(**fields) - - -def _exception_record(sync_script: ModuleType, **overrides: object) -> object: - fields: dict[str, object] = { - "event_name": "exception.recorded", - "error_type": "StaleDataError", - "error_handled": False, - "frames": APPLICATION_FRAMES, - "status_code": 500, - } - fields.update(overrides) - return _record(sync_script, **fields) - - -def _message_record(sync_script: ModuleType, **overrides: object) -> object: - fields: dict[str, object] = {"message_template": "failed to save"} - fields.update(overrides) - return _record(sync_script, **fields) - - -def _failed_request_record(sync_script: ModuleType, **overrides: object) -> object: - fields: dict[str, object] = {"event_name": "request.completed", "status_code": 500} - fields.update(overrides) - return _record(sync_script, **fields) - - -def test_primary_prefers_the_exception_record_of_a_failed_request( - sync_script: ModuleType, -) -> None: - exception = _exception_record(sync_script) - records = [ - _message_record(sync_script), - exception, - _failed_request_record(sync_script), - ] - - assert sync_script.primary_records(records) == [exception] - - -def test_primary_falls_back_to_the_message_record(sync_script: ModuleType) -> None: - message = _message_record(sync_script) - - primary = sync_script.primary_records( - [_failed_request_record(sync_script), message] - ) - - assert primary == [message] - assert primary[0].kind == "message" - - -def test_primary_falls_back_to_the_failed_request_record( - sync_script: ModuleType, -) -> None: - failed_request = _failed_request_record(sync_script) - - primary = sync_script.primary_records([failed_request]) - - assert primary == [failed_request] - assert primary[0].kind == "failed_request" - - -def test_primary_prefers_an_unhandled_exception_over_a_handled_one( - sync_script: ModuleType, -) -> None: - handled = _exception_record(sync_script, error_type="KeyError", error_handled=True) - unhandled = _exception_record( - sync_script, - timestamp=datetime(2026, 10, 20, 16, 0, 1, tzinfo=UTC), - error_type="LookupError", - ) - - assert sync_script.primary_records([handled, unhandled]) == [unhandled] - - -def test_primary_treats_a_request_completed_record_with_error_fields_as_an_exception( - sync_script: ModuleType, -) -> None: - completed = _failed_request_record( - sync_script, error_type="StaleDataError", frames=APPLICATION_FRAMES - ) - - assert completed.kind == "exception" - assert sync_script.primary_records([completed]) == [completed] - - -def test_primary_correlates_by_trace_when_the_request_id_is_missing( - sync_script: ModuleType, -) -> None: - exception = _exception_record(sync_script, request_key=None) - message = _message_record(sync_script, request_key=None) - - assert sync_script.primary_records([message, exception]) == [exception] - - -def test_primary_records_without_request_or_trace_stand_alone( - sync_script: ModuleType, -) -> None: - first = _message_record(sync_script, request_key=None, trace_key=None) - second = _message_record( - sync_script, - request_key=None, - trace_key=None, - timestamp=datetime(2026, 10, 20, 17, 0, tzinfo=UTC), - ) - - assert sync_script.primary_records([second, first]) == [first, second] - - -def test_primary_keeps_failures_of_different_services_separate( - sync_script: ModuleType, -) -> None: - api_failure = _failed_request_record(sync_script) - simulation_failure = _exception_record( - sync_script, - service_name="policyengine-simulation-entry", - frames=(), - ) - - primary = sync_script.primary_records([api_failure, simulation_failure]) - - assert primary == [api_failure, simulation_failure] - - -def test_fingerprint_exception_ignores_lines_path_prefixes_and_routes( - sync_script: ModuleType, -) -> None: - moved_frames = tuple( - frame.replace("/app/", "/srv/checkout/").replace("line 4", "line 9") - for frame in APPLICATION_FRAMES - ) - first = _exception_record(sync_script) - second = _exception_record( - sync_script, - frames=moved_frames, - route="//economy//over/", - ) - - assert sync_script.fingerprint(first, FINGERPRINT_KEY) == sync_script.fingerprint( - second, FINGERPRINT_KEY - ) - - -def test_fingerprint_exception_merges_simulation_service_versions( - sync_script: ModuleType, -) -> None: - first = _exception_record( - sync_script, service_name="policyengine-simulation-py1-2-3" - ) - second = _exception_record( - sync_script, service_name="policyengine-simulation-py1-2-4" - ) - - assert sync_script.fingerprint(first, FINGERPRINT_KEY) == sync_script.fingerprint( - second, FINGERPRINT_KEY - ) - - -@pytest.mark.parametrize( - "change", - [ - {"error_type": "ValueError"}, - {"frames": APPLICATION_FRAMES[:1]}, - {"service_name": "policyengine-simulation-entry"}, - ], -) -def test_fingerprint_exception_differs_by_type_frames_and_service( - sync_script: ModuleType, change: dict[str, object] -) -> None: - original = _exception_record(sync_script) - changed = _exception_record(sync_script, **change) - - assert sync_script.fingerprint( - original, FINGERPRINT_KEY - ) != sync_script.fingerprint(changed, FINGERPRINT_KEY) - - -def test_fingerprint_is_the_same_when_truncation_drops_the_final_traceback( - sync_script: ModuleType, -) -> None: - # The logging library keeps only the first 16,384 characters of a stack. - short_stack = _chained_stack(recursion_depth=20) - long_stack = _chained_stack(recursion_depth=120)[:16_384] - assert "CalculationError" in short_stack - assert "CalculationError" not in long_stack - - short = _exception_record( - sync_script, - error_type="CalculationError", - frames=tuple(_extract_frames(sync_script, short_stack)), - ) - long = _exception_record( - sync_script, - error_type="CalculationError", - frames=tuple(_extract_frames(sync_script, long_stack)), - stack_truncated=True, - ) - - assert sync_script.fingerprint(short, FINGERPRINT_KEY) == sync_script.fingerprint( - long, FINGERPRINT_KEY - ) - - -def test_fingerprint_depends_on_the_key(sync_script: ModuleType) -> None: - record = _exception_record(sync_script) - - assert sync_script.fingerprint(record, FINGERPRINT_KEY) != sync_script.fingerprint( - record, b"j" * 32 - ) - - -def test_fingerprint_is_16_hex_characters(sync_script: ModuleType) -> None: - value = sync_script.fingerprint(_exception_record(sync_script), FINGERPRINT_KEY) - - assert re.fullmatch(r"[0-9a-f]{16}", value) - - -def test_fingerprint_key_parts_for_each_kind(sync_script: ModuleType) -> None: - exception = _exception_record(sync_script) - message = _message_record( - sync_script, - attribute_error_type="TimeoutError", - failure_category="upstream", - ) - operation_message = _message_record( - sync_script, route=None, operation_name="simulation.dispatch" - ) - failed_request = _failed_request_record(sync_script, status_code=502) - - assert sync_script.fingerprint_parts(exception) == ( - "exception", - "policyengine-api", - "StaleDataError", - "policyengine_api/routes/report_routes.py:create_report", - "policyengine_api/services/report_service.py:create", - ) - assert sync_script.fingerprint_parts(message) == ( - "message", - "policyengine-api", - "//report", - "failed to save", - "TimeoutError", - "upstream", - ) - assert sync_script.fingerprint_parts(operation_message)[2] == "simulation.dispatch" - assert sync_script.fingerprint_parts(failed_request) == ( - "failed_request", - "policyengine-api", - "//report", - "502", - ) - - -@pytest.mark.parametrize( - "change", - [ - {"message_template": "failed to load"}, - {"route": "//calculate"}, - {"attribute_error_type": "ValueError"}, - {"failure_category": "validation"}, - ], -) -def test_fingerprint_message_differs_by_each_key_part( - sync_script: ModuleType, change: dict[str, object] -) -> None: - original = _message_record(sync_script) - changed = _message_record(sync_script, **change) - - assert sync_script.fingerprint( - original, FINGERPRINT_KEY - ) != sync_script.fingerprint(changed, FINGERPRINT_KEY) - - -def test_fingerprint_failed_request_differs_by_status(sync_script: ModuleType) -> None: - first = _failed_request_record(sync_script, status_code=500) - second = _failed_request_record(sync_script, status_code=504) - - assert sync_script.fingerprint(first, FINGERPRINT_KEY) != sync_script.fingerprint( - second, FINGERPRINT_KEY - ) - - -@pytest.mark.parametrize("value", ["", "too-short", "x" * 31]) -def test_fingerprint_key_rejects_a_secret_shorter_than_32_characters( - sync_script: ModuleType, value: str -) -> None: - with pytest.raises(ValueError, match="at least 32 characters"): - sync_script.fingerprint_key(value) - - -def test_fingerprint_key_accepts_a_64_character_hex_secret( - sync_script: ModuleType, -) -> None: - assert sync_script.fingerprint_key("ab" * 32) == ("ab" * 32).encode() - - -# Measures per group - -REPORT_DATE = date(2026, 10, 20) -# Noon Eastern on the report date: 12 hours after the window starts, and 180 -# hours after the seven days before it start. -WINDOW_NOON = datetime(2026, 10, 20, 16, 0, tzinfo=UTC) - - -def _at(hours_after_window_noon: float) -> datetime: - return WINDOW_NOON + timedelta(hours=hours_after_window_noon) - - -def _request_count( - sync_script: ModuleType, - route: str, - requests: int, - *, - service_name: str = "policyengine-api", - environment_name: str = "production", -) -> object: - return sync_script.RequestCount(service_name, environment_name, route, requests) - - -def _only_group(sync_script: ModuleType, records: list, request_counts=()) -> object: - groups = sync_script.build_groups( - records, - list(request_counts), - sync_script.reporting_window(REPORT_DATE), - FINGERPRINT_KEY, - ) - assert len(groups) == 1 - return groups[0] - - -def test_measures_count_occurrences_requests_and_observability_ids( - sync_script: ModuleType, -) -> None: - records = [ - _exception_record(sync_script, request_key="r1", observability_key="o1"), - _exception_record( - sync_script, request_key="r2", observability_key="o1", timestamp=_at(1) - ), - _exception_record( - sync_script, request_key="r3", observability_key="o2", timestamp=_at(2) - ), - _exception_record( - sync_script, request_key="s1", environment_name="staging", timestamp=_at(3) - ), - ] - - group = _only_group(sync_script, records) - - assert group.production.occurrences == 3 - assert group.production.affected_requests == 3 - assert group.production.observability_ids == 2 - assert group.staging.occurrences == 1 - assert group.staging.affected_requests == 1 - assert group.staging.observability_ids == 0 - - -def test_measures_affected_requests_fall_back_to_observability_id_then_trace( - sync_script: ModuleType, -) -> None: - records = [ - _exception_record( - sync_script, request_key=None, trace_key=None, observability_key="o1" - ), - _exception_record( - sync_script, - request_key=None, - trace_key=None, - observability_key="o1", - timestamp=_at(1), - ), - _exception_record(sync_script, request_key=None, trace_key="t1"), - _exception_record(sync_script, request_key=None, trace_key=None), - _exception_record( - sync_script, request_key=None, trace_key=None, timestamp=_at(2) - ), - ] - - group = _only_group(sync_script, records) - - assert group.production.occurrences == 5 - # o1 once, t1 once, and each record without any identifier on its own. - assert group.production.affected_requests == 4 - - -def test_measures_error_rate_uses_requests_to_the_affected_routes( - sync_script: ModuleType, -) -> None: - records = [ - _exception_record(sync_script, request_key="r1", route="//report"), - _exception_record( - sync_script, - request_key="r2", - route="//calculate", - timestamp=_at(1), - ), - _exception_record( - sync_script, - request_key="r3", - route="//calculate", - timestamp=_at(2), - ), - ] - request_counts = [ - _request_count(sync_script, "//report", 200), - _request_count(sync_script, "//calculate", 100), - _request_count(sync_script, "//metadata", 5000), - _request_count( - sync_script, "//report", 10, environment_name="staging" - ), - _request_count( - sync_script, - "//report", - 7000, - service_name="policyengine-simulation-entry", - ), - ] - - group = _only_group(sync_script, records, request_counts) - - assert group.production.route_requests == 300 - assert group.production.error_rate == pytest.approx(0.01) - assert group.staging.error_rate is None - - -def _failures_with_and_without_a_route(sync_script: ModuleType) -> list: - """Return one failed request on a route and two route-less failures.""" - return [ - _exception_record( - sync_script, request_key="r1", route="//calculate" - ), - _exception_record( - sync_script, request_key=None, trace_key="t2", route=None, timestamp=_at(1) - ), - _exception_record( - sync_script, request_key=None, trace_key="t3", route=None, timestamp=_at(2) - ), - ] - - -def test_measures_error_rate_counts_only_failures_on_routes( - sync_script: ModuleType, -) -> None: - group = _only_group( - sync_script, - _failures_with_and_without_a_route(sync_script), - [_request_count(sync_script, "//calculate", 1)], - ) - - assert group.production.affected_requests == 3 - assert group.production.route_affected_requests == 1 - assert group.production.route_requests == 1 - assert group.production.error_rate == pytest.approx(1.0) - - -def test_measures_error_rate_counts_every_simulation_service_version( - sync_script: ModuleType, -) -> None: - records = [ - _failed_request_record( - sync_script, - service_name="policyengine-simulation-py1-2-3", - route="/jobs/{job_id}", - ) - ] - request_counts = [ - _request_count( - sync_script, - "/jobs/{job_id}", - 30, - service_name="policyengine-simulation-py1-2-3", - ), - _request_count( - sync_script, - "/jobs/{job_id}", - 70, - service_name="policyengine-simulation-py1-2-4", - ), - ] - - group = _only_group(sync_script, records, request_counts) - - assert group.production.route_requests == 100 - assert group.production.error_rate == pytest.approx(0.01) - - -def test_measures_error_rate_is_unknown_without_route_requests( - sync_script: ModuleType, -) -> None: - group = _only_group( - sync_script, [_message_record(sync_script, route=None, operation_name="sync")] - ) - - assert group.production.route_requests == 0 - assert group.production.error_rate is None - - -def test_measures_hourly_rates_for_the_window_and_the_previous_seven_days( - sync_script: ModuleType, -) -> None: - window_records = [ - _message_record(sync_script, request_key=f"w{index}", timestamp=_at(index)) - for index in range(6) - ] - previous_records = [ - _message_record( - sync_script, request_key=f"p{index}", timestamp=_at(-13 - 4 * index) - ) - for index in range(42) - ] - - group = _only_group(sync_script, window_records + previous_records) - - assert group.production.occurrences == 6 - assert group.production.hourly_rate == pytest.approx(6 / 24) - assert group.production.previous_hourly_rate == pytest.approx(42 / 168) - - -def test_measures_hourly_rate_uses_the_25_hour_fall_back_day( - sync_script: ModuleType, -) -> None: - fall_back_noon = datetime(2026, 11, 1, 17, 0, tzinfo=UTC) - records = [ - _message_record( - sync_script, - request_key=f"w{index}", - timestamp=fall_back_noon + timedelta(hours=index), - ) - for index in range(5) - ] - - groups = sync_script.build_groups( - records, [], sync_script.reporting_window(date(2026, 11, 1)), FINGERPRINT_KEY - ) - - assert groups[0].production.hourly_rate == pytest.approx(5 / 25) - - -def test_measures_first_and_last_seen_cover_the_queried_eight_days( - sync_script: ModuleType, -) -> None: - records = [ - _message_record(sync_script, request_key="early", timestamp=_at(-24 * 6)), - _message_record(sync_script, request_key="latest", timestamp=_at(5)), - _message_record(sync_script, request_key="middle", timestamp=_at(1)), - ] - - group = _only_group(sync_script, records) - - assert group.production.first_seen == _at(-24 * 6) - assert group.production.last_seen == _at(5) - assert group.staging.first_seen is None - assert group.staging.last_seen is None - - -def test_measures_ignore_records_outside_the_queried_days( - sync_script: ModuleType, -) -> None: - records = [ - _message_record(sync_script, request_key="in-window"), - _message_record(sync_script, request_key="too-old", timestamp=_at(-24 * 9)), - _message_record(sync_script, request_key="too-new", timestamp=_at(24)), - ] - - group = _only_group(sync_script, records) - - assert group.production.occurrences == 1 - assert group.production.first_seen == WINDOW_NOON - - -def test_measures_report_only_groups_seen_in_the_window( - sync_script: ModuleType, -) -> None: - records = [ - _message_record(sync_script, request_key="today"), - _message_record( - sync_script, - request_key="last-week", - message_template="something else", - timestamp=_at(-48), - ), - ] - - groups = sync_script.build_groups( - records, [], sync_script.reporting_window(REPORT_DATE), FINGERPRINT_KEY - ) - - assert [group.fingerprint for group in groups] == [ - sync_script.fingerprint(records[0], FINGERPRINT_KEY) - ] - - -def test_measures_ignore_environments_other_than_production_and_staging( - sync_script: ModuleType, -) -> None: - records = [ - _message_record(sync_script, environment_name="local"), - _message_record(sync_script, environment_name=None, request_key="r2"), - ] - - groups = sync_script.build_groups( - records, [], sync_script.reporting_window(REPORT_DATE), FINGERPRINT_KEY - ) - - assert groups == [] - - -def test_measures_describe_the_group_with_its_most_frequent_production_route( - sync_script: ModuleType, -) -> None: - records = [ - _exception_record(sync_script, request_key="r1", route="//report"), - _exception_record( - sync_script, request_key="r2", route="//calculate" - ), - _exception_record( - sync_script, request_key="r3", route="//calculate" - ), - _exception_record( - sync_script, - request_key="s1", - route="//report", - environment_name="staging", - ), - _exception_record( - sync_script, - request_key="s2", - route="//report", - environment_name="staging", - ), - ] - - group = _only_group(sync_script, records) - - assert group.kind == "exception" - assert group.family == "policyengine-api" - assert group.error_type == "StaleDataError" - assert group.route == "//calculate" - - -def test_measures_describe_groups_only_with_validated_values( - sync_script: ModuleType, -) -> None: - group = _only_group( - sync_script, - [ - _exception_record( - sync_script, - error_type="SENTINEL not a class", - route="/us/policy/12345", - ) - ], - ) - - assert group.error_type == "" - assert group.route == "" - - -def test_measures_list_up_to_three_related_fingerprints_by_shared_traces( - sync_script: ModuleType, -) -> None: - base = _exception_record(sync_script, request_key="r1", trace_key="t1") - base_second = _exception_record( - sync_script, request_key="r2", trace_key="t2", timestamp=_at(1) - ) - related_twice = [ - _message_record( - sync_script, - service_name="policyengine-simulation-entry", - request_key=f"x{index}", - trace_key=trace, - ) - for index, trace in enumerate(("t1", "t2")) - ] - related_once = [ - _message_record( - sync_script, - service_name="policyengine-simulation-gateway", - request_key=f"y{index}", - trace_key="t1", - message_template=template, - ) - for index, template in enumerate(("first error", "second error", "third")) - ] - - groups = sync_script.build_groups( - [base, base_second, *related_twice, *related_once], - [], - sync_script.reporting_window(REPORT_DATE), - FINGERPRINT_KEY, - ) - by_fingerprint = {group.fingerprint: group for group in groups} - base_group = by_fingerprint[sync_script.fingerprint(base, FINGERPRINT_KEY)] - twice = sync_script.fingerprint(related_twice[0], FINGERPRINT_KEY) - once = sorted( - sync_script.fingerprint(record, FINGERPRINT_KEY) for record in related_once - ) - - assert base_group.related_fingerprints == (twice, once[0], once[1]) - - -def test_measures_sort_groups_by_production_occurrences( - sync_script: ModuleType, -) -> None: - frequent = [ - _message_record(sync_script, request_key=f"f{index}", message_template="often") - for index in range(3) - ] - staging_only = [ - _message_record( - sync_script, - request_key=f"s{index}", - message_template="staging", - environment_name="staging", - ) - for index in range(5) - ] - rare = [_message_record(sync_script, request_key="rare", message_template="rare")] - - groups = sync_script.build_groups( - staging_only + rare + frequent, - [], - sync_script.reporting_window(REPORT_DATE), - FINGERPRINT_KEY, - ) - - assert [group.production.occurrences for group in groups] == [3, 1, 0] - - -def test_measures_count_production_occurrences_since_a_moment( - sync_script: ModuleType, -) -> None: - records = [ - _exception_record(sync_script, request_key="before", timestamp=_at(-30)), - _exception_record(sync_script, request_key="r1", timestamp=_at(1)), - _exception_record(sync_script, request_key="r2", timestamp=_at(2)), - _exception_record( - sync_script, request_key="r2", trace_key="t9", timestamp=_at(3) - ), - _exception_record( - sync_script, request_key="staging", environment_name="staging" - ), - ] - - group = _only_group(sync_script, records) - - assert group.production_since(_at(-1)) == (2, 2) - assert group.production_since(_at(-48)) == (3, 3) - - -# BigQuery queries and client - -ACCESS_TOKEN = "fake-access-token" - - -class FakeTransport: - """Records each HTTP request and answers with queued responses.""" - - def __init__(self, sync_script: ModuleType, responses: list[object]) -> None: - self.sync_script = sync_script - self.responses = list(responses) - self.requests: list[object] = [] - - def __call__(self, request: object) -> object: - self.requests.append(request) - return self.responses.pop(0) - - def json_body(self, index: int) -> dict: - return json.loads(self.requests[index].body) - - -def _json_response( - sync_script: ModuleType, payload: object, status: int = 200, reason: str = "OK" -) -> object: - return sync_script.HttpResponse( - status=status, reason=reason, headers={}, body=json.dumps(payload).encode() - ) - - -def _cell(value: object) -> dict: - if isinstance(value, list): - return {"v": [{"v": item} for item in value]} - return {"v": value} - - -def _query_response( - fields: list[dict], - rows: list[list[object]], - *, - page_token: str | None = None, - job_complete: bool = True, -) -> dict: - response: dict[str, object] = { - "jobComplete": job_complete, - "jobReference": { - "projectId": "policyengine-observability", - "jobId": "job-1", - "location": "US", - }, - } - if job_complete: - response["schema"] = {"fields": fields} - response["rows"] = [{"f": [_cell(value) for value in row]} for row in rows] - if page_token: - response["pageToken"] = page_token - return response - - -STRING_FIELD = {"name": "name", "type": "STRING", "mode": "NULLABLE"} - - -def _client(sync_script: ModuleType, transport: FakeTransport) -> object: - return sync_script.BigQueryClient( - "policyengine-observability", ACCESS_TOKEN, transport - ) - - -def _sent_parameters(transport: FakeTransport, index: int = 0) -> dict: - return { - parameter["name"]: parameter - for parameter in transport.json_body(index)["queryParameters"] - } - - -def test_bigquery_error_query_request_limits_cost_and_names_its_parameters( - sync_script: ModuleType, -) -> None: - transport = FakeTransport( - sync_script, [_json_response(sync_script, _query_response([], []))] - ) - window = sync_script.reporting_window(REPORT_DATE) - - sync_script.fetch_error_records(_client(sync_script, transport), window) - - request = transport.requests[0] - body = transport.json_body(0) - parameters = _sent_parameters(transport) - assert request.method == "POST" - assert request.url == ( - "https://bigquery.googleapis.com/bigquery/v2/projects/" - "policyengine-observability/queries" - ) - assert request.headers["Authorization"] == f"Bearer {ACCESS_TOKEN}" - assert body["useLegacySql"] is False - assert body["parameterMode"] == "NAMED" - assert body["maximumBytesBilled"] == str(5 * 1024**3) - assert body["labels"] == { - "purpose": "telemetry-error-issues", - "query": "error-records", - } - assert body["defaultDataset"] == { - "projectId": "policyengine-observability", - "datasetId": "observability_logs", - } - assert body["formatOptions"] == {"useInt64Timestamp": True} - assert parameters["query_start"]["parameterType"] == {"type": "TIMESTAMP"} - assert parameters["query_start"]["parameterValue"]["value"] == ( - "2026-10-13 04:00:00+00:00" - ) - assert parameters["window_end"]["parameterValue"]["value"] == ( - "2026-10-21 04:00:00+00:00" - ) - assert parameters["frame_pattern"]["parameterValue"]["value"] == ( - sync_script.FRAME_PATTERN - ) - assert parameters["chained_exception_pattern"]["parameterValue"]["value"] == ( - sync_script.CHAINED_EXCEPTION_PATTERN - ) - for name, pattern in ( - ("double_quoted_pattern", sync_script.DOUBLE_QUOTED_PATTERN), - ("single_quoted_pattern", sync_script.SINGLE_QUOTED_PATTERN), - ("template_word_pattern", sync_script.TEMPLATE_WORD_PATTERN), - ): - assert parameters[name]["parameterValue"]["value"] == pattern - assert [ - item["value"] - for item in parameters["excluded_routes"]["parameterValue"]["arrayValues"] - ] == ["/health", "/simulation-gateway-check", "/liveness-check", "/readiness-check"] - - -def test_bigquery_error_query_filters_application_error_records( - sync_script: ModuleType, -) -> None: - sql = sync_script.ERROR_RECORDS_QUERY - - assert "FROM `_AllLogs`" in sql - assert "timestamp >= @query_start AND timestamp < @window_end" in sql - assert ( - "JSON_VALUE(json_payload, '$.schema_version') = " - "'policyengine.observability.v2'" in sql - ) - assert ( - "JSON_VALUE(json_payload, '$.\"service.namespace\"') = " - "'policyengine.api-v1'" in sql - ) - assert "severity_number >= 500" in sql - assert "NOT IN UNNEST(@excluded_routes)" in sql - assert "QUALIFY ROW_NUMBER() OVER" in sql - - -def test_bigquery_error_query_reads_stacks_and_messages_only_through_expressions( - sync_script: ModuleType, -) -> None: - sql = sync_script.ERROR_RECORDS_QUERY - stack = "JSON_VALUE(json_payload, '$.\"error.stack\"')" - - assert sql.count("error.stack") == 2 - assert ( - " REGEXP_EXTRACT_ALL(\n" - " REGEXP_REPLACE(\n" - f" {stack}, @chained_exception_pattern, ''\n" - " ),\n" - " @frame_pattern\n" - " ) AS frames" - ) in sql - assert f"LENGTH({stack}) >= 16384" in sql - assert sql.count("$.message") == 1 - assert ( - " REGEXP_REPLACE(\n" - " REGEXP_REPLACE(\n" - " JSON_VALUE(json_payload, '$.message'), " - "@double_quoted_pattern, ''\n" - " ),\n" - " @single_quoted_pattern,\n" - " r'\\1'\n" - " ),\n" - " @template_word_pattern\n" - ) in sql - assert "error.message" not in sql - assert "SELECT *" not in sql - assert "json_payload AS" not in sql - - -@pytest.mark.parametrize( - "identifier", - [ - "JSON_VALUE(json_payload, '$.\"request.id\"')", - "JSON_VALUE(json_payload, '$.attributes.observability_id')", - ], -) -def test_bigquery_error_query_hashes_identifiers( - sync_script: ModuleType, identifier: str -) -> None: - sql = sync_script.ERROR_RECORDS_QUERY - - assert sql.count(identifier) == 1 - assert f"TO_HEX(SHA256({identifier}))" in sql - - -def _message_template(sync_script: ModuleType, message: str) -> str: - """Build the template the way the query does, with RE2's ASCII word rules.""" - without_double_quotes = re.sub(sync_script.DOUBLE_QUOTED_PATTERN, "", message) - without_quotes = re.sub( - sync_script.SINGLE_QUOTED_PATTERN, r"\1", without_double_quotes - ) - words = re.findall(sync_script.TEMPLATE_WORD_PATTERN, without_quotes, re.ASCII) - return " ".join(word.lower() for word in words[:3]) - - -def test_bigquery_template_keeps_words_around_an_apostrophe( - sync_script: ModuleType, -) -> None: - templates = { - _message_template(sync_script, f"Can't load policy '{value}'") - for value in ("alpha", "beta", "gamma") - } - - assert templates == {"can t load"} - - -@pytest.mark.parametrize( - ("message", "template"), - [ - ("Failed to save 'report 12' for user", "failed to save"), - ("'SENTINEL' could not be parsed", "could not be"), - ('Missing "SENTINEL" in household input', "missing in household"), - ("Timeout after 30s calling the simulation API", "timeout after calling"), - ("The users' policies 'SENTINEL' failed", "the users policies"), - ], -) -def test_bigquery_template_drops_quoted_values_and_partial_words( - sync_script: ModuleType, message: str, template: str -) -> None: - assert _message_template(sync_script, message) == template - - -def test_bigquery_error_query_hashes_the_trace(sync_script: ModuleType) -> None: - sql = sync_script.ERROR_RECORDS_QUERY - - assert "TO_HEX(SHA256(COALESCE(" in sql - assert sql.count("$.trace_id") == 1 - - -def test_bigquery_request_count_query_counts_completed_requests_in_the_window( - sync_script: ModuleType, -) -> None: - sql = sync_script.REQUEST_COUNTS_QUERY - transport = FakeTransport( - sync_script, [_json_response(sync_script, _query_response([], []))] - ) - - sync_script.fetch_request_counts( - _client(sync_script, transport), sync_script.reporting_window(REPORT_DATE) - ) - - parameters = _sent_parameters(transport) - assert "JSON_VALUE(json_payload, '$.\"event.name\"') = 'request.completed'" in sql - assert "COUNT(*) AS requests" in sql - assert "GROUP BY service_name, environment_name, route" in sql - assert "timestamp >= @window_start AND timestamp < @window_end" in sql - assert parameters["window_start"]["parameterValue"]["value"] == ( - "2026-10-20 04:00:00+00:00" - ) - assert transport.json_body(0)["labels"]["query"] == "request-counts" - - -def test_bigquery_client_polls_until_the_job_completes(sync_script: ModuleType) -> None: - transport = FakeTransport( - sync_script, - [ - _json_response(sync_script, _query_response([], [], job_complete=False)), - _json_response(sync_script, _query_response([], [], job_complete=False)), - _json_response(sync_script, _query_response([STRING_FIELD], [["done"]])), - ], - ) - - rows = _client(sync_script, transport).query("SELECT 1", [], "test") - - assert rows == [{"name": "done"}] - poll = urlparse(transport.requests[1].url) - assert transport.requests[1].method == "GET" - assert poll.path.endswith("/projects/policyengine-observability/queries/job-1") - assert parse_qs(poll.query)["location"] == ["US"] - assert parse_qs(poll.query)["formatOptions.useInt64Timestamp"] == ["true"] - - -def test_bigquery_client_follows_page_tokens(sync_script: ModuleType) -> None: - transport = FakeTransport( - sync_script, - [ - _json_response( - sync_script, - _query_response([STRING_FIELD], [["first"]], page_token="page-2"), - ), - _json_response(sync_script, _query_response([STRING_FIELD], [["second"]])), - ], - ) - - rows = _client(sync_script, transport).query("SELECT 1", [], "test") - - assert rows == [{"name": "first"}, {"name": "second"}] - page = parse_qs(urlparse(transport.requests[1].url).query) - assert page["pageToken"] == ["page-2"] - assert page["location"] == ["US"] - - -def test_bigquery_client_gives_up_on_a_job_that_never_completes( - sync_script: ModuleType, -) -> None: - pending = _json_response(sync_script, _query_response([], [], job_complete=False)) - transport = FakeTransport(sync_script, [pending] * 200) - - with pytest.raises(sync_script.BigQueryError, match="did not complete"): - _client(sync_script, transport).query("SELECT 1", [], "test") - - -def test_bigquery_client_converts_row_values_from_strings( - sync_script: ModuleType, -) -> None: - fields = [ - {"name": "text", "type": "STRING", "mode": "NULLABLE"}, - {"name": "count", "type": "INTEGER", "mode": "NULLABLE"}, - {"name": "flag", "type": "BOOLEAN", "mode": "NULLABLE"}, - {"name": "moment", "type": "TIMESTAMP", "mode": "NULLABLE"}, - {"name": "items", "type": "STRING", "mode": "REPEATED"}, - {"name": "missing", "type": "INTEGER", "mode": "NULLABLE"}, - ] - row = ["value", "42", "true", "1792512000000001", ["a", "b"], None] - transport = FakeTransport( - sync_script, [_json_response(sync_script, _query_response(fields, [row]))] - ) - - rows = _client(sync_script, transport).query("SELECT 1", [], "test") - - assert rows == [ - { - "text": "value", - "count": 42, - "flag": True, - "moment": datetime(2026, 10, 20, 16, 0, 0, 1, tzinfo=UTC), - "items": ["a", "b"], - "missing": None, - } - ] - - -def test_bigquery_http_error_reports_only_the_status_and_reason( - sync_script: ModuleType, -) -> None: - error_body = { - "error": { - "code": 400, - "message": "SENTINEL value echoed from a row", - "errors": [{"reason": "invalidQuery", "message": "SENTINEL detail"}], - } - } - transport = FakeTransport( - sync_script, - [_json_response(sync_script, error_body, status=400, reason="Bad Request")], - ) - - with pytest.raises(sync_script.BigQueryError) as raised: - _client(sync_script, transport).query("SELECT 1", [], "test") - - assert str(raised.value) == "BigQuery request failed: HTTP 400 invalidQuery" - assert raised.value.status == 400 - assert raised.value.reason == "invalidQuery" - - -def test_bigquery_http_error_without_a_json_reason_uses_the_status_phrase( - sync_script: ModuleType, -) -> None: - response = sync_script.HttpResponse( - status=503, reason="Service Unavailable", headers={}, body=b"SENTINEL " - ) - transport = FakeTransport(sync_script, [response]) - - with pytest.raises(sync_script.BigQueryError) as raised: - _client(sync_script, transport).query("SELECT 1", [], "test") - - assert str(raised.value) == "BigQuery request failed: HTTP 503 Service Unavailable" - - -def test_bigquery_rows_become_error_records(sync_script: ModuleType) -> None: - row = { - "timestamp": WINDOW_NOON, - "service_name": "policyengine-api", - "environment_name": "production", - "event_name": "exception.recorded", - "error_type": "StaleDataError", - "error_handled": "false", - "frames": list(APPLICATION_FRAMES), - "stack_truncated": False, - "message_template": "", - "attribute_error_type": None, - "failure_category": None, - "route": "//report", - "status_code": "500", - "operation_name": None, - "request_key": "aa", - "observability_key": None, - "trace_key": "bb", - } - - record = sync_script.error_record_from_row(row) - - assert record == _exception_record(sync_script, request_key="aa", trace_key="bb") - - -def test_bigquery_rows_tolerate_missing_and_malformed_values( - sync_script: ModuleType, -) -> None: - row = { - "timestamp": WINDOW_NOON, - "service_name": None, - "environment_name": None, - "event_name": None, - "error_type": None, - "error_handled": "SENTINEL", - "frames": None, - "stack_truncated": None, - "message_template": None, - "attribute_error_type": None, - "failure_category": None, - "route": None, - "status_code": "SENTINEL", - "operation_name": None, - "request_key": None, - "observability_key": None, - "trace_key": None, - } - - record = sync_script.error_record_from_row(row) - - assert record.service_name == "-" - assert record.error_handled is None - assert record.frames == () - assert record.stack_truncated is False - assert record.status_code is None - - -def test_bigquery_rows_become_request_counts(sync_script: ModuleType) -> None: - rows = [ - { - "service_name": "policyengine-api", - "environment_name": "production", - "route": "//report", - "requests": 120, - } - ] - transport = FakeTransport( - sync_script, - [ - _json_response( - sync_script, - _query_response( - [ - {"name": "service_name", "type": "STRING"}, - {"name": "environment_name", "type": "STRING"}, - {"name": "route", "type": "STRING"}, - {"name": "requests", "type": "INTEGER"}, - ], - [list(map(str, rows[0].values()))], - ), - ) - ], - ) - - counts = sync_script.fetch_request_counts( - _client(sync_script, transport), sync_script.reporting_window(REPORT_DATE) - ) - - assert counts == [ - _request_count(sync_script, "//report", 120), - ] - - -# Issue rendering - -PROJECT_ID = "policyengine-observability" - - -def _group(sync_script: ModuleType, records: list, request_counts=()) -> object: - groups = sync_script.build_groups( - records, - list(request_counts), - sync_script.reporting_window(REPORT_DATE), - FINGERPRINT_KEY, - ) - return groups[0] - - -def _render(sync_script: ModuleType, group: object) -> tuple[str, str]: - window = sync_script.reporting_window(REPORT_DATE) - return ( - sync_script.issue_title(group), - sync_script.issue_body(group, window, PROJECT_ID), - ) - - -def _section(sync_script: ModuleType, body: str) -> str: - start = body.index(sync_script.SECTION_START) - end = body.index(sync_script.SECTION_END) + len(sync_script.SECTION_END) - return body[start:end] - - -def _logs_query(url: str) -> tuple[str, dict[str, str]]: - """Split a Logs Explorer URL into its decoded query and other settings.""" - parsed = urlsplit(url) - settings = dict( - segment.split("=", 1) - for segment in parsed.path.split(";")[1:] - if "=" in segment - ) - return unquote(settings.pop("query")), settings - - -def test_render_title_names_the_exception_and_its_most_frequent_route( - sync_script: ModuleType, -) -> None: - group = _group(sync_script, [_exception_record(sync_script)]) - - title, _ = _render(sync_script, group) - - assert title == ( - "[telemetry] policyengine-api: StaleDataError on `//report` " - f"({group.fingerprint})" - ) - - -def test_render_title_for_a_logged_error(sync_script: ModuleType) -> None: - route_group = _group(sync_script, [_message_record(sync_script)]) - operation_group = _group( - sync_script, - [ - _message_record( - sync_script, route=None, operation_name="simulation.dispatch" - ) - ], - ) - - assert sync_script.issue_title(route_group).startswith( - "[telemetry] policyengine-api: logged error on `//report` (" - ) - assert sync_script.issue_title(operation_group).startswith( - "[telemetry] policyengine-api: logged error in `simulation.dispatch` (" - ) - - -def test_render_title_for_a_failed_request_without_an_exception( - sync_script: ModuleType, -) -> None: - group = _group(sync_script, [_failed_request_record(sync_script, status_code=502)]) - - assert sync_script.issue_title(group) == ( - "[telemetry] policyengine-api: HTTP 502 without recorded exception on " - f"`//report` ({group.fingerprint})" - ) - - -def test_render_title_without_a_route(sync_script: ModuleType) -> None: - group = _group( - sync_script, - [ - _exception_record( - sync_script, - service_name="policyengine-stage12-us-worker", - route=None, - ) - ], - ) - - assert sync_script.issue_title(group) == ( - "[telemetry] policyengine-stage12-us-worker: StaleDataError " - f"({group.fingerprint})" - ) - - -def test_render_body_marked_section_holds_the_measures_table( - sync_script: ModuleType, -) -> None: - records = [ - _exception_record(sync_script, request_key="r1", observability_key="o1"), - _exception_record(sync_script, request_key="r2", timestamp=_at(2)), - _exception_record( - sync_script, request_key="s1", environment_name="staging", timestamp=_at(1) - ), - ] - group = _group( - sync_script, - records, - [_request_count(sync_script, "//report", 400)], - ) - - _, body = _render(sync_script, group) - section = _section(sync_script, body) - - assert body.index(sync_script.SECTION_START) > 0 - assert f"`{group.fingerprint}`" in section - assert "2026-10-20 00:00 EDT to 2026-10-21 00:00 EDT (24 hours)" in section - assert "| Measure | Production | Staging |" in section - assert "| Occurrences | 2 | 1 |" in section - assert "| Affected requests | 2 | 1 |" in section - assert "| Observability IDs | 1 | 0 |" in section - assert ( - "| Error rate on affected routes | 0.50% (2 of 400 requests) | n/a |" in section - ) - assert "| Occurrences per hour | 0.08 | 0.04 |" in section - assert "| Occurrences per hour, previous 7 days | 0.00 | 0.00 |" in section - assert ( - "| First seen in the last 8 days " - "| 2026-10-20 12:00 EDT | 2026-10-20 13:00 EDT |" in section - ) - assert "| Last seen | 2026-10-20 14:00 EDT | 2026-10-20 13:00 EDT |" in section - assert "Related fingerprints: none" in section - assert "https://console.cloud.google.com/logs/query;" in section - - -def test_render_error_rate_names_only_failures_on_routes( - sync_script: ModuleType, -) -> None: - group = _group( - sync_script, - _failures_with_and_without_a_route(sync_script), - [_request_count(sync_script, "//calculate", 1)], - ) - - _, body = _render(sync_script, group) - - assert "| Error rate on affected routes | 100.00% (1 of 1 requests) | n/a |" in body - - -def test_render_body_lists_related_fingerprints(sync_script: ModuleType) -> None: - records = [ - _exception_record(sync_script, trace_key="t1"), - _message_record( - sync_script, service_name="policyengine-simulation-entry", trace_key="t1" - ), - ] - groups = sync_script.build_groups( - records, [], sync_script.reporting_window(REPORT_DATE), FINGERPRINT_KEY - ) - exception_group = next(group for group in groups if group.kind == "exception") - other = exception_group.related_fingerprints[0] - - _, body = _render(sync_script, exception_group) - - assert f"Related fingerprints (sharing traces): `{other}`" in body - - -def test_render_body_state_comment_records_the_group_and_window( - sync_script: ModuleType, -) -> None: - group = _group(sync_script, [_exception_record(sync_script)]) - - _, body = _render(sync_script, group) - - assert sync_script.parse_state(body) == { - "algorithm": 1, - "fingerprint": group.fingerprint, - "kind": "exception", - "window_end": "2026-10-21T04:00:00+00:00", - } - - -def test_render_body_stays_under_the_github_limit(sync_script: ModuleType) -> None: - # The longest route that still passes validation (195 characters). - long_route = "/" + "/".join([""] * 13) - records = [ - _exception_record( - sync_script, - error_type="E" * 120, - route=long_route, - trace_key=f"t{index}", - request_key=f"r{index}", - ) - for index in range(4) - ] - records += [ - _message_record( - sync_script, - request_key=f"m{index}", - trace_key=f"t{index}", - message_template=f"template {index}", - service_name="policyengine-simulation-entry", - ) - for index in range(4) - ] - groups = sync_script.build_groups( - records, [], sync_script.reporting_window(REPORT_DATE), FINGERPRINT_KEY - ) - - for group in groups: - title, body = _render(sync_script, group) - assert len(body) < 65_536 - assert len(title) <= 256 - - -def test_render_logs_link_filters_exceptions_by_service_and_type( - sync_script: ModuleType, -) -> None: - group = _group(sync_script, [_exception_record(sync_script)]) - url = sync_script.logs_explorer_url( - group, sync_script.reporting_window(REPORT_DATE), PROJECT_ID - ) - - query, settings = _logs_query(url) - - assert urlparse(url).query == "project=policyengine-observability" - assert settings["storageScope"] == quote( - "storage,projects/policyengine-observability/locations/global/" - "buckets/policyengine-observability/views/_AllLogs", - safe="", - ) - assert query.splitlines() == [ - 'jsonPayload.schema_version="policyengine.observability.v2"', - 'jsonPayload."service.name"="policyengine-api"', - "severity>=ERROR", - 'jsonPayload."error.type"="StaleDataError"', - 'timestamp>="2026-10-20T04:00:00Z"', - 'timestamp<"2026-10-21T04:00:00Z"', - ] - for character in (" ", "\n", '"'): - assert character not in url - - -def test_render_logs_link_filters_failed_requests_by_route_and_status( - sync_script: ModuleType, -) -> None: - group = _group( - sync_script, - [ - _failed_request_record( - sync_script, - service_name="policyengine-simulation-py1-2-3", - route="/jobs/{job_id}", - status_code=504, - ) - ], - ) - - query, _ = _logs_query( - sync_script.logs_explorer_url( - group, sync_script.reporting_window(REPORT_DATE), PROJECT_ID - ) - ) - - assert query.splitlines()[1:5] == [ - 'jsonPayload."service.name"=~"^policyengine-simulation-py[0-9]+-[0-9]+-[0-9]+$"', - 'jsonPayload."event.name"="request.completed"', - 'jsonPayload."http.route"="/jobs/{job_id}"', - 'jsonPayload."http.response.status_code"=504', - ] - - -def test_render_logs_link_filters_logged_errors_by_route_or_operation( - sync_script: ModuleType, -) -> None: - window = sync_script.reporting_window(REPORT_DATE) - route_group = _group(sync_script, [_message_record(sync_script)]) - operation_group = _group( - sync_script, - [ - _message_record( - sync_script, route=None, operation_name="simulation.dispatch" - ) - ], - ) - - route_query, _ = _logs_query( - sync_script.logs_explorer_url(route_group, window, PROJECT_ID) - ) - operation_query, _ = _logs_query( - sync_script.logs_explorer_url(operation_group, window, PROJECT_ID) - ) - - assert 'jsonPayload."http.route"="//report"' in route_query - assert "severity>=ERROR" in route_query - assert 'jsonPayload."operation.name"="simulation.dispatch"' in operation_query - - -def _sentinel_row(sync_script: ModuleType, **overrides: object) -> dict[str, object]: - row: dict[str, object] = { - "timestamp": WINDOW_NOON, - "service_name": "SENTINEL service name", - "environment_name": "production", - "event_name": "SENTINEL event", - "error_type": "SENTINEL error type", - "error_handled": "SENTINEL handled", - "frames": [ - '/app/policyengine_api/SENTINEL.py", line 1, in SENTINEL_function', - "SENTINEL frame text", - ], - "stack_truncated": "SENTINEL", - "message_template": "SENTINEL message words", - "attribute_error_type": "SENTINEL attribute type", - "failure_category": "SENTINEL category", - "route": "/SENTINEL/route?query=SENTINEL", - "status_code": "SENTINEL", - "operation_name": "SENTINEL operation", - "request_key": "SENTINEL request", - "observability_key": "SENTINEL observability", - "trace_key": "SENTINEL trace", - } - row.update(overrides) - return row - - -@pytest.mark.parametrize( - "overrides", - [ - {}, - {"error_type": None}, - {"error_type": None, "event_name": "request.completed"}, - {"route": None}, - ], -) -def test_render_never_includes_values_from_records( - sync_script: ModuleType, overrides: dict[str, object] -) -> None: - record = sync_script.error_record_from_row(_sentinel_row(sync_script, **overrides)) - group = _group(sync_script, [record]) - window = sync_script.reporting_window(REPORT_DATE) - - title, body = _render(sync_script, group) - url = sync_script.logs_explorer_url(group, window, PROJECT_ID) - - for output in (title, body, url, unquote(url)): - assert "SENTINEL" not in output - - -# GitHub issue sync - -REPOSITORY = "PolicyEngine/policyengine-api" -BOT = "github-actions[bot]" - - -class FakeGitHub: - """An in-memory GitHub issues API with a clock that advances when it sleeps.""" - - def __init__( - self, - sync_script: ModuleType, - issues: list[dict] = (), - labels: tuple[str, ...] = (), - *, - drop_created_labels: bool = False, - ) -> None: - self.sync_script = sync_script - self.issues = {issue["number"]: dict(issue) for issue in issues} - self.labels = set(labels) - self.drop_created_labels = drop_created_labels - self.failures: list[object] = [] - self.requests: list[tuple[float, object]] = [] - self.comments: dict[int, list[str]] = {} - self.sleeps: list[float] = [] - self.now = 0.0 - - def sleep(self, seconds: float) -> None: - self.sleeps.append(seconds) - self.now += seconds - - def clock(self) -> float: - return self.now - - def client(self) -> object: - return self.sync_script.GitHubClient( - REPOSITORY, "fake-github-token", self, sleep=self.sleep, clock=self.clock - ) - - @property - def writes(self) -> list[tuple[float, object]]: - return [entry for entry in self.requests if entry[1].method != "GET"] - - def _respond(self, payload: object, status: int = 200) -> object: - return self.sync_script.HttpResponse( - status=status, reason="OK", headers={}, body=json.dumps(payload).encode() - ) - - def __call__(self, request: object) -> object: - self.requests.append((self.now, request)) - if request.method != "GET" and self.failures: - return self.failures.pop(0) - split = urlsplit(request.url) - path = split.path.removeprefix(f"/repos/{REPOSITORY}") - query = {name: values[0] for name, values in parse_qs(split.query).items()} - body = json.loads(request.body) if request.body else None - return self._route(request.method, path, query, body) - - def _route(self, method: str, path: str, query: dict, body: dict | None) -> object: - parts = path.strip("/").split("/") - if len(parts) > 1: - return self._route_issue(method, int(parts[1]), parts[2:], body) - if method == "GET" and parts == ["issues"]: - issues = sorted(self.issues.values(), key=lambda issue: issue["number"]) - return self._respond(self._page(issues, query)) - if method == "GET": - labels = [{"name": name} for name in sorted(self.labels)] - return self._respond(self._page(labels, query)) - if parts == ["labels"]: - # GitHub label names are unique regardless of case. - if body["name"].casefold() in {name.casefold() for name in self.labels}: - return self._respond({"message": "Validation Failed"}, status=422) - self.labels.add(body["name"]) - return self._respond({"name": body["name"]}, status=201) - return self._create(body) - - def _existing_label_names(self, requested: list[str]) -> list[str]: - """Return the stored spelling of each label, as GitHub applies them.""" - stored = {name.casefold(): name for name in self.labels} - return [stored.get(name.casefold(), name) for name in requested] - - def _route_issue( - self, method: str, number: int, rest: list[str], body: dict - ) -> object: - if method == "PATCH": - self.issues[number].update(body) - return self._respond(self.issues[number]) - if rest == ["labels"]: - names = {label["name"] for label in self.issues[number]["labels"]} - if not self.drop_created_labels: - names |= set(self._existing_label_names(body["labels"])) - self.issues[number]["labels"] = [{"name": name} for name in sorted(names)] - return self._respond(self.issues[number]["labels"]) - self.comments.setdefault(number, []).append(body["body"]) - return self._respond({"id": 1}, status=201) - - def _page(self, items: list, query: dict) -> list: - page = int(query.get("page", "1")) - size = int(query.get("per_page", "30")) - return items[(page - 1) * size : page * size] - - def _create(self, body: dict) -> object: - number = max(self.issues, default=0) + 1 - labels = ( - [] - if self.drop_created_labels - else self._existing_label_names(body["labels"]) - ) - self.issues[number] = { - "number": number, - "title": body["title"], - "body": body["body"], - "state": "open", - "state_reason": None, - "closed_at": None, - "labels": [{"name": name} for name in labels], - "user": {"login": BOT}, - } - return self._respond(self.issues[number], status=201) - - -def _bot_issue(number: int, **fields: object) -> dict: - issue = { - "number": number, - "title": f"Issue {number}", - "body": "", - "state": "open", - "state_reason": None, - "closed_at": None, - "labels": [], - "user": {"login": BOT}, - } - issue.update(fields) - return issue - - -def _failures( - sync_script: ModuleType, - count: int, - *, - request_key_prefix: str = "r", - **fields: object, -) -> list: - return [ - _exception_record( - sync_script, - request_key=f"{request_key_prefix}{index}", - timestamp=_at(index), - **fields, - ) - for index in range(count) - ] - - -def _groups(sync_script: ModuleType, records: list) -> list: - return sync_script.build_groups( - records, [], sync_script.reporting_window(REPORT_DATE), FINGERPRINT_KEY - ) - - -def _sync(sync_script: ModuleType, groups: list, github: FakeGitHub, *, dry_run=False): - return sync_script.sync_issues( - groups, - sync_script.reporting_window(REPORT_DATE), - PROJECT_ID, - github.client(), - dry_run=dry_run, - ) - - -def _existing_issue( - sync_script: ModuleType, group: object, number: int = 7, **fields: object -) -> dict: - """Return the issue the job opened for ``group`` on an earlier day.""" - earlier = sync_script.reporting_window(REPORT_DATE - timedelta(days=1)) - issue = _bot_issue( - number, - title=sync_script.issue_title(group), - body=sync_script.issue_body(group, earlier, PROJECT_ID), - ) - issue.update(fields) - return issue - - -def test_sync_lists_open_and_closed_issues_from_the_bot_page_by_page( - sync_script: ModuleType, -) -> None: - github = FakeGitHub(sync_script, [_bot_issue(number) for number in range(1, 151)]) - - _sync(sync_script, [], github) - - listing = [ - parse_qs(urlsplit(request.url).query) - for _, request in github.requests - if urlsplit(request.url).path.endswith("/issues") - ] - assert len(listing) == 2 - assert listing[0]["creator"] == [BOT] - assert listing[0]["state"] == ["all"] - assert listing[0]["per_page"] == ["100"] - assert [query["page"] for query in listing] == [["1"], ["2"]] - - -def test_sync_opens_an_issue_at_three_production_occurrences_in_two_requests( - sync_script: ModuleType, -) -> None: - eligible = _failures(sync_script, 3) - # Three failures that all affected the request with observability ID o1. - one_request = [ - _exception_record( - sync_script, - error_type="KeyError", - request_key=None, - trace_key=None, - observability_key="o1", - timestamp=_at(index), - ) - for index in range(3) - ] - too_few = _failures( - sync_script, 2, request_key_prefix="few", error_type="TypeError" - ) - groups = _groups(sync_script, eligible + one_request + too_few) - github = FakeGitHub(sync_script) - - result = _sync(sync_script, groups, github) - - created = list(github.issues.values()) - eligible_group = next( - group for group in groups if group.error_type == "StaleDataError" - ) - assert [issue["title"] for issue in created] == [ - sync_script.issue_title(eligible_group) - ] - assert created[0]["labels"] == [ - {"name": "telemetry-error"}, - {"name": "service: policyengine-api"}, - ] - actions = {action.fingerprint: action.action for action in result.actions} - assert sorted(actions.values()) == ["below-threshold", "below-threshold", "created"] - - -def test_sync_never_opens_an_issue_for_a_staging_only_group( - sync_script: ModuleType, -) -> None: - groups = _groups( - sync_script, _failures(sync_script, 10, environment_name="staging") - ) - github = FakeGitHub(sync_script) - - result = _sync(sync_script, groups, github) - - assert github.issues == {} - assert result.actions[0].action == "below-threshold" - - -def test_sync_opens_at_most_five_issues_in_order_of_production_volume( - sync_script: ModuleType, -) -> None: - records = [] - for index in range(7): - records += _failures( - sync_script, - 3 + index, - request_key_prefix=f"g{index}-", - error_type=f"Error{index}", - ) - groups = _groups(sync_script, records) - github = FakeGitHub(sync_script) - - result = _sync(sync_script, groups, github) - - titles = [issue["title"] for issue in github.issues.values()] - assert [title.split(": ")[1].split(" ")[0] for title in titles] == [ - "Error6", - "Error5", - "Error4", - "Error3", - "Error2", - ] - assert result.deferred == 2 - assert [action.action for action in result.actions].count("deferred") == 2 - - -def test_sync_replaces_only_the_marked_section_and_never_the_title( - sync_script: ModuleType, -) -> None: - group = _groups(sync_script, _failures(sync_script, 3))[0] - issue = _existing_issue(sync_script, group, title="Renamed by a person") - issue["body"] = f"Notes before.\n\n{issue['body']}\nNotes after.\n" - github = FakeGitHub(sync_script, [issue]) - - result = _sync(sync_script, [group], github) - - patch = json.loads(github.writes[0][1].body) - expected_section = sync_script.issue_section( - group, sync_script.reporting_window(REPORT_DATE), PROJECT_ID - ) - assert list(patch) == ["body"] - assert patch["body"].startswith("Notes before.\n\n") - assert patch["body"].endswith("\nNotes after.\n") - assert expected_section in patch["body"] - assert github.issues[7]["title"] == "Renamed by a person" - assert result.actions[0].action == "updated" - assert result.actions[0].issue_number == 7 - - -def test_sync_does_not_write_when_the_section_is_unchanged( - sync_script: ModuleType, -) -> None: - group = _groups(sync_script, _failures(sync_script, 3))[0] - github = FakeGitHub(sync_script, [_existing_issue(sync_script, group)]) - - _sync(sync_script, [group], github) - first_run_writes = len(github.writes) - second = _sync(sync_script, [group], github) - - assert first_run_writes == 1 - assert len(github.writes) == 1 - assert second.actions[0].action == "unchanged" - - -def test_sync_adds_a_deleted_section_back_at_the_end(sync_script: ModuleType) -> None: - group = _groups(sync_script, _failures(sync_script, 3))[0] - issue = _existing_issue(sync_script, group, body="Only human notes now.") - github = FakeGitHub(sync_script, [issue]) - - _sync(sync_script, [group], github) - - body = github.issues[7]["body"] - assert body.startswith("Only human notes now.\n\n") - assert body.rstrip().endswith(sync_script.SECTION_END) - assert len(github.issues) == 1 - - -def test_sync_ignores_issues_opened_by_other_authors(sync_script: ModuleType) -> None: - group = _groups(sync_script, _failures(sync_script, 3))[0] - person_issue = _existing_issue(sync_script, group, user={"login": "someone"}) - github = FakeGitHub(sync_script, [person_issue]) - - _sync(sync_script, [group], github) - - assert github.issues[7]["body"] == person_issue["body"] - assert len(github.issues) == 2 - - -def test_sync_reopens_a_completed_issue_when_production_failures_return( - sync_script: ModuleType, -) -> None: - group = _groups(sync_script, _failures(sync_script, 4))[0] - closed_at = (WINDOW_NOON - timedelta(days=2)).isoformat().replace("+00:00", "Z") - issue = _existing_issue( - sync_script, - group, - state="closed", - state_reason="completed", - closed_at=closed_at, - ) - github = FakeGitHub(sync_script, [issue], labels=("telemetry-error",)) - - result = _sync(sync_script, [group], github) - - reopened = github.issues[7] - assert reopened["state"] == "open" - assert reopened["state_reason"] == "reopened" - assert {"name": "regression"} in reopened["labels"] - assert "regression" in github.labels - assert github.comments[7] == [ - "Production failures returned after this issue was closed: 4 occurrences " - "in 4 requests since 2026-10-19 12:00 EDT, 24 hours after it was closed." - ] - assert result.actions[0].action == "reopened" - - -def test_sync_treats_an_issue_closed_without_a_reason_as_completed( - sync_script: ModuleType, -) -> None: - group = _groups(sync_script, _failures(sync_script, 3))[0] - closed_at = (WINDOW_NOON - timedelta(days=2)).isoformat() - issue = _existing_issue( - sync_script, group, state="closed", state_reason=None, closed_at=closed_at - ) - github = FakeGitHub(sync_script, [issue]) - - result = _sync(sync_script, [group], github) - - assert github.issues[7]["state"] == "open" - assert result.actions[0].action == "reopened" - - -def test_sync_keeps_a_completed_issue_closed_within_24_hours_of_closing( - sync_script: ModuleType, -) -> None: - group = _groups(sync_script, _failures(sync_script, 4))[0] - closed_at = (WINDOW_NOON - timedelta(hours=12)).isoformat() - issue = _existing_issue( - sync_script, - group, - state="closed", - state_reason="completed", - closed_at=closed_at, - ) - github = FakeGitHub(sync_script, [issue]) - - result = _sync(sync_script, [group], github) - - assert github.issues[7]["state"] == "closed" - assert 7 not in github.comments - assert ( - sync_script.issue_section( - group, sync_script.reporting_window(REPORT_DATE), PROJECT_ID - ) - in github.issues[7]["body"] - ) - assert result.actions[0].action == "updated" - - -@pytest.mark.parametrize("reason", ["not_planned", "duplicate"]) -def test_sync_updates_but_never_reopens_not_planned_or_duplicate_issues( - sync_script: ModuleType, reason: str -) -> None: - group = _groups(sync_script, _failures(sync_script, 10))[0] - closed_at = (WINDOW_NOON - timedelta(days=5)).isoformat() - issue = _existing_issue( - sync_script, group, state="closed", state_reason=reason, closed_at=closed_at - ) - github = FakeGitHub(sync_script, [issue]) - - result = _sync(sync_script, [group], github) - - assert github.issues[7]["state"] == "closed" - assert result.actions[0].action == "updated" - assert len(github.issues) == 1 - - -def test_sync_keeps_a_newer_report_instead_of_overwriting_it( - sync_script: ModuleType, -) -> None: - group = _groups(sync_script, _failures(sync_script, 3))[0] - later = sync_script.reporting_window(REPORT_DATE + timedelta(days=1)) - issue = _bot_issue( - 7, - title=sync_script.issue_title(group), - body=sync_script.issue_body(group, later, PROJECT_ID), - ) - github = FakeGitHub(sync_script, [issue]) - - result = _sync(sync_script, [group], github) - - assert github.writes == [] - assert result.actions[0].action == "kept-newer" - - -def test_sync_creates_missing_labels_before_opening_issues( - sync_script: ModuleType, -) -> None: - groups = _groups(sync_script, _failures(sync_script, 3)) - github = FakeGitHub(sync_script, labels=("telemetry-error",)) - - _sync(sync_script, groups, github) - - created_labels = [ - json.loads(request.body)["name"] - for _, request in github.writes - if urlsplit(request.url).path.endswith("/labels") - ] - assert created_labels == ["service: policyengine-api"] - - -def test_sync_uses_existing_labels_whatever_their_case( - sync_script: ModuleType, -) -> None: - groups = _groups(sync_script, _failures(sync_script, 3)) - github = FakeGitHub( - sync_script, labels=("Telemetry-Error", "Service: Policyengine-Api") - ) - - _sync(sync_script, groups, github) - - created_labels = [ - request - for _, request in github.writes - if urlsplit(request.url).path.endswith("/labels") - ] - assert created_labels == [] - assert github.issues[1]["labels"] == [ - {"name": "Telemetry-Error"}, - {"name": "Service: Policyengine-Api"}, - ] - - -def test_sync_reopens_with_an_existing_regression_label_of_another_case( - sync_script: ModuleType, -) -> None: - group = _groups(sync_script, _failures(sync_script, 4))[0] - closed_at = (WINDOW_NOON - timedelta(days=2)).isoformat() - issue = _existing_issue( - sync_script, - group, - state="closed", - state_reason="completed", - closed_at=closed_at, - ) - github = FakeGitHub(sync_script, [issue], labels=("Regression",)) - - result = _sync(sync_script, [group], github) - - assert result.actions[0].action == "reopened" - assert {"name": "Regression"} in github.issues[7]["labels"] - assert github.labels == {"Regression"} - - -def test_sync_adds_labels_a_create_response_is_missing( - sync_script: ModuleType, -) -> None: - groups = _groups(sync_script, _failures(sync_script, 3)) - github = FakeGitHub(sync_script, drop_created_labels=True) - - with pytest.raises(sync_script.GitHubError, match="labels"): - _sync(sync_script, groups, github) - - added = [ - request - for _, request in github.writes - if urlsplit(request.url).path.endswith("/issues/1/labels") - ] - assert len(added) == 1 - - -def test_sync_writes_one_at_a_time_at_least_a_second_apart( - sync_script: ModuleType, -) -> None: - records = [] - for index in range(3): - records += _failures( - sync_script, 3, request_key_prefix=f"g{index}-", error_type=f"Error{index}" - ) - github = FakeGitHub(sync_script) - - _sync(sync_script, _groups(sync_script, records), github) - - write_times = [moment for moment, _ in github.writes] - assert len(write_times) == 5 - assert all( - later - earlier >= 1.0 for earlier, later in itertools.pairwise(write_times) - ) - - -@pytest.mark.parametrize("status", [403, 429]) -def test_sync_waits_for_retry_after_on_rate_limits( - sync_script: ModuleType, status: int -) -> None: - groups = _groups(sync_script, _failures(sync_script, 3)) - github = FakeGitHub( - sync_script, labels=("telemetry-error", "service: policyengine-api") - ) - github.failures = [ - sync_script.HttpResponse( - status=status, reason="Forbidden", headers={"retry-after": "7"}, body=b"{}" - ) - ] - - _sync(sync_script, groups, github) - - assert 7 in github.sleeps - assert len(github.issues) == 1 - - -def _limited_github(sync_script: ModuleType, response: object) -> FakeGitHub: - """Return a fake whose first write gets ``response`` and later writes succeed.""" - github = FakeGitHub( - sync_script, labels=("telemetry-error", "service: policyengine-api") - ) - github.failures = [response] - return github - - -def test_sync_waits_a_minute_on_a_secondary_rate_limit_without_retry_after( - sync_script: ModuleType, -) -> None: - body = b'{"message": "You have exceeded a secondary rate limit. Please wait."}' - github = _limited_github( - sync_script, - sync_script.HttpResponse(status=403, reason="Forbidden", headers={}, body=body), - ) - - _sync(sync_script, _groups(sync_script, _failures(sync_script, 3)), github) - - assert 60 in github.sleeps - assert len(github.issues) == 1 - - -def test_sync_waits_until_an_exhausted_rate_limit_resets( - sync_script: ModuleType, -) -> None: - headers = { - "date": "Tue, 20 Oct 2026 23:00:00 GMT", - "x-ratelimit-remaining": "0", - "x-ratelimit-reset": str( - int(datetime(2026, 10, 20, 23, 0, 42, tzinfo=UTC).timestamp()) - ), - } - github = _limited_github( - sync_script, - sync_script.HttpResponse( - status=403, reason="Forbidden", headers=headers, body=b"{}" - ), - ) - - _sync(sync_script, _groups(sync_script, _failures(sync_script, 3)), github) - - assert 42 in github.sleeps - assert len(github.issues) == 1 - - -def test_sync_fails_instead_of_waiting_hours_for_a_rate_limit_reset( - sync_script: ModuleType, -) -> None: - headers = { - "date": "Tue, 20 Oct 2026 23:00:00 GMT", - "x-ratelimit-remaining": "0", - "x-ratelimit-reset": str( - int(datetime(2026, 10, 21, 0, 0, tzinfo=UTC).timestamp()) - ), - } - github = _limited_github( - sync_script, - sync_script.HttpResponse( - status=403, reason="Forbidden", headers=headers, body=b"{}" - ), - ) - - with pytest.raises(sync_script.GitHubError, match="HTTP 403"): - _sync(sync_script, _groups(sync_script, _failures(sync_script, 3)), github) - - assert github.sleeps == [] - - -def test_sync_stops_after_three_rate_limit_retries(sync_script: ModuleType) -> None: - groups = _groups(sync_script, _failures(sync_script, 3)) - github = FakeGitHub( - sync_script, labels=("telemetry-error", "service: policyengine-api") - ) - limited = sync_script.HttpResponse( - status=429, reason="Too Many Requests", headers={"retry-after": "2"}, body=b"{}" - ) - github.failures = [limited] * 4 - - with pytest.raises(sync_script.GitHubError) as raised: - _sync(sync_script, groups, github) - - assert str(raised.value) == "GitHub request failed: HTTP 429 Too Many Requests" - assert github.sleeps.count(2) == 3 - - -def test_sync_reports_a_forbidden_write_without_retrying( - sync_script: ModuleType, -) -> None: - groups = _groups(sync_script, _failures(sync_script, 3)) - github = FakeGitHub( - sync_script, labels=("telemetry-error", "service: policyengine-api") - ) - github.failures = [ - sync_script.HttpResponse( - status=403, - reason="Forbidden", - headers={}, - body=b'{"message": "SENTINEL Resource not accessible"}', - ) - ] - - with pytest.raises(sync_script.GitHubError) as raised: - _sync(sync_script, groups, github) - - assert str(raised.value) == "GitHub request failed: HTTP 403 Forbidden" - assert len(github.writes) == 1 - - -def test_sync_dry_run_makes_no_write_calls(sync_script: ModuleType) -> None: - eligible = _groups(sync_script, _failures(sync_script, 3))[0] - existing = _groups(sync_script, _failures(sync_script, 3, error_type="KeyError"))[0] - github = FakeGitHub(sync_script, [_existing_issue(sync_script, existing)]) - - result = _sync(sync_script, [eligible, existing], github, dry_run=True) - - assert github.writes == [] - assert sorted(action.action for action in result.actions) == ["created", "updated"] - - -# End-to-end run and public output - -ERROR_RECORD_FIELDS = [ - {"name": "timestamp", "type": "TIMESTAMP"}, - {"name": "service_name", "type": "STRING"}, - {"name": "environment_name", "type": "STRING"}, - {"name": "event_name", "type": "STRING"}, - {"name": "error_type", "type": "STRING"}, - {"name": "error_handled", "type": "STRING"}, - {"name": "frames", "type": "STRING", "mode": "REPEATED"}, - {"name": "stack_truncated", "type": "BOOLEAN"}, - {"name": "message_template", "type": "STRING"}, - {"name": "attribute_error_type", "type": "STRING"}, - {"name": "failure_category", "type": "STRING"}, - {"name": "route", "type": "STRING"}, - {"name": "status_code", "type": "STRING"}, - {"name": "operation_name", "type": "STRING"}, - {"name": "request_key", "type": "STRING"}, - {"name": "observability_key", "type": "STRING"}, - {"name": "trace_key", "type": "STRING"}, -] -REQUEST_COUNT_FIELDS = [ - {"name": "service_name", "type": "STRING"}, - {"name": "environment_name", "type": "STRING"}, - {"name": "route", "type": "STRING"}, - {"name": "requests", "type": "INTEGER"}, -] -SCHEDULED_RUN_TIME = datetime(2026, 10, 21, 0, 45, tzinfo=EASTERN) - - -def _microseconds(moment: datetime) -> str: - return str((moment - datetime(1970, 1, 1, tzinfo=UTC)) // timedelta(microseconds=1)) - - -def _error_row(**overrides: object) -> list[object]: - values: dict[str, object] = { - "timestamp": _microseconds(WINDOW_NOON), - "service_name": "policyengine-api", - "environment_name": "production", - "event_name": "exception.recorded", - "error_type": "StaleDataError", - "error_handled": "false", - "frames": list(APPLICATION_FRAMES), - "stack_truncated": "false", - "message_template": None, - "attribute_error_type": None, - "failure_category": None, - "route": "//report", - "status_code": "500", - "operation_name": None, - "request_key": "r1", - "observability_key": None, - "trace_key": "t1", - } - values.update(overrides) - return [values[field["name"]] for field in ERROR_RECORD_FIELDS] - - -class FakeBigQuery: - """Answers the error-record and request-count queries with fixed rows.""" - - def __init__( - self, - sync_script: ModuleType, - error_rows: list[list[object]], - count_rows: list[list[object]] = (), - ) -> None: - self.sync_script = sync_script - self.error_rows = error_rows - self.count_rows = list(count_rows) - self.requests: list[object] = [] - - def __call__(self, request: object) -> object: - self.requests.append(request) - label = json.loads(request.body)["labels"]["query"] - if label == "error-records": - response = _query_response(ERROR_RECORD_FIELDS, self.error_rows) - else: - response = _query_response(REQUEST_COUNT_FIELDS, self.count_rows) - return _json_response(self.sync_script, response) - - def json_body(self, index: int) -> dict: - return json.loads(self.requests[index].body) - - -def _environment(tmp_path: Path, **overrides: str) -> dict[str, str]: - environment = { - "GOOGLE_OAUTH_ACCESS_TOKEN": ACCESS_TOKEN, - "GITHUB_TOKEN": "fake-github-token", - "GITHUB_REPOSITORY": REPOSITORY, - "TELEMETRY_ERROR_FINGERPRINT_KEY": "f" * 64, - "GITHUB_STEP_SUMMARY": str(tmp_path / "summary.md"), - } - environment.update(overrides) - return {name: value for name, value in environment.items() if value} - - -def _run_main( - sync_script: ModuleType, - tmp_path: Path, - bigquery: object, - github: FakeGitHub, - *arguments: str, - environment: dict[str, str] | None = None, -) -> tuple[int, str, str]: - stdout, stderr = io.StringIO(), io.StringIO() - runtime = sync_script.Runtime( - environ=_environment(tmp_path) if environment is None else environment, - bigquery_transport=bigquery, - github_transport=github, - now=lambda: SCHEDULED_RUN_TIME, - sleep=github.sleep, - clock=github.clock, - stdout=stdout, - stderr=stderr, - ) - exit_code = sync_script.main( - ["--project-id", PROJECT_ID, *arguments], runtime=runtime - ) - return exit_code, stdout.getvalue(), stderr.getvalue() - - -def _three_failures(**overrides: object) -> list[list[object]]: - return [ - _error_row( - request_key=f"r{index}", - timestamp=_microseconds(_at(index)), - **overrides, - ) - for index in range(3) - ] - - -def test_main_queries_groups_and_opens_issues_for_the_previous_day( - sync_script: ModuleType, tmp_path: Path -) -> None: - bigquery = FakeBigQuery( - sync_script, - _three_failures(), - [["policyengine-api", "production", "//report", "600"]], - ) - github = FakeGitHub(sync_script) - - exit_code, stdout, stderr = _run_main(sync_script, tmp_path, bigquery, github) - - assert exit_code == 0, stderr - issue = github.issues[1] - assert "StaleDataError" in issue["title"] - assert ( - "| Error rate on affected routes | 0.50% (3 of 600 requests) | n/a |" - in (issue["body"]) - ) - window_end = _sent_parameters(bigquery)["window_end"]["parameterValue"] - assert window_end["value"] == "2026-10-21 04:00:00+00:00" - summary = (tmp_path / "summary.md").read_text() - fingerprint = issue["title"].rsplit("(", 1)[1].rstrip(")") - assert ( - f"| `{fingerprint}` | exception | `policyengine-api` | 3 | 0 | #1 | created |" - in summary - ) - assert summary in stdout - - -def test_main_dry_run_reports_actions_without_writing( - sync_script: ModuleType, tmp_path: Path -) -> None: - github = FakeGitHub(sync_script) - - exit_code, stdout, _ = _run_main( - sync_script, - tmp_path, - FakeBigQuery(sync_script, _three_failures()), - github, - "--dry-run", - ) - - assert exit_code == 0 - assert github.writes == [] - assert "Dry run: no issues were changed." in stdout - assert "| - | created |" in stdout - - -def test_main_reports_the_requested_day( - sync_script: ModuleType, tmp_path: Path -) -> None: - bigquery = FakeBigQuery(sync_script, []) - - exit_code, stdout, _ = _run_main( - sync_script, - tmp_path, - bigquery, - FakeGitHub(sync_script), - "--report-date", - "2026-10-15", - ) - - assert exit_code == 0 - window_end = _sent_parameters(bigquery)["window_end"]["parameterValue"] - assert window_end["value"] == "2026-10-16 04:00:00+00:00" - assert "## Telemetry error issues for 2026-10-15" in stdout - assert "No error groups had failures in the window." in stdout - - -def test_main_rejects_a_day_that_has_not_ended( - sync_script: ModuleType, tmp_path: Path -) -> None: - exit_code, stdout, stderr = _run_main( - sync_script, - tmp_path, - FakeBigQuery(sync_script, []), - FakeGitHub(sync_script), - "--report-date", - "2026-10-21", - ) - - assert exit_code == 1 - assert stdout == "" - assert stderr == ( - "configuration failed: ConfigurationError " - "(2026-10-21 has not ended in US Eastern time)\n" - ) - - -def test_main_names_missing_environment_variables_without_values( - sync_script: ModuleType, tmp_path: Path -) -> None: - environment = _environment( - tmp_path, GITHUB_TOKEN="", TELEMETRY_ERROR_FINGERPRINT_KEY="" - ) - - exit_code, _, stderr = _run_main( - sync_script, - tmp_path, - FakeBigQuery(sync_script, []), - FakeGitHub(sync_script), - environment=environment, - ) - - assert exit_code == 1 - assert stderr == ( - "configuration failed: ConfigurationError (missing environment variables: " - "GITHUB_TOKEN, TELEMETRY_ERROR_FINGERPRINT_KEY)\n" - ) - - -def test_main_rejects_a_short_fingerprint_key_without_printing_it( - sync_script: ModuleType, tmp_path: Path -) -> None: - environment = _environment( - tmp_path, TELEMETRY_ERROR_FINGERPRINT_KEY="SENTINEL-short-key" - ) - - exit_code, stdout, stderr = _run_main( - sync_script, - tmp_path, - FakeBigQuery(sync_script, []), - FakeGitHub(sync_script), - environment=environment, - ) - - assert exit_code == 1 - assert "at least 32 characters" in stderr - assert "SENTINEL" not in stdout + stderr - - -def test_main_reports_an_api_failure_by_stage_class_and_status_only( - sync_script: ModuleType, tmp_path: Path -) -> None: - error_body = { - "error": { - "message": "SENTINEL echoed value", - "errors": [{"reason": "backendError", "message": "SENTINEL"}], - } - } - failing = FakeTransport( - sync_script, - [_json_response(sync_script, error_body, status=500, reason="SENTINEL")], - ) - - exit_code, stdout, stderr = _run_main( - sync_script, tmp_path, failing, FakeGitHub(sync_script) - ) - - assert exit_code == 1 - assert stdout == "" - assert stderr == ( - "query error records failed: BigQueryError (HTTP 500 backendError)\n" - ) - - -def test_main_reports_an_unexpected_exception_by_class_only( - sync_script: ModuleType, tmp_path: Path -) -> None: - def broken_transport(request: object) -> object: - message = "SENTINEL connection detail" - raise RuntimeError(message) - - exit_code, _, stderr = _run_main( - sync_script, tmp_path, broken_transport, FakeGitHub(sync_script) - ) - - assert exit_code == 1 - assert stderr == "query error records failed: RuntimeError\n" - - -def test_main_reports_a_github_failure_by_stage( - sync_script: ModuleType, tmp_path: Path -) -> None: - github = FakeGitHub(sync_script) - github.failures = [ - sync_script.HttpResponse( - status=403, reason="Forbidden", headers={}, body=b'{"message": "SENTINEL"}' - ) - ] - - exit_code, _, stderr = _run_main( - sync_script, tmp_path, FakeBigQuery(sync_script, _three_failures()), github - ) - - assert exit_code == 1 - assert stderr == "sync issues failed: GitHubError (HTTP 403 Forbidden)\n" - - -@pytest.mark.parametrize( - "overrides", - [ - {}, - {"error_type": None}, - {"error_type": None, "event_name": "request.completed"}, - ], -) -def test_main_never_publishes_values_from_records( - sync_script: ModuleType, tmp_path: Path, overrides: dict[str, object] -) -> None: - sentinel_values = { - name: value - for name, value in _sentinel_row(sync_script, **overrides).items() - if name not in {"timestamp", "environment_name", "frames", "stack_truncated"} - } - sentinel_values["stack_truncated"] = "true" - sentinel_values["frames"] = ['/app/SENTINEL.py", line 1, in SENTINEL'] - rows = [ - _error_row( - **{ - **sentinel_values, - "request_key": f"SENTINEL request {index}", - "timestamp": _microseconds(_at(index)), - } - ) - for index in range(3) - ] - count_rows = [["SENTINEL service", "production", "SENTINEL route", "10"]] - github = FakeGitHub(sync_script) - - exit_code, stdout, stderr = _run_main( - sync_script, tmp_path, FakeBigQuery(sync_script, rows, count_rows), github - ) - - assert exit_code == 0, stderr - assert len(github.issues) == 1 - published = [ - stdout, - stderr, - (tmp_path / "summary.md").read_text(), - *(issue["title"] for issue in github.issues.values()), - *(issue["body"] for issue in github.issues.values()), - ] - for text in published: - assert "SENTINEL" not in unquote(text) - - -# Workflow and wrapper - -WORKFLOW_PATH = REPO / ".github" / "workflows" / "telemetry-error-issues.yml" -WRAPPER_PATH = REPO / ".github" / "scripts" / "sync_telemetry_error_issues.sh" -WRAPPER_ENVIRONMENT = { - "GOOGLE_OAUTH_ACCESS_TOKEN": "fake-access-token", - "GITHUB_TOKEN": "fake-github-token", - "GITHUB_REPOSITORY": REPOSITORY, - "TELEMETRY_ERROR_FINGERPRINT_KEY": "f" * 64, - "OBSERVABILITY_PROJECT_ID": PROJECT_ID, -} - - -def _workflow() -> dict: - return yaml.safe_load(WORKFLOW_PATH.read_text(encoding="utf-8")) - - -def _sync_job() -> dict: - return _workflow()["jobs"]["sync"] - - -def _step(name: str) -> dict: - return next(step for step in _sync_job()["steps"] if step["name"] == name) - - -def test_workflow_runs_at_00_45_eastern_and_on_request() -> None: - # PyYAML reads the bare key "on" as the boolean True. - triggers = _workflow()[True] - - assert triggers["schedule"] == [ - {"cron": "45 0 * * *", "timezone": "America/New_York"} - ] - inputs = triggers["workflow_dispatch"]["inputs"] - assert inputs["dry_run"]["type"] == "boolean" - assert inputs["dry_run"]["default"] is True - assert inputs["report_date"]["type"] == "string" - assert inputs["report_date"]["default"] == "" - - -def test_workflow_job_is_limited_to_master_in_its_environment() -> None: - workflow = _workflow() - job = _sync_job() - - assert workflow["concurrency"] == { - "group": "telemetry-error-issues", - "cancel-in-progress": False, - } - assert job["if"] == "github.repository == 'PolicyEngine/policyengine-api'" - assert job["environment"] == "telemetry-error-issues" - assert job["timeout-minutes"] == 15 - assert job["permissions"] == { - "contents": "read", - "issues": "write", - "id-token": "write", - } - assert job["steps"][0] == { - "name": "Require master branch", - "if": "github.ref != 'refs/heads/master'", - "run": "exit 1", - } - - -def test_workflow_authenticates_as_the_report_service_account() -> None: - auth = _step("Authenticate to Google Cloud") - - assert auth["id"] == "auth" - assert auth["uses"] == "google-github-actions/auth@v2" - assert auth["with"] == { - "workload_identity_provider": "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}", - "service_account": "${{ vars.TELEMETRY_ERROR_ISSUES_SERVICE_ACCOUNT }}", - "token_format": "access_token", - "create_credentials_file": False, - } - assert _step("Set up Python")["with"] == {"python-version": "3.12"} - - -def test_workflow_passes_settings_to_the_wrapper() -> None: - sync = _step("Sync telemetry error issues") - - assert sync["run"] == "bash .github/scripts/sync_telemetry_error_issues.sh" - assert sync["env"] == { - "GOOGLE_OAUTH_ACCESS_TOKEN": "${{ steps.auth.outputs.access_token }}", - "GITHUB_TOKEN": "${{ github.token }}", - "TELEMETRY_ERROR_FINGERPRINT_KEY": ( - "${{ secrets.TELEMETRY_ERROR_FINGERPRINT_KEY }}" - ), - "OBSERVABILITY_PROJECT_ID": "${{ vars.OBSERVABILITY_PROJECT_ID }}", - # A scheduled run has no inputs, so it is never a dry run. - "DRY_RUN": "${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}", - "REPORT_DATE": "${{ inputs.report_date }}", - } - - -def _run_wrapper( - tmp_path: Path, environment: dict[str, str] -) -> tuple[subprocess.CompletedProcess[str], list[str] | None]: - """Run the wrapper with a fake python3 that records its arguments.""" - bin_dir = tmp_path / "bin" - bin_dir.mkdir() - calls = tmp_path / "python-arguments.txt" - fake_python = bin_dir / "python3" - fake_python.write_text(f'#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "{calls}"\n') - fake_python.chmod(0o755) - result = subprocess.run( - ["bash", str(WRAPPER_PATH)], - cwd=REPO, - env={"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", **environment}, - text=True, - capture_output=True, - check=False, - ) - arguments = calls.read_text().splitlines() if calls.exists() else None - return result, arguments - - -def test_wrapper_passes_bash_syntax_checking() -> None: - result = subprocess.run( - ["bash", "-n", str(WRAPPER_PATH)], capture_output=True, text=True, check=False - ) - - assert result.returncode == 0, result.stderr - - -def test_wrapper_runs_the_report_for_the_project(tmp_path: Path) -> None: - result, arguments = _run_wrapper(tmp_path, WRAPPER_ENVIRONMENT) - - assert result.returncode == 0, result.stderr - assert arguments == [str(SCRIPT_PATH), "--project-id", PROJECT_ID] - - -def test_wrapper_masks_the_access_token_and_fingerprint_key(tmp_path: Path) -> None: - result, _ = _run_wrapper(tmp_path, WRAPPER_ENVIRONMENT) - - assert result.stdout.splitlines() == [ - "::add-mask::fake-access-token", - f"::add-mask::{'f' * 64}", - ] - - -def test_wrapper_adds_dry_run_and_report_date_when_set(tmp_path: Path) -> None: - environment = { - **WRAPPER_ENVIRONMENT, - "DRY_RUN": "true", - "REPORT_DATE": "2026-10-15", - } - - result, arguments = _run_wrapper(tmp_path, environment) - - assert result.returncode == 0, result.stderr - assert arguments == [ - str(SCRIPT_PATH), - "--project-id", - PROJECT_ID, - "--dry-run", - "--report-date", - "2026-10-15", - ] - - -def test_wrapper_runs_live_when_dry_run_is_false(tmp_path: Path) -> None: - environment = {**WRAPPER_ENVIRONMENT, "DRY_RUN": "false", "REPORT_DATE": ""} - - _, arguments = _run_wrapper(tmp_path, environment) - - assert arguments == [str(SCRIPT_PATH), "--project-id", PROJECT_ID] - - -@pytest.mark.parametrize("missing", sorted(WRAPPER_ENVIRONMENT)) -def test_wrapper_fails_before_running_when_a_variable_is_missing( - tmp_path: Path, missing: str -) -> None: - environment = { - name: value for name, value in WRAPPER_ENVIRONMENT.items() if name != missing - } - - result, arguments = _run_wrapper(tmp_path, environment) - - assert result.returncode != 0 - assert f"{missing} is required" in result.stderr - assert arguments is None - assert result.stdout == "" From f67d4b4e950f60ea68fdbb8a8ebd392a2210c9af Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:59:20 +0400 Subject: [PATCH 17/18] Read the telemetry log names and namespace from repository variables The report had the deployment's names typed into the code: the service namespace in both SQL queries, the linked dataset and its view in the BigQuery requests, and the bucket, its location and the view in the Logs Explorer link. They now come from repository variables, so moving the logs needs no code change: OBSERVABILITY_LOG_BUCKET, OBSERVABILITY_LOG_BUCKET_LOCATION, OBSERVABILITY_LOG_DATASET, OBSERVABILITY_LOG_VIEW and OBSERVABILITY_SERVICE_NAMESPACE, the same namespace variable the API reads. The workflow passes each one to the wrapper, the wrapper requires each one, and read_settings reads them into a LogSource with the project ID. The namespace goes to BigQuery as a named query parameter. BigQuery can't take a dataset or view name as a parameter, so each name, and the project ID, must match a strict identifier pattern. The run checks them when it reads the settings, failing with a configuration error that names the setting but not its value, and again just before a name goes into a query, a job configuration or a link. Anything else is refused, which keeps SQL out of the queries. The rules that decide which group a failure belongs to stay in code, because changing one changes fingerprints. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.sh | 5 ++ .../telemetry_error_issues/bigquery.py | 58 +++++++++++------- .github/scripts/telemetry_error_issues/cli.py | 45 ++++++++++++-- .../telemetry_error_issues/log_source.py | 49 +++++++++++++++ .../telemetry_error_issues/rendering.py | 44 ++++++++++---- .../scripts/telemetry_error_issues/sync.py | 13 ++-- .github/workflows/telemetry-error-issues.yml | 5 ++ gcp/observability/README.md | 23 +++++++- .../telemetry_error_issues/github_support.py | 4 +- .../telemetry_error_issues/report_support.py | 17 ++++++ .../telemetry_error_issues/test_bigquery.py | 57 ++++++++++++++++-- .../telemetry_error_issues/test_end_to_end.py | 59 +++++++++++++++++++ .../telemetry_error_issues/test_rendering.py | 54 ++++++++++++++--- .../unit/telemetry_error_issues/test_sync.py | 10 ++-- .../test_workflow_and_wrapper.py | 15 ++++- 15 files changed, 395 insertions(+), 63 deletions(-) create mode 100644 .github/scripts/telemetry_error_issues/log_source.py diff --git a/.github/scripts/sync_telemetry_error_issues.sh b/.github/scripts/sync_telemetry_error_issues.sh index 00032e369..ac6660742 100644 --- a/.github/scripts/sync_telemetry_error_issues.sh +++ b/.github/scripts/sync_telemetry_error_issues.sh @@ -9,6 +9,11 @@ set -euo pipefail : "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" : "${TELEMETRY_ERROR_FINGERPRINT_KEY:?TELEMETRY_ERROR_FINGERPRINT_KEY is required}" : "${OBSERVABILITY_PROJECT_ID:?OBSERVABILITY_PROJECT_ID is required}" +: "${OBSERVABILITY_LOG_BUCKET:?OBSERVABILITY_LOG_BUCKET is required}" +: "${OBSERVABILITY_LOG_BUCKET_LOCATION:?OBSERVABILITY_LOG_BUCKET_LOCATION is required}" +: "${OBSERVABILITY_LOG_DATASET:?OBSERVABILITY_LOG_DATASET is required}" +: "${OBSERVABILITY_LOG_VIEW:?OBSERVABILITY_LOG_VIEW is required}" +: "${OBSERVABILITY_SERVICE_NAMESPACE:?OBSERVABILITY_SERVICE_NAMESPACE is required}" # Hide both values in the job log even if a later command prints them. echo "::add-mask::${GOOGLE_OAUTH_ACCESS_TOKEN}" diff --git a/.github/scripts/telemetry_error_issues/bigquery.py b/.github/scripts/telemetry_error_issues/bigquery.py index b2a9d614a..18775a785 100644 --- a/.github/scripts/telemetry_error_issues/bigquery.py +++ b/.github/scripts/telemetry_error_issues/bigquery.py @@ -9,6 +9,11 @@ from urllib.parse import quote, urlencode from telemetry_error_issues.groups import RequestCount +from telemetry_error_issues.log_source import ( + DATASET_PATTERN, + LOG_VIEW_PATTERN, + checked_identifier, +) from telemetry_error_issues.normalization import MISSING from telemetry_error_issues.records import ErrorRecord from telemetry_error_issues.transport import ( @@ -21,6 +26,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Mapping + from telemetry_error_issues.log_source import LogSource from telemetry_error_issues.window import ReportingWindow # Captures 'path", line N, in function' from each frame header of a Python @@ -49,7 +55,6 @@ TEMPLATE_WORD_PATTERN = r"\b[A-Za-z]+\b" BIGQUERY_API = "https://bigquery.googleapis.com/bigquery/v2" -LINKED_DATASET = "observability_logs" MAXIMUM_BYTES_BILLED = 5 * 1024**3 QUERY_WAIT_MILLISECONDS = 30_000 MAX_QUERY_POLLS = 40 @@ -62,10 +67,12 @@ "/readiness-check", ) -# Each query reads the linked dataset's _AllLogs view. Only code-defined or -# normalized values leave BigQuery: stacks only as frame headers, messages -# only as templates (see DOUBLE_QUOTED_PATTERN), and identifiers only as -# SHA-256 hashes. Duplicate deliveries of one log entry are removed. +# Each query reads the linked dataset's log view, whose name replaces +# {log_view} once it has passed LOG_VIEW_PATTERN: BigQuery takes no table name +# as a parameter. Only code-defined or normalized values leave BigQuery: stacks +# only as frame headers, messages only as templates (see DOUBLE_QUOTED_PATTERN), +# and identifiers only as SHA-256 hashes. Duplicate deliveries of one log entry +# are removed. ERROR_RECORDS_QUERY = r""" SELECT timestamp, @@ -114,10 +121,10 @@ JSON_VALUE(json_payload, '$.trace_id'), REGEXP_EXTRACT(trace, r'/traces/([0-9A-Fa-f]+)$') ))) AS trace_key -FROM `_AllLogs` +FROM `{log_view}` WHERE timestamp >= @query_start AND timestamp < @window_end AND JSON_VALUE(json_payload, '$.schema_version') = 'policyengine.observability.v2' - AND JSON_VALUE(json_payload, '$."service.namespace"') = 'policyengine.api-v1' + AND JSON_VALUE(json_payload, '$."service.namespace"') = @service_namespace AND severity_number >= 500 AND COALESCE(JSON_VALUE(json_payload, '$."http.route"'), '') NOT IN UNNEST(@excluded_routes) @@ -132,10 +139,10 @@ JSON_VALUE(json_payload, '$."service.name"') AS service_name, JSON_VALUE(json_payload, '$."deployment.environment.name"') AS environment_name, JSON_VALUE(json_payload, '$."http.route"') AS route - FROM `_AllLogs` + FROM `{log_view}` WHERE timestamp >= @window_start AND timestamp < @window_end AND JSON_VALUE(json_payload, '$.schema_version') = 'policyengine.observability.v2' - AND JSON_VALUE(json_payload, '$."service.namespace"') = 'policyengine.api-v1' + AND JSON_VALUE(json_payload, '$."service.namespace"') = @service_namespace AND JSON_VALUE(json_payload, '$."event.name"') = 'request.completed' AND COALESCE(JSON_VALUE(json_payload, '$."http.route"'), '') NOT IN UNNEST(@excluded_routes) @@ -206,17 +213,16 @@ class BigQueryClient: def __init__( self, - project_id: str, + log_source: LogSource, access_token: str, transport: Callable[[HttpRequest], HttpResponse] = urllib_transport, ) -> None: - """Run jobs in ``project_id`` against its linked log dataset.""" - self.project_id = project_id + """Run jobs in the log source's project against its linked dataset.""" + self.log_source = log_source self._access_token = access_token self._transport = transport - self._queries_url = ( - f"{BIGQUERY_API}/projects/{quote(project_id, safe='')}/queries" - ) + project_id = quote(log_source.project_id, safe="") + self._queries_url = f"{BIGQUERY_API}/projects/{project_id}/queries" def query( self, sql: str, parameters: list[dict[str, Any]], label: str @@ -228,8 +234,10 @@ def query( "parameterMode": "NAMED", "queryParameters": parameters, "defaultDataset": { - "projectId": self.project_id, - "datasetId": LINKED_DATASET, + "projectId": self.log_source.project_id, + "datasetId": checked_identifier( + self.log_source.dataset, DATASET_PATTERN, "the log dataset" + ), }, "maximumBytesBilled": str(MAXIMUM_BYTES_BILLED), "labels": {"purpose": "telemetry-error-issues", "query": label}, @@ -286,6 +294,12 @@ def _timestamp_parameter(name: str, moment: datetime) -> dict[str, Any]: } +def _query_text(template: str, log_source: LogSource) -> str: + """Name the log view in ``template``, once the name has passed its pattern.""" + view = checked_identifier(log_source.view, LOG_VIEW_PATTERN, "the log view") + return template.format(log_view=view) + + def _query_parameters( timestamps: Mapping[str, datetime], strings: Mapping[str, str] ) -> list[dict[str, Any]]: @@ -365,6 +379,7 @@ def fetch_error_records( parameters = _query_parameters( {"query_start": window.lookback_start, "window_end": window.end}, { + "service_namespace": client.log_source.service_namespace, "frame_pattern": FRAME_PATTERN, "chained_exception_pattern": CHAINED_EXCEPTION_PATTERN, "double_quoted_pattern": DOUBLE_QUOTED_PATTERN, @@ -372,7 +387,8 @@ def fetch_error_records( "template_word_pattern": TEMPLATE_WORD_PATTERN, }, ) - rows = client.query(ERROR_RECORDS_QUERY, parameters, "error-records") + sql = _query_text(ERROR_RECORDS_QUERY, client.log_source) + rows = client.query(sql, parameters, "error-records") return [error_record_from_row(row) for row in rows] @@ -381,9 +397,11 @@ def fetch_request_counts( ) -> list[RequestCount]: """Return the requests each service completed per route in the window.""" parameters = _query_parameters( - {"window_start": window.start, "window_end": window.end}, {} + {"window_start": window.start, "window_end": window.end}, + {"service_namespace": client.log_source.service_namespace}, ) - rows = client.query(REQUEST_COUNTS_QUERY, parameters, "request-counts") + sql = _query_text(REQUEST_COUNTS_QUERY, client.log_source) + rows = client.query(sql, parameters, "request-counts") return [ RequestCount( service_name=_text(row["service_name"]) or MISSING, diff --git a/.github/scripts/telemetry_error_issues/cli.py b/.github/scripts/telemetry_error_issues/cli.py index 95e10d042..a9f8bb0a1 100644 --- a/.github/scripts/telemetry_error_issues/cli.py +++ b/.github/scripts/telemetry_error_issues/cli.py @@ -26,6 +26,15 @@ ) from telemetry_error_issues.github_client import GitHubClient from telemetry_error_issues.groups import build_groups +from telemetry_error_issues.log_source import ( + DATASET_PATTERN, + LOCATION_PATTERN, + LOG_BUCKET_PATTERN, + LOG_VIEW_PATTERN, + PROJECT_ID_PATTERN, + LogSource, + checked_identifier, +) from telemetry_error_issues.normalization import MISSING from telemetry_error_issues.records import fingerprint_key from telemetry_error_issues.rendering import code, window_text @@ -48,11 +57,22 @@ StageResult = TypeVar("StageResult") +# The repository variables that name the logs the run reads, each with the +# pattern its value must match. They say where the logs are; none of them +# changes how failures are grouped. +LOG_NAME_VARIABLES = { + "OBSERVABILITY_LOG_BUCKET": LOG_BUCKET_PATTERN, + "OBSERVABILITY_LOG_BUCKET_LOCATION": LOCATION_PATTERN, + "OBSERVABILITY_LOG_DATASET": DATASET_PATTERN, + "OBSERVABILITY_LOG_VIEW": LOG_VIEW_PATTERN, +} REQUIRED_ENVIRONMENT_VARIABLES = ( "GOOGLE_OAUTH_ACCESS_TOKEN", "GITHUB_TOKEN", "GITHUB_REPOSITORY", "TELEMETRY_ERROR_FINGERPRINT_KEY", + *LOG_NAME_VARIABLES, + "OBSERVABILITY_SERVICE_NAMESPACE", ) REPOSITORY_PATTERN = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") @@ -131,11 +151,12 @@ class Settings(NamedTuple): github_credential: str repository: str fingerprint_key: bytes + log_source: LogSource summary_path: str | None -def read_settings(environ: Mapping[str, str]) -> Settings: - """Read and check the environment variables the run needs.""" +def read_settings(environ: Mapping[str, str], project_id: str) -> Settings: + """Read and check the environment variables the run needs, and its project.""" missing = [name for name in REQUIRED_ENVIRONMENT_VARIABLES if not environ.get(name)] if missing: msg = f"missing environment variables: {', '.join(missing)}" @@ -145,13 +166,25 @@ def read_settings(environ: Mapping[str, str]) -> Settings: raise ConfigurationError(msg) try: key = fingerprint_key(environ["TELEMETRY_ERROR_FINGERPRINT_KEY"]) + checked_identifier(project_id, PROJECT_ID_PATTERN, "--project-id") + for name, pattern in LOG_NAME_VARIABLES.items(): + checked_identifier(environ[name], pattern, name) except ValueError as error: raise ConfigurationError(str(error)) from None + log_source = LogSource( + project_id=project_id, + bucket=environ["OBSERVABILITY_LOG_BUCKET"], + bucket_location=environ["OBSERVABILITY_LOG_BUCKET_LOCATION"], + dataset=environ["OBSERVABILITY_LOG_DATASET"], + view=environ["OBSERVABILITY_LOG_VIEW"], + service_namespace=environ["OBSERVABILITY_SERVICE_NAMESPACE"], + ) return Settings( google_access_token=environ["GOOGLE_OAUTH_ACCESS_TOKEN"], github_credential=environ["GITHUB_TOKEN"], repository=environ["GITHUB_REPOSITORY"], fingerprint_key=key, + log_source=log_source, summary_path=environ.get("GITHUB_STEP_SUMMARY") or None, ) @@ -220,12 +253,14 @@ def _append_summary(path: str, summary: str) -> None: def _report(arguments: argparse.Namespace, runtime: Runtime) -> str: - settings = _stage("configuration", lambda: read_settings(runtime.environ)) + settings = _stage( + "configuration", lambda: read_settings(runtime.environ, arguments.project_id) + ) window = _stage( "configuration", lambda: _report_window(arguments.report_date, runtime.now()) ) bigquery = BigQueryClient( - arguments.project_id, settings.google_access_token, runtime.bigquery_transport + settings.log_source, settings.google_access_token, runtime.bigquery_transport ) records = _stage( "query error records", lambda: fetch_error_records(bigquery, window) @@ -247,7 +282,7 @@ def _report(arguments: argparse.Namespace, runtime: Runtime) -> str: result = _stage( "sync issues", lambda: sync_issues( - groups, window, arguments.project_id, github, dry_run=arguments.dry_run + groups, window, settings.log_source, github, dry_run=arguments.dry_run ), ) summary = render_summary(result, window, dry_run=arguments.dry_run) diff --git a/.github/scripts/telemetry_error_issues/log_source.py b/.github/scripts/telemetry_error_issues/log_source.py new file mode 100644 index 000000000..090a94aca --- /dev/null +++ b/.github/scripts/telemetry_error_issues/log_source.py @@ -0,0 +1,49 @@ +"""Where the central telemetry logs are, as the deployment's settings name them. + +The names come from repository variables. BigQuery can't take a dataset or a +table name as a query parameter, so each name must match a strict identifier +pattern when the settings are read, and again just before it goes into a +query, a job configuration or a Logs Explorer link. A name that doesn't match +is refused, which keeps SQL out of the query. +""" + +from __future__ import annotations + +import re +from typing import NamedTuple + +# Each pattern allows only ASCII letters, digits and the separators the +# resource's own naming rules allow, and only within their length limits. +PROJECT_ID_PATTERN = re.compile(r"[a-z][a-z0-9-]{4,28}[a-z0-9]") +LOG_BUCKET_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,99}") +LOCATION_PATTERN = re.compile(r"[a-z][a-z0-9-]{0,62}") +DATASET_PATTERN = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,1023}") +# Cloud Logging gives a linked dataset's view the name of the log view. +LOG_VIEW_PATTERN = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,99}") + + +class LogSource(NamedTuple): + """The project, log bucket, linked dataset and view the run reads. + + ``service_namespace`` selects the application's records. It reaches BigQuery + only as a query parameter, so it needs no identifier pattern. + """ + + project_id: str + bucket: str + bucket_location: str + dataset: str + view: str + service_namespace: str + + +def checked_identifier(value: str, pattern: re.Pattern[str], setting: str) -> str: + """Return ``value`` if all of it matches ``pattern``, else raise ValueError. + + The error names the setting, never the value, because a value that fails + the check might be anything. + """ + if pattern.fullmatch(value) is None: + msg = f"{setting} is not a valid name" + raise ValueError(msg) + return value diff --git a/.github/scripts/telemetry_error_issues/rendering.py b/.github/scripts/telemetry_error_issues/rendering.py index 286fddea3..fd7c0635c 100644 --- a/.github/scripts/telemetry_error_issues/rendering.py +++ b/.github/scripts/telemetry_error_issues/rendering.py @@ -8,12 +8,20 @@ from typing import TYPE_CHECKING, Any from urllib.parse import quote, urlencode +from telemetry_error_issues.log_source import ( + LOCATION_PATTERN, + LOG_BUCKET_PATTERN, + LOG_VIEW_PATTERN, + PROJECT_ID_PATTERN, + checked_identifier, +) from telemetry_error_issues.normalization import MISSING, UNRECOGNIZED from telemetry_error_issues.records import FINGERPRINT_ALGORITHM from telemetry_error_issues.window import REPORTING_TIME_ZONE, ReportingWindow if TYPE_CHECKING: from telemetry_error_issues.groups import EnvironmentMeasures, ErrorGroup + from telemetry_error_issues.log_source import LogSource ISSUE_TITLE_PREFIX = "[telemetry]" MAX_TITLE_LENGTH = 256 @@ -22,8 +30,6 @@ STATE_PATTERN = re.compile(r"") NOT_AVAILABLE = "n/a" LOGS_EXPLORER_URL = "https://console.cloud.google.com/logs/query" -LOG_BUCKET = "policyengine-observability" -LOG_VIEW = "_AllLogs" SCHEMA_VERSION = "policyengine.observability.v2" SIMULATION_FAMILY = "policyengine-simulation-py" @@ -93,8 +99,21 @@ def _logs_filters(group: ErrorGroup) -> list[str]: return filters +def _log_view_path(log_source: LogSource) -> str: + """Return the log view's resource name, once each part has passed its pattern.""" + project_id = checked_identifier( + log_source.project_id, PROJECT_ID_PATTERN, "the project ID" + ) + location = checked_identifier( + log_source.bucket_location, LOCATION_PATTERN, "the log bucket location" + ) + bucket = checked_identifier(log_source.bucket, LOG_BUCKET_PATTERN, "the log bucket") + view = checked_identifier(log_source.view, LOG_VIEW_PATTERN, "the log view") + return f"projects/{project_id}/locations/{location}/buckets/{bucket}/views/{view}" + + def logs_explorer_url( - group: ErrorGroup, window: ReportingWindow, project_id: str + group: ErrorGroup, window: ReportingWindow, log_source: LogSource ) -> str: """Return a Logs Explorer link to the group's records in the window.""" query = "\n".join( @@ -104,9 +123,8 @@ def logs_explorer_url( f'timestamp<"{_logs_timestamp(window.end)}"', ] ) - view = ( - f"projects/{project_id}/locations/global/buckets/{LOG_BUCKET}/views/{LOG_VIEW}" - ) + view = _log_view_path(log_source) + project_id = log_source.project_id settings = {"query": query, "storageScope": f"storage,{view}"} encoded = "".join( f";{name}={quote(value, safe='')}" for name, value in settings.items() @@ -202,7 +220,9 @@ def parse_state(body: str) -> dict[str, Any] | None: return state if isinstance(state, dict) else None -def issue_section(group: ErrorGroup, window: ReportingWindow, project_id: str) -> str: +def issue_section( + group: ErrorGroup, window: ReportingWindow, log_source: LogSource +) -> str: """Return the marked section of the issue body, which each run replaces.""" related = ", ".join(map(code, group.related_fingerprints)) lines = [ @@ -222,8 +242,8 @@ def issue_section(group: ErrorGroup, window: ReportingWindow, project_id: str) - "", ( f"[Matching log records in Logs Explorer]" - f"({logs_explorer_url(group, window, project_id)}), for the window only. " - f"It needs read access to the `{project_id}` project." + f"({logs_explorer_url(group, window, log_source)}), for the window only. " + f"It needs read access to the `{log_source.project_id}` project." ), "", _state_comment(group, window), @@ -232,11 +252,13 @@ def issue_section(group: ErrorGroup, window: ReportingWindow, project_id: str) - return "\n".join(lines) -def issue_body(group: ErrorGroup, window: ReportingWindow, project_id: str) -> str: +def issue_body( + group: ErrorGroup, window: ReportingWindow, log_source: LogSource +) -> str: """Return a new issue's body: a short note, then the marked section.""" note = ( "The daily telemetry error report opened this issue for a production " "error group. Each run replaces the marked section below with the " "latest day's measures; text outside it is kept." ) - return f"{note}\n\n{issue_section(group, window, project_id)}\n" + return f"{note}\n\n{issue_section(group, window, log_source)}\n" diff --git a/.github/scripts/telemetry_error_issues/sync.py b/.github/scripts/telemetry_error_issues/sync.py index 2f1ac46eb..0baf6b7a1 100644 --- a/.github/scripts/telemetry_error_issues/sync.py +++ b/.github/scripts/telemetry_error_issues/sync.py @@ -23,6 +23,7 @@ from collections.abc import Iterable, Mapping from telemetry_error_issues.groups import ErrorGroup + from telemetry_error_issues.log_source import LogSource from telemetry_error_issues.window import ReportingWindow BOT_LOGIN = "github-actions[bot]" @@ -147,14 +148,14 @@ def __init__( self, github: GitHubClient, window: ReportingWindow, - project_id: str, + log_source: LogSource, *, dry_run: bool, ) -> None: """Sync the groups of ``window``; a dry run reads but never writes.""" self._github = github self._window = window - self._project_id = project_id + self._log_source = log_source self._dry_run = dry_run # Case-folded names: GitHub label names are unique regardless of case. self._known_labels: set[str] | None = None @@ -210,7 +211,7 @@ def _open(self, group: ErrorGroup) -> int | None: self._ensure_labels(labels) issue = self._github.create_issue( issue_title(group), - issue_body(group, self._window, self._project_id), + issue_body(group, self._window, self._log_source), labels, ) number = issue["number"] @@ -239,7 +240,7 @@ def _update(self, group: ErrorGroup, issue: Mapping[str, Any]) -> str: if reported_end is not None and reported_end > self._window.end: return "kept-newer" new_body = replace_section( - body, issue_section(group, self._window, self._project_id) + body, issue_section(group, self._window, self._log_source) ) since = self._reopen_since(issue) if since is not None and _meets_thresholds(*group.production_since(since)): @@ -272,11 +273,11 @@ def _reopen( def sync_issues( groups: Iterable[ErrorGroup], window: ReportingWindow, - project_id: str, + log_source: LogSource, github: GitHubClient, *, dry_run: bool, ) -> SyncResult: """Create or update the issue for each group; a dry run makes no changes.""" - synchronizer = IssueSynchronizer(github, window, project_id, dry_run=dry_run) + synchronizer = IssueSynchronizer(github, window, log_source, dry_run=dry_run) return synchronizer.run(groups) diff --git a/.github/workflows/telemetry-error-issues.yml b/.github/workflows/telemetry-error-issues.yml index fc36e0020..fa0fd1c26 100644 --- a/.github/workflows/telemetry-error-issues.yml +++ b/.github/workflows/telemetry-error-issues.yml @@ -60,6 +60,11 @@ jobs: GITHUB_TOKEN: ${{ github.token }} TELEMETRY_ERROR_FINGERPRINT_KEY: ${{ secrets.TELEMETRY_ERROR_FINGERPRINT_KEY }} OBSERVABILITY_PROJECT_ID: ${{ vars.OBSERVABILITY_PROJECT_ID }} + OBSERVABILITY_LOG_BUCKET: ${{ vars.OBSERVABILITY_LOG_BUCKET }} + OBSERVABILITY_LOG_BUCKET_LOCATION: ${{ vars.OBSERVABILITY_LOG_BUCKET_LOCATION }} + OBSERVABILITY_LOG_DATASET: ${{ vars.OBSERVABILITY_LOG_DATASET }} + OBSERVABILITY_LOG_VIEW: ${{ vars.OBSERVABILITY_LOG_VIEW }} + OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }} DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} REPORT_DATE: ${{ inputs.report_date }} run: bash .github/scripts/sync_telemetry_error_issues.sh diff --git a/gcp/observability/README.md b/gcp/observability/README.md index bc6943414..19e081b35 100644 --- a/gcp/observability/README.md +++ b/gcp/observability/README.md @@ -190,6 +190,11 @@ GOOGLE_OAUTH_ACCESS_TOKEN="$(gcloud auth print-access-token)" \ GITHUB_TOKEN="$(gh auth token)" \ GITHUB_REPOSITORY=PolicyEngine/policyengine-api \ TELEMETRY_ERROR_FINGERPRINT_KEY="$(openssl rand -hex 32)" \ +OBSERVABILITY_LOG_BUCKET=policyengine-observability \ +OBSERVABILITY_LOG_BUCKET_LOCATION=global \ +OBSERVABILITY_LOG_DATASET=observability_logs \ +OBSERVABILITY_LOG_VIEW=_AllLogs \ +OBSERVABILITY_SERVICE_NAMESPACE=policyengine.api-v1 \ python3 .github/scripts/sync_telemetry_error_issues.py \ --project-id policyengine-observability --dry-run ``` @@ -207,7 +212,23 @@ python3 .github/scripts/sync_telemetry_error_issues.py \ | Environment secret `TELEMETRY_ERROR_FINGERPRINT_KEY` | The 32-byte hexadecimal HMAC key for fingerprints | The workflow also uses the repository secret `GCP_WORKLOAD_IDENTITY_PROVIDER` -and the repository variable `OBSERVABILITY_PROJECT_ID`. +and these repository variables, which say where the logs are: + +| Repository variable | Value | Used for | +| --- | --- | --- | +| `OBSERVABILITY_PROJECT_ID` | `policyengine-observability` | The project that runs the query jobs and holds the bucket | +| `OBSERVABILITY_LOG_BUCKET` | `policyengine-observability` | The log bucket, in the Logs Explorer link | +| `OBSERVABILITY_LOG_BUCKET_LOCATION` | `global` | The bucket's location, in the Logs Explorer link | +| `OBSERVABILITY_LOG_DATASET` | `observability_logs` | The linked BigQuery dataset the queries read | +| `OBSERVABILITY_LOG_VIEW` | `_AllLogs` | The log view, which is also the name of the linked dataset's view | +| `OBSERVABILITY_SERVICE_NAMESPACE` | `policyengine.api-v1` | The `service.namespace` of the records to report; the API reads the same variable | + +Each name must match a strict identifier pattern +(`.github/scripts/telemetry_error_issues/log_source.py`), because BigQuery +can't take a dataset or view name as a query parameter; the run refuses any +other value. The namespace is sent as a query parameter. The rules that decide +which group a failure belongs to stay in code, because changing one changes +fingerprints. These resources were created on 2026-10-20: diff --git a/tests/unit/telemetry_error_issues/github_support.py b/tests/unit/telemetry_error_issues/github_support.py index 614a99e24..d77182967 100644 --- a/tests/unit/telemetry_error_issues/github_support.py +++ b/tests/unit/telemetry_error_issues/github_support.py @@ -13,7 +13,7 @@ from tests.unit.telemetry_error_issues.report_support import ( FINGERPRINT_KEY, - PROJECT_ID, + LOG_SOURCE, REPORT_DATE, exception_record, hours_after_window_noon, @@ -175,7 +175,7 @@ def sync(groups: list, github: FakeGitHub, *, dry_run=False): return sync_issues( groups, reporting_window(REPORT_DATE), - PROJECT_ID, + LOG_SOURCE, github.client(), dry_run=dry_run, ) diff --git a/tests/unit/telemetry_error_issues/report_support.py b/tests/unit/telemetry_error_issues/report_support.py index a597af8e4..439d57ad0 100644 --- a/tests/unit/telemetry_error_issues/report_support.py +++ b/tests/unit/telemetry_error_issues/report_support.py @@ -8,6 +8,7 @@ from telemetry_error_issues.bigquery import CHAINED_EXCEPTION_PATTERN, FRAME_PATTERN from telemetry_error_issues.groups import RequestCount +from telemetry_error_issues.log_source import LogSource from telemetry_error_issues.records import ErrorRecord EASTERN = ZoneInfo("America/New_York") @@ -142,6 +143,22 @@ def failures_with_and_without_a_route() -> list: PROJECT_ID = "policyengine-observability" +# The deployment's values of the repository variables that name the logs. +LOG_SOURCE = LogSource( + project_id=PROJECT_ID, + bucket="policyengine-observability", + bucket_location="global", + dataset="observability_logs", + view="_AllLogs", + service_namespace="policyengine.api-v1", +) +LOG_SOURCE_ENVIRONMENT = { + "OBSERVABILITY_LOG_BUCKET": LOG_SOURCE.bucket, + "OBSERVABILITY_LOG_BUCKET_LOCATION": LOG_SOURCE.bucket_location, + "OBSERVABILITY_LOG_DATASET": LOG_SOURCE.dataset, + "OBSERVABILITY_LOG_VIEW": LOG_SOURCE.view, + "OBSERVABILITY_SERVICE_NAMESPACE": LOG_SOURCE.service_namespace, +} def sentinel_row(**overrides: object) -> dict[str, object]: diff --git a/tests/unit/telemetry_error_issues/test_bigquery.py b/tests/unit/telemetry_error_issues/test_bigquery.py index 166776e91..2c43c7b6a 100644 --- a/tests/unit/telemetry_error_issues/test_bigquery.py +++ b/tests/unit/telemetry_error_issues/test_bigquery.py @@ -33,6 +33,7 @@ ) from tests.unit.telemetry_error_issues.report_support import ( APPLICATION_FRAMES, + LOG_SOURCE, REPORT_DATE, WINDOW_NOON, exception_record, @@ -42,8 +43,8 @@ STRING_FIELD = {"name": "name", "type": "STRING", "mode": "NULLABLE"} -def _client(transport: FakeTransport) -> object: - return BigQueryClient("policyengine-observability", ACCESS_TOKEN, transport) +def _client(transport: FakeTransport, log_source: object = LOG_SOURCE) -> object: + return BigQueryClient(log_source, ACCESS_TOKEN, transport) def test_bigquery_error_query_request_limits_cost_and_names_its_parameters() -> None: @@ -73,6 +74,10 @@ def test_bigquery_error_query_request_limits_cost_and_names_its_parameters() -> "datasetId": "observability_logs", } assert body["formatOptions"] == {"useInt64Timestamp": True} + assert "FROM `_AllLogs`" in body["query"] + assert parameters["service_namespace"]["parameterValue"]["value"] == ( + "policyengine.api-v1" + ) assert parameters["query_start"]["parameterType"] == {"type": "TIMESTAMP"} assert parameters["query_start"]["parameterValue"]["value"] == ( "2026-10-13 04:00:00+00:00" @@ -96,19 +101,57 @@ def test_bigquery_error_query_request_limits_cost_and_names_its_parameters() -> ] == ["/health", "/simulation-gateway-check", "/liveness-check", "/readiness-check"] +def test_bigquery_queries_name_the_configured_dataset_and_view() -> None: + transport = FakeTransport([json_response(query_response([], []))] * 2) + log_source = LOG_SOURCE._replace(dataset="linked_logs", view="_Requests") + client = _client(transport, log_source) + + fetch_error_records(client, reporting_window(REPORT_DATE)) + fetch_request_counts(client, reporting_window(REPORT_DATE)) + + for index in range(2): + body = transport.json_body(index) + assert body["defaultDataset"]["datasetId"] == "linked_logs" + assert "FROM `_Requests`" in body["query"] + assert "_AllLogs" not in body["query"] + + +@pytest.mark.parametrize( + "unsafe_names", + [ + {"view": "_AllLogs` WHERE TRUE --"}, + {"view": "_AllLogs\n"}, + {"view": ""}, + {"dataset": "observability_logs.other"}, + {"dataset": "logs; DROP TABLE logs"}, + ], +) +def test_bigquery_refuses_a_dataset_or_view_that_is_not_an_identifier( + unsafe_names: dict[str, str], +) -> None: + transport = FakeTransport([json_response(query_response([], []))]) + client = _client(transport, LOG_SOURCE._replace(**unsafe_names)) + + with pytest.raises(ValueError, match="is not a valid name"): + fetch_error_records(client, reporting_window(REPORT_DATE)) + + assert transport.requests == [] + + def test_bigquery_error_query_filters_application_error_records() -> None: sql = ERROR_RECORDS_QUERY - assert "FROM `_AllLogs`" in sql + assert "FROM `{log_view}`" in sql assert "timestamp >= @query_start AND timestamp < @window_end" in sql assert ( "JSON_VALUE(json_payload, '$.schema_version') = " "'policyengine.observability.v2'" in sql ) assert ( - "JSON_VALUE(json_payload, '$.\"service.namespace\"') = " - "'policyengine.api-v1'" in sql + "JSON_VALUE(json_payload, '$.\"service.namespace\"') = @service_namespace" + in sql ) + assert "policyengine.api-v1" not in sql assert "severity_number >= 500" in sql assert "NOT IN UNNEST(@excluded_routes)" in sql assert "QUALIFY ROW_NUMBER() OVER" in sql @@ -213,6 +256,10 @@ def test_bigquery_request_count_query_counts_completed_requests_in_the_window() assert parameters["window_start"]["parameterValue"]["value"] == ( "2026-10-20 04:00:00+00:00" ) + assert parameters["service_namespace"]["parameterValue"]["value"] == ( + "policyengine.api-v1" + ) + assert "FROM `_AllLogs`" in transport.json_body(0)["query"] assert transport.json_body(0)["labels"]["query"] == "request-counts" diff --git a/tests/unit/telemetry_error_issues/test_end_to_end.py b/tests/unit/telemetry_error_issues/test_end_to_end.py index 090d7c9c3..c7fbaab8e 100644 --- a/tests/unit/telemetry_error_issues/test_end_to_end.py +++ b/tests/unit/telemetry_error_issues/test_end_to_end.py @@ -23,6 +23,7 @@ from tests.unit.telemetry_error_issues.report_support import ( APPLICATION_FRAMES, EASTERN, + LOG_SOURCE_ENVIRONMENT, PROJECT_ID, WINDOW_NOON, hours_after_window_noon, @@ -120,6 +121,7 @@ def _environment(tmp_path: Path, **overrides: str) -> dict[str, str]: "GITHUB_REPOSITORY": REPOSITORY, "TELEMETRY_ERROR_FINGERPRINT_KEY": "f" * 64, "GITHUB_STEP_SUMMARY": str(tmp_path / "summary.md"), + **LOG_SOURCE_ENVIRONMENT, } environment.update(overrides) return {name: value for name, value in environment.items() if value} @@ -278,6 +280,63 @@ def test_main_rejects_a_short_fingerprint_key_without_printing_it( assert "SENTINEL" not in stdout + stderr +def test_main_names_missing_log_settings(tmp_path: Path) -> None: + environment = _environment(tmp_path, **dict.fromkeys(LOG_SOURCE_ENVIRONMENT, "")) + + exit_code, _, stderr = _run_main( + tmp_path, FakeBigQuery([]), FakeGitHub(), environment=environment + ) + + assert exit_code == 1 + assert stderr == ( + "configuration failed: ConfigurationError (missing environment variables: " + f"{', '.join(LOG_SOURCE_ENVIRONMENT)})\n" + ) + + +@pytest.mark.parametrize( + ("setting", "value"), + [ + ("OBSERVABILITY_LOG_BUCKET", "SENTINEL bucket"), + ("OBSERVABILITY_LOG_BUCKET_LOCATION", "SENTINEL/location"), + ("OBSERVABILITY_LOG_DATASET", "SENTINEL`; DROP TABLE logs"), + ("OBSERVABILITY_LOG_VIEW", "_AllLogs` WHERE SENTINEL"), + ], +) +def test_main_refuses_a_log_name_that_is_not_an_identifier( + tmp_path: Path, setting: str, value: str +) -> None: + bigquery = FakeBigQuery([]) + + exit_code, stdout, stderr = _run_main( + tmp_path, + bigquery, + FakeGitHub(), + environment=_environment(tmp_path, **{setting: value}), + ) + + assert exit_code == 1 + assert stdout == "" + assert stderr == ( + f"configuration failed: ConfigurationError ({setting} is not a valid name)\n" + ) + assert bigquery.requests == [] + + +def test_main_refuses_a_project_id_that_is_not_an_identifier(tmp_path: Path) -> None: + bigquery = FakeBigQuery([]) + + exit_code, _, stderr = _run_main( + tmp_path, bigquery, FakeGitHub(), "--project-id", "SENTINEL project" + ) + + assert exit_code == 1 + assert stderr == ( + "configuration failed: ConfigurationError (--project-id is not a valid name)\n" + ) + assert bigquery.requests == [] + + def test_main_reports_an_api_failure_by_stage_class_and_status_only( tmp_path: Path, ) -> None: diff --git a/tests/unit/telemetry_error_issues/test_rendering.py b/tests/unit/telemetry_error_issues/test_rendering.py index d1d442a06..475c0eda4 100644 --- a/tests/unit/telemetry_error_issues/test_rendering.py +++ b/tests/unit/telemetry_error_issues/test_rendering.py @@ -19,7 +19,7 @@ from tests.unit.telemetry_error_issues.report_support import ( FINGERPRINT_KEY, - PROJECT_ID, + LOG_SOURCE, REPORT_DATE, exception_record, failed_request_record, @@ -45,7 +45,7 @@ def _render(group: object) -> tuple[str, str]: window = reporting_window(REPORT_DATE) return ( issue_title(group), - issue_body(group, window, PROJECT_ID), + issue_body(group, window, LOG_SOURCE), ) @@ -224,7 +224,7 @@ def test_render_body_stays_under_the_github_limit() -> None: def test_render_logs_link_filters_exceptions_by_service_and_type() -> None: group = _group([exception_record()]) - url = logs_explorer_url(group, reporting_window(REPORT_DATE), PROJECT_ID) + url = logs_explorer_url(group, reporting_window(REPORT_DATE), LOG_SOURCE) query, settings = _logs_query(url) @@ -246,6 +246,46 @@ def test_render_logs_link_filters_exceptions_by_service_and_type() -> None: assert character not in url +def test_render_logs_link_names_the_configured_bucket_location_and_view() -> None: + group = _group([exception_record()]) + log_source = LOG_SOURCE._replace( + project_id="other-project", + bucket="central-logs", + bucket_location="us-central1", + view="_Requests", + ) + + url = logs_explorer_url(group, reporting_window(REPORT_DATE), log_source) + + _, settings = _logs_query(url) + assert urlparse(url).query == "project=other-project" + assert settings["storageScope"] == quote( + "storage,projects/other-project/locations/us-central1/" + "buckets/central-logs/views/_Requests", + safe="", + ) + + +@pytest.mark.parametrize( + "unsafe_names", + [ + {"project_id": "SENTINEL project"}, + {"bucket": "logs/views/other"}, + {"bucket_location": "global/../other"}, + {"view": "_AllLogs?project=other"}, + ], +) +def test_render_logs_link_refuses_names_that_are_not_identifiers( + unsafe_names: dict[str, str], +) -> None: + group = _group([exception_record()]) + + with pytest.raises(ValueError, match="is not a valid name"): + logs_explorer_url( + group, reporting_window(REPORT_DATE), LOG_SOURCE._replace(**unsafe_names) + ) + + def test_render_logs_link_filters_failed_requests_by_route_and_status() -> None: group = _group( [ @@ -258,7 +298,7 @@ def test_render_logs_link_filters_failed_requests_by_route_and_status() -> None: ) query, _ = _logs_query( - logs_explorer_url(group, reporting_window(REPORT_DATE), PROJECT_ID) + logs_explorer_url(group, reporting_window(REPORT_DATE), LOG_SOURCE) ) assert query.splitlines()[1:5] == [ @@ -276,9 +316,9 @@ def test_render_logs_link_filters_logged_errors_by_route_or_operation() -> None: [message_record(route=None, operation_name="simulation.dispatch")], ) - route_query, _ = _logs_query(logs_explorer_url(route_group, window, PROJECT_ID)) + route_query, _ = _logs_query(logs_explorer_url(route_group, window, LOG_SOURCE)) operation_query, _ = _logs_query( - logs_explorer_url(operation_group, window, PROJECT_ID) + logs_explorer_url(operation_group, window, LOG_SOURCE) ) assert 'jsonPayload."http.route"="//report"' in route_query @@ -303,7 +343,7 @@ def test_render_never_includes_values_from_records( window = reporting_window(REPORT_DATE) title, body = _render(group) - url = logs_explorer_url(group, window, PROJECT_ID) + url = logs_explorer_url(group, window, LOG_SOURCE) for output in (title, body, url, unquote(url)): assert "SENTINEL" not in output diff --git a/tests/unit/telemetry_error_issues/test_sync.py b/tests/unit/telemetry_error_issues/test_sync.py index 5115d65f1..67cbb2448 100644 --- a/tests/unit/telemetry_error_issues/test_sync.py +++ b/tests/unit/telemetry_error_issues/test_sync.py @@ -24,7 +24,7 @@ sync, ) from tests.unit.telemetry_error_issues.report_support import ( - PROJECT_ID, + LOG_SOURCE, REPORT_DATE, WINDOW_NOON, exception_record, @@ -38,7 +38,7 @@ def _existing_issue(group: object, number: int = 7, **fields: object) -> dict: issue = bot_issue( number, title=issue_title(group), - body=issue_body(group, earlier, PROJECT_ID), + body=issue_body(group, earlier, LOG_SOURCE), ) issue.update(fields) return issue @@ -120,7 +120,7 @@ def test_sync_replaces_only_the_marked_section_and_never_the_title() -> None: result = sync([group], github) patch = json.loads(github.writes[0][1].body) - expected_section = issue_section(group, reporting_window(REPORT_DATE), PROJECT_ID) + expected_section = issue_section(group, reporting_window(REPORT_DATE), LOG_SOURCE) assert list(patch) == ["body"] assert patch["body"].startswith("Notes before.\n\n") assert patch["body"].endswith("\nNotes after.\n") @@ -222,7 +222,7 @@ def test_sync_keeps_a_completed_issue_closed_within_24_hours_of_closing() -> Non assert github.issues[7]["state"] == "closed" assert 7 not in github.comments assert ( - issue_section(group, reporting_window(REPORT_DATE), PROJECT_ID) + issue_section(group, reporting_window(REPORT_DATE), LOG_SOURCE) in github.issues[7]["body"] ) assert result.actions[0].action == "updated" @@ -252,7 +252,7 @@ def test_sync_keeps_a_newer_report_instead_of_overwriting_it() -> None: issue = bot_issue( 7, title=issue_title(group), - body=issue_body(group, later, PROJECT_ID), + body=issue_body(group, later, LOG_SOURCE), ) github = FakeGitHub([issue]) diff --git a/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py b/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py index e7a1c778f..b4d4ddd05 100644 --- a/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py +++ b/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py @@ -11,7 +11,10 @@ import yaml from tests.unit.telemetry_error_issues.github_support import REPOSITORY -from tests.unit.telemetry_error_issues.report_support import PROJECT_ID +from tests.unit.telemetry_error_issues.report_support import ( + LOG_SOURCE_ENVIRONMENT, + PROJECT_ID, +) REPO = Path(__file__).resolve().parents[3] SCRIPT_PATH = REPO / ".github" / "scripts" / "sync_telemetry_error_issues.py" @@ -23,6 +26,7 @@ "GITHUB_REPOSITORY": REPOSITORY, "TELEMETRY_ERROR_FINGERPRINT_KEY": "f" * 64, "OBSERVABILITY_PROJECT_ID": PROJECT_ID, + **LOG_SOURCE_ENVIRONMENT, } @@ -100,6 +104,15 @@ def test_workflow_passes_settings_to_the_wrapper() -> None: "${{ secrets.TELEMETRY_ERROR_FINGERPRINT_KEY }}" ), "OBSERVABILITY_PROJECT_ID": "${{ vars.OBSERVABILITY_PROJECT_ID }}", + "OBSERVABILITY_LOG_BUCKET": "${{ vars.OBSERVABILITY_LOG_BUCKET }}", + "OBSERVABILITY_LOG_BUCKET_LOCATION": ( + "${{ vars.OBSERVABILITY_LOG_BUCKET_LOCATION }}" + ), + "OBSERVABILITY_LOG_DATASET": "${{ vars.OBSERVABILITY_LOG_DATASET }}", + "OBSERVABILITY_LOG_VIEW": "${{ vars.OBSERVABILITY_LOG_VIEW }}", + "OBSERVABILITY_SERVICE_NAMESPACE": ( + "${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}" + ), # A scheduled run has no inputs, so it is never a dry run. "DRY_RUN": "${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}", "REPORT_DATE": "${{ inputs.report_date }}", From 238ae7b03f0d14d580ba5b575cd1946b5b48cac1 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:10:36 +0400 Subject: [PATCH 18/18] Query BigQuery with google-cloud-bigquery instead of our own REST client The report submitted each query job, polled until it finished, converted every cell from its REST string and parsed error bodies itself. The client library now does all of that. What stays the same: the two queries, standard SQL, the 5 GiB maximum_bytes_billed limit, the labels, the ErrorRecord and RequestCount results, and a time limit of 1,230 seconds, the 41 waits of 30 seconds the polling loop allowed. Credentials still come from the access token google-github-actions/auth makes, with no credentials file: the client gets a google.oauth2.credentials.Credentials built from it. Rows are read by iterating query_job.result(), never with to_dataframe or to_arrow, which may use the Storage Read API that the service account has no role for. The library's error messages can include the query text and error details, and this repository is public. A library error becomes a BigQueryError that holds only the HTTP status and a short reason code, so a failed stage still prints only the stage, the exception class and the status and reason. The tests replace the library client through an injected factory and check that marker text in a library error never reaches the log or the job summary. google-cloud-bigquery is a locked telemetry-error-issues dependency group, so the Cloud Run image doesn't install it. uv.lock gains four packages and changes no others. The workflow sets up the pinned uv and the wrapper runs the script with uv run --frozen --only-group, so the job installs only the group. make test and make debug-test add the group to the uv run that runs pytest, which installs the library for CI's test job; make install stays the locked development environment that tests/unit/test_locked_test_environments.py expects. Co-Authored-By: Claude Opus 5.5 --- .../scripts/sync_telemetry_error_issues.sh | 8 +- .../telemetry_error_issues/bigquery.py | 216 ++++++--------- .github/scripts/telemetry_error_issues/cli.py | 18 +- .github/workflows/telemetry-error-issues.yml | 9 + Makefile | 4 +- gcp/observability/README.md | 10 +- pyproject.toml | 7 + .../bigquery_support.py | 120 ++++---- .../telemetry_error_issues/test_bigquery.py | 260 +++++++----------- .../telemetry_error_issues/test_end_to_end.py | 174 ++++++------ .../test_workflow_and_wrapper.py | 48 +++- uv.lock | 89 ++++++ 12 files changed, 509 insertions(+), 454 deletions(-) diff --git a/.github/scripts/sync_telemetry_error_issues.sh b/.github/scripts/sync_telemetry_error_issues.sh index ac6660742..5062e36bd 100644 --- a/.github/scripts/sync_telemetry_error_issues.sh +++ b/.github/scripts/sync_telemetry_error_issues.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Run the daily telemetry error report with the settings the workflow passes -# in the environment. DRY_RUN=true adds --dry-run; a non-empty REPORT_DATE +# in the environment, through uv (the workflow sets it up). DRY_RUN=true adds --dry-run; a non-empty REPORT_DATE # (YYYY-MM-DD, an Eastern calendar day) replaces the default previous day. set -euo pipefail @@ -28,4 +28,8 @@ if [[ -n "${REPORT_DATE:-}" ]]; then fi script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -exec python3 "${script_directory}/sync_telemetry_error_issues.py" "${arguments[@]}" +repository_root="$(cd "${script_directory}/../.." && pwd)" +# Run in an environment that holds only the telemetry-error-issues dependency +# group, at the versions uv.lock pins, not the API's own dependencies. +exec uv run --frozen --only-group telemetry-error-issues --project "${repository_root}" \ + python "${script_directory}/sync_telemetry_error_issues.py" "${arguments[@]}" diff --git a/.github/scripts/telemetry_error_issues/bigquery.py b/.github/scripts/telemetry_error_issues/bigquery.py index 18775a785..35e0d25e0 100644 --- a/.github/scripts/telemetry_error_issues/bigquery.py +++ b/.github/scripts/telemetry_error_issues/bigquery.py @@ -1,13 +1,21 @@ -"""Query error records and request counts through the BigQuery REST API.""" +"""Query error records and request counts with the BigQuery client library. + +The library starts each job, waits for it, reads every page of rows over the +REST API and converts each value to its Python type. Rows are read by +iterating the job's result, never with ``to_dataframe`` or ``to_arrow``, +which may use the BigQuery Storage Read API: the workflow's service account +has no role that can open a read session. +""" from __future__ import annotations -import json -from datetime import UTC, datetime, timedelta +from datetime import datetime from http import HTTPStatus -from typing import TYPE_CHECKING, Any -from urllib.parse import quote, urlencode +from typing import TYPE_CHECKING +from google.api_core.exceptions import GoogleAPICallError +from google.cloud import bigquery +from google.oauth2.credentials import Credentials from telemetry_error_issues.groups import RequestCount from telemetry_error_issues.log_source import ( DATASET_PATTERN, @@ -16,15 +24,10 @@ ) from telemetry_error_issues.normalization import MISSING from telemetry_error_issues.records import ErrorRecord -from telemetry_error_issues.transport import ( - ApiError, - HttpRequest, - HttpResponse, - urllib_transport, -) +from telemetry_error_issues.transport import ERROR_REASON_PATTERN, ApiError if TYPE_CHECKING: - from collections.abc import Callable, Mapping + from collections.abc import Callable, Mapping, Sequence from telemetry_error_issues.log_source import LogSource from telemetry_error_issues.window import ReportingWindow @@ -54,11 +57,10 @@ SINGLE_QUOTED_PATTERN = r"(^|[^A-Za-z0-9])'[^']*'" TEMPLATE_WORD_PATTERN = r"\b[A-Za-z]+\b" -BIGQUERY_API = "https://bigquery.googleapis.com/bigquery/v2" MAXIMUM_BYTES_BILLED = 5 * 1024**3 -QUERY_WAIT_MILLISECONDS = 30_000 -MAX_QUERY_POLLS = 40 -UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=UTC) +# How long to wait for a query job to finish: 41 waits of 30 seconds, the +# limit the report had when it polled the REST API itself. +QUERY_TIMEOUT_SECONDS = 41 * 30 HEALTH_CHECK_ROUTES = ( "/health", @@ -163,135 +165,86 @@ class BigQueryError(ApiError): class BigQueryTimeoutError(BigQueryError): - """A BigQuery query job didn't complete within the polling limit.""" + """A BigQuery query job didn't complete within the time limit.""" -def _bigquery_boolean(text: str) -> bool: - return text == "true" +QueryParameter = bigquery.ScalarQueryParameter | bigquery.ArrayQueryParameter -def _bigquery_timestamp(text: str) -> datetime: - """Convert microseconds since the epoch, as ``useInt64Timestamp`` returns.""" - return UNIX_EPOCH + timedelta(microseconds=int(text)) +def library_client(project_id: str, access_token: str) -> bigquery.Client: + """Return a library client that runs jobs in ``project_id`` as the token's owner. + google-github-actions/auth makes the access token and writes no credentials + file, so the client gets the token itself and never looks for other + credentials. + """ + credentials = Credentials(token=access_token) + return bigquery.Client(project=project_id, credentials=credentials) -CELL_CONVERTERS: dict[str, Callable[[str], object]] = { - "INTEGER": int, - "INT64": int, - "FLOAT": float, - "FLOAT64": float, - "BOOLEAN": _bigquery_boolean, - "BOOL": _bigquery_boolean, - "TIMESTAMP": _bigquery_timestamp, -} +def _error_reason(errors: Sequence[object], status: int) -> str: + """Return the error's reason code, else the HTTP reason phrase. -def _convert_cell(field: Mapping[str, Any], value: object) -> object: - """Convert one BigQuery REST value, which arrives as a string, to Python.""" - if value is None: - return None - if field.get("mode") == "REPEATED" and isinstance(value, list): - item_field = {**field, "mode": "NULLABLE"} - return [_convert_cell(item_field, item["v"]) for item in value] - converter = CELL_CONVERTERS.get(str(field.get("type")), str) - return converter(str(value)) + The library's message can repeat the query and values from the request, + so only a short code-defined reason such as ``invalidQuery`` is kept. + """ + reason = errors[0].get("reason") if errors and isinstance(errors[0], dict) else None + if isinstance(reason, str) and ERROR_REASON_PATTERN.match(reason): + return reason + try: + return HTTPStatus(status).phrase + except ValueError: + return "unknown" -def _convert_rows(response: Mapping[str, Any]) -> list[dict[str, object]]: - fields = response["schema"]["fields"] - return [ - { - field["name"]: _convert_cell(field, cell["v"]) - for field, cell in zip(fields, row["f"], strict=True) - } - for row in response.get("rows", []) - ] +def _bigquery_error(error: GoogleAPICallError) -> BigQueryError: + """Describe a library error by its HTTP status and reason, never its message.""" + if error.code is None: + return BigQueryError("BigQuery request failed") + status = int(error.code) + reason = _error_reason(error.errors, status) + message = f"BigQuery request failed: HTTP {status} {reason}" + return BigQueryError(message, status=status, reason=reason) class BigQueryClient: - """Runs parameterized standard SQL queries through the BigQuery REST API.""" + """Runs parameterized standard SQL queries against the linked log dataset.""" def __init__( self, log_source: LogSource, access_token: str, - transport: Callable[[HttpRequest], HttpResponse] = urllib_transport, + client_factory: Callable[[str, str], bigquery.Client] = library_client, ) -> None: """Run jobs in the log source's project against its linked dataset.""" self.log_source = log_source - self._access_token = access_token - self._transport = transport - project_id = quote(log_source.project_id, safe="") - self._queries_url = f"{BIGQUERY_API}/projects/{project_id}/queries" + self._client = client_factory(log_source.project_id, access_token) def query( - self, sql: str, parameters: list[dict[str, Any]], label: str + self, sql: str, parameters: list[QueryParameter], label: str ) -> list[dict[str, object]]: - """Run ``sql`` and return every row as a dictionary of converted values.""" - body = { - "query": sql, - "useLegacySql": False, - "parameterMode": "NAMED", - "queryParameters": parameters, - "defaultDataset": { - "projectId": self.log_source.project_id, - "datasetId": checked_identifier( - self.log_source.dataset, DATASET_PATTERN, "the log dataset" - ), - }, - "maximumBytesBilled": str(MAXIMUM_BYTES_BILLED), - "labels": {"purpose": "telemetry-error-issues", "query": label}, - "timeoutMs": QUERY_WAIT_MILLISECONDS, - "formatOptions": {"useInt64Timestamp": True}, - } - response = self._send("POST", self._queries_url, body) - job = response["jobReference"] - for _ in range(MAX_QUERY_POLLS): - if response.get("jobComplete"): - break - response = self._results(job, page_token=None) - else: + """Run ``sql`` and return every row as a dictionary of its values.""" + dataset = checked_identifier( + self.log_source.dataset, DATASET_PATTERN, "the log dataset" + ) + job_config = bigquery.QueryJobConfig( + query_parameters=parameters, + default_dataset=bigquery.DatasetReference( + self.log_source.project_id, dataset + ), + maximum_bytes_billed=MAXIMUM_BYTES_BILLED, + use_legacy_sql=False, + labels={"purpose": "telemetry-error-issues", "query": label}, + ) + try: + job = self._client.query(sql, job_config=job_config) + rows = job.result(timeout=QUERY_TIMEOUT_SECONDS) + return [dict(row.items()) for row in rows] + except GoogleAPICallError as error: + raise _bigquery_error(error) from error + except TimeoutError as error: msg = "BigQuery query did not complete in time" - raise BigQueryTimeoutError(msg) - rows = _convert_rows(response) - while response.get("pageToken"): - response = self._results(job, page_token=response["pageToken"]) - rows.extend(_convert_rows(response)) - return rows - - def _results( - self, job: Mapping[str, str], page_token: str | None - ) -> dict[str, Any]: - parameters = { - "location": job["location"], - "timeoutMs": str(QUERY_WAIT_MILLISECONDS), - "formatOptions.useInt64Timestamp": "true", - } - if page_token: - parameters["pageToken"] = page_token - job_id = quote(job["jobId"], safe="") - url = f"{self._queries_url}/{job_id}?{urlencode(parameters)}" - return self._send("GET", url, None) - - def _send( - self, method: str, url: str, body: Mapping[str, object] | None - ) -> dict[str, Any]: - headers = {"Authorization": f"Bearer {self._access_token}"} - if body is not None: - headers["Content-Type"] = "application/json" - encoded = None if body is None else json.dumps(body).encode() - response = self._transport(HttpRequest(method, url, headers, encoded)) - if response.status >= HTTPStatus.BAD_REQUEST: - raise BigQueryError.from_response(response) - return json.loads(response.body) - - -def _timestamp_parameter(name: str, moment: datetime) -> dict[str, Any]: - return { - "name": name, - "parameterType": {"type": "TIMESTAMP"}, - "parameterValue": {"value": moment.astimezone(UTC).isoformat(sep=" ")}, - } + raise BigQueryTimeoutError(msg) from error def _query_text(template: str, log_source: LogSource) -> str: @@ -302,26 +255,19 @@ def _query_text(template: str, log_source: LogSource) -> str: def _query_parameters( timestamps: Mapping[str, datetime], strings: Mapping[str, str] -) -> list[dict[str, Any]]: - parameters = [ - _timestamp_parameter(name, value) for name, value in timestamps.items() +) -> list[QueryParameter]: + parameters: list[QueryParameter] = [ + bigquery.ScalarQueryParameter(name, "TIMESTAMP", value) + for name, value in timestamps.items() ] parameters.extend( - { - "name": name, - "parameterType": {"type": "STRING"}, - "parameterValue": {"value": value}, - } + bigquery.ScalarQueryParameter(name, "STRING", value) for name, value in strings.items() ) parameters.append( - { - "name": "excluded_routes", - "parameterType": {"type": "ARRAY", "arrayType": {"type": "STRING"}}, - "parameterValue": { - "arrayValues": [{"value": route} for route in HEALTH_CHECK_ROUTES] - }, - } + bigquery.ArrayQueryParameter( + "excluded_routes", "STRING", list(HEALTH_CHECK_ROUTES) + ) ) return parameters diff --git a/.github/scripts/telemetry_error_issues/cli.py b/.github/scripts/telemetry_error_issues/cli.py index a9f8bb0a1..eb6529444 100644 --- a/.github/scripts/telemetry_error_issues/cli.py +++ b/.github/scripts/telemetry_error_issues/cli.py @@ -23,6 +23,7 @@ BigQueryClient, fetch_error_records, fetch_request_counts, + library_client, ) from telemetry_error_issues.github_client import GitHubClient from telemetry_error_issues.groups import build_groups @@ -55,6 +56,8 @@ from collections.abc import Callable, Mapping from typing import TextIO + from google.cloud import bigquery + StageResult = TypeVar("StageResult") # The repository variables that name the logs the run reads, each with the @@ -132,7 +135,7 @@ class Runtime: environ: Mapping[str, str] stdout: TextIO stderr: TextIO - bigquery_transport: Callable[[HttpRequest], HttpResponse] = urllib_transport + bigquery_client_factory: Callable[[str, str], bigquery.Client] = library_client github_transport: Callable[[HttpRequest], HttpResponse] = urllib_transport now: Callable[[], datetime] = _current_time sleep: Callable[[float], None] = time.sleep @@ -259,14 +262,19 @@ def _report(arguments: argparse.Namespace, runtime: Runtime) -> str: window = _stage( "configuration", lambda: _report_window(arguments.report_date, runtime.now()) ) - bigquery = BigQueryClient( - settings.log_source, settings.google_access_token, runtime.bigquery_transport + bigquery_client = _stage( + "configuration", + lambda: BigQueryClient( + settings.log_source, + settings.google_access_token, + runtime.bigquery_client_factory, + ), ) records = _stage( - "query error records", lambda: fetch_error_records(bigquery, window) + "query error records", lambda: fetch_error_records(bigquery_client, window) ) request_counts = _stage( - "query request counts", lambda: fetch_request_counts(bigquery, window) + "query request counts", lambda: fetch_request_counts(bigquery_client, window) ) groups = _stage( "group errors", diff --git a/.github/workflows/telemetry-error-issues.yml b/.github/workflows/telemetry-error-issues.yml index fa0fd1c26..f2f549634 100644 --- a/.github/workflows/telemetry-error-issues.yml +++ b/.github/workflows/telemetry-error-issues.yml @@ -36,6 +36,10 @@ jobs: contents: read issues: write id-token: write + env: + # uv installs uv.lock on the Cloud Run image's Python, not the 3.14 + # that .python-version selects. + UV_PYTHON: "3.12" steps: - name: Require master branch if: github.ref != 'refs/heads/master' @@ -46,6 +50,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" + # The same uv version as the Cloud Run image and the release job. + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.13.4" - name: Authenticate to Google Cloud id: auth uses: google-github-actions/auth@v2 diff --git a/Makefile b/Makefile index 17743258d..7b13feb2c 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test-env-vars: uv run --frozen --extra dev pytest tests/env_variables test: - MAX_HOUSEHOLDS=1000 uv run --frozen --extra dev coverage run -a --branch -m pytest tests/to_refactor tests/unit tests/contract tests/integration/test_budget_window_in_flight_dedupe.py --disable-pytest-warnings + MAX_HOUSEHOLDS=1000 uv run --frozen --extra dev --group telemetry-error-issues coverage run -a --branch -m pytest tests/to_refactor tests/unit tests/contract tests/integration/test_budget_window_in_flight_dedupe.py --disable-pytest-warnings uv run --frozen --extra dev coverage xml -i quality-guards: @@ -31,7 +31,7 @@ typecheck-v2: uv run --frozen --extra dev mypy debug-test: - MAX_HOUSEHOLDS=1000 FLASK_DEBUG=1 uv run --frozen --extra dev pytest -vv --durations=0 tests + MAX_HOUSEHOLDS=1000 FLASK_DEBUG=1 uv run --frozen --extra dev --group telemetry-error-issues pytest -vv --durations=0 tests format: ruff format . diff --git a/gcp/observability/README.md b/gcp/observability/README.md index 19e081b35..bd6d060fa 100644 --- a/gcp/observability/README.md +++ b/gcp/observability/README.md @@ -195,10 +195,18 @@ OBSERVABILITY_LOG_BUCKET_LOCATION=global \ OBSERVABILITY_LOG_DATASET=observability_logs \ OBSERVABILITY_LOG_VIEW=_AllLogs \ OBSERVABILITY_SERVICE_NAMESPACE=policyengine.api-v1 \ -python3 .github/scripts/sync_telemetry_error_issues.py \ +uv run --frozen --only-group telemetry-error-issues \ + python .github/scripts/sync_telemetry_error_issues.py \ --project-id policyengine-observability --dry-run ``` +The workflow runs the script the same way, through +`.github/scripts/sync_telemetry_error_issues.sh`: in an environment holding +only the `telemetry-error-issues` dependency group from `pyproject.toml`, +which is `google-cloud-bigquery` at the version `uv.lock` pins. The queries +read rows by iterating each job's result over the BigQuery REST API, never +through the Storage Read API, which the service account has no role for. + ### Required infrastructure | Resource | Purpose | diff --git a/pyproject.toml b/pyproject.toml index 9805eaf0f..ff73d1fde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,13 @@ dev = [ "towncrier>=24.8.0", ] +[dependency-groups] +# Only the daily telemetry error issues workflow needs these. It installs this +# group alone (uv run --frozen --only-group), not the API's dependencies. +telemetry-error-issues = [ + "google-cloud-bigquery>=3.30,<4", +] + [tool.hatch.build.targets.wheel] packages = ["policyengine_api"] diff --git a/tests/unit/telemetry_error_issues/bigquery_support.py b/tests/unit/telemetry_error_issues/bigquery_support.py index beb1a784b..dc8ae8714 100644 --- a/tests/unit/telemetry_error_issues/bigquery_support.py +++ b/tests/unit/telemetry_error_issues/bigquery_support.py @@ -1,66 +1,68 @@ -"""A fake HTTP transport and BigQuery REST API responses.""" +"""A fake BigQuery client library client, with rows as the library returns them.""" from __future__ import annotations -import json - -from telemetry_error_issues.transport import HttpResponse +from google.cloud.bigquery.table import Row ACCESS_TOKEN = "fake-access-token" -class FakeTransport: - """Records each HTTP request and answers with queued responses.""" - - def __init__(self, responses: list[object]) -> None: - self.responses = list(responses) - self.requests: list[object] = [] - - def __call__(self, request: object) -> object: - self.requests.append(request) - return self.responses.pop(0) - - def json_body(self, index: int) -> dict: - return json.loads(self.requests[index].body) - - -def json_response(payload: object, status: int = 200, reason: str = "OK") -> object: - return HttpResponse( - status=status, reason=reason, headers={}, body=json.dumps(payload).encode() - ) - - -def _cell(value: object) -> dict: - if isinstance(value, list): - return {"v": [{"v": item} for item in value]} - return {"v": value} - - -def query_response( - fields: list[dict], - rows: list[list[object]], - *, - page_token: str | None = None, - job_complete: bool = True, -) -> dict: - response: dict[str, object] = { - "jobComplete": job_complete, - "jobReference": { - "projectId": "policyengine-observability", - "jobId": "job-1", - "location": "US", - }, - } - if job_complete: - response["schema"] = {"fields": fields} - response["rows"] = [{"f": [_cell(value) for value in row]} for row in rows] - if page_token: - response["pageToken"] = page_token - return response - - -def sent_parameters(transport: FakeTransport, index: int = 0) -> dict: - return { - parameter["name"]: parameter - for parameter in transport.json_body(index)["queryParameters"] - } +def library_rows(rows: list[dict[str, object]]) -> list[Row]: + """Return each dictionary as a ``Row``, the type the client library yields.""" + return [ + Row(tuple(row.values()), {name: index for index, name in enumerate(row)}) + for row in rows + ] + + +class FakeQueryJob: + """A finished query job: its result yields the rows, or raises the error.""" + + def __init__( + self, client: FakeBigQueryClient, rows: list[Row], error: Exception | None + ) -> None: + self.client = client + self.rows = rows + self.error = error + + def result(self, timeout: float | None = None) -> list[Row]: + self.client.result_timeouts.append(timeout) + if self.error is not None: + raise self.error + return self.rows + + +class FakeBigQueryClient: + """Stands in for ``bigquery.Client``, recording each query and its job config. + + Each query returns the rows given for its ``query`` label, or raises + ``error`` when the test reads its result, as a failed job does. + """ + + def __init__( + self, + rows_by_label: dict[str, list[dict[str, object]]] | None = None, + *, + error: Exception | None = None, + ) -> None: + self.rows_by_label = rows_by_label or {} + self.error = error + self.created_with: list[tuple[str, str]] = [] + self.queries: list[tuple[str, object]] = [] + self.result_timeouts: list[float | None] = [] + + def factory(self, project_id: str, access_token: str) -> FakeBigQueryClient: + """Replace ``library_client``: record the arguments and return this fake.""" + self.created_with.append((project_id, access_token)) + return self + + def query(self, sql: str, job_config: object) -> FakeQueryJob: + self.queries.append((sql, job_config)) + rows = self.rows_by_label.get(job_config.labels["query"], []) + return FakeQueryJob(self, library_rows(rows), self.error) + + +def sent_parameters(client: FakeBigQueryClient, index: int = 0) -> dict[str, object]: + """Return the query parameters of the ``index``-th query, by name.""" + _, job_config = client.queries[index] + return {parameter.name: parameter for parameter in job_config.query_parameters} diff --git a/tests/unit/telemetry_error_issues/test_bigquery.py b/tests/unit/telemetry_error_issues/test_bigquery.py index 2c43c7b6a..5a0fde716 100644 --- a/tests/unit/telemetry_error_issues/test_bigquery.py +++ b/tests/unit/telemetry_error_issues/test_bigquery.py @@ -4,31 +4,32 @@ import re from datetime import UTC, datetime -from urllib.parse import parse_qs, urlparse import pytest +from google.api_core.exceptions import BadRequest, Forbidden, ServiceUnavailable +from google.cloud import bigquery from telemetry_error_issues.bigquery import ( CHAINED_EXCEPTION_PATTERN, DOUBLE_QUOTED_PATTERN, ERROR_RECORDS_QUERY, FRAME_PATTERN, + QUERY_TIMEOUT_SECONDS, REQUEST_COUNTS_QUERY, SINGLE_QUOTED_PATTERN, TEMPLATE_WORD_PATTERN, BigQueryClient, BigQueryError, + BigQueryTimeoutError, error_record_from_row, fetch_error_records, fetch_request_counts, + library_client, ) -from telemetry_error_issues.transport import HttpResponse from telemetry_error_issues.window import reporting_window from tests.unit.telemetry_error_issues.bigquery_support import ( ACCESS_TOKEN, - FakeTransport, - json_response, - query_response, + FakeBigQueryClient, sent_parameters, ) from tests.unit.telemetry_error_issues.report_support import ( @@ -40,80 +41,69 @@ request_count, ) -STRING_FIELD = {"name": "name", "type": "STRING", "mode": "NULLABLE"} +def _client(fake: FakeBigQueryClient, log_source: object = LOG_SOURCE) -> object: + return BigQueryClient(log_source, ACCESS_TOKEN, fake.factory) -def _client(transport: FakeTransport, log_source: object = LOG_SOURCE) -> object: - return BigQueryClient(log_source, ACCESS_TOKEN, transport) + +def _string_parameter(name: str, value: str) -> bigquery.ScalarQueryParameter: + return bigquery.ScalarQueryParameter(name, "STRING", value) def test_bigquery_error_query_request_limits_cost_and_names_its_parameters() -> None: - transport = FakeTransport([json_response(query_response([], []))]) - window = reporting_window(REPORT_DATE) - - fetch_error_records(_client(transport), window) - - request = transport.requests[0] - body = transport.json_body(0) - parameters = sent_parameters(transport) - assert request.method == "POST" - assert request.url == ( - "https://bigquery.googleapis.com/bigquery/v2/projects/" - "policyengine-observability/queries" - ) - assert request.headers["Authorization"] == f"Bearer {ACCESS_TOKEN}" - assert body["useLegacySql"] is False - assert body["parameterMode"] == "NAMED" - assert body["maximumBytesBilled"] == str(5 * 1024**3) - assert body["labels"] == { + fake = FakeBigQueryClient() + + fetch_error_records(_client(fake), reporting_window(REPORT_DATE)) + + sql, job_config = fake.queries[0] + parameters = sent_parameters(fake) + assert fake.created_with == [("policyengine-observability", ACCESS_TOKEN)] + assert job_config.use_legacy_sql is False + assert job_config.maximum_bytes_billed == 5 * 1024**3 + assert job_config.labels == { "purpose": "telemetry-error-issues", "query": "error-records", } - assert body["defaultDataset"] == { - "projectId": "policyengine-observability", - "datasetId": "observability_logs", - } - assert body["formatOptions"] == {"useInt64Timestamp": True} - assert "FROM `_AllLogs`" in body["query"] - assert parameters["service_namespace"]["parameterValue"]["value"] == ( - "policyengine.api-v1" + assert job_config.default_dataset == bigquery.DatasetReference( + "policyengine-observability", "observability_logs" ) - assert parameters["query_start"]["parameterType"] == {"type": "TIMESTAMP"} - assert parameters["query_start"]["parameterValue"]["value"] == ( - "2026-10-13 04:00:00+00:00" + assert "FROM `_AllLogs`" in sql + assert parameters["query_start"] == bigquery.ScalarQueryParameter( + "query_start", "TIMESTAMP", datetime(2026, 10, 13, 4, 0, tzinfo=UTC) ) - assert parameters["window_end"]["parameterValue"]["value"] == ( - "2026-10-21 04:00:00+00:00" + assert parameters["window_end"] == bigquery.ScalarQueryParameter( + "window_end", "TIMESTAMP", datetime(2026, 10, 21, 4, 0, tzinfo=UTC) ) - assert parameters["frame_pattern"]["parameterValue"]["value"] == (FRAME_PATTERN) - assert parameters["chained_exception_pattern"]["parameterValue"]["value"] == ( - CHAINED_EXCEPTION_PATTERN + assert parameters["service_namespace"] == _string_parameter( + "service_namespace", "policyengine.api-v1" ) for name, pattern in ( + ("frame_pattern", FRAME_PATTERN), + ("chained_exception_pattern", CHAINED_EXCEPTION_PATTERN), ("double_quoted_pattern", DOUBLE_QUOTED_PATTERN), ("single_quoted_pattern", SINGLE_QUOTED_PATTERN), ("template_word_pattern", TEMPLATE_WORD_PATTERN), ): - assert parameters[name]["parameterValue"]["value"] == pattern - assert [ - item["value"] - for item in parameters["excluded_routes"]["parameterValue"]["arrayValues"] - ] == ["/health", "/simulation-gateway-check", "/liveness-check", "/readiness-check"] + assert parameters[name] == _string_parameter(name, pattern) + assert parameters["excluded_routes"] == bigquery.ArrayQueryParameter( + "excluded_routes", + "STRING", + ["/health", "/simulation-gateway-check", "/liveness-check", "/readiness-check"], + ) def test_bigquery_queries_name_the_configured_dataset_and_view() -> None: - transport = FakeTransport([json_response(query_response([], []))] * 2) + fake = FakeBigQueryClient() log_source = LOG_SOURCE._replace(dataset="linked_logs", view="_Requests") - client = _client(transport, log_source) + client = _client(fake, log_source) fetch_error_records(client, reporting_window(REPORT_DATE)) fetch_request_counts(client, reporting_window(REPORT_DATE)) - for index in range(2): - body = transport.json_body(index) - assert body["defaultDataset"]["datasetId"] == "linked_logs" - assert "FROM `_Requests`" in body["query"] - assert "_AllLogs" not in body["query"] + for sql, job_config in fake.queries: + assert job_config.default_dataset.dataset_id == "linked_logs" + assert "FROM `_Requests`" in sql + assert "_AllLogs" not in sql @pytest.mark.parametrize( @@ -129,13 +119,13 @@ def test_bigquery_queries_name_the_configured_dataset_and_view() -> None: def test_bigquery_refuses_a_dataset_or_view_that_is_not_an_identifier( unsafe_names: dict[str, str], ) -> None: - transport = FakeTransport([json_response(query_response([], []))]) - client = _client(transport, LOG_SOURCE._replace(**unsafe_names)) + fake = FakeBigQueryClient() + client = _client(fake, LOG_SOURCE._replace(**unsafe_names)) with pytest.raises(ValueError, match="is not a valid name"): fetch_error_records(client, reporting_window(REPORT_DATE)) - assert transport.requests == [] + assert fake.queries == [] def test_bigquery_error_query_filters_application_error_records() -> None: @@ -244,126 +234,86 @@ def test_bigquery_error_query_hashes_the_trace() -> None: def test_bigquery_request_count_query_counts_completed_requests_in_the_window() -> None: sql = REQUEST_COUNTS_QUERY - transport = FakeTransport([json_response(query_response([], []))]) + fake = FakeBigQueryClient() - fetch_request_counts(_client(transport), reporting_window(REPORT_DATE)) + fetch_request_counts(_client(fake), reporting_window(REPORT_DATE)) - parameters = sent_parameters(transport) + sent_sql, job_config = fake.queries[0] + parameters = sent_parameters(fake) assert "JSON_VALUE(json_payload, '$.\"event.name\"') = 'request.completed'" in sql assert "COUNT(*) AS requests" in sql assert "GROUP BY service_name, environment_name, route" in sql assert "timestamp >= @window_start AND timestamp < @window_end" in sql - assert parameters["window_start"]["parameterValue"]["value"] == ( - "2026-10-20 04:00:00+00:00" + assert parameters["window_start"] == bigquery.ScalarQueryParameter( + "window_start", "TIMESTAMP", datetime(2026, 10, 20, 4, 0, tzinfo=UTC) ) - assert parameters["service_namespace"]["parameterValue"]["value"] == ( - "policyengine.api-v1" + assert parameters["service_namespace"] == _string_parameter( + "service_namespace", "policyengine.api-v1" ) - assert "FROM `_AllLogs`" in transport.json_body(0)["query"] - assert transport.json_body(0)["labels"]["query"] == "request-counts" + assert "FROM `_AllLogs`" in sent_sql + assert job_config.labels["query"] == "request-counts" -def test_bigquery_client_polls_until_the_job_completes() -> None: - transport = FakeTransport( - [ - json_response(query_response([], [], job_complete=False)), - json_response(query_response([], [], job_complete=False)), - json_response(query_response([STRING_FIELD], [["done"]])), - ], - ) +def test_bigquery_client_reads_every_row_the_job_yields() -> None: + rows = [{"name": "first", "count": 1}, {"name": "second", "count": 2}] + fake = FakeBigQueryClient({"test": rows}) - rows = _client(transport).query("SELECT 1", [], "test") + result = _client(fake).query("SELECT 1", [], "test") - assert rows == [{"name": "done"}] - poll = urlparse(transport.requests[1].url) - assert transport.requests[1].method == "GET" - assert poll.path.endswith("/projects/policyengine-observability/queries/job-1") - assert parse_qs(poll.query)["location"] == ["US"] - assert parse_qs(poll.query)["formatOptions.useInt64Timestamp"] == ["true"] + assert result == rows + # 41 waits of 30 seconds, the limit of the REST client this replaced. + assert fake.result_timeouts == [QUERY_TIMEOUT_SECONDS] == [1_230] -def test_bigquery_client_follows_page_tokens() -> None: - transport = FakeTransport( - [ - json_response( - query_response([STRING_FIELD], [["first"]], page_token="page-2"), - ), - json_response(query_response([STRING_FIELD], [["second"]])), - ], - ) +def test_bigquery_client_reports_a_job_that_does_not_finish_in_time() -> None: + fake = FakeBigQueryClient(error=TimeoutError()) - rows = _client(transport).query("SELECT 1", [], "test") + with pytest.raises(BigQueryTimeoutError, match="did not complete"): + _client(fake).query("SELECT 1", [], "test") - assert rows == [{"name": "first"}, {"name": "second"}] - page = parse_qs(urlparse(transport.requests[1].url).query) - assert page["pageToken"] == ["page-2"] - assert page["location"] == ["US"] +def test_bigquery_library_error_reports_only_the_status_and_reason() -> None: + error = BadRequest( + "SENTINEL value echoed from a row", + errors=[{"reason": "invalidQuery", "message": "SENTINEL detail"}], + ) + fake = FakeBigQueryClient(error=error) -def test_bigquery_client_gives_up_on_a_job_that_never_completes() -> None: - pending = json_response(query_response([], [], job_complete=False)) - transport = FakeTransport([pending] * 200) + with pytest.raises(BigQueryError) as raised: + _client(fake).query("SELECT 1", [], "test") - with pytest.raises(BigQueryError, match="did not complete"): - _client(transport).query("SELECT 1", [], "test") + assert str(raised.value) == "BigQuery request failed: HTTP 400 invalidQuery" + assert raised.value.status == 400 + assert raised.value.reason == "invalidQuery" -def test_bigquery_client_converts_row_values_from_strings() -> None: - fields = [ - {"name": "text", "type": "STRING", "mode": "NULLABLE"}, - {"name": "count", "type": "INTEGER", "mode": "NULLABLE"}, - {"name": "flag", "type": "BOOLEAN", "mode": "NULLABLE"}, - {"name": "moment", "type": "TIMESTAMP", "mode": "NULLABLE"}, - {"name": "items", "type": "STRING", "mode": "REPEATED"}, - {"name": "missing", "type": "INTEGER", "mode": "NULLABLE"}, - ] - row = ["value", "42", "true", "1792512000000001", ["a", "b"], None] - transport = FakeTransport([json_response(query_response(fields, [row]))]) +def test_bigquery_library_error_without_a_reason_code_uses_the_status_phrase() -> None: + fake = FakeBigQueryClient(error=ServiceUnavailable("SENTINEL ")) - rows = _client(transport).query("SELECT 1", [], "test") + with pytest.raises(BigQueryError) as raised: + _client(fake).query("SELECT 1", [], "test") - assert rows == [ - { - "text": "value", - "count": 42, - "flag": True, - "moment": datetime(2026, 10, 20, 16, 0, 0, 1, tzinfo=UTC), - "items": ["a", "b"], - "missing": None, - } - ] + assert str(raised.value) == "BigQuery request failed: HTTP 503 Service Unavailable" -def test_bigquery_http_error_reports_only_the_status_and_reason() -> None: - error_body = { - "error": { - "code": 400, - "message": "SENTINEL value echoed from a row", - "errors": [{"reason": "invalidQuery", "message": "SENTINEL detail"}], - } - } - transport = FakeTransport( - [json_response(error_body, status=400, reason="Bad Request")], - ) +@pytest.mark.parametrize("reason", ["SENTINEL: 'value'", "a" * 61, 7]) +def test_bigquery_library_error_hides_a_reason_that_is_not_a_short_code( + reason: object, +) -> None: + fake = FakeBigQueryClient(error=Forbidden("SENTINEL", errors=[{"reason": reason}])) with pytest.raises(BigQueryError) as raised: - _client(transport).query("SELECT 1", [], "test") + _client(fake).query("SELECT 1", [], "test") - assert str(raised.value) == "BigQuery request failed: HTTP 400 invalidQuery" - assert raised.value.status == 400 - assert raised.value.reason == "invalidQuery" + assert str(raised.value) == "BigQuery request failed: HTTP 403 Forbidden" -def test_bigquery_http_error_without_a_json_reason_uses_the_status_phrase() -> None: - response = HttpResponse( - status=503, reason="Service Unavailable", headers={}, body=b"SENTINEL " - ) - transport = FakeTransport([response]) +def test_bigquery_library_client_sends_the_access_token_for_the_project() -> None: + client = library_client("policyengine-observability", ACCESS_TOKEN) - with pytest.raises(BigQueryError) as raised: - _client(transport).query("SELECT 1", [], "test") - - assert str(raised.value) == "BigQuery request failed: HTTP 503 Service Unavailable" + assert isinstance(client, bigquery.Client) + assert client.project == "policyengine-observability" + assert client._credentials.token == ACCESS_TOKEN def test_bigquery_rows_become_error_records() -> None: @@ -431,23 +381,9 @@ def test_bigquery_rows_become_request_counts() -> None: "requests": 120, } ] - transport = FakeTransport( - [ - json_response( - query_response( - [ - {"name": "service_name", "type": "STRING"}, - {"name": "environment_name", "type": "STRING"}, - {"name": "route", "type": "STRING"}, - {"name": "requests", "type": "INTEGER"}, - ], - [list(map(str, rows[0].values()))], - ), - ) - ], - ) + fake = FakeBigQueryClient({"request-counts": rows}) - counts = fetch_request_counts(_client(transport), reporting_window(REPORT_DATE)) + counts = fetch_request_counts(_client(fake), reporting_window(REPORT_DATE)) assert counts == [ request_count("//report", 120), diff --git a/tests/unit/telemetry_error_issues/test_end_to_end.py b/tests/unit/telemetry_error_issues/test_end_to_end.py index c7fbaab8e..b9d0dff98 100644 --- a/tests/unit/telemetry_error_issues/test_end_to_end.py +++ b/tests/unit/telemetry_error_issues/test_end_to_end.py @@ -3,20 +3,18 @@ from __future__ import annotations import io -import json -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from typing import TYPE_CHECKING from urllib.parse import unquote import pytest +from google.api_core.exceptions import BadRequest, Forbidden, InternalServerError from telemetry_error_issues.cli import Runtime, main from telemetry_error_issues.transport import HttpResponse from tests.unit.telemetry_error_issues.bigquery_support import ( ACCESS_TOKEN, - FakeTransport, - json_response, - query_response, + FakeBigQueryClient, sent_parameters, ) from tests.unit.telemetry_error_issues.github_support import REPOSITORY, FakeGitHub @@ -33,48 +31,40 @@ if TYPE_CHECKING: from pathlib import Path -ERROR_RECORD_FIELDS = [ - {"name": "timestamp", "type": "TIMESTAMP"}, - {"name": "service_name", "type": "STRING"}, - {"name": "environment_name", "type": "STRING"}, - {"name": "event_name", "type": "STRING"}, - {"name": "error_type", "type": "STRING"}, - {"name": "error_handled", "type": "STRING"}, - {"name": "frames", "type": "STRING", "mode": "REPEATED"}, - {"name": "stack_truncated", "type": "BOOLEAN"}, - {"name": "message_template", "type": "STRING"}, - {"name": "attribute_error_type", "type": "STRING"}, - {"name": "failure_category", "type": "STRING"}, - {"name": "route", "type": "STRING"}, - {"name": "status_code", "type": "STRING"}, - {"name": "operation_name", "type": "STRING"}, - {"name": "request_key", "type": "STRING"}, - {"name": "observability_key", "type": "STRING"}, - {"name": "trace_key", "type": "STRING"}, -] -REQUEST_COUNT_FIELDS = [ - {"name": "service_name", "type": "STRING"}, - {"name": "environment_name", "type": "STRING"}, - {"name": "route", "type": "STRING"}, - {"name": "requests", "type": "INTEGER"}, -] +ERROR_RECORD_COLUMNS = ( + "timestamp", + "service_name", + "environment_name", + "event_name", + "error_type", + "error_handled", + "frames", + "stack_truncated", + "message_template", + "attribute_error_type", + "failure_category", + "route", + "status_code", + "operation_name", + "request_key", + "observability_key", + "trace_key", +) +REQUEST_COUNT_COLUMNS = ("service_name", "environment_name", "route", "requests") SCHEDULED_RUN_TIME = datetime(2026, 10, 21, 0, 45, tzinfo=EASTERN) -def _microseconds(moment: datetime) -> str: - return str((moment - datetime(1970, 1, 1, tzinfo=UTC)) // timedelta(microseconds=1)) - - -def _error_row(**overrides: object) -> list[object]: +def _error_row(**overrides: object) -> dict[str, object]: + """Return an error-record row with the types the client library gives it.""" values: dict[str, object] = { - "timestamp": _microseconds(WINDOW_NOON), + "timestamp": WINDOW_NOON, "service_name": "policyengine-api", "environment_name": "production", "event_name": "exception.recorded", "error_type": "StaleDataError", "error_handled": "false", "frames": list(APPLICATION_FRAMES), - "stack_truncated": "false", + "stack_truncated": False, "message_template": None, "attribute_error_type": None, "failure_category": None, @@ -86,32 +76,29 @@ def _error_row(**overrides: object) -> list[object]: "trace_key": "t1", } values.update(overrides) - return [values[field["name"]] for field in ERROR_RECORD_FIELDS] + return {column: values[column] for column in ERROR_RECORD_COLUMNS} -class FakeBigQuery: +class FakeBigQuery(FakeBigQueryClient): """Answers the error-record and request-count queries with fixed rows.""" def __init__( self, - error_rows: list[list[object]], + error_rows: list[dict[str, object]], count_rows: list[list[object]] = (), + *, + error: Exception | None = None, ) -> None: - self.error_rows = error_rows - self.count_rows = list(count_rows) - self.requests: list[object] = [] - - def __call__(self, request: object) -> object: - self.requests.append(request) - label = json.loads(request.body)["labels"]["query"] - if label == "error-records": - response = query_response(ERROR_RECORD_FIELDS, self.error_rows) - else: - response = query_response(REQUEST_COUNT_FIELDS, self.count_rows) - return json_response(response) - - def json_body(self, index: int) -> dict: - return json.loads(self.requests[index].body) + super().__init__( + { + "error-records": error_rows, + "request-counts": [ + dict(zip(REQUEST_COUNT_COLUMNS, row, strict=True)) + for row in count_rows + ], + }, + error=error, + ) def _environment(tmp_path: Path, **overrides: str) -> dict[str, str]: @@ -137,7 +124,7 @@ def _run_main( stdout, stderr = io.StringIO(), io.StringIO() runtime = Runtime( environ=_environment(tmp_path) if environment is None else environment, - bigquery_transport=bigquery, + bigquery_client_factory=bigquery.factory, github_transport=github, now=lambda: SCHEDULED_RUN_TIME, sleep=github.sleep, @@ -149,11 +136,11 @@ def _run_main( return exit_code, stdout.getvalue(), stderr.getvalue() -def _three_failures(**overrides: object) -> list[list[object]]: +def _three_failures(**overrides: object) -> list[dict[str, object]]: return [ _error_row( request_key=f"r{index}", - timestamp=_microseconds(hours_after_window_noon(index)), + timestamp=hours_after_window_noon(index), **overrides, ) for index in range(3) @@ -165,7 +152,7 @@ def test_main_queries_groups_and_opens_issues_for_the_previous_day( ) -> None: bigquery = FakeBigQuery( _three_failures(), - [["policyengine-api", "production", "//report", "600"]], + [["policyengine-api", "production", "//report", 600]], ) github = FakeGitHub() @@ -178,8 +165,8 @@ def test_main_queries_groups_and_opens_issues_for_the_previous_day( "| Error rate on affected routes | 0.50% (3 of 600 requests) | n/a |" in (issue["body"]) ) - window_end = sent_parameters(bigquery)["window_end"]["parameterValue"] - assert window_end["value"] == "2026-10-21 04:00:00+00:00" + window_end = sent_parameters(bigquery)["window_end"] + assert window_end.value == datetime(2026, 10, 21, 4, 0, tzinfo=UTC) summary = (tmp_path / "summary.md").read_text() fingerprint = issue["title"].rsplit("(", 1)[1].rstrip(")") assert ( @@ -217,8 +204,8 @@ def test_main_reports_the_requested_day(tmp_path: Path) -> None: ) assert exit_code == 0 - window_end = sent_parameters(bigquery)["window_end"]["parameterValue"] - assert window_end["value"] == "2026-10-16 04:00:00+00:00" + window_end = sent_parameters(bigquery)["window_end"] + assert window_end.value == datetime(2026, 10, 16, 4, 0, tzinfo=UTC) assert "## Telemetry error issues for 2026-10-15" in stdout assert "No error groups had failures in the window." in stdout @@ -320,7 +307,7 @@ def test_main_refuses_a_log_name_that_is_not_an_identifier( assert stderr == ( f"configuration failed: ConfigurationError ({setting} is not a valid name)\n" ) - assert bigquery.requests == [] + assert bigquery.queries == [] def test_main_refuses_a_project_id_that_is_not_an_identifier(tmp_path: Path) -> None: @@ -334,21 +321,17 @@ def test_main_refuses_a_project_id_that_is_not_an_identifier(tmp_path: Path) -> assert stderr == ( "configuration failed: ConfigurationError (--project-id is not a valid name)\n" ) - assert bigquery.requests == [] + assert bigquery.queries == [] def test_main_reports_an_api_failure_by_stage_class_and_status_only( tmp_path: Path, ) -> None: - error_body = { - "error": { - "message": "SENTINEL echoed value", - "errors": [{"reason": "backendError", "message": "SENTINEL"}], - } - } - failing = FakeTransport( - [json_response(error_body, status=500, reason="SENTINEL")], + error = InternalServerError( + "SENTINEL echoed value", + errors=[{"reason": "backendError", "message": "SENTINEL"}], ) + failing = FakeBigQuery([], error=error) exit_code, stdout, stderr = _run_main(tmp_path, failing, FakeGitHub()) @@ -359,12 +342,43 @@ def test_main_reports_an_api_failure_by_stage_class_and_status_only( ) +@pytest.mark.parametrize( + ("error", "expected"), + [ + ( + BadRequest( + "SENTINEL message about the query", + errors=[{"reason": "invalidQuery", "message": "SENTINEL detail"}], + ), + "query error records failed: BigQueryError (HTTP 400 invalidQuery)\n", + ), + ( + Forbidden( + "SENTINEL access denied", errors=[{"reason": "SENTINEL: 'value'"}] + ), + "query error records failed: BigQueryError (HTTP 403 Forbidden)\n", + ), + ], +) +def test_main_never_prints_a_client_library_error_message( + tmp_path: Path, error: Exception, expected: str +) -> None: + # The library adds the query text to a failed job's error. + error.debug_message = "SENTINEL query text" + bigquery = FakeBigQuery(_three_failures(), error=error) + + exit_code, stdout, stderr = _run_main(tmp_path, bigquery, FakeGitHub()) + + assert exit_code == 1 + assert stdout == "" + assert stderr == expected + assert not (tmp_path / "summary.md").exists() + + def test_main_reports_an_unexpected_exception_by_class_only(tmp_path: Path) -> None: - def broken_transport(request: object) -> object: - message = "SENTINEL connection detail" - raise RuntimeError(message) + broken = FakeBigQuery([], error=RuntimeError("SENTINEL connection detail")) - exit_code, _, stderr = _run_main(tmp_path, broken_transport, FakeGitHub()) + exit_code, _, stderr = _run_main(tmp_path, broken, FakeGitHub()) assert exit_code == 1 assert stderr == "query error records failed: RuntimeError\n" @@ -400,19 +414,19 @@ def test_main_never_publishes_values_from_records( for name, value in sentinel_row(**overrides).items() if name not in {"timestamp", "environment_name", "frames", "stack_truncated"} } - sentinel_values["stack_truncated"] = "true" + sentinel_values["stack_truncated"] = True sentinel_values["frames"] = ['/app/SENTINEL.py", line 1, in SENTINEL'] rows = [ _error_row( **{ **sentinel_values, "request_key": f"SENTINEL request {index}", - "timestamp": _microseconds(hours_after_window_noon(index)), + "timestamp": hours_after_window_noon(index), } ) for index in range(3) ] - count_rows = [["SENTINEL service", "production", "SENTINEL route", "10"]] + count_rows = [["SENTINEL service", "production", "SENTINEL route", 10]] github = FakeGitHub() exit_code, stdout, stderr = _run_main( diff --git a/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py b/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py index b4d4ddd05..b85ca91e2 100644 --- a/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py +++ b/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py @@ -5,6 +5,7 @@ import os import subprocess import sys +import tomllib from pathlib import Path import pytest @@ -20,6 +21,18 @@ SCRIPT_PATH = REPO / ".github" / "scripts" / "sync_telemetry_error_issues.py" WORKFLOW_PATH = REPO / ".github" / "workflows" / "telemetry-error-issues.yml" WRAPPER_PATH = REPO / ".github" / "scripts" / "sync_telemetry_error_issues.sh" +DEPENDENCY_GROUP = "telemetry-error-issues" +# The wrapper runs the script in an environment of only its locked group. +UV_RUN_ARGUMENTS = [ + "run", + "--frozen", + "--only-group", + DEPENDENCY_GROUP, + "--project", + str(REPO), + "python", + str(SCRIPT_PATH), +] WRAPPER_ENVIRONMENT = { "GOOGLE_OAUTH_ACCESS_TOKEN": "fake-access-token", "GITHUB_TOKEN": "fake-github-token", @@ -79,6 +92,25 @@ def test_workflow_job_is_limited_to_master_in_its_environment() -> None: } +def test_workflow_runs_the_report_with_the_pinned_uv_and_python() -> None: + assert _sync_job()["env"] == {"UV_PYTHON": "3.12"} + assert _step("Set up uv") == { + "name": "Set up uv", + "uses": "astral-sh/setup-uv@v6", + "with": {"version": "0.13.4"}, + } + + +def test_wrapper_dependency_group_holds_the_bigquery_client_library() -> None: + pyproject = tomllib.loads((REPO / "pyproject.toml").read_text(encoding="utf-8")) + + group = pyproject["dependency-groups"][DEPENDENCY_GROUP] + + assert [requirement.split(">")[0] for requirement in group] == [ + "google-cloud-bigquery" + ] + + def test_workflow_authenticates_as_the_report_service_account() -> None: auth = _step("Authenticate to Google Cloud") @@ -122,13 +154,13 @@ def test_workflow_passes_settings_to_the_wrapper() -> None: def _run_wrapper( tmp_path: Path, environment: dict[str, str] ) -> tuple[subprocess.CompletedProcess[str], list[str] | None]: - """Run the wrapper with a fake python3 that records its arguments.""" + """Run the wrapper with a fake uv that records its arguments.""" bin_dir = tmp_path / "bin" bin_dir.mkdir() - calls = tmp_path / "python-arguments.txt" - fake_python = bin_dir / "python3" - fake_python.write_text(f'#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "{calls}"\n') - fake_python.chmod(0o755) + calls = tmp_path / "uv-arguments.txt" + fake_uv = bin_dir / "uv" + fake_uv.write_text(f'#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "{calls}"\n') + fake_uv.chmod(0o755) result = subprocess.run( ["bash", str(WRAPPER_PATH)], cwd=REPO, @@ -153,7 +185,7 @@ def test_wrapper_runs_the_report_for_the_project(tmp_path: Path) -> None: result, arguments = _run_wrapper(tmp_path, WRAPPER_ENVIRONMENT) assert result.returncode == 0, result.stderr - assert arguments == [str(SCRIPT_PATH), "--project-id", PROJECT_ID] + assert arguments == [*UV_RUN_ARGUMENTS, "--project-id", PROJECT_ID] def test_wrapper_masks_the_access_token_and_fingerprint_key(tmp_path: Path) -> None: @@ -176,7 +208,7 @@ def test_wrapper_adds_dry_run_and_report_date_when_set(tmp_path: Path) -> None: assert result.returncode == 0, result.stderr assert arguments == [ - str(SCRIPT_PATH), + *UV_RUN_ARGUMENTS, "--project-id", PROJECT_ID, "--dry-run", @@ -190,7 +222,7 @@ def test_wrapper_runs_live_when_dry_run_is_false(tmp_path: Path) -> None: _, arguments = _run_wrapper(tmp_path, environment) - assert arguments == [str(SCRIPT_PATH), "--project-id", PROJECT_ID] + assert arguments == [*UV_RUN_ARGUMENTS, "--project-id", PROJECT_ID] @pytest.mark.parametrize("missing", sorted(WRAPPER_ENVIRONMENT)) diff --git a/uv.lock b/uv.lock index 84a152e4e..beb814a87 100644 --- a/uv.lock +++ b/uv.lock @@ -1033,6 +1033,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/76/d241a5c927433420507215df6cac1b1fa4ac0ba7a794df42a84326c68da8/google_auth-2.49.2-py3-none-any.whl", hash = "sha256:c2720924dfc82dedb962c9f52cabb2ab16714fd0a6a707e40561d217574ed6d5", size = 240638, upload-time = "2026-04-10T00:41:14.501Z" }, ] +[package.optional-dependencies] +pyopenssl = [ + { name = "pyopenssl" }, +] + [[package]] name = "google-cloud-appengine-logging" version = "1.9.0" @@ -1062,6 +1067,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/40/79fa535b6e3321d5e07b2a9ab4bb63860d3fea12230c765837881348003c/google_cloud_audit_log-0.5.0-py3-none-any.whl", hash = "sha256:3f4632f25bf67446fa9085c52868f3cb42fb1afbab9489ba8978e30991afc79f", size = 44862, upload-time = "2026-03-30T22:47:57.533Z" }, ] +[[package]] +name = "google-cloud-bigquery" +version = "3.46.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth", extra = ["pyopenssl"] }, + { name = "google-cloud-core" }, + { name = "google-resumable-media" }, + { name = "packaging" }, + { name = "python-dateutil" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/df/1b6f6fa4c119babcbbd002826a0a24ac287066808fbc873e7733b5ecc2bf/google_cloud_bigquery-3.46.2.tar.gz", hash = "sha256:a87e2e8026133e709deab9ebf537a6a2eee3fbcf4befa334a2f9f6164435b331", size = 530538, upload-time = "2026-10-15T17:49:37.245Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/ba/e89552c32a1cd5483261483c303da560e6504365d7314ecd88216ff9e119/google_cloud_bigquery-3.46.2-py3-none-any.whl", hash = "sha256:5e9a4e15f778cc00ee88167de9934a4c1bbb7fc20058e8f573f2bfdaf0d99052", size = 269195, upload-time = "2026-10-15T17:49:06.152Z" }, +] + [[package]] name = "google-cloud-core" version = "2.5.1" @@ -1113,6 +1136,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/4b/6dd1e2efd9a2e73aa847fd455a1ce375d8d3cba1a2c4f7fd69f9bf0b9dce/google_cloud_secret_manager-2.27.0-py3-none-any.whl", hash = "sha256:e5540bece65a3ad720146f3b438973faf9315109b3ffa012a58711843047a3dc", size = 225577, upload-time = "2026-03-26T22:15:19.622Z" }, ] +[[package]] +name = "google-crc32c" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/25/9cb0c1c31c45b893eb8f11ae70b3f4309432d59b5acaebca5dbe791729a4/google_crc32c-1.9.0.tar.gz", hash = "sha256:7b8c84c3d159ab6817fe3f74e6e6cef099c3f95dcec3abc0d8afb1404642efbe", size = 14857, upload-time = "2026-09-24T21:39:32.067Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/55/a2f07f15e624f0de79359b1a6c1deb59ec5061bd3b38744b3b2849400662/google_crc32c-1.9.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:457d0d9a4718fd52b1494eac5c200ad25beeadbdc91843d550a003910838589f", size = 31958, upload-time = "2026-09-24T21:19:00.994Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b3/923743597b774bbcf12a7c3e00e48d745e15fd616ad7489a40a63fff8f2f/google_crc32c-1.9.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:ccfe40021fd6afe23361175cf7551e3cef5fd34dc1ebe319f14993a83579e0eb", size = 31805, upload-time = "2026-09-24T21:22:25.019Z" }, + { url = "https://files.pythonhosted.org/packages/df/a6/4d0352fe889663e0d81cea7fc664ec9158727384de4a44ab10e9967a7682/google_crc32c-1.9.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fbef61a3794e011c65fb4396a196cf123a7f474fe5a443db8e5dd7d751b9e6d4", size = 38119, upload-time = "2026-09-24T21:38:06.634Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e3/26685384e4b66ff0928d9566ef6110a7df76029175a1842329d7e3515f10/google_crc32c-1.9.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86764b99e7a607830d93cb5b75e0ec3ff6cb06d3c274624418473cee701900d4", size = 36677, upload-time = "2026-09-24T21:38:08.082Z" }, + { url = "https://files.pythonhosted.org/packages/cb/ce/4e90102e84880e97d3cf935f2672ecd29191bdeacf57f01740f92debda00/google_crc32c-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:43a2dc26f9be213fbe0b4fc4a1088c5d45cbfcb3247420ccc820f0fc3edeea86", size = 35096, upload-time = "2026-09-24T21:39:28.201Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5d/0730e1b3a14d054d1466f2fec88dadf978509c749a3d96d8b069cc56d38a/google_crc32c-1.9.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:53fdafef58e230d0c946ab5f8446d123d9f548230a73b29c8b41c9546f268bc1", size = 31961, upload-time = "2026-09-24T21:19:01.724Z" }, + { url = "https://files.pythonhosted.org/packages/dd/32/d085abaf2fd907121975b92245bb3480fb8be40c37d03f9d6c41857f84c3/google_crc32c-1.9.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:8b91f41645b15a720357183fa5716682ada441873e3c462c15f9714be36f146b", size = 31804, upload-time = "2026-09-24T21:22:25.81Z" }, + { url = "https://files.pythonhosted.org/packages/94/78/dd1935432337e5da7af391a6fc9f161c1c8e9b9002a402b9190135fe1b59/google_crc32c-1.9.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:16865b477d7941712cb0e0aad8ad4815e984fb5fc16d3fdaef7d986e26e53c95", size = 37775, upload-time = "2026-09-24T21:38:09.249Z" }, + { url = "https://files.pythonhosted.org/packages/9e/43/9db03635bb10188d93dcbab9baa2a8670a0da4e868b4370cdbd98d65fed8/google_crc32c-1.9.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3abb18297d9ef0ab120531838be0e6d68c9fa876570e11c229c48f2edac23ce7", size = 36347, upload-time = "2026-09-24T21:38:10.141Z" }, + { url = "https://files.pythonhosted.org/packages/cf/eb/94dee516c846bd9382c3f566d8f8e5fb9e90599e45afeb697f9fc2533528/google_crc32c-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:fb63a8d7fa2e95dcff1ca16af2f4d88b526fa5ff72d1696285884ac2d49b6963", size = 35097, upload-time = "2026-09-24T21:39:28.934Z" }, + { url = "https://files.pythonhosted.org/packages/3f/34/cb484e8b6174f130f8c6dc79c733a9dd8869b410ad6511fb6104c46b973a/google_crc32c-1.9.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:f1dc17d987ddcc5eba12a7ce48f0eb93141dea236b170c1101151396edf2f0cf", size = 31960, upload-time = "2026-09-24T21:19:02.454Z" }, + { url = "https://files.pythonhosted.org/packages/af/25/3e8e567bd48448e225ea27318ccf2b94e05124e7b8b97b13eaec9e127199/google_crc32c-1.9.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:f894a2877650b56201d26a012a257b76d54a68834dc3913a93830ca8a047b075", size = 31803, upload-time = "2026-09-24T21:22:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/f0/18/bee0dd59ae622482dc6463636c79e4bde7c954d061c859c9256362c9931a/google_crc32c-1.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4488f1553a9ab7e86cdedc833374a7e904031803b995dc0bd0be48c271fa6556", size = 37776, upload-time = "2026-09-24T21:38:11.056Z" }, + { url = "https://files.pythonhosted.org/packages/fd/b6/e76e80fed5f2558273c7839e622f98095c9b36c719c7147e38e3c055cb70/google_crc32c-1.9.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0568b17ed90ac596f29400d99e243fd0cc6276766183def888d1bf8d1dc13827", size = 36350, upload-time = "2026-09-24T21:38:12.138Z" }, + { url = "https://files.pythonhosted.org/packages/87/34/165542bfa99dfef91a76471cc48cce74b8ff4e295722896087ab2b8e8611/google_crc32c-1.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:8583ec21d56b565d68ab2963cc7e21b3b271247c29b04286068255ef65f221bd", size = 35090, upload-time = "2026-09-24T21:39:29.764Z" }, + { url = "https://files.pythonhosted.org/packages/8f/eb/43ea41f4061a1cad87b2b6559c98e960e45bf551fe66f83d833b98aaf0c9/google_crc32c-1.9.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:6a3b2c8a343c570ed8100a7627c20badfd92c6caa2067093a86be45af27f5b1b", size = 31963, upload-time = "2026-09-24T21:19:03.208Z" }, + { url = "https://files.pythonhosted.org/packages/45/d2/a968c0c29ccd2b0c980ff4f9e3f7035cee28c23a1c57541825cc8221858c/google_crc32c-1.9.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:13179f7e3282617923e957b8e54b8f9c3968030f48640a9f47fd7c5c38c4a215", size = 31805, upload-time = "2026-09-24T21:22:27.917Z" }, + { url = "https://files.pythonhosted.org/packages/03/73/388e493d6c3e252e37165d22efe5a1361f872a24425391b999822861b23a/google_crc32c-1.9.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:265233aff33d835f5b909584fe36ab29647b598c271b661a300001099109e53e", size = 37925, upload-time = "2026-09-24T21:38:13.32Z" }, + { url = "https://files.pythonhosted.org/packages/98/36/190d32caa363ef25d685f422ed1bbf93ff1140fb22fd4d90f24cec209977/google_crc32c-1.9.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dee799544cae42a42b17a88e38b59cf2c271051dc001da2117a8ff240ffa0548", size = 36495, upload-time = "2026-09-24T21:38:14.211Z" }, + { url = "https://files.pythonhosted.org/packages/d3/fd/81cefea6adae7bd92abb23d4567d199f6485a20ec0a305ca5fa04c52b9c5/google_crc32c-1.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:af73200fa9791ccd380f3598235dba8d82b8af0905df045b3dc60b59836e8ddd", size = 35535, upload-time = "2026-09-24T21:39:30.52Z" }, + { url = "https://files.pythonhosted.org/packages/c5/18/19d4f17f3f33f8fdffcb3e1e69219d6f7ec2c359c160867b04dac1d0a64d/google_crc32c-1.9.0-cp315-cp315-macosx_12_0_arm64.whl", hash = "sha256:e6e8be8a94436079cb5340f6d495d9d7ba30124d8b952703994c739c7c06e236", size = 31977, upload-time = "2026-09-24T21:19:03.976Z" }, + { url = "https://files.pythonhosted.org/packages/81/b4/8010372c4b46f2ee2352dfdb630c397570cd85522a315df024ad2f9459aa/google_crc32c-1.9.0-cp315-cp315-macosx_12_0_x86_64.whl", hash = "sha256:f2b64641bca27497b986b9d87883014035aa904cb4fa333407c6752b3afee9ba", size = 31809, upload-time = "2026-09-24T21:22:29.1Z" }, + { url = "https://files.pythonhosted.org/packages/c5/f8/7e33845d6b90ce1cf37cfabf25cb859277c7d3533ef1b6b1e1ca58581549/google_crc32c-1.9.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f97c3806dcea41c29c04965347b0e12481561b75e0045dc7a4f69d75dec5d9b1", size = 38359, upload-time = "2026-09-24T21:38:14.983Z" }, + { url = "https://files.pythonhosted.org/packages/36/ff/556b2423f449a7515af6b8222a4d7833cbe09ff3e8d2f0b80471f5f6d02e/google_crc32c-1.9.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0abe7e202c25909869c35672ab0f2fe748a7acf276eb78577332a7c38999740f", size = 36858, upload-time = "2026-09-24T21:38:15.799Z" }, + { url = "https://files.pythonhosted.org/packages/40/71/4733f1b7c921d04a2bb9b9916cf66498bf7ad0860a06289413830da83192/google_crc32c-1.9.0-cp315-cp315-win_amd64.whl", hash = "sha256:5695c8b9327e040b2aba12c6659b0acb5995314ef0af0192da66e662e011103b", size = 35534, upload-time = "2026-09-24T21:39:31.337Z" }, +] + +[[package]] +name = "google-resumable-media" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-crc32c" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cf/64/df6a482d5aa39d7f7be186d892377d605cae6c88525fd851d456e8bbe9c9/google_resumable_media-2.11.0.tar.gz", hash = "sha256:febd83686752799661b4de575f0b993c5c25c349a5362556fc4d7be164056a37", size = 2164914, upload-time = "2026-09-29T19:26:13.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/2e/4f0a152f2e576e496f31ba1c3c62ed174a8878d06008916a7edc58b1bb28/google_resumable_media-2.11.0-py3-none-any.whl", hash = "sha256:f43d15e6a7f818f762eaead0f369c551f8275a4179c9d6225d0d259f49b87b5d", size = 81515, upload-time = "2026-09-29T19:25:47.31Z" }, +] + [[package]] name = "googleapis-common-protos" version = "1.74.0" @@ -2833,6 +2901,11 @@ dev = [ { name = "towncrier" }, ] +[package.dev-dependencies] +telemetry-error-issues = [ + { name = "google-cloud-bigquery" }, +] + [package.metadata] requires-dist = [ { name = "a2wsgi", specifier = ">=1.10,<2" }, @@ -2883,6 +2956,9 @@ requires-dist = [ ] provides-extras = ["dev"] +[package.metadata.requires-dev] +telemetry-error-issues = [{ name = "google-cloud-bigquery", specifier = ">=3.30,<4" }] + [[package]] name = "policyengine-canada" version = "0.96.3" @@ -3513,6 +3589,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl", hash = "sha256:e6b1d89711dd51f8f74b1631fe08f039e7d76cf67a42a323d3178f0f25762ed9", size = 45300, upload-time = "2025-08-24T12:55:53.394Z" }, ] +[[package]] +name = "pyopenssl" +version = "26.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/51/27a5ad5f939d08f690a326ef9582cda7140555180db71695f6fb747d6a36/pyopenssl-26.2.0.tar.gz", hash = "sha256:8c6fcecd1183a7fc897548dfe388b0cdb7f37e018200d8409cf33959dbe35387", size = 182195, upload-time = "2026-05-04T23:06:09.72Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/b8/a0e2790ae249d6f38c9f66de7a211621a7ab2650217bcd04e1262f578a56/pyopenssl-26.2.0-py3-none-any.whl", hash = "sha256:4f9d971bc5298b8bc1fab282803da04bf000c755d4ad9d99b52de2569ca19a70", size = 55823, upload-time = "2026-05-04T23:06:08.395Z" }, +] + [[package]] name = "pyproject-hooks" version = "1.2.0"