diff --git a/.github/scripts/sync_telemetry_error_issues.py b/.github/scripts/sync_telemetry_error_issues.py new file mode 100644 index 000000000..9b80188ae --- /dev/null +++ b/.github/scripts/sync_telemetry_error_issues.py @@ -0,0 +1,11 @@ +"""Report each day's telemetry error groups as GitHub issues. + +The code is in the ``telemetry_error_issues`` package beside this file. Python +puts the directory of a script it runs by path first on ``sys.path``, so the +package imports from any working directory. +""" + +from telemetry_error_issues.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/sync_telemetry_error_issues.sh b/.github/scripts/sync_telemetry_error_issues.sh new file mode 100644 index 000000000..5062e36bd --- /dev/null +++ b/.github/scripts/sync_telemetry_error_issues.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Run the daily telemetry error report with the settings the workflow passes +# in the environment, through uv (the workflow sets it up). DRY_RUN=true adds --dry-run; a non-empty REPORT_DATE +# (YYYY-MM-DD, an Eastern calendar day) replaces the default previous day. +set -euo pipefail + +: "${GOOGLE_OAUTH_ACCESS_TOKEN:?GOOGLE_OAUTH_ACCESS_TOKEN is required}" +: "${GITHUB_TOKEN:?GITHUB_TOKEN is required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" +: "${TELEMETRY_ERROR_FINGERPRINT_KEY:?TELEMETRY_ERROR_FINGERPRINT_KEY is required}" +: "${OBSERVABILITY_PROJECT_ID:?OBSERVABILITY_PROJECT_ID is required}" +: "${OBSERVABILITY_LOG_BUCKET:?OBSERVABILITY_LOG_BUCKET is required}" +: "${OBSERVABILITY_LOG_BUCKET_LOCATION:?OBSERVABILITY_LOG_BUCKET_LOCATION is required}" +: "${OBSERVABILITY_LOG_DATASET:?OBSERVABILITY_LOG_DATASET is required}" +: "${OBSERVABILITY_LOG_VIEW:?OBSERVABILITY_LOG_VIEW is required}" +: "${OBSERVABILITY_SERVICE_NAMESPACE:?OBSERVABILITY_SERVICE_NAMESPACE is required}" + +# Hide both values in the job log even if a later command prints them. +echo "::add-mask::${GOOGLE_OAUTH_ACCESS_TOKEN}" +echo "::add-mask::${TELEMETRY_ERROR_FINGERPRINT_KEY}" + +arguments=(--project-id "${OBSERVABILITY_PROJECT_ID}") +if [[ "${DRY_RUN:-false}" == "true" ]]; then + arguments+=(--dry-run) +fi +if [[ -n "${REPORT_DATE:-}" ]]; then + arguments+=(--report-date "${REPORT_DATE}") +fi + +script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repository_root="$(cd "${script_directory}/../.." && pwd)" +# Run in an environment that holds only the telemetry-error-issues dependency +# group, at the versions uv.lock pins, not the API's own dependencies. +exec uv run --frozen --only-group telemetry-error-issues --project "${repository_root}" \ + python "${script_directory}/sync_telemetry_error_issues.py" "${arguments[@]}" diff --git a/.github/scripts/telemetry_error_issues/__init__.py b/.github/scripts/telemetry_error_issues/__init__.py new file mode 100644 index 000000000..0f7cb75ee --- /dev/null +++ b/.github/scripts/telemetry_error_issues/__init__.py @@ -0,0 +1,13 @@ +"""Report each day's telemetry error groups as GitHub issues. + +Each run reads one US Eastern calendar day of error records from the central +Cloud Logging bucket through its linked BigQuery dataset. It groups the records +by a keyed fingerprint, measures each group in production and staging, and +creates or updates one GitHub issue per production error group. + +Workflow logs, job summaries and issues in this repository are public. The +script therefore publishes only fingerprints, counts, code-defined names and a +Logs Explorer link, never error messages, stack text or identifiers. + +``sync_telemetry_error_issues.py`` beside this package runs ``cli.main``. +""" diff --git a/.github/scripts/telemetry_error_issues/bigquery.py b/.github/scripts/telemetry_error_issues/bigquery.py new file mode 100644 index 000000000..35e0d25e0 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/bigquery.py @@ -0,0 +1,359 @@ +"""Query error records and request counts with the BigQuery client library. + +The library starts each job, waits for it, reads every page of rows over the +REST API and converts each value to its Python type. Rows are read by +iterating the job's result, never with ``to_dataframe`` or ``to_arrow``, +which may use the BigQuery Storage Read API: the workflow's service account +has no role that can open a read session. +""" + +from __future__ import annotations + +from datetime import datetime +from http import HTTPStatus +from typing import TYPE_CHECKING + +from google.api_core.exceptions import GoogleAPICallError +from google.cloud import bigquery +from google.oauth2.credentials import Credentials +from telemetry_error_issues.groups import RequestCount +from telemetry_error_issues.log_source import ( + DATASET_PATTERN, + LOG_VIEW_PATTERN, + checked_identifier, +) +from telemetry_error_issues.normalization import MISSING +from telemetry_error_issues.records import ErrorRecord +from telemetry_error_issues.transport import ERROR_REASON_PATTERN, ApiError + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping, Sequence + + from telemetry_error_issues.log_source import LogSource + from telemetry_error_issues.window import ReportingWindow + +# Captures 'path", line N, in function' from each frame header of a Python +# traceback, including indented and "|"-prefixed exception group frames. It has +# one capturing group because BigQuery's REGEXP_EXTRACT_ALL allows only one, +# and the trailing newline skips a header that stack truncation cut short. +FRAME_PATTERN = r'File "([^"\n]+", line [0-9]+, in [^\s]+)\n' +# Matches everything after the first traceback of a chained exception, so the +# query can remove it before extracting frames. The logging library keeps only +# the start of a long stack, which can drop the final traceback; the first +# traceback is always kept, so frames from it give an exception the same +# fingerprint however long its stack is. +CHAINED_EXCEPTION_PATTERN = ( + r"(?s)\n\n(?:The above exception was the direct cause of the following " + r"exception|During handling of the above exception, another exception " + r"occurred):\n.*" +) + +# A message's template is its first three whole words after removing quoted +# values, so values don't split one error into many groups. A single quote +# opens a value only after the start of the message or a character that is not +# a letter or digit; otherwise it is an apostrophe, as in "can't" or "users'". +# The query passes these to REGEXP_REPLACE and REGEXP_EXTRACT_ALL. +DOUBLE_QUOTED_PATTERN = r'"[^"]*"' +SINGLE_QUOTED_PATTERN = r"(^|[^A-Za-z0-9])'[^']*'" +TEMPLATE_WORD_PATTERN = r"\b[A-Za-z]+\b" + +MAXIMUM_BYTES_BILLED = 5 * 1024**3 +# How long to wait for a query job to finish: 41 waits of 30 seconds, the +# limit the report had when it polled the REST API itself. +QUERY_TIMEOUT_SECONDS = 41 * 30 + +HEALTH_CHECK_ROUTES = ( + "/health", + "/simulation-gateway-check", + "/liveness-check", + "/readiness-check", +) + +# Each query reads the linked dataset's log view, whose name replaces +# {log_view} once it has passed LOG_VIEW_PATTERN: BigQuery takes no table name +# as a parameter. Only code-defined or normalized values leave BigQuery: stacks +# only as frame headers, messages only as templates (see DOUBLE_QUOTED_PATTERN), +# and identifiers only as SHA-256 hashes. Duplicate deliveries of one log entry +# are removed. +ERROR_RECORDS_QUERY = r""" +SELECT + timestamp, + JSON_VALUE(json_payload, '$."service.name"') AS service_name, + JSON_VALUE(json_payload, '$."deployment.environment.name"') AS environment_name, + JSON_VALUE(json_payload, '$."event.name"') AS event_name, + JSON_VALUE(json_payload, '$."error.type"') AS error_type, + JSON_VALUE(json_payload, '$."error.handled"') AS error_handled, + REGEXP_EXTRACT_ALL( + REGEXP_REPLACE( + JSON_VALUE(json_payload, '$."error.stack"'), @chained_exception_pattern, '' + ), + @frame_pattern + ) AS frames, + COALESCE(LENGTH(JSON_VALUE(json_payload, '$."error.stack"')) >= 16384, FALSE) + AS stack_truncated, + ARRAY_TO_STRING( + ARRAY( + SELECT LOWER(word) + FROM UNNEST( + REGEXP_EXTRACT_ALL( + REGEXP_REPLACE( + REGEXP_REPLACE( + JSON_VALUE(json_payload, '$.message'), @double_quoted_pattern, '' + ), + @single_quoted_pattern, + r'\1' + ), + @template_word_pattern + ) + ) AS word WITH OFFSET AS position + WHERE position < 3 + ORDER BY position + ), + ' ' + ) AS message_template, + JSON_VALUE(json_payload, '$.attributes.error_type') AS attribute_error_type, + JSON_VALUE(json_payload, '$.attributes.failure_category') AS failure_category, + JSON_VALUE(json_payload, '$."http.route"') AS route, + JSON_VALUE(json_payload, '$."http.response.status_code"') AS status_code, + JSON_VALUE(json_payload, '$."operation.name"') AS operation_name, + TO_HEX(SHA256(JSON_VALUE(json_payload, '$."request.id"'))) AS request_key, + TO_HEX(SHA256(JSON_VALUE(json_payload, '$.attributes.observability_id'))) + AS observability_key, + TO_HEX(SHA256(COALESCE( + JSON_VALUE(json_payload, '$.trace_id'), + REGEXP_EXTRACT(trace, r'/traces/([0-9A-Fa-f]+)$') + ))) AS trace_key +FROM `{log_view}` +WHERE timestamp >= @query_start AND timestamp < @window_end + AND JSON_VALUE(json_payload, '$.schema_version') = 'policyengine.observability.v2' + AND JSON_VALUE(json_payload, '$."service.namespace"') = @service_namespace + AND severity_number >= 500 + AND COALESCE(JSON_VALUE(json_payload, '$."http.route"'), '') + NOT IN UNNEST(@excluded_routes) +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY insert_id, timestamp ORDER BY receive_timestamp +) = 1 +""" + +REQUEST_COUNTS_QUERY = r""" +WITH completed_requests AS ( + SELECT + JSON_VALUE(json_payload, '$."service.name"') AS service_name, + JSON_VALUE(json_payload, '$."deployment.environment.name"') AS environment_name, + JSON_VALUE(json_payload, '$."http.route"') AS route + FROM `{log_view}` + WHERE timestamp >= @window_start AND timestamp < @window_end + AND JSON_VALUE(json_payload, '$.schema_version') = 'policyengine.observability.v2' + AND JSON_VALUE(json_payload, '$."service.namespace"') = @service_namespace + AND JSON_VALUE(json_payload, '$."event.name"') = 'request.completed' + AND COALESCE(JSON_VALUE(json_payload, '$."http.route"'), '') + NOT IN UNNEST(@excluded_routes) + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY insert_id, timestamp ORDER BY receive_timestamp + ) = 1 +) +SELECT service_name, environment_name, route, COUNT(*) AS requests +FROM completed_requests +GROUP BY service_name, environment_name, route +""" + + +class BigQueryError(ApiError): + """A BigQuery request failed.""" + + service = "BigQuery" + + +class BigQueryTimeoutError(BigQueryError): + """A BigQuery query job didn't complete within the time limit.""" + + +QueryParameter = bigquery.ScalarQueryParameter | bigquery.ArrayQueryParameter + + +def library_client(project_id: str, access_token: str) -> bigquery.Client: + """Return a library client that runs jobs in ``project_id`` as the token's owner. + + google-github-actions/auth makes the access token and writes no credentials + file, so the client gets the token itself and never looks for other + credentials. + """ + credentials = Credentials(token=access_token) + return bigquery.Client(project=project_id, credentials=credentials) + + +def _error_reason(errors: Sequence[object], status: int) -> str: + """Return the error's reason code, else the HTTP reason phrase. + + The library's message can repeat the query and values from the request, + so only a short code-defined reason such as ``invalidQuery`` is kept. + """ + reason = errors[0].get("reason") if errors and isinstance(errors[0], dict) else None + if isinstance(reason, str) and ERROR_REASON_PATTERN.match(reason): + return reason + try: + return HTTPStatus(status).phrase + except ValueError: + return "unknown" + + +def _bigquery_error(error: GoogleAPICallError) -> BigQueryError: + """Describe a library error by its HTTP status and reason, never its message.""" + if error.code is None: + return BigQueryError("BigQuery request failed") + status = int(error.code) + reason = _error_reason(error.errors, status) + message = f"BigQuery request failed: HTTP {status} {reason}" + return BigQueryError(message, status=status, reason=reason) + + +class BigQueryClient: + """Runs parameterized standard SQL queries against the linked log dataset.""" + + def __init__( + self, + log_source: LogSource, + access_token: str, + client_factory: Callable[[str, str], bigquery.Client] = library_client, + ) -> None: + """Run jobs in the log source's project against its linked dataset.""" + self.log_source = log_source + self._client = client_factory(log_source.project_id, access_token) + + def query( + self, sql: str, parameters: list[QueryParameter], label: str + ) -> list[dict[str, object]]: + """Run ``sql`` and return every row as a dictionary of its values.""" + dataset = checked_identifier( + self.log_source.dataset, DATASET_PATTERN, "the log dataset" + ) + job_config = bigquery.QueryJobConfig( + query_parameters=parameters, + default_dataset=bigquery.DatasetReference( + self.log_source.project_id, dataset + ), + maximum_bytes_billed=MAXIMUM_BYTES_BILLED, + use_legacy_sql=False, + labels={"purpose": "telemetry-error-issues", "query": label}, + ) + try: + job = self._client.query(sql, job_config=job_config) + rows = job.result(timeout=QUERY_TIMEOUT_SECONDS) + return [dict(row.items()) for row in rows] + except GoogleAPICallError as error: + raise _bigquery_error(error) from error + except TimeoutError as error: + msg = "BigQuery query did not complete in time" + raise BigQueryTimeoutError(msg) from error + + +def _query_text(template: str, log_source: LogSource) -> str: + """Name the log view in ``template``, once the name has passed its pattern.""" + view = checked_identifier(log_source.view, LOG_VIEW_PATTERN, "the log view") + return template.format(log_view=view) + + +def _query_parameters( + timestamps: Mapping[str, datetime], strings: Mapping[str, str] +) -> list[QueryParameter]: + parameters: list[QueryParameter] = [ + bigquery.ScalarQueryParameter(name, "TIMESTAMP", value) + for name, value in timestamps.items() + ] + parameters.extend( + bigquery.ScalarQueryParameter(name, "STRING", value) + for name, value in strings.items() + ) + parameters.append( + bigquery.ArrayQueryParameter( + "excluded_routes", "STRING", list(HEALTH_CHECK_ROUTES) + ) + ) + return parameters + + +def _text(value: object) -> str | None: + return None if value is None or value == "" else str(value) + + +def _integer(value: object) -> int | None: + if isinstance(value, int) and not isinstance(value, bool): + return value + if isinstance(value, str) and value.isascii() and value.isdigit(): + return int(value) + return None + + +def _boolean(value: object) -> bool | None: + if isinstance(value, bool): + return value + return {"true": True, "false": False}.get(str(value)) + + +def error_record_from_row(row: Mapping[str, object]) -> ErrorRecord: + """Build an ``ErrorRecord`` from one row of ``ERROR_RECORDS_QUERY``.""" + timestamp = row["timestamp"] + if not isinstance(timestamp, datetime): + msg = "an error record has no timestamp" + raise TypeError(msg) + frames = row["frames"] + return ErrorRecord( + timestamp=timestamp, + service_name=_text(row["service_name"]) or MISSING, + environment_name=_text(row["environment_name"]), + event_name=_text(row["event_name"]), + error_type=_text(row["error_type"]), + error_handled=_boolean(row["error_handled"]), + frames=tuple(map(str, frames)) if isinstance(frames, list) else (), + stack_truncated=row["stack_truncated"] is True, + message_template=_text(row["message_template"]), + attribute_error_type=_text(row["attribute_error_type"]), + failure_category=_text(row["failure_category"]), + route=_text(row["route"]), + status_code=_integer(row["status_code"]), + operation_name=_text(row["operation_name"]), + request_key=_text(row["request_key"]), + observability_key=_text(row["observability_key"]), + trace_key=_text(row["trace_key"]), + ) + + +def fetch_error_records( + client: BigQueryClient, window: ReportingWindow +) -> list[ErrorRecord]: + """Return the ERROR records of the window and the seven days before it.""" + parameters = _query_parameters( + {"query_start": window.lookback_start, "window_end": window.end}, + { + "service_namespace": client.log_source.service_namespace, + "frame_pattern": FRAME_PATTERN, + "chained_exception_pattern": CHAINED_EXCEPTION_PATTERN, + "double_quoted_pattern": DOUBLE_QUOTED_PATTERN, + "single_quoted_pattern": SINGLE_QUOTED_PATTERN, + "template_word_pattern": TEMPLATE_WORD_PATTERN, + }, + ) + sql = _query_text(ERROR_RECORDS_QUERY, client.log_source) + rows = client.query(sql, parameters, "error-records") + return [error_record_from_row(row) for row in rows] + + +def fetch_request_counts( + client: BigQueryClient, window: ReportingWindow +) -> list[RequestCount]: + """Return the requests each service completed per route in the window.""" + parameters = _query_parameters( + {"window_start": window.start, "window_end": window.end}, + {"service_namespace": client.log_source.service_namespace}, + ) + sql = _query_text(REQUEST_COUNTS_QUERY, client.log_source) + rows = client.query(sql, parameters, "request-counts") + return [ + RequestCount( + service_name=_text(row["service_name"]) or MISSING, + environment_name=_text(row["environment_name"]), + route=_text(row["route"]), + requests=_integer(row["requests"]) or 0, + ) + for row in rows + ] diff --git a/.github/scripts/telemetry_error_issues/cli.py b/.github/scripts/telemetry_error_issues/cli.py new file mode 100644 index 000000000..eb6529444 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/cli.py @@ -0,0 +1,317 @@ +"""Report each day's telemetry error groups as GitHub issues. + +The command line: it parses the arguments, reads the settings from the +environment, runs each stage of the report and writes the job summary. +``parse_arguments`` uses the first line of this docstring as the description +that ``--help`` prints. +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +import time +from collections import Counter +from dataclasses import dataclass +from datetime import UTC, date, datetime +from pathlib import Path +from typing import TYPE_CHECKING, NamedTuple, TypeVar + +from telemetry_error_issues.bigquery import ( + BigQueryClient, + fetch_error_records, + fetch_request_counts, + library_client, +) +from telemetry_error_issues.github_client import GitHubClient +from telemetry_error_issues.groups import build_groups +from telemetry_error_issues.log_source import ( + DATASET_PATTERN, + LOCATION_PATTERN, + LOG_BUCKET_PATTERN, + LOG_VIEW_PATTERN, + PROJECT_ID_PATTERN, + LogSource, + checked_identifier, +) +from telemetry_error_issues.normalization import MISSING +from telemetry_error_issues.records import fingerprint_key +from telemetry_error_issues.rendering import code, window_text +from telemetry_error_issues.sync import SyncResult, sync_issues +from telemetry_error_issues.transport import ( + ApiError, + HttpRequest, + HttpResponse, + urllib_transport, +) +from telemetry_error_issues.window import ( + ReportingWindow, + default_report_date, + reporting_window, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + from typing import TextIO + + from google.cloud import bigquery + +StageResult = TypeVar("StageResult") + +# The repository variables that name the logs the run reads, each with the +# pattern its value must match. They say where the logs are; none of them +# changes how failures are grouped. +LOG_NAME_VARIABLES = { + "OBSERVABILITY_LOG_BUCKET": LOG_BUCKET_PATTERN, + "OBSERVABILITY_LOG_BUCKET_LOCATION": LOCATION_PATTERN, + "OBSERVABILITY_LOG_DATASET": DATASET_PATTERN, + "OBSERVABILITY_LOG_VIEW": LOG_VIEW_PATTERN, +} +REQUIRED_ENVIRONMENT_VARIABLES = ( + "GOOGLE_OAUTH_ACCESS_TOKEN", + "GITHUB_TOKEN", + "GITHUB_REPOSITORY", + "TELEMETRY_ERROR_FINGERPRINT_KEY", + *LOG_NAME_VARIABLES, + "OBSERVABILITY_SERVICE_NAMESPACE", +) +REPOSITORY_PATTERN = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") + + +def _report_date_argument(value: str) -> date | None: + if not value: + return None + try: + return date.fromisoformat(value) + except ValueError as error: + msg = "expected an Eastern calendar date as YYYY-MM-DD" + raise argparse.ArgumentTypeError(msg) from error + + +def parse_arguments(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the command line; argparse exits with code 2 on invalid input.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--project-id", + required=True, + help="Google Cloud project that holds the linked BigQuery dataset", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="query and group errors, but make no changes to GitHub issues", + ) + parser.add_argument( + "--report-date", + type=_report_date_argument, + default=None, + help="Eastern calendar day to report (default: the previous day)", + ) + return parser.parse_args(argv) + + +class ConfigurationError(Exception): + """The run's settings are missing or invalid. The text names only settings.""" + + +class StageFailedError(Exception): + """A stage of the run failed. + + The text holds only the stage, the exception class, and for an API error + the HTTP status and reason, so it can be printed in a public log. + """ + + +def _current_time() -> datetime: + return datetime.now(UTC) + + +@dataclass(frozen=True) +class Runtime: + """What a run reads from outside the process, so tests can replace it.""" + + environ: Mapping[str, str] + stdout: TextIO + stderr: TextIO + bigquery_client_factory: Callable[[str, str], bigquery.Client] = library_client + github_transport: Callable[[HttpRequest], HttpResponse] = urllib_transport + now: Callable[[], datetime] = _current_time + sleep: Callable[[float], None] = time.sleep + clock: Callable[[], float] = time.monotonic + + @classmethod + def from_process(cls) -> Runtime: + """Use the process environment, standard streams and the network.""" + return cls(environ=os.environ, stdout=sys.stdout, stderr=sys.stderr) + + +class Settings(NamedTuple): + """Settings read from the environment; none of them is ever printed.""" + + google_access_token: str + github_credential: str + repository: str + fingerprint_key: bytes + log_source: LogSource + summary_path: str | None + + +def read_settings(environ: Mapping[str, str], project_id: str) -> Settings: + """Read and check the environment variables the run needs, and its project.""" + missing = [name for name in REQUIRED_ENVIRONMENT_VARIABLES if not environ.get(name)] + if missing: + msg = f"missing environment variables: {', '.join(missing)}" + raise ConfigurationError(msg) + if not REPOSITORY_PATTERN.match(environ["GITHUB_REPOSITORY"]): + msg = "GITHUB_REPOSITORY must be owner/name" + raise ConfigurationError(msg) + try: + key = fingerprint_key(environ["TELEMETRY_ERROR_FINGERPRINT_KEY"]) + checked_identifier(project_id, PROJECT_ID_PATTERN, "--project-id") + for name, pattern in LOG_NAME_VARIABLES.items(): + checked_identifier(environ[name], pattern, name) + except ValueError as error: + raise ConfigurationError(str(error)) from None + log_source = LogSource( + project_id=project_id, + bucket=environ["OBSERVABILITY_LOG_BUCKET"], + bucket_location=environ["OBSERVABILITY_LOG_BUCKET_LOCATION"], + dataset=environ["OBSERVABILITY_LOG_DATASET"], + view=environ["OBSERVABILITY_LOG_VIEW"], + service_namespace=environ["OBSERVABILITY_SERVICE_NAMESPACE"], + ) + return Settings( + google_access_token=environ["GOOGLE_OAUTH_ACCESS_TOKEN"], + github_credential=environ["GITHUB_TOKEN"], + repository=environ["GITHUB_REPOSITORY"], + fingerprint_key=key, + log_source=log_source, + summary_path=environ.get("GITHUB_STEP_SUMMARY") or None, + ) + + +def _report_window(report_date: date | None, now: datetime) -> ReportingWindow: + window = reporting_window(report_date or default_report_date(now)) + if window.end > now: + msg = f"{window.start.date().isoformat()} has not ended in US Eastern time" + raise ConfigurationError(msg) + return window + + +def _failure_description(stage: str, error: Exception) -> str: + detail = "" + if isinstance(error, ConfigurationError): + detail = f" ({error})" + elif isinstance(error, ApiError) and error.status is not None: + detail = f" (HTTP {error.status} {error.reason})" + return f"{stage} failed: {type(error).__name__}{detail}" + + +def _stage(name: str, operation: Callable[[], StageResult]) -> StageResult: + """Run one stage, replacing any failure with a description safe to print.""" + try: + return operation() + except Exception as error: + raise StageFailedError(_failure_description(name, error)) from error + + +def render_summary( + result: SyncResult, window: ReportingWindow, *, dry_run: bool +) -> str: + """Return the job summary: fingerprints, counts, issue numbers and actions.""" + lines = [ + f"## Telemetry error issues for {window.start.date().isoformat()}", + "", + f"Window: {window_text(window)}.", + ] + if dry_run: + lines.append("Dry run: no issues were changed.") + if not result.actions: + lines += ["", "No error groups had failures in the window."] + return "\n".join(lines) + totals = Counter(action.action for action in result.actions) + described = ", ".join(f"{count} {name}" for name, count in sorted(totals.items())) + lines += [ + "", + f"{len(result.actions)} error groups: {described}.", + "", + "| Fingerprint | Kind | Service | Production | Staging | Issue | Action |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] + for action in result.actions: + issue = MISSING if action.issue_number is None else f"#{action.issue_number}" + lines.append( + f"| {code(action.fingerprint)} | {action.kind} | {code(action.family)} " + f"| {action.production_occurrences:,} | {action.staging_occurrences:,} " + f"| {issue} | {action.action} |" + ) + return "\n".join(lines) + + +def _append_summary(path: str, summary: str) -> None: + with Path(path).open("a", encoding="utf-8") as summary_file: + summary_file.write(f"{summary}\n") + + +def _report(arguments: argparse.Namespace, runtime: Runtime) -> str: + settings = _stage( + "configuration", lambda: read_settings(runtime.environ, arguments.project_id) + ) + window = _stage( + "configuration", lambda: _report_window(arguments.report_date, runtime.now()) + ) + bigquery_client = _stage( + "configuration", + lambda: BigQueryClient( + settings.log_source, + settings.google_access_token, + runtime.bigquery_client_factory, + ), + ) + records = _stage( + "query error records", lambda: fetch_error_records(bigquery_client, window) + ) + request_counts = _stage( + "query request counts", lambda: fetch_request_counts(bigquery_client, window) + ) + groups = _stage( + "group errors", + lambda: build_groups(records, request_counts, window, settings.fingerprint_key), + ) + github = GitHubClient( + settings.repository, + settings.github_credential, + runtime.github_transport, + sleep=runtime.sleep, + clock=runtime.clock, + ) + result = _stage( + "sync issues", + lambda: sync_issues( + groups, window, settings.log_source, github, dry_run=arguments.dry_run + ), + ) + summary = render_summary(result, window, dry_run=arguments.dry_run) + summary_path = settings.summary_path + if summary_path: + _stage("write job summary", lambda: _append_summary(summary_path, summary)) + return summary + + +def main(argv: list[str] | None = None, runtime: Runtime | None = None) -> int: + """Run one report and return the process exit code. + + A failed stage prints one line naming the stage and exception class, never + the exception's message, and returns 1. + """ + arguments = parse_arguments(argv) + runtime = runtime or Runtime.from_process() + try: + summary = _report(arguments, runtime) + except StageFailedError as failure: + print(failure, file=runtime.stderr) + return 1 + print(summary, file=runtime.stdout) + return 0 diff --git a/.github/scripts/telemetry_error_issues/github_client.py b/.github/scripts/telemetry_error_issues/github_client.py new file mode 100644 index 000000000..ee1642847 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/github_client.py @@ -0,0 +1,208 @@ +"""A client for one GitHub repository's issues and labels, with rate limiting.""" + +from __future__ import annotations + +import json +import time +from email.utils import parsedate_to_datetime +from http import HTTPStatus +from typing import TYPE_CHECKING, Any +from urllib.parse import urlencode + +from telemetry_error_issues.transport import ( + ApiError, + HttpRequest, + HttpResponse, + urllib_transport, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + +GITHUB_API = "https://api.github.com" +GITHUB_API_VERSION = "2022-11-28" +GITHUB_PAGE_SIZE = 100 + +# GitHub asks clients to send content-creating requests one at a time, at +# least a second apart, and to wait as its rate-limit responses say. +MIN_SECONDS_BETWEEN_WRITES = 1.0 +MAX_RATE_LIMIT_RETRIES = 3 +DEFAULT_RATE_LIMIT_WAIT_SECONDS = 60 +MAX_RATE_LIMIT_WAIT_SECONDS = 300 +RATE_LIMIT_STATUSES = frozenset({HTTPStatus.FORBIDDEN, HTTPStatus.TOO_MANY_REQUESTS}) + + +class GitHubError(ApiError): + """A GitHub request failed.""" + + service = "GitHub" + + +class GitHubLabelError(GitHubError): + """GitHub didn't apply the labels the job asked for.""" + + +def _seconds_until_reset(response: HttpResponse) -> int | None: + """Return the seconds from the response's ``date`` to ``x-ratelimit-reset``.""" + reset = response.headers.get("x-ratelimit-reset", "") + sent = response.headers.get("date") + if not (reset.isascii() and reset.isdigit()) or not sent: + return None + try: + sent_at = parsedate_to_datetime(sent) + except (TypeError, ValueError): + return None + return max(int(reset) - int(sent_at.timestamp()), 0) + + +def _rate_limit_wait(response: HttpResponse) -> int | None: + """Return the seconds to wait before retrying a rate-limited request. + + Follows GitHub's guidance: wait for ``retry-after``; when no requests + remain, wait until ``x-ratelimit-reset``; otherwise wait a minute. A 403 + whose headers and message don't mention a rate limit is a permission + failure, so it isn't retried. The message is only searched, never printed. + """ + if response.status not in RATE_LIMIT_STATUSES: + return None + retry_after = response.headers.get("retry-after", "") + if retry_after.isascii() and retry_after.isdigit(): + seconds = int(retry_after) + elif response.headers.get("x-ratelimit-remaining") == "0": + until_reset = _seconds_until_reset(response) + seconds = ( + DEFAULT_RATE_LIMIT_WAIT_SECONDS if until_reset is None else until_reset + ) + elif ( + response.status == HTTPStatus.TOO_MANY_REQUESTS + or b"rate limit" in response.body.lower() + ): + seconds = DEFAULT_RATE_LIMIT_WAIT_SECONDS + else: + return None + return seconds if seconds <= MAX_RATE_LIMIT_WAIT_SECONDS else None + + +def _json_object(response: HttpResponse) -> dict[str, Any]: + payload = json.loads(response.body) + if not isinstance(payload, dict): + msg = "GitHub returned an unexpected response" + raise GitHubError(msg) + return payload + + +def _json_list(response: HttpResponse) -> list[dict[str, Any]]: + payload = json.loads(response.body) + if not isinstance(payload, list): + msg = "GitHub returned an unexpected response" + raise GitHubError(msg) + return payload + + +class GitHubClient: + """Reads and writes one repository's issues and labels. + + Writes go one at a time, at least a second apart. A rate-limited request is + retried after the wait GitHub asks for, up to three times. + """ + + def __init__( + self, + repository: str, + credential: str, + transport: Callable[[HttpRequest], HttpResponse] = urllib_transport, + *, + sleep: Callable[[float], None] = time.sleep, + clock: Callable[[], float] = time.monotonic, + ) -> None: + """Send requests for ``repository`` (``owner/name``) with ``credential``.""" + self._repository_url = f"{GITHUB_API}/repos/{repository}" + self._headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {credential}", + "User-Agent": "policyengine-telemetry-error-issues", + "X-GitHub-Api-Version": GITHUB_API_VERSION, + } + self._transport = transport + self._sleep = sleep + self._clock = clock + self._last_write: float | None = None + + def list_issues(self, creator: str) -> list[dict[str, Any]]: + """Return the open and closed issues ``creator`` opened, without PRs.""" + issues = self._pages("/issues", {"state": "all", "creator": creator}) + return [issue for issue in issues if "pull_request" not in issue] + + def list_labels(self) -> set[str]: + """Return the names of the repository's labels.""" + return {label["name"] for label in self._pages("/labels", {})} + + def create_label(self, name: str, color: str, description: str) -> None: + """Create a label.""" + self._send( + "POST", + "/labels", + {"name": name, "color": color, "description": description}, + ) + + def create_issue(self, title: str, body: str, labels: list[str]) -> dict[str, Any]: + """Open an issue and return it as GitHub describes it.""" + payload = {"title": title, "body": body, "labels": labels} + return _json_object(self._send("POST", "/issues", payload)) + + def update_issue(self, number: int, fields: Mapping[str, object]) -> None: + """Change an issue's fields, such as its body or state.""" + self._send("PATCH", f"/issues/{number}", fields) + + def add_labels(self, number: int, labels: list[str]) -> set[str]: + """Add labels to an issue and return all the labels it now has.""" + response = self._send("POST", f"/issues/{number}/labels", {"labels": labels}) + return {label["name"] for label in _json_list(response)} + + def add_comment(self, number: int, body: str) -> None: + """Comment on an issue.""" + self._send("POST", f"/issues/{number}/comments", {"body": body}) + + def _pages(self, path: str, parameters: Mapping[str, str]) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + page = 1 + while True: + query = urlencode( + {**parameters, "per_page": GITHUB_PAGE_SIZE, "page": page} + ) + batch = _json_list(self._send("GET", f"{path}?{query}", None)) + items.extend(batch) + if len(batch) < GITHUB_PAGE_SIZE: + return items + page += 1 + + def _wait_between_writes(self) -> None: + if self._last_write is None: + return + elapsed = self._clock() - self._last_write + if elapsed < MIN_SECONDS_BETWEEN_WRITES: + self._sleep(MIN_SECONDS_BETWEEN_WRITES - elapsed) + + def _send( + self, method: str, path: str, body: Mapping[str, object] | None + ) -> HttpResponse: + headers = dict(self._headers) + encoded = None + if body is not None: + headers["Content-Type"] = "application/json" + encoded = json.dumps(body).encode() + request = HttpRequest(method, f"{self._repository_url}{path}", headers, encoded) + is_write = method != "GET" + for retry in range(MAX_RATE_LIMIT_RETRIES + 1): + if is_write: + self._wait_between_writes() + response = self._transport(request) + if is_write: + self._last_write = self._clock() + wait = _rate_limit_wait(response) + if wait is None or retry == MAX_RATE_LIMIT_RETRIES: + break + self._sleep(wait) + if response.status >= HTTPStatus.BAD_REQUEST: + raise GitHubError.from_response(response) + return response diff --git a/.github/scripts/telemetry_error_issues/groups.py b/.github/scripts/telemetry_error_issues/groups.py new file mode 100644 index 000000000..2d04f4080 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/groups.py @@ -0,0 +1,271 @@ +"""Group failures by fingerprint, and measure each group in each environment.""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass +from typing import TYPE_CHECKING, NamedTuple + +from telemetry_error_issues.normalization import ( + environment_class, + public_error_type, + public_family, + public_operation, + public_route, + public_status, + service_family, +) +from telemetry_error_issues.records import ErrorRecord, fingerprint, primary_records + +if TYPE_CHECKING: + from collections.abc import Iterable + from datetime import datetime + + from telemetry_error_issues.window import ReportingWindow + +REPORTED_ENVIRONMENTS = ("production", "staging") +MAX_RELATED_FINGERPRINTS = 3 + + +class RequestCount(NamedTuple): + """Requests that one service completed on one route during the window.""" + + service_name: str + environment_name: str | None + route: str | None + requests: int + + +class Occurrence(NamedTuple): + """One failure in an error group.""" + + timestamp: datetime + environment: str + route: str | None + affected_key: str + observability_key: str | None + trace_key: str | None + + +@dataclass(frozen=True) +class EnvironmentMeasures: + """An error group's measures in one environment. + + Counts and rates cover the window; first and last seen cover the window + and the seven days before it. + """ + + occurrences: int + affected_requests: int + observability_ids: int + route_affected_requests: int + route_requests: int + error_rate: float | None + hourly_rate: float + previous_hourly_rate: float + first_seen: datetime | None + last_seen: datetime | None + + +@dataclass(frozen=True) +class ErrorGroup: + """Failures that share a fingerprint, described only by validated values.""" + + fingerprint: str + kind: str + family: str + error_type: str + route: str + status_code: int | None + operation: str + production: EnvironmentMeasures + staging: EnvironmentMeasures + related_fingerprints: tuple[str, ...] + occurrences: tuple[Occurrence, ...] + + def production_since(self, moment: datetime) -> tuple[int, int]: + """Return production occurrences and affected requests from ``moment``.""" + recent = [ + occurrence + for occurrence in self.occurrences + if occurrence.environment == "production" and occurrence.timestamp >= moment + ] + return len(recent), len({occurrence.affected_key for occurrence in recent}) + + +def _occurrence(record: ErrorRecord, position: int) -> Occurrence: + """Describe a failure, identifying what it affected as precisely as possible. + + The affected request is the request ID, else the observability ID, else the + trace. A failure with none of them counts as its own request. + """ + if record.request_key: + affected_key = f"request:{record.request_key}" + elif record.observability_key: + affected_key = f"observability:{record.observability_key}" + elif record.trace_key: + affected_key = f"trace:{record.trace_key}" + else: + affected_key = f"record:{position}" + return Occurrence( + timestamp=record.timestamp, + environment=record.environment, + route=record.route, + affected_key=affected_key, + observability_key=record.observability_key, + trace_key=record.trace_key, + ) + + +class RouteRequests: + """Requests completed in the window, per service family, environment and route.""" + + def __init__(self, request_counts: Iterable[RequestCount]) -> None: + """Add up the request counts, merging versioned simulation services.""" + self._totals: Counter[tuple[str, str, str | None]] = Counter() + for count in request_counts: + environment = environment_class(count.environment_name) + family = service_family(count.service_name) + self._totals[family, environment, count.route] += count.requests + + def total(self, family: str, environment: str, routes: Iterable[str]) -> int: + """Return the requests one service family completed on the given routes.""" + return sum(self._totals[family, environment, route] for route in routes) + + +def _related_fingerprints( + occurrences_by_group: dict[str, list[Occurrence]], +) -> dict[str, tuple[str, ...]]: + """For each group, the groups that share the most traces with it.""" + groups_by_trace: dict[str, set[str]] = {} + for group, occurrences in occurrences_by_group.items(): + for occurrence in occurrences: + if occurrence.trace_key: + groups_by_trace.setdefault(occurrence.trace_key, set()).add(group) + related: dict[str, tuple[str, ...]] = {} + for group, occurrences in occurrences_by_group.items(): + shared_traces: Counter[str] = Counter() + traces = {occurrence.trace_key for occurrence in occurrences} + for trace in traces - {None}: + shared_traces.update(groups_by_trace[trace] - {group}) + ranked = sorted(shared_traces, key=lambda other: (-shared_traces[other], other)) + related[group] = tuple(ranked[:MAX_RELATED_FINGERPRINTS]) + return related + + +def _environment_measures( + occurrences: list[Occurrence], + environment: str, + family: str, + route_requests: RouteRequests, + window: ReportingWindow, +) -> EnvironmentMeasures: + in_environment = [item for item in occurrences if item.environment == environment] + in_window = [item for item in in_environment if item.timestamp >= window.start] + before_window = len(in_environment) - len(in_window) + affected_requests = len({item.affected_key for item in in_window}) + # The error rate compares failed requests with completed requests on the + # same routes, so failures without a route, such as background + # operations, are left out of both. + on_routes = [item for item in in_window if item.route] + route_affected_requests = len({item.affected_key for item in on_routes}) + routes = {item.route for item in on_routes} + requests = route_requests.total(family, environment, routes) + timestamps = [item.timestamp for item in in_environment] + return EnvironmentMeasures( + occurrences=len(in_window), + affected_requests=affected_requests, + observability_ids=len( + {item.observability_key for item in in_window if item.observability_key} + ), + route_affected_requests=route_affected_requests, + route_requests=requests, + error_rate=route_affected_requests / requests if requests else None, + hourly_rate=len(in_window) / window.hours, + previous_hourly_rate=before_window / window.lookback_hours, + first_seen=min(timestamps, default=None), + last_seen=max(timestamps, default=None), + ) + + +def _most_frequent_route(occurrences: list[Occurrence], window: ReportingWindow) -> str: + """Return the route most failures in the window used, preferring production.""" + in_window = [item for item in occurrences if item.timestamp >= window.start] + production = [item for item in in_window if item.environment == "production"] + routes = Counter(item.route for item in production or in_window) + route, _ = min(routes.items(), key=lambda item: (-item[1], item[0] or "")) + return public_route(route) + + +def _error_group( + group_fingerprint: str, + members: list[tuple[ErrorRecord, Occurrence]], + related_fingerprints: tuple[str, ...], + route_requests: RouteRequests, + window: ReportingWindow, +) -> ErrorGroup: + example = members[0][0] + occurrences = [occurrence for _, occurrence in members] + measures = { + environment: _environment_measures( + occurrences, environment, example.family, route_requests, window + ) + for environment in REPORTED_ENVIRONMENTS + } + return ErrorGroup( + fingerprint=group_fingerprint, + kind=example.kind, + family=public_family(example.family), + error_type=public_error_type(example.error_type), + route=_most_frequent_route(occurrences, window), + status_code=public_status(example.status_code), + operation=public_operation(example.operation_name), + production=measures["production"], + staging=measures["staging"], + related_fingerprints=related_fingerprints, + occurrences=tuple(occurrences), + ) + + +def build_groups( + records: Iterable[ErrorRecord], + request_counts: Iterable[RequestCount], + window: ReportingWindow, + key: bytes, +) -> list[ErrorGroup]: + """Group the failures in production and staging and measure each group. + + Only groups with a failure in the window are returned, most production + occurrences first. + """ + in_scope = [ + record + for record in records + if record.environment in REPORTED_ENVIRONMENTS + and window.lookback_start <= record.timestamp < window.end + ] + members_by_group: dict[str, list[tuple[ErrorRecord, Occurrence]]] = {} + for position, record in enumerate(primary_records(in_scope)): + members_by_group.setdefault(fingerprint(record, key), []).append( + (record, _occurrence(record, position)) + ) + related = _related_fingerprints( + { + group: [occurrence for _, occurrence in members] + for group, members in members_by_group.items() + } + ) + route_requests = RouteRequests(request_counts) + groups = [ + _error_group(group, members, related[group], route_requests, window) + for group, members in members_by_group.items() + if any(occurrence.timestamp >= window.start for _, occurrence in members) + ] + return sorted( + groups, + key=lambda group: ( + -group.production.occurrences, + -group.staging.occurrences, + group.fingerprint, + ), + ) diff --git a/.github/scripts/telemetry_error_issues/log_source.py b/.github/scripts/telemetry_error_issues/log_source.py new file mode 100644 index 000000000..090a94aca --- /dev/null +++ b/.github/scripts/telemetry_error_issues/log_source.py @@ -0,0 +1,49 @@ +"""Where the central telemetry logs are, as the deployment's settings name them. + +The names come from repository variables. BigQuery can't take a dataset or a +table name as a query parameter, so each name must match a strict identifier +pattern when the settings are read, and again just before it goes into a +query, a job configuration or a Logs Explorer link. A name that doesn't match +is refused, which keeps SQL out of the query. +""" + +from __future__ import annotations + +import re +from typing import NamedTuple + +# Each pattern allows only ASCII letters, digits and the separators the +# resource's own naming rules allow, and only within their length limits. +PROJECT_ID_PATTERN = re.compile(r"[a-z][a-z0-9-]{4,28}[a-z0-9]") +LOG_BUCKET_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,99}") +LOCATION_PATTERN = re.compile(r"[a-z][a-z0-9-]{0,62}") +DATASET_PATTERN = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,1023}") +# Cloud Logging gives a linked dataset's view the name of the log view. +LOG_VIEW_PATTERN = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,99}") + + +class LogSource(NamedTuple): + """The project, log bucket, linked dataset and view the run reads. + + ``service_namespace`` selects the application's records. It reaches BigQuery + only as a query parameter, so it needs no identifier pattern. + """ + + project_id: str + bucket: str + bucket_location: str + dataset: str + view: str + service_namespace: str + + +def checked_identifier(value: str, pattern: re.Pattern[str], setting: str) -> str: + """Return ``value`` if all of it matches ``pattern``, else raise ValueError. + + The error names the setting, never the value, because a value that fails + the check might be anything. + """ + if pattern.fullmatch(value) is None: + msg = f"{setting} is not a valid name" + raise ValueError(msg) + return value diff --git a/.github/scripts/telemetry_error_issues/normalization.py b/.github/scripts/telemetry_error_issues/normalization.py new file mode 100644 index 000000000..3bc2737c3 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/normalization.py @@ -0,0 +1,168 @@ +"""Normalize service, environment and source names, and validate public values. + +Each ``public_*`` function returns a value only if it matches the pattern of a +code-defined name, and a placeholder otherwise, so that free text from a record +never reaches an issue, a log or a job summary. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + +# Shown in place of a value that is missing, or that fails validation and so +# might not be code-defined. +MISSING = "-" +UNRECOGNIZED = "" + +PRODUCTION_ENVIRONMENT_NAMES = frozenset({"production", "main"}) +STAGING_ENVIRONMENT_NAMES = frozenset({"staging"}) +VERSIONED_SIMULATION_SERVICE = re.compile(r"^(policyengine-simulation-py)\d+-\d+-\d+$") + +FRAME_PARTS = re.compile(r'^(?P[^"\n]+)", line [0-9]+, in (?P\S+)$') +FINGERPRINT_FRAME_COUNT = 3 + +INSTALLED_PACKAGE_PREFIX = re.compile(r"^.*/(?:site|dist)-packages/") +STANDARD_LIBRARY_PREFIX = re.compile(r"^.*/lib/python3(?:\.\d+)?/") +APPLICATION_PACKAGES = ( + r"(?:policyengine_api|policyengine_fastapi" + r"|policyengine_simulation_[a-z0-9_]+|policyengine_stage12_[a-z0-9_]+)/" +) +APPLICATION_PATH = re.compile(rf"^{APPLICATION_PACKAGES}") +APPLICATION_PATH_IN_ABSOLUTE_PATH = re.compile(rf"(?:^|/)({APPLICATION_PACKAGES}.*)$") +FALLBACK_PATH_COMPONENTS = 2 + +ERROR_TYPE_PATTERN = re.compile(r"^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*$", re.ASCII) +MAX_ERROR_TYPE_LENGTH = 120 +UNMATCHED_ROUTE = "" +MAX_ROUTE_LENGTH = 200 +ROUTE_PARAMETER = re.compile( + r"^(?:<(?:[a-z]+:)?[A-Za-z_]\w*>|\{[A-Za-z_]\w*(?::[a-z]+)?\})$", re.ASCII +) +ROUTE_LITERAL = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,39}$") +# Literal route segments are words such as "v2" or "user-policies". A segment +# with more digits than this is more likely a record ID than code. +MAX_ROUTE_LITERAL_DIGITS = 2 +FAMILY_PATTERN = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") +MAX_FAMILY_LENGTH = 63 +OPERATION_PATTERN = re.compile(r"^[A-Za-z_][\w.:/-]*$", re.ASCII) +MAX_OPERATION_LENGTH = 100 +LOWEST_HTTP_STATUS = 100 +HIGHEST_HTTP_STATUS = 599 + + +def service_family(service_name: str) -> str: + """Return the service name, merging versioned simulation services into one.""" + return VERSIONED_SIMULATION_SERVICE.sub(r"\1", service_name) + + +def environment_class(environment_name: str | None) -> str: + """Classify a deployment environment name as production, staging or other. + + The Modal gateway reports its production environment as ``main``. + """ + if environment_name in PRODUCTION_ENVIRONMENT_NAMES: + return "production" + if environment_name in STAGING_ENVIRONMENT_NAMES: + return "staging" + return "other" + + +def normalize_path(path: str) -> str: + """Return a source path without its installation or checkout prefix.""" + installed = INSTALLED_PACKAGE_PREFIX.sub("", path, count=1) + if installed != path: + return installed + application = APPLICATION_PATH_IN_ABSOLUTE_PATH.search(path) + if application: + return application.group(1) + standard_library = STANDARD_LIBRARY_PREFIX.sub("", path, count=1) + if standard_library != path: + return standard_library + return "/".join(path.split("/")[-FALLBACK_PATH_COMPONENTS:]) + + +def frame_key(frame: str) -> str | None: + """Return ``path:function`` for one ``FRAME_PATTERN`` match, without its line.""" + parts = FRAME_PARTS.match(frame) + if parts is None: + return None + return f"{normalize_path(parts['path'])}:{parts['function']}" + + +def fingerprint_frames(frames: Iterable[str] | None) -> tuple[str, ...]: + """Choose the frames that identify where an exception was raised. + + ``frames`` come from the first traceback of the stack only (see + ``CHAINED_EXCEPTION_PATTERN``); for a chained exception that is its first + cause. These are the three innermost frames in application packages. + Without application frames they are the three innermost frames of any + kind, and without a stack, a placeholder. + """ + keys = [key for key in map(frame_key, frames or ()) if key is not None] + if not keys: + return (MISSING,) + application_keys = [key for key in keys if APPLICATION_PATH.match(key)] + return tuple((application_keys or keys)[-FINGERPRINT_FRAME_COUNT:]) + + +def _validated(value: str | None, pattern: re.Pattern[str], max_length: int) -> str: + if not value: + return MISSING + if len(value) <= max_length and pattern.fullmatch(value): + return value + return UNRECOGNIZED + + +def public_error_type(value: str | None) -> str: + """Return an exception class name, or a placeholder for anything else.""" + return _validated(value, ERROR_TYPE_PATTERN, MAX_ERROR_TYPE_LENGTH) + + +def public_family(value: str | None) -> str: + """Return a service name or family, or a placeholder for anything else.""" + return _validated(value, FAMILY_PATTERN, MAX_FAMILY_LENGTH) + + +def public_operation(value: str | None) -> str: + """Return a code-defined operation name, or a placeholder for anything else.""" + return _validated(value, OPERATION_PATTERN, MAX_OPERATION_LENGTH) + + +def _route_segment_is_code(segment: str) -> bool: + if ROUTE_PARAMETER.match(segment): + return True + digits = sum(character.isdigit() for character in segment) + return bool(ROUTE_LITERAL.match(segment)) and digits <= MAX_ROUTE_LITERAL_DIGITS + + +def public_route(value: str | None) -> str: + """Return a route template, or a placeholder for anything else. + + A path that holds a record ID, a query string or free text is not a + template, so it never reaches public output. + """ + if not value: + return MISSING + if value == UNMATCHED_ROUTE: + return value + if len(value) > MAX_ROUTE_LENGTH or not value.startswith("/"): + return UNRECOGNIZED + segments = [segment for segment in value.split("/") if segment] + if all(map(_route_segment_is_code, segments)): + return value + return UNRECOGNIZED + + +def public_status(value: object) -> int | None: + """Return an HTTP status code as an integer, or None for anything else.""" + if isinstance(value, bool): + return None + if isinstance(value, str) and value.isascii() and value.isdigit(): + value = int(value) + if isinstance(value, int) and LOWEST_HTTP_STATUS <= value <= HIGHEST_HTTP_STATUS: + return value + return None diff --git a/.github/scripts/telemetry_error_issues/records.py b/.github/scripts/telemetry_error_issues/records.py new file mode 100644 index 000000000..d685968f6 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/records.py @@ -0,0 +1,166 @@ +"""Error records, the one record chosen per failure, and their fingerprints.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from telemetry_error_issues.normalization import ( + MISSING, + environment_class, + fingerprint_frames, + service_family, +) + +if TYPE_CHECKING: + from collections.abc import Iterable + from datetime import datetime + +REQUEST_COMPLETED_EVENT = "request.completed" +# Changing how fingerprints are computed changes every fingerprint, and the +# job matches existing issues only by the current algorithm. A change to it +# must say how existing issues are carried over. +FINGERPRINT_ALGORITHM = 1 +FINGERPRINT_LENGTH = 16 +MIN_FINGERPRINT_KEY_LENGTH = 32 + + +@dataclass(frozen=True) +class ErrorRecord: + """One ERROR record from the central log bucket, as the query returns it. + + The query returns identifiers only as SHA-256 hashes, stacks only as frame + headers, and messages only as their first few words. + """ + + timestamp: datetime + service_name: str + environment_name: str | None + event_name: str | None + error_type: str | None + error_handled: bool | None + frames: tuple[str, ...] + stack_truncated: bool + message_template: str | None + attribute_error_type: str | None + failure_category: str | None + route: str | None + status_code: int | None + operation_name: str | None + request_key: str | None + observability_key: str | None + trace_key: str | None + + @property + def family(self) -> str: + """Return the service family that produced the record.""" + return service_family(self.service_name) + + @property + def environment(self) -> str: + """Return production, staging or other.""" + return environment_class(self.environment_name) + + @property + def kind(self) -> str: + """Return exception, message or failed_request. + + A record with an exception type is an exception, even a + ``request.completed`` record. Any other ``request.completed`` record is + a request that failed without a recorded exception. + """ + if self.error_type: + return "exception" + if self.event_name == REQUEST_COMPLETED_EVENT: + return "failed_request" + return "message" + + +def _primary_preference(record: ErrorRecord) -> tuple[int, int, datetime]: + """Sort key: exceptions before messages before request completions. + + Among exceptions, an unhandled one (the cause of an error response) comes + first; otherwise the earliest record does. + """ + if record.event_name == REQUEST_COMPLETED_EVENT: + source_rank = 2 + elif record.error_type: + source_rank = 0 + else: + source_rank = 1 + handled_rank = 0 if record.error_handled is False else 1 + return source_rank, handled_rank, record.timestamp + + +def primary_records(records: Iterable[ErrorRecord]) -> list[ErrorRecord]: + """Return one record per failure, in time order. + + Records of one service that share a request ID, or a trace when there is no + request ID, describe one failure. A record with neither is its own failure. + """ + failures: dict[tuple[str, str], list[ErrorRecord]] = {} + standalone: list[ErrorRecord] = [] + for record in records: + correlation = record.request_key or record.trace_key + if correlation is None: + standalone.append(record) + else: + failures.setdefault((record.family, correlation), []).append(record) + chosen = [min(group, key=_primary_preference) for group in failures.values()] + return sorted([*chosen, *standalone], key=lambda record: record.timestamp) + + +def fingerprint_parts(record: ErrorRecord) -> tuple[str, ...]: + """Return the kind and the values that identify the record's error group. + + An exception's route is left out, so one shared failure doesn't open an + issue for every route that reaches it. + """ + kind = record.kind + if kind == "exception": + return ( + kind, + record.family, + record.error_type or MISSING, + *fingerprint_frames(record.frames), + ) + if kind == "message": + return ( + kind, + record.family, + record.route or record.operation_name or MISSING, + record.message_template or MISSING, + record.attribute_error_type or MISSING, + record.failure_category or MISSING, + ) + status = MISSING if record.status_code is None else str(record.status_code) + return (kind, record.family, record.route or MISSING, status) + + +def fingerprint_key(secret: str) -> bytes: + """Return the HMAC key, rejecting a secret too short to be random.""" + if len(secret) < MIN_FINGERPRINT_KEY_LENGTH: + msg = ( + "the fingerprint key must be at least " + f"{MIN_FINGERPRINT_KEY_LENGTH} characters" + ) + raise ValueError(msg) + return secret.encode() + + +def fingerprint(record: ErrorRecord, key: bytes) -> str: + """Return the record's error group as 16 hex characters of HMAC-SHA256. + + The key keeps anyone from confirming a guessed message or stack frame by + hashing it and comparing the result with a public fingerprint. + """ + payload = json.dumps( + [FINGERPRINT_ALGORITHM, *fingerprint_parts(record)], + ensure_ascii=False, + separators=(",", ":"), + ) + digest = hmac.new(key, payload.encode(), hashlib.sha256).hexdigest() + return digest[:FINGERPRINT_LENGTH] diff --git a/.github/scripts/telemetry_error_issues/rendering.py b/.github/scripts/telemetry_error_issues/rendering.py new file mode 100644 index 000000000..fd7c0635c --- /dev/null +++ b/.github/scripts/telemetry_error_issues/rendering.py @@ -0,0 +1,264 @@ +"""Issue titles and bodies: the measures table, the state and a Logs Explorer link.""" + +from __future__ import annotations + +import json +import re +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from urllib.parse import quote, urlencode + +from telemetry_error_issues.log_source import ( + LOCATION_PATTERN, + LOG_BUCKET_PATTERN, + LOG_VIEW_PATTERN, + PROJECT_ID_PATTERN, + checked_identifier, +) +from telemetry_error_issues.normalization import MISSING, UNRECOGNIZED +from telemetry_error_issues.records import FINGERPRINT_ALGORITHM +from telemetry_error_issues.window import REPORTING_TIME_ZONE, ReportingWindow + +if TYPE_CHECKING: + from telemetry_error_issues.groups import EnvironmentMeasures, ErrorGroup + from telemetry_error_issues.log_source import LogSource + +ISSUE_TITLE_PREFIX = "[telemetry]" +MAX_TITLE_LENGTH = 256 +SECTION_START = "" +SECTION_END = "" +STATE_PATTERN = re.compile(r"") +NOT_AVAILABLE = "n/a" +LOGS_EXPLORER_URL = "https://console.cloud.google.com/logs/query" +SCHEMA_VERSION = "policyengine.observability.v2" +SIMULATION_FAMILY = "policyengine-simulation-py" + + +def code(value: str) -> str: + """Return ``value`` as Markdown inline code.""" + return f"`{value}`" + + +def _place(group: ErrorGroup) -> str: + """Describe where the group fails: its route, else its operation, else nothing.""" + if group.route != MISSING: + return f" on {code(group.route)}" + if group.kind == "message" and group.operation != MISSING: + return f" in {code(group.operation)}" + return "" + + +def _status_text(group: ErrorGroup) -> str: + return "HTTP error" if group.status_code is None else f"HTTP {group.status_code}" + + +def _summary(group: ErrorGroup) -> str: + if group.kind == "exception": + return group.error_type + if group.kind == "message": + return "logged error" + return f"{_status_text(group)} without recorded exception" + + +def issue_title(group: ErrorGroup) -> str: + """Return the issue title, which names the fingerprint for matching. + + The route is left out when the title would exceed GitHub's length limit. + """ + lead = f"{ISSUE_TITLE_PREFIX} {group.family}: {_summary(group)}" + suffix = f" ({group.fingerprint})" + title = f"{lead}{_place(group)}{suffix}" + return title if len(title) <= MAX_TITLE_LENGTH else f"{lead}{suffix}" + + +def _logs_timestamp(moment: datetime) -> str: + return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _logs_filters(group: ErrorGroup) -> list[str]: + """Return logging query clauses that select the group's records.""" + filters = [f'jsonPayload.schema_version="{SCHEMA_VERSION}"'] + if group.family == SIMULATION_FAMILY: + filters.append( + f'jsonPayload."service.name"=~"^{SIMULATION_FAMILY}[0-9]+-[0-9]+-[0-9]+$"' + ) + elif group.family not in {MISSING, UNRECOGNIZED}: + filters.append(f'jsonPayload."service.name"="{group.family}"') + if group.kind == "failed_request": + filters.append('jsonPayload."event.name"="request.completed"') + else: + filters.append("severity>=ERROR") + if group.kind == "exception" and group.error_type not in {MISSING, UNRECOGNIZED}: + filters.append(f'jsonPayload."error.type"="{group.error_type}"') + if group.kind != "exception" and group.route not in {MISSING, UNRECOGNIZED}: + filters.append(f'jsonPayload."http.route"="{group.route}"') + elif group.kind == "message" and group.operation not in {MISSING, UNRECOGNIZED}: + filters.append(f'jsonPayload."operation.name"="{group.operation}"') + if group.kind == "failed_request" and group.status_code is not None: + filters.append(f'jsonPayload."http.response.status_code"={group.status_code}') + return filters + + +def _log_view_path(log_source: LogSource) -> str: + """Return the log view's resource name, once each part has passed its pattern.""" + project_id = checked_identifier( + log_source.project_id, PROJECT_ID_PATTERN, "the project ID" + ) + location = checked_identifier( + log_source.bucket_location, LOCATION_PATTERN, "the log bucket location" + ) + bucket = checked_identifier(log_source.bucket, LOG_BUCKET_PATTERN, "the log bucket") + view = checked_identifier(log_source.view, LOG_VIEW_PATTERN, "the log view") + return f"projects/{project_id}/locations/{location}/buckets/{bucket}/views/{view}" + + +def logs_explorer_url( + group: ErrorGroup, window: ReportingWindow, log_source: LogSource +) -> str: + """Return a Logs Explorer link to the group's records in the window.""" + query = "\n".join( + [ + *_logs_filters(group), + f'timestamp>="{_logs_timestamp(window.start)}"', + f'timestamp<"{_logs_timestamp(window.end)}"', + ] + ) + view = _log_view_path(log_source) + project_id = log_source.project_id + settings = {"query": query, "storageScope": f"storage,{view}"} + encoded = "".join( + f";{name}={quote(value, safe='')}" for name, value in settings.items() + ) + return f"{LOGS_EXPLORER_URL}{encoded}?{urlencode({'project': project_id})}" + + +def eastern_time(moment: datetime | None) -> str: + """Return ``moment`` in US Eastern time, such as 2026-10-20 12:00 EDT, or n/a.""" + if moment is None: + return NOT_AVAILABLE + return moment.astimezone(REPORTING_TIME_ZONE).strftime("%Y-%m-%d %H:%M %Z") + + +def window_text(window: ReportingWindow) -> str: + """Describe the window by its Eastern start and end, and its length in hours.""" + return ( + f"{eastern_time(window.start)} to {eastern_time(window.end)} " + f"({window.hours:.0f} hours)" + ) + + +def _error_rate(measures: EnvironmentMeasures) -> str: + if measures.error_rate is None: + return NOT_AVAILABLE + return ( + f"{measures.error_rate:.2%} ({measures.route_affected_requests:,} of " + f"{measures.route_requests:,} requests)" + ) + + +def _measures_table(group: ErrorGroup) -> list[str]: + rows = [ + ("Occurrences", lambda measures: f"{measures.occurrences:,}"), + ("Affected requests", lambda measures: f"{measures.affected_requests:,}"), + ("Observability IDs", lambda measures: f"{measures.observability_ids:,}"), + ("Error rate on affected routes", _error_rate), + ("Occurrences per hour", lambda measures: f"{measures.hourly_rate:.2f}"), + ( + "Occurrences per hour, previous 7 days", + lambda measures: f"{measures.previous_hourly_rate:.2f}", + ), + ( + "First seen in the last 8 days", + lambda measures: eastern_time(measures.first_seen), + ), + ("Last seen", lambda measures: eastern_time(measures.last_seen)), + ] + return [ + "| Measure | Production | Staging |", + "| --- | --- | --- |", + *( + f"| {name} | {describe(group.production)} | {describe(group.staging)} |" + for name, describe in rows + ), + ] + + +def _group_details(group: ErrorGroup) -> list[str]: + details = [ + f"- Kind: {group.kind} (fingerprint algorithm {FINGERPRINT_ALGORITHM})", + f"- Service: {code(group.family)}", + ] + if group.kind == "exception": + details.append(f"- Exception type: {code(group.error_type)}") + if group.kind == "failed_request": + details.append(f"- Response: {_status_text(group)}") + if group.kind == "message" and group.operation != MISSING: + details.append(f"- Operation: {code(group.operation)}") + details.append(f"- Most frequent route: {code(group.route)}") + return details + + +def _state_comment(group: ErrorGroup, window: ReportingWindow) -> str: + state = { + "algorithm": FINGERPRINT_ALGORITHM, + "fingerprint": group.fingerprint, + "kind": group.kind, + "window_end": window.end.astimezone(UTC).isoformat(), + } + return f"" + + +def parse_state(body: str) -> dict[str, Any] | None: + """Return the state the job last wrote into an issue body, if it is there.""" + match = STATE_PATTERN.search(body) + if match is None: + return None + try: + state = json.loads(match.group(1)) + except ValueError: + return None + return state if isinstance(state, dict) else None + + +def issue_section( + group: ErrorGroup, window: ReportingWindow, log_source: LogSource +) -> str: + """Return the marked section of the issue body, which each run replaces.""" + related = ", ".join(map(code, group.related_fingerprints)) + lines = [ + SECTION_START, + f"### Error group {code(group.fingerprint)}", + "", + *_group_details(group), + f"- Window: {window_text(window)}", + "", + *_measures_table(group), + "", + ( + f"Related fingerprints (sharing traces): {related}" + if related + else "Related fingerprints: none" + ), + "", + ( + f"[Matching log records in Logs Explorer]" + f"({logs_explorer_url(group, window, log_source)}), for the window only. " + f"It needs read access to the `{log_source.project_id}` project." + ), + "", + _state_comment(group, window), + SECTION_END, + ] + return "\n".join(lines) + + +def issue_body( + group: ErrorGroup, window: ReportingWindow, log_source: LogSource +) -> str: + """Return a new issue's body: a short note, then the marked section.""" + note = ( + "The daily telemetry error report opened this issue for a production " + "error group. Each run replaces the marked section below with the " + "latest day's measures; text outside it is kept." + ) + return f"{note}\n\n{issue_section(group, window, log_source)}\n" diff --git a/.github/scripts/telemetry_error_issues/sync.py b/.github/scripts/telemetry_error_issues/sync.py new file mode 100644 index 000000000..0baf6b7a1 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/sync.py @@ -0,0 +1,283 @@ +"""Create, update and reopen one GitHub issue per production error group.""" + +from __future__ import annotations + +import re +from datetime import datetime, timedelta +from typing import TYPE_CHECKING, Any, NamedTuple + +from telemetry_error_issues.github_client import GitHubClient, GitHubLabelError +from telemetry_error_issues.records import FINGERPRINT_ALGORITHM +from telemetry_error_issues.rendering import ( + ISSUE_TITLE_PREFIX, + SECTION_END, + SECTION_START, + eastern_time, + issue_body, + issue_section, + issue_title, + parse_state, +) + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + + from telemetry_error_issues.groups import ErrorGroup + from telemetry_error_issues.log_source import LogSource + from telemetry_error_issues.window import ReportingWindow + +BOT_LOGIN = "github-actions[bot]" +TITLE_FINGERPRINT = re.compile(r"\(([0-9a-f]{16})\)$") +BASE_LABEL = "telemetry-error" +REGRESSION_LABEL = "regression" +SERVICE_LABEL_PREFIX = "service: " +MAX_LABEL_LENGTH = 50 +LABEL_DETAILS = { + BASE_LABEL: ("b60205", "Opened by the daily telemetry error report"), + REGRESSION_LABEL: ("d93f0b", "An error that returned after its issue was closed"), +} +SERVICE_LABEL_DETAILS = ("1d76db", "Service that produced a telemetry error group") +# A production group opens an issue only when it fails this often, in this many +# distinct requests, during the window; the same thresholds reopen an issue. +MIN_PRODUCTION_OCCURRENCES = 3 +MIN_PRODUCTION_REQUESTS = 2 +MAX_NEW_ISSUES_PER_RUN = 5 +# Failures in the first day after an issue is closed may come from code that +# hasn't been deployed yet, so they never reopen it. +REOPEN_DELAY = timedelta(hours=24) +REOPENABLE_CLOSE_REASONS = frozenset({None, "completed"}) + + +class SyncAction(NamedTuple): + """What one run did, or in a dry run would do, for one error group.""" + + fingerprint: str + kind: str + family: str + production_occurrences: int + staging_occurrences: int + issue_number: int | None + action: str + + +class SyncResult(NamedTuple): + """Every group's action, and how many new issues waited for a later run.""" + + actions: list[SyncAction] + deferred: int + + +def replace_section(body: str, section: str) -> str: + """Replace the marked section of ``body``, or add it at the end if missing.""" + start = body.find(SECTION_START) + end = body.find(SECTION_END, start) if start >= 0 else -1 + if end >= 0: + return f"{body[:start]}{section}{body[end + len(SECTION_END) :]}" + kept = body.rstrip() + return f"{kept}\n\n{section}\n" if kept else f"{section}\n" + + +def _issue_fingerprint(issue: Mapping[str, Any]) -> str | None: + """Return the fingerprint from the state comment, or else from the title.""" + state = parse_state(issue.get("body") or "") + if state and state.get("algorithm") == FINGERPRINT_ALGORITHM: + fingerprint_value = state.get("fingerprint") + if isinstance(fingerprint_value, str): + return fingerprint_value + title = issue.get("title") or "" + match = TITLE_FINGERPRINT.search(title) + if match and title.startswith(ISSUE_TITLE_PREFIX): + return match.group(1) + return None + + +def _issues_by_fingerprint( + issues: Iterable[Mapping[str, Any]], +) -> dict[str, Mapping[str, Any]]: + """Index the bot's issues by fingerprint, preferring open, then newest.""" + index: dict[str, Mapping[str, Any]] = {} + for issue in issues: + if (issue.get("user") or {}).get("login") != BOT_LOGIN: + continue + group = _issue_fingerprint(issue) + if group is None: + continue + rank = (issue.get("state") == "open", issue.get("number", 0)) + current = index.get(group) + if current is None or rank > ( + current.get("state") == "open", + current["number"], + ): + index[group] = issue + return index + + +def _meets_thresholds(occurrences: int, affected_requests: int) -> bool: + return ( + occurrences >= MIN_PRODUCTION_OCCURRENCES + and affected_requests >= MIN_PRODUCTION_REQUESTS + ) + + +def _folded(names: Iterable[str]) -> set[str]: + """Return label names case-folded, as GitHub compares them.""" + return {name.casefold() for name in names} + + +def _issue_labels(group: ErrorGroup) -> list[str]: + service_label = f"{SERVICE_LABEL_PREFIX}{group.family}" + if len(service_label) > MAX_LABEL_LENGTH: + return [BASE_LABEL] + return [BASE_LABEL, service_label] + + +def _parse_time(value: object) -> datetime | None: + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value) + except ValueError: + return None + return parsed if parsed.tzinfo is not None else None + + +class IssueSynchronizer: + """Creates and updates one issue per production error group.""" + + def __init__( + self, + github: GitHubClient, + window: ReportingWindow, + log_source: LogSource, + *, + dry_run: bool, + ) -> None: + """Sync the groups of ``window``; a dry run reads but never writes.""" + self._github = github + self._window = window + self._log_source = log_source + self._dry_run = dry_run + # Case-folded names: GitHub label names are unique regardless of case. + self._known_labels: set[str] | None = None + + def run(self, groups: Iterable[ErrorGroup]) -> SyncResult: + """Sync each group, opening at most five issues, largest groups first.""" + issues = _issues_by_fingerprint(self._github.list_issues(BOT_LOGIN)) + actions: list[SyncAction] = [] + opened = 0 + for group in groups: + issue = issues.get(group.fingerprint) + number: int | None = None + if issue is not None: + number = issue["number"] + action = self._update(group, issue) + elif not _meets_thresholds( + group.production.occurrences, group.production.affected_requests + ): + action = "below-threshold" + elif opened >= MAX_NEW_ISSUES_PER_RUN: + action = "deferred" + else: + number = self._open(group) + opened += 1 + action = "created" + actions.append( + SyncAction( + fingerprint=group.fingerprint, + kind=group.kind, + family=group.family, + production_occurrences=group.production.occurrences, + staging_occurrences=group.staging.occurrences, + issue_number=number, + action=action, + ) + ) + deferred = sum(action.action == "deferred" for action in actions) + return SyncResult(actions, deferred) + + def _ensure_labels(self, names: Iterable[str]) -> None: + if self._known_labels is None: + self._known_labels = _folded(self._github.list_labels()) + for name in names: + if name.casefold() not in self._known_labels: + color, description = LABEL_DETAILS.get(name, SERVICE_LABEL_DETAILS) + self._github.create_label(name, color, description) + self._known_labels.add(name.casefold()) + + def _open(self, group: ErrorGroup) -> int | None: + if self._dry_run: + return None + labels = _issue_labels(group) + self._ensure_labels(labels) + issue = self._github.create_issue( + issue_title(group), + issue_body(group, self._window, self._log_source), + labels, + ) + number = issue["number"] + applied = _folded(label["name"] for label in issue.get("labels", [])) + missing = [label for label in labels if label.casefold() not in applied] + if missing and not _folded(missing) <= _folded( + self._github.add_labels(number, missing) + ): + msg = f"GitHub did not apply the labels to issue #{number}" + raise GitHubLabelError(msg) + return number + + def _reopen_since(self, issue: Mapping[str, Any]) -> datetime | None: + """Return when regressions count from, if the issue should reopen.""" + if issue.get("state") != "closed": + return None + if issue.get("state_reason") not in REOPENABLE_CLOSE_REASONS: + return None + closed_at = _parse_time(issue.get("closed_at")) + return None if closed_at is None else closed_at + REOPEN_DELAY + + def _update(self, group: ErrorGroup, issue: Mapping[str, Any]) -> str: + body = issue.get("body") or "" + state = parse_state(body) + reported_end = _parse_time(state.get("window_end")) if state else None + if reported_end is not None and reported_end > self._window.end: + return "kept-newer" + new_body = replace_section( + body, issue_section(group, self._window, self._log_source) + ) + since = self._reopen_since(issue) + if since is not None and _meets_thresholds(*group.production_since(since)): + if not self._dry_run: + self._reopen(issue["number"], new_body, group, since) + return "reopened" + if new_body == body: + return "unchanged" + if not self._dry_run: + self._github.update_issue(issue["number"], {"body": new_body}) + return "updated" + + def _reopen( + self, number: int, body: str, group: ErrorGroup, since: datetime + ) -> None: + occurrences, affected_requests = group.production_since(since) + self._ensure_labels([REGRESSION_LABEL]) + self._github.update_issue( + number, {"body": body, "state": "open", "state_reason": "reopened"} + ) + self._github.add_labels(number, [REGRESSION_LABEL]) + self._github.add_comment( + number, + "Production failures returned after this issue was closed: " + f"{occurrences:,} occurrences in {affected_requests:,} requests since " + f"{eastern_time(since)}, 24 hours after it was closed.", + ) + + +def sync_issues( + groups: Iterable[ErrorGroup], + window: ReportingWindow, + log_source: LogSource, + github: GitHubClient, + *, + dry_run: bool, +) -> SyncResult: + """Create or update the issue for each group; a dry run makes no changes.""" + synchronizer = IssueSynchronizer(github, window, log_source, dry_run=dry_run) + return synchronizer.run(groups) diff --git a/.github/scripts/telemetry_error_issues/transport.py b/.github/scripts/telemetry_error_issues/transport.py new file mode 100644 index 000000000..14c69497a --- /dev/null +++ b/.github/scripts/telemetry_error_issues/transport.py @@ -0,0 +1,96 @@ +"""HTTP requests and responses, the standard library transport, and API errors.""" + +from __future__ import annotations + +import json +import re +from typing import TYPE_CHECKING, NamedTuple +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +if TYPE_CHECKING: + from collections.abc import Mapping + +HTTP_TIMEOUT_SECONDS = 60 +# A short code from an API error, such as "invalidQuery" or "Not Found". The +# error's message is never used, because it can repeat values from a record. +ERROR_REASON_PATTERN = re.compile(r"^[A-Za-z][A-Za-z ]{0,59}$") + + +class HttpRequest(NamedTuple): + """An HTTP request for a transport to send.""" + + method: str + url: str + headers: dict[str, str] + body: bytes | None + + +class HttpResponse(NamedTuple): + """An HTTP response, including error responses. Header names are lowercase.""" + + status: int + reason: str + headers: Mapping[str, str] + body: bytes + + +def urllib_transport(request: HttpRequest) -> HttpResponse: + """Send a request with the standard library, returning HTTP errors too.""" + outgoing = Request( + request.url, + data=request.body, + headers=request.headers, + method=request.method, + ) + try: + response = urlopen(outgoing, timeout=HTTP_TIMEOUT_SECONDS) + except HTTPError as error: + with error: + return HttpResponse( + status=error.code, + reason=str(error.reason), + headers={name.lower(): value for name, value in error.headers.items()}, + body=error.read(), + ) + with response: + return HttpResponse( + status=response.status, + reason=response.reason, + headers={name.lower(): value for name, value in response.headers.items()}, + body=response.read(), + ) + + +class ApiError(Exception): + """An API request failed. The text holds only code-defined values.""" + + service = "API" + + def __init__( + self, message: str, *, status: int | None = None, reason: str | None = None + ) -> None: + """Keep the HTTP status and reason for the caller's summary.""" + super().__init__(message) + self.status = status + self.reason = reason + + @classmethod + def from_response(cls, response: HttpResponse) -> ApiError: + """Describe an error response by its status and reason, never its body.""" + reason = _error_reason(response) + message = f"{cls.service} request failed: HTTP {response.status} {reason}" + return cls(message, status=response.status, reason=reason) + + +def _error_reason(response: HttpResponse) -> str: + """Return a Google API error's reason code, else the HTTP reason phrase.""" + try: + payload = json.loads(response.body) + reason = payload["error"]["errors"][0]["reason"] + except (ValueError, KeyError, IndexError, TypeError): + reason = None + for candidate in (reason, response.reason): + if isinstance(candidate, str) and ERROR_REASON_PATTERN.match(candidate): + return candidate + return "unknown" diff --git a/.github/scripts/telemetry_error_issues/window.py b/.github/scripts/telemetry_error_issues/window.py new file mode 100644 index 000000000..5145fc1a0 --- /dev/null +++ b/.github/scripts/telemetry_error_issues/window.py @@ -0,0 +1,57 @@ +"""The reporting window: one US Eastern calendar day, and the week before it.""" + +from __future__ import annotations + +from datetime import UTC, date, datetime, timedelta +from datetime import time as time_of_day +from typing import NamedTuple +from zoneinfo import ZoneInfo + +REPORTING_TIME_ZONE = ZoneInfo("America/New_York") +MIDNIGHT = time_of_day(0, 0) + +# Each run also reads the seven days before the window, for comparison rates, +# first-seen times and regressions. +LOOKBACK = timedelta(days=7) + + +class ReportingWindow(NamedTuple): + """One Eastern calendar day, as the half-open interval [start, end).""" + + start: datetime + end: datetime + + @property + def hours(self) -> float: + """Return the elapsed hours: 23 or 25 on the days the clocks change.""" + return _hours_between(self.start, self.end) + + @property + def lookback_start(self) -> datetime: + """Return midnight Eastern seven days before the window starts.""" + return self.start - LOOKBACK + + @property + def lookback_hours(self) -> float: + """Return the elapsed hours of the seven days before the window.""" + return _hours_between(self.lookback_start, self.start) + + +def _hours_between(start: datetime, end: datetime) -> float: + # Subtracting two datetimes in the same zone ignores a clock change, so + # compare them in UTC. + return (end.astimezone(UTC) - start.astimezone(UTC)) / timedelta(hours=1) + + +def reporting_window(report_date: date) -> ReportingWindow: + """Return the window from midnight Eastern on ``report_date`` to the next.""" + next_date = report_date + timedelta(days=1) + return ReportingWindow( + start=datetime.combine(report_date, MIDNIGHT, tzinfo=REPORTING_TIME_ZONE), + end=datetime.combine(next_date, MIDNIGHT, tzinfo=REPORTING_TIME_ZONE), + ) + + +def default_report_date(now: datetime) -> date: + """Return the most recent Eastern calendar day that ended before ``now``.""" + return now.astimezone(REPORTING_TIME_ZONE).date() - timedelta(days=1) diff --git a/.github/workflows/telemetry-error-issues.yml b/.github/workflows/telemetry-error-issues.yml new file mode 100644 index 000000000..f2f549634 --- /dev/null +++ b/.github/workflows/telemetry-error-issues.yml @@ -0,0 +1,79 @@ +name: Telemetry error issues + +# Each day at 00:45 US Eastern, report the previous Eastern calendar day's +# error groups from the central telemetry log bucket as GitHub issues. 00:45 +# is never skipped or repeated when the clocks change, and it avoids the +# delays GitHub has for schedules at the start of an hour. +on: + schedule: + - cron: "45 0 * * *" + timezone: "America/New_York" + workflow_dispatch: + inputs: + dry_run: + description: "Query and group errors without changing any issue" + type: boolean + default: true + report_date: + description: "Eastern calendar day to report, as YYYY-MM-DD (default: the previous day)" + type: string + default: "" + +concurrency: + group: telemetry-error-issues + cancel-in-progress: false + +jobs: + sync: + name: Sync telemetry error issues + if: github.repository == 'PolicyEngine/policyengine-api' + runs-on: ubuntu-latest + # Deployments to this environment are limited to master, which keeps other + # branches from authenticating as its service account. + environment: telemetry-error-issues + timeout-minutes: 15 + permissions: + contents: read + issues: write + id-token: write + env: + # uv installs uv.lock on the Cloud Run image's Python, not the 3.14 + # that .python-version selects. + UV_PYTHON: "3.12" + steps: + - name: Require master branch + if: github.ref != 'refs/heads/master' + run: exit 1 + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + # The same uv version as the Cloud Run image and the release job. + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.13.4" + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.TELEMETRY_ERROR_ISSUES_SERVICE_ACCOUNT }} + token_format: access_token + create_credentials_file: false + - name: Sync telemetry error issues + env: + GOOGLE_OAUTH_ACCESS_TOKEN: ${{ steps.auth.outputs.access_token }} + GITHUB_TOKEN: ${{ github.token }} + TELEMETRY_ERROR_FINGERPRINT_KEY: ${{ secrets.TELEMETRY_ERROR_FINGERPRINT_KEY }} + OBSERVABILITY_PROJECT_ID: ${{ vars.OBSERVABILITY_PROJECT_ID }} + OBSERVABILITY_LOG_BUCKET: ${{ vars.OBSERVABILITY_LOG_BUCKET }} + OBSERVABILITY_LOG_BUCKET_LOCATION: ${{ vars.OBSERVABILITY_LOG_BUCKET_LOCATION }} + OBSERVABILITY_LOG_DATASET: ${{ vars.OBSERVABILITY_LOG_DATASET }} + OBSERVABILITY_LOG_VIEW: ${{ vars.OBSERVABILITY_LOG_VIEW }} + OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} + REPORT_DATE: ${{ inputs.report_date }} + run: bash .github/scripts/sync_telemetry_error_issues.sh diff --git a/Makefile b/Makefile index 17743258d..7b13feb2c 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test-env-vars: uv run --frozen --extra dev pytest tests/env_variables test: - MAX_HOUSEHOLDS=1000 uv run --frozen --extra dev coverage run -a --branch -m pytest tests/to_refactor tests/unit tests/contract tests/integration/test_budget_window_in_flight_dedupe.py --disable-pytest-warnings + MAX_HOUSEHOLDS=1000 uv run --frozen --extra dev --group telemetry-error-issues coverage run -a --branch -m pytest tests/to_refactor tests/unit tests/contract tests/integration/test_budget_window_in_flight_dedupe.py --disable-pytest-warnings uv run --frozen --extra dev coverage xml -i quality-guards: @@ -31,7 +31,7 @@ typecheck-v2: uv run --frozen --extra dev mypy debug-test: - MAX_HOUSEHOLDS=1000 FLASK_DEBUG=1 uv run --frozen --extra dev pytest -vv --durations=0 tests + MAX_HOUSEHOLDS=1000 FLASK_DEBUG=1 uv run --frozen --extra dev --group telemetry-error-issues pytest -vv --durations=0 tests format: ruff format . diff --git a/changelog.d/telemetry-error-issues.added.md b/changelog.d/telemetry-error-issues.added.md new file mode 100644 index 000000000..d712f6a11 --- /dev/null +++ b/changelog.d/telemetry-error-issues.added.md @@ -0,0 +1 @@ +A daily workflow reports the previous day's production error groups from the central telemetry logs as GitHub issues, with production and staging counts and no error messages or stack text. diff --git a/gcp/observability/README.md b/gcp/observability/README.md index 695a57ff7..bd6d060fa 100644 --- a/gcp/observability/README.md +++ b/gcp/observability/README.md @@ -131,6 +131,139 @@ Compare the deployed interval with the recorded pre-fix baseline: 46 collector metric export errors comprising 863 rejected points. Trace export had no corresponding rejected points in that interval. +## Daily telemetry error issues + +The `Telemetry error issues` workflow +(`.github/workflows/telemetry-error-issues.yml`) runs at 00:45 US Eastern each +day. It reports the previous Eastern calendar day, from midnight to midnight, +which lasts 23 or 25 hours on the days the clocks change. It runs +`.github/scripts/sync_telemetry_error_issues.py`, a short entry point that calls +the `telemetry_error_issues` package beside it in `.github/scripts/`. The package's +`cli` module reads the arguments and settings and runs each step; `bigquery` +runs the queries, `records` and `groups` choose, fingerprint and measure the +failures, `rendering` writes the issue text, and `sync` updates the issues +through `github_client`. Each run: + +1. Queries the central bucket through its linked BigQuery dataset + `observability_logs`. It reads the `_AllLogs` view for ERROR records with + the `policyengine.api-v1` service namespace in the report day and the seven + days before it, excluding the health-check routes. It also counts + `request.completed` records per route in the report day, for error rates. + Each query is limited to 5 GiB billed. +2. Chooses one record per failed request and service: the exception record, + else the error message, else the failed `request.completed` record. +3. Groups failures by a 16-character HMAC-SHA256 fingerprint of the service + family and either the exception type and innermost application stack + frames, the route and message template, or the route and HTTP status. + Frames come from the first traceback of the stack, which for a chained + exception is its first cause. The logging library keeps only the first + 16,384 characters of a stack, so a long cause can drop the final + traceback, but the first one is always kept. +4. Creates or updates one issue per group in this repository, labelled + `telemetry-error` and `service: `. A group opens an issue + only after 3 production failures in at least 2 requests in one day, and at + most 5 issues open per run. Staging counts appear in each issue but never + open one. An issue closed as completed reopens, with the `regression` + label, when production failures that start 24 hours after it closed meet + the same thresholds. + +This repository is public, so issues, workflow logs and job summaries hold +only fingerprints, counts, service names, exception class names, route +templates and a Logs Explorer link. Error messages, stack text and +identifiers stay in BigQuery: the queries return stacks only as file and +function names, messages only as their first three words, and identifiers +only as hashes. A failed run prints only the failed stage, the exception class +and the HTTP status. + +### Running it by hand + +Dispatch the workflow from `master`. `dry_run` defaults to `true`: the job +queries and groups errors and writes the actions it would take to the job +summary, without changing any issue. `report_date` (`YYYY-MM-DD`) reports a +different Eastern day, such as one a scheduled run missed. + +To run it locally with your own credentials, use a throwaway fingerprint key. +Its fingerprints won't match existing issues. + +```bash +GOOGLE_OAUTH_ACCESS_TOKEN="$(gcloud auth print-access-token)" \ +GITHUB_TOKEN="$(gh auth token)" \ +GITHUB_REPOSITORY=PolicyEngine/policyengine-api \ +TELEMETRY_ERROR_FINGERPRINT_KEY="$(openssl rand -hex 32)" \ +OBSERVABILITY_LOG_BUCKET=policyengine-observability \ +OBSERVABILITY_LOG_BUCKET_LOCATION=global \ +OBSERVABILITY_LOG_DATASET=observability_logs \ +OBSERVABILITY_LOG_VIEW=_AllLogs \ +OBSERVABILITY_SERVICE_NAMESPACE=policyengine.api-v1 \ +uv run --frozen --only-group telemetry-error-issues \ + python .github/scripts/sync_telemetry_error_issues.py \ + --project-id policyengine-observability --dry-run +``` + +The workflow runs the script the same way, through +`.github/scripts/sync_telemetry_error_issues.sh`: in an environment holding +only the `telemetry-error-issues` dependency group from `pyproject.toml`, +which is `google-cloud-bigquery` at the version `uv.lock` pins. The queries +read rows by iterating each job's result over the BigQuery REST API, never +through the Storage Read API, which the service account has no role for. + +### Required infrastructure + +| Resource | Purpose | +| --- | --- | +| Linked BigQuery dataset `observability_logs` on the `policyengine-observability` bucket | SQL access to the bucket's log views | +| Service account `telemetry-error-issues@policyengine-observability.iam.gserviceaccount.com` | The identity the workflow queries BigQuery as | +| `roles/bigquery.jobUser` on the project, and `roles/bigquery.dataViewer` on the linked dataset | Run query jobs and read the log views | +| `roles/iam.workloadIdentityUser` on the service account for `principal://iam.googleapis.com/projects/389282473430/locations/global/workloadIdentityPools/github/subject/repo:PolicyEngine/policyengine-api:environment:telemetry-error-issues` | Lets only jobs in the `telemetry-error-issues` GitHub environment act as the service account | +| GitHub environment `telemetry-error-issues`, with deployments limited to `master` | Keeps other branches from acting as the service account | +| Environment variable `TELEMETRY_ERROR_ISSUES_SERVICE_ACCOUNT` | The service account's email address | +| Environment secret `TELEMETRY_ERROR_FINGERPRINT_KEY` | The 32-byte hexadecimal HMAC key for fingerprints | + +The workflow also uses the repository secret `GCP_WORKLOAD_IDENTITY_PROVIDER` +and these repository variables, which say where the logs are: + +| Repository variable | Value | Used for | +| --- | --- | --- | +| `OBSERVABILITY_PROJECT_ID` | `policyengine-observability` | The project that runs the query jobs and holds the bucket | +| `OBSERVABILITY_LOG_BUCKET` | `policyengine-observability` | The log bucket, in the Logs Explorer link | +| `OBSERVABILITY_LOG_BUCKET_LOCATION` | `global` | The bucket's location, in the Logs Explorer link | +| `OBSERVABILITY_LOG_DATASET` | `observability_logs` | The linked BigQuery dataset the queries read | +| `OBSERVABILITY_LOG_VIEW` | `_AllLogs` | The log view, which is also the name of the linked dataset's view | +| `OBSERVABILITY_SERVICE_NAMESPACE` | `policyengine.api-v1` | The `service.namespace` of the records to report; the API reads the same variable | + +Each name must match a strict identifier pattern +(`.github/scripts/telemetry_error_issues/log_source.py`), because BigQuery +can't take a dataset or view name as a query parameter; the run refuses any +other value. The namespace is sent as a query parameter. The rules that decide +which group a failure belongs to stay in code, because changing one changes +fingerprints. + +These resources were created on 2026-10-20: + +- The linked dataset is in the BigQuery `US` location and holds one view, + `_AllLogs`. +- `roles/bigquery.dataViewer` was granted on the dataset alone with + `GRANT ... ON SCHEMA`; `bq update --source` would replace the dataset's + access list. The service account needs no Cloud Logging role. +- A dry run as the service account, for 2026-10-19, read 1,648 ERROR records + across eight days and found 15 error groups, 5 of them above the issue + thresholds. The queries scanned about 0.18 GiB of error records and + 0.02 GiB of request counts, so a daily run stays well inside BigQuery's free + monthly allowance. + +Never rotate `TELEMETRY_ERROR_FINGERPRINT_KEY` on its own. Every fingerprint +would change, so the next run would open a second issue for each group that +already has one. A key change needs a plan for moving the existing issues to +their new fingerprints, as does any change to `FINGERPRINT_ALGORITHM` in the +script. + +### Known gap: Cloud Run platform failures + +The source sinks forward only records in the application log format +(`schema_version = policyengine.observability.v2`). Cloud Run request logs +are not forwarded, so requests that time out, run out of memory or crash the +container before the application logs anything never reach these reports. + ## Rollback 1. Remove the OTel endpoint from participating service configuration. diff --git a/pyproject.toml b/pyproject.toml index 9805eaf0f..ff73d1fde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,13 @@ dev = [ "towncrier>=24.8.0", ] +[dependency-groups] +# Only the daily telemetry error issues workflow needs these. It installs this +# group alone (uv run --frozen --only-group), not the API's dependencies. +telemetry-error-issues = [ + "google-cloud-bigquery>=3.30,<4", +] + [tool.hatch.build.targets.wheel] packages = ["policyengine_api"] diff --git a/tests/unit/telemetry_error_issues/__init__.py b/tests/unit/telemetry_error_issues/__init__.py new file mode 100644 index 000000000..72527b225 --- /dev/null +++ b/tests/unit/telemetry_error_issues/__init__.py @@ -0,0 +1 @@ +"""Tests for the telemetry error issues package under .github/scripts.""" diff --git a/tests/unit/telemetry_error_issues/bigquery_support.py b/tests/unit/telemetry_error_issues/bigquery_support.py new file mode 100644 index 000000000..dc8ae8714 --- /dev/null +++ b/tests/unit/telemetry_error_issues/bigquery_support.py @@ -0,0 +1,68 @@ +"""A fake BigQuery client library client, with rows as the library returns them.""" + +from __future__ import annotations + +from google.cloud.bigquery.table import Row + +ACCESS_TOKEN = "fake-access-token" + + +def library_rows(rows: list[dict[str, object]]) -> list[Row]: + """Return each dictionary as a ``Row``, the type the client library yields.""" + return [ + Row(tuple(row.values()), {name: index for index, name in enumerate(row)}) + for row in rows + ] + + +class FakeQueryJob: + """A finished query job: its result yields the rows, or raises the error.""" + + def __init__( + self, client: FakeBigQueryClient, rows: list[Row], error: Exception | None + ) -> None: + self.client = client + self.rows = rows + self.error = error + + def result(self, timeout: float | None = None) -> list[Row]: + self.client.result_timeouts.append(timeout) + if self.error is not None: + raise self.error + return self.rows + + +class FakeBigQueryClient: + """Stands in for ``bigquery.Client``, recording each query and its job config. + + Each query returns the rows given for its ``query`` label, or raises + ``error`` when the test reads its result, as a failed job does. + """ + + def __init__( + self, + rows_by_label: dict[str, list[dict[str, object]]] | None = None, + *, + error: Exception | None = None, + ) -> None: + self.rows_by_label = rows_by_label or {} + self.error = error + self.created_with: list[tuple[str, str]] = [] + self.queries: list[tuple[str, object]] = [] + self.result_timeouts: list[float | None] = [] + + def factory(self, project_id: str, access_token: str) -> FakeBigQueryClient: + """Replace ``library_client``: record the arguments and return this fake.""" + self.created_with.append((project_id, access_token)) + return self + + def query(self, sql: str, job_config: object) -> FakeQueryJob: + self.queries.append((sql, job_config)) + rows = self.rows_by_label.get(job_config.labels["query"], []) + return FakeQueryJob(self, library_rows(rows), self.error) + + +def sent_parameters(client: FakeBigQueryClient, index: int = 0) -> dict[str, object]: + """Return the query parameters of the ``index``-th query, by name.""" + _, job_config = client.queries[index] + return {parameter.name: parameter for parameter in job_config.query_parameters} diff --git a/tests/unit/telemetry_error_issues/conftest.py b/tests/unit/telemetry_error_issues/conftest.py new file mode 100644 index 000000000..c0a2b5304 --- /dev/null +++ b/tests/unit/telemetry_error_issues/conftest.py @@ -0,0 +1,13 @@ +"""Make the telemetry_error_issues package importable, as running the script does. + +Python puts the directory of a script it runs by path first on ``sys.path``, +which is how ``sync_telemetry_error_issues.py`` imports the package beside it. +""" + +import sys +from pathlib import Path + +SCRIPTS_DIRECTORY = Path(__file__).resolve().parents[3] / ".github" / "scripts" + +if str(SCRIPTS_DIRECTORY) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIRECTORY)) diff --git a/tests/unit/telemetry_error_issues/github_support.py b/tests/unit/telemetry_error_issues/github_support.py new file mode 100644 index 000000000..d77182967 --- /dev/null +++ b/tests/unit/telemetry_error_issues/github_support.py @@ -0,0 +1,181 @@ +"""An in-memory GitHub issues API, and helpers that sync issues against it.""" + +from __future__ import annotations + +import json +from urllib.parse import parse_qs, urlsplit + +from telemetry_error_issues.github_client import GitHubClient +from telemetry_error_issues.groups import build_groups +from telemetry_error_issues.sync import sync_issues +from telemetry_error_issues.transport import HttpResponse +from telemetry_error_issues.window import reporting_window + +from tests.unit.telemetry_error_issues.report_support import ( + FINGERPRINT_KEY, + LOG_SOURCE, + REPORT_DATE, + exception_record, + hours_after_window_noon, +) + +REPOSITORY = "PolicyEngine/policyengine-api" +BOT = "github-actions[bot]" + + +class FakeGitHub: + """An in-memory GitHub issues API with a clock that advances when it sleeps.""" + + def __init__( + self, + issues: list[dict] = (), + labels: tuple[str, ...] = (), + *, + drop_created_labels: bool = False, + ) -> None: + self.issues = {issue["number"]: dict(issue) for issue in issues} + self.labels = set(labels) + self.drop_created_labels = drop_created_labels + self.failures: list[object] = [] + self.requests: list[tuple[float, object]] = [] + self.comments: dict[int, list[str]] = {} + self.sleeps: list[float] = [] + self.now = 0.0 + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += seconds + + def clock(self) -> float: + return self.now + + def client(self) -> object: + return GitHubClient( + REPOSITORY, "fake-github-token", self, sleep=self.sleep, clock=self.clock + ) + + @property + def writes(self) -> list[tuple[float, object]]: + return [entry for entry in self.requests if entry[1].method != "GET"] + + def _respond(self, payload: object, status: int = 200) -> object: + return HttpResponse( + status=status, reason="OK", headers={}, body=json.dumps(payload).encode() + ) + + def __call__(self, request: object) -> object: + self.requests.append((self.now, request)) + if request.method != "GET" and self.failures: + return self.failures.pop(0) + split = urlsplit(request.url) + path = split.path.removeprefix(f"/repos/{REPOSITORY}") + query = {name: values[0] for name, values in parse_qs(split.query).items()} + body = json.loads(request.body) if request.body else None + return self._route(request.method, path, query, body) + + def _route(self, method: str, path: str, query: dict, body: dict | None) -> object: + parts = path.strip("/").split("/") + if len(parts) > 1: + return self._route_issue(method, int(parts[1]), parts[2:], body) + if method == "GET" and parts == ["issues"]: + issues = sorted(self.issues.values(), key=lambda issue: issue["number"]) + return self._respond(self._page(issues, query)) + if method == "GET": + labels = [{"name": name} for name in sorted(self.labels)] + return self._respond(self._page(labels, query)) + if parts == ["labels"]: + # GitHub label names are unique regardless of case. + if body["name"].casefold() in {name.casefold() for name in self.labels}: + return self._respond({"message": "Validation Failed"}, status=422) + self.labels.add(body["name"]) + return self._respond({"name": body["name"]}, status=201) + return self._create(body) + + def _existing_label_names(self, requested: list[str]) -> list[str]: + """Return the stored spelling of each label, as GitHub applies them.""" + stored = {name.casefold(): name for name in self.labels} + return [stored.get(name.casefold(), name) for name in requested] + + def _route_issue( + self, method: str, number: int, rest: list[str], body: dict + ) -> object: + if method == "PATCH": + self.issues[number].update(body) + return self._respond(self.issues[number]) + if rest == ["labels"]: + names = {label["name"] for label in self.issues[number]["labels"]} + if not self.drop_created_labels: + names |= set(self._existing_label_names(body["labels"])) + self.issues[number]["labels"] = [{"name": name} for name in sorted(names)] + return self._respond(self.issues[number]["labels"]) + self.comments.setdefault(number, []).append(body["body"]) + return self._respond({"id": 1}, status=201) + + def _page(self, items: list, query: dict) -> list: + page = int(query.get("page", "1")) + size = int(query.get("per_page", "30")) + return items[(page - 1) * size : page * size] + + def _create(self, body: dict) -> object: + number = max(self.issues, default=0) + 1 + labels = ( + [] + if self.drop_created_labels + else self._existing_label_names(body["labels"]) + ) + self.issues[number] = { + "number": number, + "title": body["title"], + "body": body["body"], + "state": "open", + "state_reason": None, + "closed_at": None, + "labels": [{"name": name} for name in labels], + "user": {"login": BOT}, + } + return self._respond(self.issues[number], status=201) + + +def bot_issue(number: int, **fields: object) -> dict: + issue = { + "number": number, + "title": f"Issue {number}", + "body": "", + "state": "open", + "state_reason": None, + "closed_at": None, + "labels": [], + "user": {"login": BOT}, + } + issue.update(fields) + return issue + + +def failures( + count: int, + *, + request_key_prefix: str = "r", + **fields: object, +) -> list: + return [ + exception_record( + request_key=f"{request_key_prefix}{index}", + timestamp=hours_after_window_noon(index), + **fields, + ) + for index in range(count) + ] + + +def groups_for_report_date(records: list) -> list: + return build_groups(records, [], reporting_window(REPORT_DATE), FINGERPRINT_KEY) + + +def sync(groups: list, github: FakeGitHub, *, dry_run=False): + return sync_issues( + groups, + reporting_window(REPORT_DATE), + LOG_SOURCE, + github.client(), + dry_run=dry_run, + ) diff --git a/tests/unit/telemetry_error_issues/report_support.py b/tests/unit/telemetry_error_issues/report_support.py new file mode 100644 index 000000000..439d57ad0 --- /dev/null +++ b/tests/unit/telemetry_error_issues/report_support.py @@ -0,0 +1,188 @@ +"""Records, stacks, times and settings shared by several test modules.""" + +from __future__ import annotations + +import re +from datetime import UTC, date, datetime, timedelta +from zoneinfo import ZoneInfo + +from telemetry_error_issues.bigquery import CHAINED_EXCEPTION_PATTERN, FRAME_PATTERN +from telemetry_error_issues.groups import RequestCount +from telemetry_error_issues.log_source import LogSource +from telemetry_error_issues.records import ErrorRecord + +EASTERN = ZoneInfo("America/New_York") + + +def extract_frames(stack: str) -> list[str]: + """Extract frames as the query does: REGEXP_REPLACE, then REGEXP_EXTRACT_ALL.""" + first_traceback = re.sub(CHAINED_EXCEPTION_PATTERN, "", stack) + return re.findall(FRAME_PATTERN, first_traceback) + + +def chained_stack(recursion_depth: int) -> str: + """Return a chained traceback whose cause recursed ``recursion_depth`` times.""" + library_frame = ( + ' File "/venv/lib/python3.12/site-packages/policyengine_core/simulation.py", ' + "line 500, in calculate\n return self._calculate(variable_name, period)\n" + ) + cause = ( + "Traceback (most recent call last):\n" + ' File "/app/policyengine_api/services/household.py", line 40, in calculate\n' + " simulation.calculate(name)\n" + f"{library_frame * recursion_depth}" + "ValueError: SENTINEL cause\n" + ) + final = ( + "Traceback (most recent call last):\n" + ' File "/app/policyengine_api/routes/household_routes.py", line 12, in get\n' + " return service.calculate()\n" + ' File "/app/policyengine_api/services/household.py", line 44, in calculate\n' + " raise CalculationError(name) from error\n" + "CalculationError: SENTINEL final\n" + ) + separator = ( + "\nThe above exception was the direct cause of the following exception:\n\n" + ) + return f"{cause}{separator}{final}" + + +FINGERPRINT_KEY = b"k" * 32 +APPLICATION_FRAMES = ( + '/app/policyengine_api/routes/report_routes.py", line 41, in create_report', + '/app/policyengine_api/services/report_service.py", line 88, in create', + '/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 43, in flush', +) + + +def _record(**overrides: object) -> object: + fields: dict[str, object] = { + "timestamp": datetime(2026, 10, 20, 16, 0, tzinfo=UTC), + "service_name": "policyengine-api", + "environment_name": "production", + "event_name": None, + "error_type": None, + "error_handled": None, + "frames": (), + "stack_truncated": False, + "message_template": None, + "attribute_error_type": None, + "failure_category": None, + "route": "//report", + "status_code": None, + "operation_name": None, + "request_key": "request-1", + "observability_key": None, + "trace_key": "trace-1", + } + fields.update(overrides) + return ErrorRecord(**fields) + + +def exception_record(**overrides: object) -> object: + fields: dict[str, object] = { + "event_name": "exception.recorded", + "error_type": "StaleDataError", + "error_handled": False, + "frames": APPLICATION_FRAMES, + "status_code": 500, + } + fields.update(overrides) + return _record(**fields) + + +def message_record(**overrides: object) -> object: + fields: dict[str, object] = {"message_template": "failed to save"} + fields.update(overrides) + return _record(**fields) + + +def failed_request_record(**overrides: object) -> object: + fields: dict[str, object] = {"event_name": "request.completed", "status_code": 500} + fields.update(overrides) + return _record(**fields) + + +REPORT_DATE = date(2026, 10, 20) +# Noon Eastern on the report date: 12 hours after the window starts, and 180 +# hours after the seven days before it start. +WINDOW_NOON = datetime(2026, 10, 20, 16, 0, tzinfo=UTC) + + +def hours_after_window_noon(hours: float) -> datetime: + return WINDOW_NOON + timedelta(hours=hours) + + +def request_count( + route: str, + requests: int, + *, + service_name: str = "policyengine-api", + environment_name: str = "production", +) -> object: + return RequestCount(service_name, environment_name, route, requests) + + +def failures_with_and_without_a_route() -> list: + """Return one failed request on a route and two route-less failures.""" + return [ + exception_record(request_key="r1", route="//calculate"), + exception_record( + request_key=None, + trace_key="t2", + route=None, + timestamp=hours_after_window_noon(1), + ), + exception_record( + request_key=None, + trace_key="t3", + route=None, + timestamp=hours_after_window_noon(2), + ), + ] + + +PROJECT_ID = "policyengine-observability" +# The deployment's values of the repository variables that name the logs. +LOG_SOURCE = LogSource( + project_id=PROJECT_ID, + bucket="policyengine-observability", + bucket_location="global", + dataset="observability_logs", + view="_AllLogs", + service_namespace="policyengine.api-v1", +) +LOG_SOURCE_ENVIRONMENT = { + "OBSERVABILITY_LOG_BUCKET": LOG_SOURCE.bucket, + "OBSERVABILITY_LOG_BUCKET_LOCATION": LOG_SOURCE.bucket_location, + "OBSERVABILITY_LOG_DATASET": LOG_SOURCE.dataset, + "OBSERVABILITY_LOG_VIEW": LOG_SOURCE.view, + "OBSERVABILITY_SERVICE_NAMESPACE": LOG_SOURCE.service_namespace, +} + + +def sentinel_row(**overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "timestamp": WINDOW_NOON, + "service_name": "SENTINEL service name", + "environment_name": "production", + "event_name": "SENTINEL event", + "error_type": "SENTINEL error type", + "error_handled": "SENTINEL handled", + "frames": [ + '/app/policyengine_api/SENTINEL.py", line 1, in SENTINEL_function', + "SENTINEL frame text", + ], + "stack_truncated": "SENTINEL", + "message_template": "SENTINEL message words", + "attribute_error_type": "SENTINEL attribute type", + "failure_category": "SENTINEL category", + "route": "/SENTINEL/route?query=SENTINEL", + "status_code": "SENTINEL", + "operation_name": "SENTINEL operation", + "request_key": "SENTINEL request", + "observability_key": "SENTINEL observability", + "trace_key": "SENTINEL trace", + } + row.update(overrides) + return row diff --git a/tests/unit/telemetry_error_issues/test_bigquery.py b/tests/unit/telemetry_error_issues/test_bigquery.py new file mode 100644 index 000000000..5a0fde716 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_bigquery.py @@ -0,0 +1,390 @@ +"""Tests for telemetry_error_issues.bigquery: the queries, client and rows.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime + +import pytest +from google.api_core.exceptions import BadRequest, Forbidden, ServiceUnavailable +from google.cloud import bigquery +from telemetry_error_issues.bigquery import ( + CHAINED_EXCEPTION_PATTERN, + DOUBLE_QUOTED_PATTERN, + ERROR_RECORDS_QUERY, + FRAME_PATTERN, + QUERY_TIMEOUT_SECONDS, + REQUEST_COUNTS_QUERY, + SINGLE_QUOTED_PATTERN, + TEMPLATE_WORD_PATTERN, + BigQueryClient, + BigQueryError, + BigQueryTimeoutError, + error_record_from_row, + fetch_error_records, + fetch_request_counts, + library_client, +) +from telemetry_error_issues.window import reporting_window + +from tests.unit.telemetry_error_issues.bigquery_support import ( + ACCESS_TOKEN, + FakeBigQueryClient, + sent_parameters, +) +from tests.unit.telemetry_error_issues.report_support import ( + APPLICATION_FRAMES, + LOG_SOURCE, + REPORT_DATE, + WINDOW_NOON, + exception_record, + request_count, +) + + +def _client(fake: FakeBigQueryClient, log_source: object = LOG_SOURCE) -> object: + return BigQueryClient(log_source, ACCESS_TOKEN, fake.factory) + + +def _string_parameter(name: str, value: str) -> bigquery.ScalarQueryParameter: + return bigquery.ScalarQueryParameter(name, "STRING", value) + + +def test_bigquery_error_query_request_limits_cost_and_names_its_parameters() -> None: + fake = FakeBigQueryClient() + + fetch_error_records(_client(fake), reporting_window(REPORT_DATE)) + + sql, job_config = fake.queries[0] + parameters = sent_parameters(fake) + assert fake.created_with == [("policyengine-observability", ACCESS_TOKEN)] + assert job_config.use_legacy_sql is False + assert job_config.maximum_bytes_billed == 5 * 1024**3 + assert job_config.labels == { + "purpose": "telemetry-error-issues", + "query": "error-records", + } + assert job_config.default_dataset == bigquery.DatasetReference( + "policyengine-observability", "observability_logs" + ) + assert "FROM `_AllLogs`" in sql + assert parameters["query_start"] == bigquery.ScalarQueryParameter( + "query_start", "TIMESTAMP", datetime(2026, 10, 13, 4, 0, tzinfo=UTC) + ) + assert parameters["window_end"] == bigquery.ScalarQueryParameter( + "window_end", "TIMESTAMP", datetime(2026, 10, 21, 4, 0, tzinfo=UTC) + ) + assert parameters["service_namespace"] == _string_parameter( + "service_namespace", "policyengine.api-v1" + ) + for name, pattern in ( + ("frame_pattern", FRAME_PATTERN), + ("chained_exception_pattern", CHAINED_EXCEPTION_PATTERN), + ("double_quoted_pattern", DOUBLE_QUOTED_PATTERN), + ("single_quoted_pattern", SINGLE_QUOTED_PATTERN), + ("template_word_pattern", TEMPLATE_WORD_PATTERN), + ): + assert parameters[name] == _string_parameter(name, pattern) + assert parameters["excluded_routes"] == bigquery.ArrayQueryParameter( + "excluded_routes", + "STRING", + ["/health", "/simulation-gateway-check", "/liveness-check", "/readiness-check"], + ) + + +def test_bigquery_queries_name_the_configured_dataset_and_view() -> None: + fake = FakeBigQueryClient() + log_source = LOG_SOURCE._replace(dataset="linked_logs", view="_Requests") + client = _client(fake, log_source) + + fetch_error_records(client, reporting_window(REPORT_DATE)) + fetch_request_counts(client, reporting_window(REPORT_DATE)) + + for sql, job_config in fake.queries: + assert job_config.default_dataset.dataset_id == "linked_logs" + assert "FROM `_Requests`" in sql + assert "_AllLogs" not in sql + + +@pytest.mark.parametrize( + "unsafe_names", + [ + {"view": "_AllLogs` WHERE TRUE --"}, + {"view": "_AllLogs\n"}, + {"view": ""}, + {"dataset": "observability_logs.other"}, + {"dataset": "logs; DROP TABLE logs"}, + ], +) +def test_bigquery_refuses_a_dataset_or_view_that_is_not_an_identifier( + unsafe_names: dict[str, str], +) -> None: + fake = FakeBigQueryClient() + client = _client(fake, LOG_SOURCE._replace(**unsafe_names)) + + with pytest.raises(ValueError, match="is not a valid name"): + fetch_error_records(client, reporting_window(REPORT_DATE)) + + assert fake.queries == [] + + +def test_bigquery_error_query_filters_application_error_records() -> None: + sql = ERROR_RECORDS_QUERY + + assert "FROM `{log_view}`" in sql + assert "timestamp >= @query_start AND timestamp < @window_end" in sql + assert ( + "JSON_VALUE(json_payload, '$.schema_version') = " + "'policyengine.observability.v2'" in sql + ) + assert ( + "JSON_VALUE(json_payload, '$.\"service.namespace\"') = @service_namespace" + in sql + ) + assert "policyengine.api-v1" not in sql + assert "severity_number >= 500" in sql + assert "NOT IN UNNEST(@excluded_routes)" in sql + assert "QUALIFY ROW_NUMBER() OVER" in sql + + +def test_bigquery_error_query_reads_stacks_and_messages_only_through_expressions(): + sql = ERROR_RECORDS_QUERY + stack = "JSON_VALUE(json_payload, '$.\"error.stack\"')" + + assert sql.count("error.stack") == 2 + assert ( + " REGEXP_EXTRACT_ALL(\n" + " REGEXP_REPLACE(\n" + f" {stack}, @chained_exception_pattern, ''\n" + " ),\n" + " @frame_pattern\n" + " ) AS frames" + ) in sql + assert f"LENGTH({stack}) >= 16384" in sql + assert sql.count("$.message") == 1 + assert ( + " REGEXP_REPLACE(\n" + " REGEXP_REPLACE(\n" + " JSON_VALUE(json_payload, '$.message'), " + "@double_quoted_pattern, ''\n" + " ),\n" + " @single_quoted_pattern,\n" + " r'\\1'\n" + " ),\n" + " @template_word_pattern\n" + ) in sql + assert "error.message" not in sql + assert "SELECT *" not in sql + assert "json_payload AS" not in sql + + +@pytest.mark.parametrize( + "identifier", + [ + "JSON_VALUE(json_payload, '$.\"request.id\"')", + "JSON_VALUE(json_payload, '$.attributes.observability_id')", + ], +) +def test_bigquery_error_query_hashes_identifiers(identifier: str) -> None: + sql = ERROR_RECORDS_QUERY + + assert sql.count(identifier) == 1 + assert f"TO_HEX(SHA256({identifier}))" in sql + + +def _message_template(message: str) -> str: + """Build the template the way the query does, with RE2's ASCII word rules.""" + without_double_quotes = re.sub(DOUBLE_QUOTED_PATTERN, "", message) + without_quotes = re.sub(SINGLE_QUOTED_PATTERN, r"\1", without_double_quotes) + words = re.findall(TEMPLATE_WORD_PATTERN, without_quotes, re.ASCII) + return " ".join(word.lower() for word in words[:3]) + + +def test_bigquery_template_keeps_words_around_an_apostrophe() -> None: + templates = { + _message_template(f"Can't load policy '{value}'") + for value in ("alpha", "beta", "gamma") + } + + assert templates == {"can t load"} + + +@pytest.mark.parametrize( + ("message", "template"), + [ + ("Failed to save 'report 12' for user", "failed to save"), + ("'SENTINEL' could not be parsed", "could not be"), + ('Missing "SENTINEL" in household input', "missing in household"), + ("Timeout after 30s calling the simulation API", "timeout after calling"), + ("The users' policies 'SENTINEL' failed", "the users policies"), + ], +) +def test_bigquery_template_drops_quoted_values_and_partial_words( + message: str, template: str +) -> None: + assert _message_template(message) == template + + +def test_bigquery_error_query_hashes_the_trace() -> None: + sql = ERROR_RECORDS_QUERY + + assert "TO_HEX(SHA256(COALESCE(" in sql + assert sql.count("$.trace_id") == 1 + + +def test_bigquery_request_count_query_counts_completed_requests_in_the_window() -> None: + sql = REQUEST_COUNTS_QUERY + fake = FakeBigQueryClient() + + fetch_request_counts(_client(fake), reporting_window(REPORT_DATE)) + + sent_sql, job_config = fake.queries[0] + parameters = sent_parameters(fake) + assert "JSON_VALUE(json_payload, '$.\"event.name\"') = 'request.completed'" in sql + assert "COUNT(*) AS requests" in sql + assert "GROUP BY service_name, environment_name, route" in sql + assert "timestamp >= @window_start AND timestamp < @window_end" in sql + assert parameters["window_start"] == bigquery.ScalarQueryParameter( + "window_start", "TIMESTAMP", datetime(2026, 10, 20, 4, 0, tzinfo=UTC) + ) + assert parameters["service_namespace"] == _string_parameter( + "service_namespace", "policyengine.api-v1" + ) + assert "FROM `_AllLogs`" in sent_sql + assert job_config.labels["query"] == "request-counts" + + +def test_bigquery_client_reads_every_row_the_job_yields() -> None: + rows = [{"name": "first", "count": 1}, {"name": "second", "count": 2}] + fake = FakeBigQueryClient({"test": rows}) + + result = _client(fake).query("SELECT 1", [], "test") + + assert result == rows + # 41 waits of 30 seconds, the limit of the REST client this replaced. + assert fake.result_timeouts == [QUERY_TIMEOUT_SECONDS] == [1_230] + + +def test_bigquery_client_reports_a_job_that_does_not_finish_in_time() -> None: + fake = FakeBigQueryClient(error=TimeoutError()) + + with pytest.raises(BigQueryTimeoutError, match="did not complete"): + _client(fake).query("SELECT 1", [], "test") + + +def test_bigquery_library_error_reports_only_the_status_and_reason() -> None: + error = BadRequest( + "SENTINEL value echoed from a row", + errors=[{"reason": "invalidQuery", "message": "SENTINEL detail"}], + ) + fake = FakeBigQueryClient(error=error) + + with pytest.raises(BigQueryError) as raised: + _client(fake).query("SELECT 1", [], "test") + + assert str(raised.value) == "BigQuery request failed: HTTP 400 invalidQuery" + assert raised.value.status == 400 + assert raised.value.reason == "invalidQuery" + + +def test_bigquery_library_error_without_a_reason_code_uses_the_status_phrase() -> None: + fake = FakeBigQueryClient(error=ServiceUnavailable("SENTINEL ")) + + with pytest.raises(BigQueryError) as raised: + _client(fake).query("SELECT 1", [], "test") + + assert str(raised.value) == "BigQuery request failed: HTTP 503 Service Unavailable" + + +@pytest.mark.parametrize("reason", ["SENTINEL: 'value'", "a" * 61, 7]) +def test_bigquery_library_error_hides_a_reason_that_is_not_a_short_code( + reason: object, +) -> None: + fake = FakeBigQueryClient(error=Forbidden("SENTINEL", errors=[{"reason": reason}])) + + with pytest.raises(BigQueryError) as raised: + _client(fake).query("SELECT 1", [], "test") + + assert str(raised.value) == "BigQuery request failed: HTTP 403 Forbidden" + + +def test_bigquery_library_client_sends_the_access_token_for_the_project() -> None: + client = library_client("policyengine-observability", ACCESS_TOKEN) + + assert isinstance(client, bigquery.Client) + assert client.project == "policyengine-observability" + assert client._credentials.token == ACCESS_TOKEN + + +def test_bigquery_rows_become_error_records() -> None: + row = { + "timestamp": WINDOW_NOON, + "service_name": "policyengine-api", + "environment_name": "production", + "event_name": "exception.recorded", + "error_type": "StaleDataError", + "error_handled": "false", + "frames": list(APPLICATION_FRAMES), + "stack_truncated": False, + "message_template": "", + "attribute_error_type": None, + "failure_category": None, + "route": "//report", + "status_code": "500", + "operation_name": None, + "request_key": "aa", + "observability_key": None, + "trace_key": "bb", + } + + record = error_record_from_row(row) + + assert record == exception_record(request_key="aa", trace_key="bb") + + +def test_bigquery_rows_tolerate_missing_and_malformed_values() -> None: + row = { + "timestamp": WINDOW_NOON, + "service_name": None, + "environment_name": None, + "event_name": None, + "error_type": None, + "error_handled": "SENTINEL", + "frames": None, + "stack_truncated": None, + "message_template": None, + "attribute_error_type": None, + "failure_category": None, + "route": None, + "status_code": "SENTINEL", + "operation_name": None, + "request_key": None, + "observability_key": None, + "trace_key": None, + } + + record = error_record_from_row(row) + + assert record.service_name == "-" + assert record.error_handled is None + assert record.frames == () + assert record.stack_truncated is False + assert record.status_code is None + + +def test_bigquery_rows_become_request_counts() -> None: + rows = [ + { + "service_name": "policyengine-api", + "environment_name": "production", + "route": "//report", + "requests": 120, + } + ] + fake = FakeBigQueryClient({"request-counts": rows}) + + counts = fetch_request_counts(_client(fake), reporting_window(REPORT_DATE)) + + assert counts == [ + request_count("//report", 120), + ] diff --git a/tests/unit/telemetry_error_issues/test_cli.py b/tests/unit/telemetry_error_issues/test_cli.py new file mode 100644 index 000000000..d3a955331 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_cli.py @@ -0,0 +1,56 @@ +"""Tests for telemetry_error_issues.cli: the command-line arguments.""" + +from __future__ import annotations + +from datetime import date + +import pytest +from telemetry_error_issues.cli import main, parse_arguments + + +def test_cli_parses_project_dry_run_and_report_date() -> None: + arguments = parse_arguments( + [ + "--project-id", + "policyengine-observability", + "--dry-run", + "--report-date", + "2026-10-19", + ] + ) + + assert arguments.project_id == "policyengine-observability" + assert arguments.dry_run is True + assert arguments.report_date == date(2026, 10, 19) + + +def test_cli_defaults_to_a_live_run_of_the_previous_day() -> None: + arguments = parse_arguments(["--project-id", "policyengine-observability"]) + + assert arguments.dry_run is False + assert arguments.report_date is None + + +def test_cli_treats_an_empty_report_date_as_unset() -> None: + arguments = parse_arguments( + ["--project-id", "policyengine-observability", "--report-date", ""] + ) + + assert arguments.report_date is None + + +@pytest.mark.parametrize( + "argv", + [ + [], + ["--dry-run"], + ["--project-id", "policyengine-observability", "--report-date", "20 Oct"], + ], +) +def test_cli_rejects_missing_project_or_invalid_date_with_exit_code_2( + argv: list[str], +) -> None: + with pytest.raises(SystemExit) as raised: + main(argv) + + assert raised.value.code == 2 diff --git a/tests/unit/telemetry_error_issues/test_end_to_end.py b/tests/unit/telemetry_error_issues/test_end_to_end.py new file mode 100644 index 000000000..b9d0dff98 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_end_to_end.py @@ -0,0 +1,446 @@ +"""End-to-end tests of telemetry_error_issues.cli.main and its public output.""" + +from __future__ import annotations + +import io +from datetime import UTC, datetime +from typing import TYPE_CHECKING +from urllib.parse import unquote + +import pytest +from google.api_core.exceptions import BadRequest, Forbidden, InternalServerError +from telemetry_error_issues.cli import Runtime, main +from telemetry_error_issues.transport import HttpResponse + +from tests.unit.telemetry_error_issues.bigquery_support import ( + ACCESS_TOKEN, + FakeBigQueryClient, + sent_parameters, +) +from tests.unit.telemetry_error_issues.github_support import REPOSITORY, FakeGitHub +from tests.unit.telemetry_error_issues.report_support import ( + APPLICATION_FRAMES, + EASTERN, + LOG_SOURCE_ENVIRONMENT, + PROJECT_ID, + WINDOW_NOON, + hours_after_window_noon, + sentinel_row, +) + +if TYPE_CHECKING: + from pathlib import Path + +ERROR_RECORD_COLUMNS = ( + "timestamp", + "service_name", + "environment_name", + "event_name", + "error_type", + "error_handled", + "frames", + "stack_truncated", + "message_template", + "attribute_error_type", + "failure_category", + "route", + "status_code", + "operation_name", + "request_key", + "observability_key", + "trace_key", +) +REQUEST_COUNT_COLUMNS = ("service_name", "environment_name", "route", "requests") +SCHEDULED_RUN_TIME = datetime(2026, 10, 21, 0, 45, tzinfo=EASTERN) + + +def _error_row(**overrides: object) -> dict[str, object]: + """Return an error-record row with the types the client library gives it.""" + values: dict[str, object] = { + "timestamp": WINDOW_NOON, + "service_name": "policyengine-api", + "environment_name": "production", + "event_name": "exception.recorded", + "error_type": "StaleDataError", + "error_handled": "false", + "frames": list(APPLICATION_FRAMES), + "stack_truncated": False, + "message_template": None, + "attribute_error_type": None, + "failure_category": None, + "route": "//report", + "status_code": "500", + "operation_name": None, + "request_key": "r1", + "observability_key": None, + "trace_key": "t1", + } + values.update(overrides) + return {column: values[column] for column in ERROR_RECORD_COLUMNS} + + +class FakeBigQuery(FakeBigQueryClient): + """Answers the error-record and request-count queries with fixed rows.""" + + def __init__( + self, + error_rows: list[dict[str, object]], + count_rows: list[list[object]] = (), + *, + error: Exception | None = None, + ) -> None: + super().__init__( + { + "error-records": error_rows, + "request-counts": [ + dict(zip(REQUEST_COUNT_COLUMNS, row, strict=True)) + for row in count_rows + ], + }, + error=error, + ) + + +def _environment(tmp_path: Path, **overrides: str) -> dict[str, str]: + environment = { + "GOOGLE_OAUTH_ACCESS_TOKEN": ACCESS_TOKEN, + "GITHUB_TOKEN": "fake-github-token", + "GITHUB_REPOSITORY": REPOSITORY, + "TELEMETRY_ERROR_FINGERPRINT_KEY": "f" * 64, + "GITHUB_STEP_SUMMARY": str(tmp_path / "summary.md"), + **LOG_SOURCE_ENVIRONMENT, + } + environment.update(overrides) + return {name: value for name, value in environment.items() if value} + + +def _run_main( + tmp_path: Path, + bigquery: object, + github: FakeGitHub, + *arguments: str, + environment: dict[str, str] | None = None, +) -> tuple[int, str, str]: + stdout, stderr = io.StringIO(), io.StringIO() + runtime = Runtime( + environ=_environment(tmp_path) if environment is None else environment, + bigquery_client_factory=bigquery.factory, + github_transport=github, + now=lambda: SCHEDULED_RUN_TIME, + sleep=github.sleep, + clock=github.clock, + stdout=stdout, + stderr=stderr, + ) + exit_code = main(["--project-id", PROJECT_ID, *arguments], runtime=runtime) + return exit_code, stdout.getvalue(), stderr.getvalue() + + +def _three_failures(**overrides: object) -> list[dict[str, object]]: + return [ + _error_row( + request_key=f"r{index}", + timestamp=hours_after_window_noon(index), + **overrides, + ) + for index in range(3) + ] + + +def test_main_queries_groups_and_opens_issues_for_the_previous_day( + tmp_path: Path, +) -> None: + bigquery = FakeBigQuery( + _three_failures(), + [["policyengine-api", "production", "//report", 600]], + ) + github = FakeGitHub() + + exit_code, stdout, stderr = _run_main(tmp_path, bigquery, github) + + assert exit_code == 0, stderr + issue = github.issues[1] + assert "StaleDataError" in issue["title"] + assert ( + "| Error rate on affected routes | 0.50% (3 of 600 requests) | n/a |" + in (issue["body"]) + ) + window_end = sent_parameters(bigquery)["window_end"] + assert window_end.value == datetime(2026, 10, 21, 4, 0, tzinfo=UTC) + summary = (tmp_path / "summary.md").read_text() + fingerprint = issue["title"].rsplit("(", 1)[1].rstrip(")") + assert ( + f"| `{fingerprint}` | exception | `policyengine-api` | 3 | 0 | #1 | created |" + in summary + ) + assert summary in stdout + + +def test_main_dry_run_reports_actions_without_writing(tmp_path: Path) -> None: + github = FakeGitHub() + + exit_code, stdout, _ = _run_main( + tmp_path, + FakeBigQuery(_three_failures()), + github, + "--dry-run", + ) + + assert exit_code == 0 + assert github.writes == [] + assert "Dry run: no issues were changed." in stdout + assert "| - | created |" in stdout + + +def test_main_reports_the_requested_day(tmp_path: Path) -> None: + bigquery = FakeBigQuery([]) + + exit_code, stdout, _ = _run_main( + tmp_path, + bigquery, + FakeGitHub(), + "--report-date", + "2026-10-15", + ) + + assert exit_code == 0 + window_end = sent_parameters(bigquery)["window_end"] + assert window_end.value == datetime(2026, 10, 16, 4, 0, tzinfo=UTC) + assert "## Telemetry error issues for 2026-10-15" in stdout + assert "No error groups had failures in the window." in stdout + + +def test_main_rejects_a_day_that_has_not_ended(tmp_path: Path) -> None: + exit_code, stdout, stderr = _run_main( + tmp_path, + FakeBigQuery([]), + FakeGitHub(), + "--report-date", + "2026-10-21", + ) + + assert exit_code == 1 + assert stdout == "" + assert stderr == ( + "configuration failed: ConfigurationError " + "(2026-10-21 has not ended in US Eastern time)\n" + ) + + +def test_main_names_missing_environment_variables_without_values( + tmp_path: Path, +) -> None: + environment = _environment( + tmp_path, GITHUB_TOKEN="", TELEMETRY_ERROR_FINGERPRINT_KEY="" + ) + + exit_code, _, stderr = _run_main( + tmp_path, + FakeBigQuery([]), + FakeGitHub(), + environment=environment, + ) + + assert exit_code == 1 + assert stderr == ( + "configuration failed: ConfigurationError (missing environment variables: " + "GITHUB_TOKEN, TELEMETRY_ERROR_FINGERPRINT_KEY)\n" + ) + + +def test_main_rejects_a_short_fingerprint_key_without_printing_it( + tmp_path: Path, +) -> None: + environment = _environment( + tmp_path, TELEMETRY_ERROR_FINGERPRINT_KEY="SENTINEL-short-key" + ) + + exit_code, stdout, stderr = _run_main( + tmp_path, + FakeBigQuery([]), + FakeGitHub(), + environment=environment, + ) + + assert exit_code == 1 + assert "at least 32 characters" in stderr + assert "SENTINEL" not in stdout + stderr + + +def test_main_names_missing_log_settings(tmp_path: Path) -> None: + environment = _environment(tmp_path, **dict.fromkeys(LOG_SOURCE_ENVIRONMENT, "")) + + exit_code, _, stderr = _run_main( + tmp_path, FakeBigQuery([]), FakeGitHub(), environment=environment + ) + + assert exit_code == 1 + assert stderr == ( + "configuration failed: ConfigurationError (missing environment variables: " + f"{', '.join(LOG_SOURCE_ENVIRONMENT)})\n" + ) + + +@pytest.mark.parametrize( + ("setting", "value"), + [ + ("OBSERVABILITY_LOG_BUCKET", "SENTINEL bucket"), + ("OBSERVABILITY_LOG_BUCKET_LOCATION", "SENTINEL/location"), + ("OBSERVABILITY_LOG_DATASET", "SENTINEL`; DROP TABLE logs"), + ("OBSERVABILITY_LOG_VIEW", "_AllLogs` WHERE SENTINEL"), + ], +) +def test_main_refuses_a_log_name_that_is_not_an_identifier( + tmp_path: Path, setting: str, value: str +) -> None: + bigquery = FakeBigQuery([]) + + exit_code, stdout, stderr = _run_main( + tmp_path, + bigquery, + FakeGitHub(), + environment=_environment(tmp_path, **{setting: value}), + ) + + assert exit_code == 1 + assert stdout == "" + assert stderr == ( + f"configuration failed: ConfigurationError ({setting} is not a valid name)\n" + ) + assert bigquery.queries == [] + + +def test_main_refuses_a_project_id_that_is_not_an_identifier(tmp_path: Path) -> None: + bigquery = FakeBigQuery([]) + + exit_code, _, stderr = _run_main( + tmp_path, bigquery, FakeGitHub(), "--project-id", "SENTINEL project" + ) + + assert exit_code == 1 + assert stderr == ( + "configuration failed: ConfigurationError (--project-id is not a valid name)\n" + ) + assert bigquery.queries == [] + + +def test_main_reports_an_api_failure_by_stage_class_and_status_only( + tmp_path: Path, +) -> None: + error = InternalServerError( + "SENTINEL echoed value", + errors=[{"reason": "backendError", "message": "SENTINEL"}], + ) + failing = FakeBigQuery([], error=error) + + exit_code, stdout, stderr = _run_main(tmp_path, failing, FakeGitHub()) + + assert exit_code == 1 + assert stdout == "" + assert stderr == ( + "query error records failed: BigQueryError (HTTP 500 backendError)\n" + ) + + +@pytest.mark.parametrize( + ("error", "expected"), + [ + ( + BadRequest( + "SENTINEL message about the query", + errors=[{"reason": "invalidQuery", "message": "SENTINEL detail"}], + ), + "query error records failed: BigQueryError (HTTP 400 invalidQuery)\n", + ), + ( + Forbidden( + "SENTINEL access denied", errors=[{"reason": "SENTINEL: 'value'"}] + ), + "query error records failed: BigQueryError (HTTP 403 Forbidden)\n", + ), + ], +) +def test_main_never_prints_a_client_library_error_message( + tmp_path: Path, error: Exception, expected: str +) -> None: + # The library adds the query text to a failed job's error. + error.debug_message = "SENTINEL query text" + bigquery = FakeBigQuery(_three_failures(), error=error) + + exit_code, stdout, stderr = _run_main(tmp_path, bigquery, FakeGitHub()) + + assert exit_code == 1 + assert stdout == "" + assert stderr == expected + assert not (tmp_path / "summary.md").exists() + + +def test_main_reports_an_unexpected_exception_by_class_only(tmp_path: Path) -> None: + broken = FakeBigQuery([], error=RuntimeError("SENTINEL connection detail")) + + exit_code, _, stderr = _run_main(tmp_path, broken, FakeGitHub()) + + assert exit_code == 1 + assert stderr == "query error records failed: RuntimeError\n" + + +def test_main_reports_a_github_failure_by_stage(tmp_path: Path) -> None: + github = FakeGitHub() + github.failures = [ + HttpResponse( + status=403, reason="Forbidden", headers={}, body=b'{"message": "SENTINEL"}' + ) + ] + + exit_code, _, stderr = _run_main(tmp_path, FakeBigQuery(_three_failures()), github) + + assert exit_code == 1 + assert stderr == "sync issues failed: GitHubError (HTTP 403 Forbidden)\n" + + +@pytest.mark.parametrize( + "overrides", + [ + {}, + {"error_type": None}, + {"error_type": None, "event_name": "request.completed"}, + ], +) +def test_main_never_publishes_values_from_records( + tmp_path: Path, overrides: dict[str, object] +) -> None: + sentinel_values = { + name: value + for name, value in sentinel_row(**overrides).items() + if name not in {"timestamp", "environment_name", "frames", "stack_truncated"} + } + sentinel_values["stack_truncated"] = True + sentinel_values["frames"] = ['/app/SENTINEL.py", line 1, in SENTINEL'] + rows = [ + _error_row( + **{ + **sentinel_values, + "request_key": f"SENTINEL request {index}", + "timestamp": hours_after_window_noon(index), + } + ) + for index in range(3) + ] + count_rows = [["SENTINEL service", "production", "SENTINEL route", 10]] + github = FakeGitHub() + + exit_code, stdout, stderr = _run_main( + tmp_path, FakeBigQuery(rows, count_rows), github + ) + + assert exit_code == 0, stderr + assert len(github.issues) == 1 + published = [ + stdout, + stderr, + (tmp_path / "summary.md").read_text(), + *(issue["title"] for issue in github.issues.values()), + *(issue["body"] for issue in github.issues.values()), + ] + for text in published: + assert "SENTINEL" not in unquote(text) diff --git a/tests/unit/telemetry_error_issues/test_github_client.py b/tests/unit/telemetry_error_issues/test_github_client.py new file mode 100644 index 000000000..318a00ffe --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_github_client.py @@ -0,0 +1,159 @@ +"""Tests for telemetry_error_issues.github_client: paging, write spacing, retries.""" + +from __future__ import annotations + +import itertools +from datetime import UTC, datetime +from urllib.parse import parse_qs, urlsplit + +import pytest +from telemetry_error_issues.github_client import GitHubError +from telemetry_error_issues.transport import HttpResponse + +from tests.unit.telemetry_error_issues.github_support import ( + BOT, + FakeGitHub, + bot_issue, + failures, + groups_for_report_date, + sync, +) + + +def test_sync_lists_open_and_closed_issues_from_the_bot_page_by_page() -> None: + github = FakeGitHub([bot_issue(number) for number in range(1, 151)]) + + sync([], github) + + listing = [ + parse_qs(urlsplit(request.url).query) + for _, request in github.requests + if urlsplit(request.url).path.endswith("/issues") + ] + assert len(listing) == 2 + assert listing[0]["creator"] == [BOT] + assert listing[0]["state"] == ["all"] + assert listing[0]["per_page"] == ["100"] + assert [query["page"] for query in listing] == [["1"], ["2"]] + + +def test_sync_writes_one_at_a_time_at_least_a_second_apart() -> None: + records = [] + for index in range(3): + records += failures( + 3, request_key_prefix=f"g{index}-", error_type=f"Error{index}" + ) + github = FakeGitHub() + + sync(groups_for_report_date(records), github) + + write_times = [moment for moment, _ in github.writes] + assert len(write_times) == 5 + assert all( + later - earlier >= 1.0 for earlier, later in itertools.pairwise(write_times) + ) + + +@pytest.mark.parametrize("status", [403, 429]) +def test_sync_waits_for_retry_after_on_rate_limits(status: int) -> None: + groups = groups_for_report_date(failures(3)) + github = FakeGitHub(labels=("telemetry-error", "service: policyengine-api")) + github.failures = [ + HttpResponse( + status=status, reason="Forbidden", headers={"retry-after": "7"}, body=b"{}" + ) + ] + + sync(groups, github) + + assert 7 in github.sleeps + assert len(github.issues) == 1 + + +def _limited_github(response: object) -> FakeGitHub: + """Return a fake whose first write gets ``response`` and later writes succeed.""" + github = FakeGitHub(labels=("telemetry-error", "service: policyengine-api")) + github.failures = [response] + return github + + +def test_sync_waits_a_minute_on_a_secondary_rate_limit_without_retry_after() -> None: + body = b'{"message": "You have exceeded a secondary rate limit. Please wait."}' + github = _limited_github( + HttpResponse(status=403, reason="Forbidden", headers={}, body=body), + ) + + sync(groups_for_report_date(failures(3)), github) + + assert 60 in github.sleeps + assert len(github.issues) == 1 + + +def test_sync_waits_until_an_exhausted_rate_limit_resets() -> None: + headers = { + "date": "Tue, 20 Oct 2026 23:00:00 GMT", + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": str( + int(datetime(2026, 10, 20, 23, 0, 42, tzinfo=UTC).timestamp()) + ), + } + github = _limited_github( + HttpResponse(status=403, reason="Forbidden", headers=headers, body=b"{}"), + ) + + sync(groups_for_report_date(failures(3)), github) + + assert 42 in github.sleeps + assert len(github.issues) == 1 + + +def test_sync_fails_instead_of_waiting_hours_for_a_rate_limit_reset() -> None: + headers = { + "date": "Tue, 20 Oct 2026 23:00:00 GMT", + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": str( + int(datetime(2026, 10, 21, 0, 0, tzinfo=UTC).timestamp()) + ), + } + github = _limited_github( + HttpResponse(status=403, reason="Forbidden", headers=headers, body=b"{}"), + ) + + with pytest.raises(GitHubError, match="HTTP 403"): + sync(groups_for_report_date(failures(3)), github) + + assert github.sleeps == [] + + +def test_sync_stops_after_three_rate_limit_retries() -> None: + groups = groups_for_report_date(failures(3)) + github = FakeGitHub(labels=("telemetry-error", "service: policyengine-api")) + limited = HttpResponse( + status=429, reason="Too Many Requests", headers={"retry-after": "2"}, body=b"{}" + ) + github.failures = [limited] * 4 + + with pytest.raises(GitHubError) as raised: + sync(groups, github) + + assert str(raised.value) == "GitHub request failed: HTTP 429 Too Many Requests" + assert github.sleeps.count(2) == 3 + + +def test_sync_reports_a_forbidden_write_without_retrying() -> None: + groups = groups_for_report_date(failures(3)) + github = FakeGitHub(labels=("telemetry-error", "service: policyengine-api")) + github.failures = [ + HttpResponse( + status=403, + reason="Forbidden", + headers={}, + body=b'{"message": "SENTINEL Resource not accessible"}', + ) + ] + + with pytest.raises(GitHubError) as raised: + sync(groups, github) + + assert str(raised.value) == "GitHub request failed: HTTP 403 Forbidden" + assert len(github.writes) == 1 diff --git a/tests/unit/telemetry_error_issues/test_groups.py b/tests/unit/telemetry_error_issues/test_groups.py new file mode 100644 index 000000000..41807ac43 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_groups.py @@ -0,0 +1,378 @@ +"""Tests for telemetry_error_issues.groups: grouping failures and their measures.""" + +from __future__ import annotations + +from datetime import UTC, date, datetime, timedelta + +import pytest +from telemetry_error_issues.groups import build_groups +from telemetry_error_issues.records import fingerprint +from telemetry_error_issues.window import reporting_window + +from tests.unit.telemetry_error_issues.report_support import ( + FINGERPRINT_KEY, + REPORT_DATE, + WINDOW_NOON, + exception_record, + failed_request_record, + failures_with_and_without_a_route, + hours_after_window_noon, + message_record, + request_count, +) + + +def _only_group(records: list, request_counts=()) -> object: + groups = build_groups( + records, + list(request_counts), + reporting_window(REPORT_DATE), + FINGERPRINT_KEY, + ) + assert len(groups) == 1 + return groups[0] + + +def test_measures_count_occurrences_requests_and_observability_ids() -> None: + records = [ + exception_record(request_key="r1", observability_key="o1"), + exception_record( + request_key="r2", + observability_key="o1", + timestamp=hours_after_window_noon(1), + ), + exception_record( + request_key="r3", + observability_key="o2", + timestamp=hours_after_window_noon(2), + ), + exception_record( + request_key="s1", + environment_name="staging", + timestamp=hours_after_window_noon(3), + ), + ] + + group = _only_group(records) + + assert group.production.occurrences == 3 + assert group.production.affected_requests == 3 + assert group.production.observability_ids == 2 + assert group.staging.occurrences == 1 + assert group.staging.affected_requests == 1 + assert group.staging.observability_ids == 0 + + +def test_measures_affected_requests_fall_back_to_observability_id_then_trace() -> None: + records = [ + exception_record(request_key=None, trace_key=None, observability_key="o1"), + exception_record( + request_key=None, + trace_key=None, + observability_key="o1", + timestamp=hours_after_window_noon(1), + ), + exception_record(request_key=None, trace_key="t1"), + exception_record(request_key=None, trace_key=None), + exception_record( + request_key=None, trace_key=None, timestamp=hours_after_window_noon(2) + ), + ] + + group = _only_group(records) + + assert group.production.occurrences == 5 + # o1 once, t1 once, and each record without any identifier on its own. + assert group.production.affected_requests == 4 + + +def test_measures_error_rate_uses_requests_to_the_affected_routes() -> None: + records = [ + exception_record(request_key="r1", route="//report"), + exception_record( + request_key="r2", + route="//calculate", + timestamp=hours_after_window_noon(1), + ), + exception_record( + request_key="r3", + route="//calculate", + timestamp=hours_after_window_noon(2), + ), + ] + request_counts = [ + request_count("//report", 200), + request_count("//calculate", 100), + request_count("//metadata", 5000), + request_count("//report", 10, environment_name="staging"), + request_count( + "//report", + 7000, + service_name="policyengine-simulation-entry", + ), + ] + + group = _only_group(records, request_counts) + + assert group.production.route_requests == 300 + assert group.production.error_rate == pytest.approx(0.01) + assert group.staging.error_rate is None + + +def test_measures_error_rate_counts_only_failures_on_routes() -> None: + group = _only_group( + failures_with_and_without_a_route(), + [request_count("//calculate", 1)], + ) + + assert group.production.affected_requests == 3 + assert group.production.route_affected_requests == 1 + assert group.production.route_requests == 1 + assert group.production.error_rate == pytest.approx(1.0) + + +def test_measures_error_rate_counts_every_simulation_service_version() -> None: + records = [ + failed_request_record( + service_name="policyengine-simulation-py1-2-3", + route="/jobs/{job_id}", + ) + ] + request_counts = [ + request_count( + "/jobs/{job_id}", + 30, + service_name="policyengine-simulation-py1-2-3", + ), + request_count( + "/jobs/{job_id}", + 70, + service_name="policyengine-simulation-py1-2-4", + ), + ] + + group = _only_group(records, request_counts) + + assert group.production.route_requests == 100 + assert group.production.error_rate == pytest.approx(0.01) + + +def test_measures_error_rate_is_unknown_without_route_requests() -> None: + group = _only_group([message_record(route=None, operation_name="sync")]) + + assert group.production.route_requests == 0 + assert group.production.error_rate is None + + +def test_measures_hourly_rates_for_the_window_and_the_previous_seven_days() -> None: + window_records = [ + message_record( + request_key=f"w{index}", timestamp=hours_after_window_noon(index) + ) + for index in range(6) + ] + previous_records = [ + message_record( + request_key=f"p{index}", timestamp=hours_after_window_noon(-13 - 4 * index) + ) + for index in range(42) + ] + + group = _only_group(window_records + previous_records) + + assert group.production.occurrences == 6 + assert group.production.hourly_rate == pytest.approx(6 / 24) + assert group.production.previous_hourly_rate == pytest.approx(42 / 168) + + +def test_measures_hourly_rate_uses_the_25_hour_fall_back_day() -> None: + fall_back_noon = datetime(2026, 11, 1, 17, 0, tzinfo=UTC) + records = [ + message_record( + request_key=f"w{index}", + timestamp=fall_back_noon + timedelta(hours=index), + ) + for index in range(5) + ] + + groups = build_groups( + records, [], reporting_window(date(2026, 11, 1)), FINGERPRINT_KEY + ) + + assert groups[0].production.hourly_rate == pytest.approx(5 / 25) + + +def test_measures_first_and_last_seen_cover_the_queried_eight_days() -> None: + records = [ + message_record(request_key="early", timestamp=hours_after_window_noon(-24 * 6)), + message_record(request_key="latest", timestamp=hours_after_window_noon(5)), + message_record(request_key="middle", timestamp=hours_after_window_noon(1)), + ] + + group = _only_group(records) + + assert group.production.first_seen == hours_after_window_noon(-24 * 6) + assert group.production.last_seen == hours_after_window_noon(5) + assert group.staging.first_seen is None + assert group.staging.last_seen is None + + +def test_measures_ignore_records_outside_the_queried_days() -> None: + records = [ + message_record(request_key="in-window"), + message_record( + request_key="too-old", timestamp=hours_after_window_noon(-24 * 9) + ), + message_record(request_key="too-new", timestamp=hours_after_window_noon(24)), + ] + + group = _only_group(records) + + assert group.production.occurrences == 1 + assert group.production.first_seen == WINDOW_NOON + + +def test_measures_report_only_groups_seen_in_the_window() -> None: + records = [ + message_record(request_key="today"), + message_record( + request_key="last-week", + message_template="something else", + timestamp=hours_after_window_noon(-48), + ), + ] + + groups = build_groups(records, [], reporting_window(REPORT_DATE), FINGERPRINT_KEY) + + assert [group.fingerprint for group in groups] == [ + fingerprint(records[0], FINGERPRINT_KEY) + ] + + +def test_measures_ignore_environments_other_than_production_and_staging() -> None: + records = [ + message_record(environment_name="local"), + message_record(environment_name=None, request_key="r2"), + ] + + groups = build_groups(records, [], reporting_window(REPORT_DATE), FINGERPRINT_KEY) + + assert groups == [] + + +def test_measures_describe_the_group_with_its_most_frequent_production_route() -> None: + records = [ + exception_record(request_key="r1", route="//report"), + exception_record(request_key="r2", route="//calculate"), + exception_record(request_key="r3", route="//calculate"), + exception_record( + request_key="s1", + route="//report", + environment_name="staging", + ), + exception_record( + request_key="s2", + route="//report", + environment_name="staging", + ), + ] + + group = _only_group(records) + + assert group.kind == "exception" + assert group.family == "policyengine-api" + assert group.error_type == "StaleDataError" + assert group.route == "//calculate" + + +def test_measures_describe_groups_only_with_validated_values() -> None: + group = _only_group( + [ + exception_record( + error_type="SENTINEL not a class", + route="/us/policy/12345", + ) + ], + ) + + assert group.error_type == "" + assert group.route == "" + + +def test_measures_list_up_to_three_related_fingerprints_by_shared_traces() -> None: + base = exception_record(request_key="r1", trace_key="t1") + base_second = exception_record( + request_key="r2", trace_key="t2", timestamp=hours_after_window_noon(1) + ) + related_twice = [ + message_record( + service_name="policyengine-simulation-entry", + request_key=f"x{index}", + trace_key=trace, + ) + for index, trace in enumerate(("t1", "t2")) + ] + related_once = [ + message_record( + service_name="policyengine-simulation-gateway", + request_key=f"y{index}", + trace_key="t1", + message_template=template, + ) + for index, template in enumerate(("first error", "second error", "third")) + ] + + groups = build_groups( + [base, base_second, *related_twice, *related_once], + [], + reporting_window(REPORT_DATE), + FINGERPRINT_KEY, + ) + by_fingerprint = {group.fingerprint: group for group in groups} + base_group = by_fingerprint[fingerprint(base, FINGERPRINT_KEY)] + twice = fingerprint(related_twice[0], FINGERPRINT_KEY) + once = sorted(fingerprint(record, FINGERPRINT_KEY) for record in related_once) + + assert base_group.related_fingerprints == (twice, once[0], once[1]) + + +def test_measures_sort_groups_by_production_occurrences() -> None: + frequent = [ + message_record(request_key=f"f{index}", message_template="often") + for index in range(3) + ] + staging_only = [ + message_record( + request_key=f"s{index}", + message_template="staging", + environment_name="staging", + ) + for index in range(5) + ] + rare = [message_record(request_key="rare", message_template="rare")] + + groups = build_groups( + staging_only + rare + frequent, + [], + reporting_window(REPORT_DATE), + FINGERPRINT_KEY, + ) + + assert [group.production.occurrences for group in groups] == [3, 1, 0] + + +def test_measures_count_production_occurrences_since_a_moment() -> None: + records = [ + exception_record(request_key="before", timestamp=hours_after_window_noon(-30)), + exception_record(request_key="r1", timestamp=hours_after_window_noon(1)), + exception_record(request_key="r2", timestamp=hours_after_window_noon(2)), + exception_record( + request_key="r2", trace_key="t9", timestamp=hours_after_window_noon(3) + ), + exception_record(request_key="staging", environment_name="staging"), + ] + + group = _only_group(records) + + assert group.production_since(hours_after_window_noon(-1)) == (2, 2) + assert group.production_since(hours_after_window_noon(-48)) == (3, 3) diff --git a/tests/unit/telemetry_error_issues/test_normalization.py b/tests/unit/telemetry_error_issues/test_normalization.py new file mode 100644 index 000000000..f035b2b7c --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_normalization.py @@ -0,0 +1,328 @@ +"""Tests for telemetry_error_issues.normalization: names, frames and public values.""" + +from __future__ import annotations + +import re + +import pytest +from telemetry_error_issues import normalization +from telemetry_error_issues.bigquery import FRAME_PATTERN +from telemetry_error_issues.normalization import ( + fingerprint_frames, + frame_key, + normalize_path, + public_error_type, + public_family, + public_operation, + public_route, + public_status, + service_family, +) + +from tests.unit.telemetry_error_issues.report_support import ( + chained_stack, + extract_frames, +) + +STANDARD_STACK = """\ +Traceback (most recent call last): + File "/venv/lib/python3.12/site-packages/flask/app.py", line 880, in dispatch_request + rv = self.dispatch_request() + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/policyengine_api/routes/report_routes.py", line 41, in create_report + return service.create(payload) + File "/app/policyengine_api/services/report_service.py", line 88, in create + session.flush() + File "/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 43, in flush + self._flush(objects) +sqlalchemy.orm.exc.StaleDataError: SENTINEL message about 'reports' +""" + +CHAINED_STACK = """\ +Traceback (most recent call last): + File "/app/policyengine_api/services/cache.py", line 12, in read + return store[key] +KeyError: 'SENTINEL key' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/app/policyengine_api/routes/policy_routes.py", line 30, in get_policy + return service.get(policy_id) + File "/app/policyengine_api/services/policy_service.py", line 55, in get + raise LookupError(policy_id) +LookupError: SENTINEL policy +""" + +EXCEPTION_GROUP_STACK = """\ + + Exception Group Traceback (most recent call last): + | File "/root/policyengine_simulation_gateway/app.py", line 10, in run + | raise ExceptionGroup("SENTINEL group", errors) + | ExceptionGroup: SENTINEL group (1 sub-exception) + +-+---------------- 1 ---------------- + | Traceback (most recent call last): + | File "/root/policyengine_simulation_gateway/worker.py", line 22, in step + | raise ValueError("SENTINEL value") + | ValueError: SENTINEL value + +------------------------------------ +""" + + +@pytest.mark.parametrize( + ("service_name", "family"), + [ + ("policyengine-simulation-py1-2-3", "policyengine-simulation-py"), + ("policyengine-simulation-py0-412-16", "policyengine-simulation-py"), + ("policyengine-api", "policyengine-api"), + ("policyengine-simulation-gateway", "policyengine-simulation-gateway"), + ("policyengine-stage12-us-worker", "policyengine-stage12-us-worker"), + ], +) +def test_normalize_service_family_merges_simulation_versions( + service_name: str, family: str +) -> None: + assert service_family(service_name) == family + + +@pytest.mark.parametrize( + ("environment_name", "environment_class"), + [ + ("production", "production"), + ("main", "production"), + ("staging", "staging"), + ("local", "other"), + ("", "other"), + (None, "other"), + ], +) +def test_normalize_environment_maps_modal_main_to_production( + environment_name: str | None, environment_class: str +) -> None: + assert normalization.environment_class(environment_name) == environment_class + + +def test_frames_pattern_has_one_capturing_group_for_bigquery() -> None: + assert re.compile(FRAME_PATTERN).groups == 1 + + +def test_frames_pattern_extracts_frame_headers_without_source_or_message() -> None: + frames = extract_frames(STANDARD_STACK) + + assert [frame_key(frame) for frame in frames] == [ + "flask/app.py:dispatch_request", + "policyengine_api/routes/report_routes.py:create_report", + "policyengine_api/services/report_service.py:create", + "sqlalchemy/orm/session.py:flush", + ] + assert not any("SENTINEL" in frame or "\n" in frame for frame in frames) + + +def test_frames_chained_traceback_contributes_only_its_first_traceback() -> None: + frames = extract_frames(CHAINED_STACK) + + assert [frame_key(frame) for frame in frames] == [ + "policyengine_api/services/cache.py:read", + ] + + +def test_frames_explicitly_chained_traceback_contributes_only_its_cause() -> None: + frames = extract_frames(chained_stack(recursion_depth=2)) + + assert [frame_key(frame) for frame in frames] == [ + "policyengine_api/services/household.py:calculate", + "policyengine_core/simulation.py:calculate", + "policyengine_core/simulation.py:calculate", + ] + + +def test_frames_pattern_reads_exception_group_tracebacks() -> None: + frames = extract_frames(EXCEPTION_GROUP_STACK) + + assert [frame_key(frame) for frame in frames] == [ + "policyengine_simulation_gateway/app.py:run", + "policyengine_simulation_gateway/worker.py:step", + ] + + +def test_frames_pattern_skips_a_frame_header_cut_off_by_truncation() -> None: + cut_at = STANDARD_STACK.index("create_report") + len("create_re") + frames = extract_frames(STANDARD_STACK[:cut_at]) + + assert [frame_key(frame) for frame in frames] == [ + "flask/app.py:dispatch_request", + ] + + +@pytest.mark.parametrize( + ("path", "normalized"), + [ + ( + "/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/session.py", + "sqlalchemy/orm/session.py", + ), + ("/usr/lib/python3/dist-packages/yaml/loader.py", "yaml/loader.py"), + ("/usr/local/lib/python3.13/asyncio/tasks.py", "asyncio/tasks.py"), + ("/app/policyengine_api/services/x.py", "policyengine_api/services/x.py"), + ( + "/home/runner/work/api/api/policyengine_api/services/x.py", + "policyengine_api/services/x.py", + ), + ( + "/root/policyengine_simulation_gateway/app.py", + "policyengine_simulation_gateway/app.py", + ), + ("/pkg/synchronicity/combined_types.py", "synchronicity/combined_types.py"), + ("", ""), + ], +) +def test_frames_paths_drop_installation_prefixes(path: str, normalized: str) -> None: + assert normalize_path(path) == normalized + + +def test_frames_keys_ignore_line_numbers() -> None: + first = frame_key('/app/policyengine_api/x.py", line 10, in handler') + second = frame_key('/srv/policyengine_api/x.py", line 99, in handler') + + assert first == second == "policyengine_api/x.py:handler" + + +def test_frames_choose_the_three_innermost_application_frames() -> None: + frames = [ + '/app/policyengine_api/routes/a.py", line 1, in outer', + '/app/policyengine_api/services/b.py", line 2, in middle', + '/app/policyengine_api/services/c.py", line 3, in inner', + '/opt/venv/lib/python3.12/site-packages/sqlalchemy/orm/d.py", line 4, in lib', + '/app/policyengine_api/services/e.py", line 5, in callback', + '/venv/lib/python3.12/site-packages/sqlalchemy/orm/f.py", line 6, in raises', + ] + + assert fingerprint_frames(frames) == ( + "policyengine_api/services/b.py:middle", + "policyengine_api/services/c.py:inner", + "policyengine_api/services/e.py:callback", + ) + + +def test_frames_count_simulation_and_stage12_packages_as_application_code() -> None: + frames = [ + '/pkg/modal/_runtime.py", line 1, in run', + '/root/policyengine_simulation_executor/run.py", line 2, in execute', + '/root/policyengine_stage12_persistence/store.py", line 3, in save', + '/pkg/synchronicity/combined_types.py", line 4, in __call__', + ] + + assert fingerprint_frames(frames) == ( + "policyengine_simulation_executor/run.py:execute", + "policyengine_stage12_persistence/store.py:save", + ) + + +def test_frames_without_application_frames_use_the_innermost_frames() -> None: + frames = [ + '/usr/local/lib/python3.13/asyncio/a.py", line 1, in first', + '/pkg/modal/b.py", line 2, in second', + '/pkg/modal/c.py", line 3, in third', + '/pkg/synchronicity/d.py", line 4, in fourth', + ] + + assert fingerprint_frames(frames) == ( + "modal/b.py:second", + "modal/c.py:third", + "synchronicity/d.py:fourth", + ) + + +@pytest.mark.parametrize("frames", [[], None, ["not a frame header"]]) +def test_frames_without_a_stack_use_a_placeholder(frames: list[str] | None) -> None: + assert fingerprint_frames(frames) == ("-",) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("StaleDataError", "StaleDataError"), + ("sqlalchemy.orm.exc.StaleDataError", "sqlalchemy.orm.exc.StaleDataError"), + ("SENTINEL message with spaces", ""), + ("Error: 'quoted'", ""), + ("E" * 200, ""), + (None, "-"), + ("", "-"), + ], +) +def test_public_error_type_allows_only_class_names( + value: str | None, expected: str +) -> None: + assert public_error_type(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ( + "//economy//over/", + "//economy//over/", + ), + ("/v2/user-policies", "/v2/user-policies"), + ("/jobs/{job_id}", "/jobs/{job_id}"), + ("", ""), + ("/us/policy/12345", ""), + ("/v2/households/3f2b1c9e-8d7a-4c1b-9e2f-0a1b2c3d4e5f", ""), + ("/us/report?token=SENTINEL", ""), + ("SENTINEL route", ""), + ("/" + "a" * 300, ""), + (None, "-"), + ("", "-"), + ], +) +def test_public_route_allows_only_route_templates( + value: str | None, expected: str +) -> None: + assert public_route(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("policyengine-api", "policyengine-api"), + ("policyengine-simulation-py", "policyengine-simulation-py"), + ("SENTINEL family", ""), + ("Policyengine_API", ""), + (None, "-"), + ], +) +def test_public_family_allows_only_service_names( + value: str | None, expected: str +) -> None: + assert public_family(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("simulation.dispatch", "simulation.dispatch"), + ("stage12/run_comparison", "stage12/run_comparison"), + ("SENTINEL operation", ""), + (None, "-"), + ], +) +def test_public_operation_allows_only_operation_names( + value: str | None, expected: str +) -> None: + assert public_operation(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (500, 500), + ("503", 503), + (99, None), + (600, None), + ("SENTINEL", None), + (None, None), + ], +) +def test_public_status_allows_only_http_status_codes( + value: object, expected: int | None +) -> None: + assert public_status(value) == expected diff --git a/tests/unit/telemetry_error_issues/test_records.py b/tests/unit/telemetry_error_issues/test_records.py new file mode 100644 index 000000000..cc5105b2a --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_records.py @@ -0,0 +1,246 @@ +"""Tests for telemetry_error_issues.records: primary records and fingerprints.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime + +import pytest +from telemetry_error_issues.records import ( + fingerprint, + fingerprint_key, + fingerprint_parts, + primary_records, +) + +from tests.unit.telemetry_error_issues.report_support import ( + APPLICATION_FRAMES, + FINGERPRINT_KEY, + chained_stack, + exception_record, + extract_frames, + failed_request_record, + message_record, +) + + +def test_primary_prefers_the_exception_record_of_a_failed_request() -> None: + exception = exception_record() + records = [ + message_record(), + exception, + failed_request_record(), + ] + + assert primary_records(records) == [exception] + + +def test_primary_falls_back_to_the_message_record() -> None: + message = message_record() + + primary = primary_records([failed_request_record(), message]) + + assert primary == [message] + assert primary[0].kind == "message" + + +def test_primary_falls_back_to_the_failed_request_record() -> None: + failed_request = failed_request_record() + + primary = primary_records([failed_request]) + + assert primary == [failed_request] + assert primary[0].kind == "failed_request" + + +def test_primary_prefers_an_unhandled_exception_over_a_handled_one() -> None: + handled = exception_record(error_type="KeyError", error_handled=True) + unhandled = exception_record( + timestamp=datetime(2026, 10, 20, 16, 0, 1, tzinfo=UTC), + error_type="LookupError", + ) + + assert primary_records([handled, unhandled]) == [unhandled] + + +def test_primary_treats_a_request_completed_record_with_error_fields_as_an_exception(): + completed = failed_request_record( + error_type="StaleDataError", frames=APPLICATION_FRAMES + ) + + assert completed.kind == "exception" + assert primary_records([completed]) == [completed] + + +def test_primary_correlates_by_trace_when_the_request_id_is_missing() -> None: + exception = exception_record(request_key=None) + message = message_record(request_key=None) + + assert primary_records([message, exception]) == [exception] + + +def test_primary_records_without_request_or_trace_stand_alone() -> None: + first = message_record(request_key=None, trace_key=None) + second = message_record( + request_key=None, + trace_key=None, + timestamp=datetime(2026, 10, 20, 17, 0, tzinfo=UTC), + ) + + assert primary_records([second, first]) == [first, second] + + +def test_primary_keeps_failures_of_different_services_separate() -> None: + api_failure = failed_request_record() + simulation_failure = exception_record( + service_name="policyengine-simulation-entry", + frames=(), + ) + + primary = primary_records([api_failure, simulation_failure]) + + assert primary == [api_failure, simulation_failure] + + +def test_fingerprint_exception_ignores_lines_path_prefixes_and_routes() -> None: + moved_frames = tuple( + frame.replace("/app/", "/srv/checkout/").replace("line 4", "line 9") + for frame in APPLICATION_FRAMES + ) + first = exception_record() + second = exception_record( + frames=moved_frames, + route="//economy//over/", + ) + + assert fingerprint(first, FINGERPRINT_KEY) == fingerprint(second, FINGERPRINT_KEY) + + +def test_fingerprint_exception_merges_simulation_service_versions() -> None: + first = exception_record(service_name="policyengine-simulation-py1-2-3") + second = exception_record(service_name="policyengine-simulation-py1-2-4") + + assert fingerprint(first, FINGERPRINT_KEY) == fingerprint(second, FINGERPRINT_KEY) + + +@pytest.mark.parametrize( + "change", + [ + {"error_type": "ValueError"}, + {"frames": APPLICATION_FRAMES[:1]}, + {"service_name": "policyengine-simulation-entry"}, + ], +) +def test_fingerprint_exception_differs_by_type_frames_and_service( + change: dict[str, object], +) -> None: + original = exception_record() + changed = exception_record(**change) + + assert fingerprint(original, FINGERPRINT_KEY) != fingerprint( + changed, FINGERPRINT_KEY + ) + + +def test_fingerprint_is_the_same_when_truncation_drops_the_final_traceback() -> None: + # The logging library keeps only the first 16,384 characters of a stack. + short_stack = chained_stack(recursion_depth=20) + long_stack = chained_stack(recursion_depth=120)[:16_384] + assert "CalculationError" in short_stack + assert "CalculationError" not in long_stack + + short = exception_record( + error_type="CalculationError", + frames=tuple(extract_frames(short_stack)), + ) + long = exception_record( + error_type="CalculationError", + frames=tuple(extract_frames(long_stack)), + stack_truncated=True, + ) + + assert fingerprint(short, FINGERPRINT_KEY) == fingerprint(long, FINGERPRINT_KEY) + + +def test_fingerprint_depends_on_the_key() -> None: + record = exception_record() + + assert fingerprint(record, FINGERPRINT_KEY) != fingerprint(record, b"j" * 32) + + +def test_fingerprint_is_16_hex_characters() -> None: + value = fingerprint(exception_record(), FINGERPRINT_KEY) + + assert re.fullmatch(r"[0-9a-f]{16}", value) + + +def test_fingerprint_key_parts_for_each_kind() -> None: + exception = exception_record() + message = message_record( + attribute_error_type="TimeoutError", + failure_category="upstream", + ) + operation_message = message_record(route=None, operation_name="simulation.dispatch") + failed_request = failed_request_record(status_code=502) + + assert fingerprint_parts(exception) == ( + "exception", + "policyengine-api", + "StaleDataError", + "policyengine_api/routes/report_routes.py:create_report", + "policyengine_api/services/report_service.py:create", + ) + assert fingerprint_parts(message) == ( + "message", + "policyengine-api", + "//report", + "failed to save", + "TimeoutError", + "upstream", + ) + assert fingerprint_parts(operation_message)[2] == "simulation.dispatch" + assert fingerprint_parts(failed_request) == ( + "failed_request", + "policyengine-api", + "//report", + "502", + ) + + +@pytest.mark.parametrize( + "change", + [ + {"message_template": "failed to load"}, + {"route": "//calculate"}, + {"attribute_error_type": "ValueError"}, + {"failure_category": "validation"}, + ], +) +def test_fingerprint_message_differs_by_each_key_part( + change: dict[str, object], +) -> None: + original = message_record() + changed = message_record(**change) + + assert fingerprint(original, FINGERPRINT_KEY) != fingerprint( + changed, FINGERPRINT_KEY + ) + + +def test_fingerprint_failed_request_differs_by_status() -> None: + first = failed_request_record(status_code=500) + second = failed_request_record(status_code=504) + + assert fingerprint(first, FINGERPRINT_KEY) != fingerprint(second, FINGERPRINT_KEY) + + +@pytest.mark.parametrize("value", ["", "too-short", "x" * 31]) +def test_fingerprint_key_rejects_a_secret_shorter_than_32_characters( + value: str, +) -> None: + with pytest.raises(ValueError, match="at least 32 characters"): + fingerprint_key(value) + + +def test_fingerprint_key_accepts_a_64_character_hex_secret() -> None: + assert fingerprint_key("ab" * 32) == ("ab" * 32).encode() diff --git a/tests/unit/telemetry_error_issues/test_rendering.py b/tests/unit/telemetry_error_issues/test_rendering.py new file mode 100644 index 000000000..475c0eda4 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_rendering.py @@ -0,0 +1,349 @@ +"""Tests for telemetry_error_issues.rendering: issue titles and bodies.""" + +from __future__ import annotations + +from urllib.parse import quote, unquote, urlparse, urlsplit + +import pytest +from telemetry_error_issues.bigquery import error_record_from_row +from telemetry_error_issues.groups import build_groups +from telemetry_error_issues.rendering import ( + SECTION_END, + SECTION_START, + issue_body, + issue_title, + logs_explorer_url, + parse_state, +) +from telemetry_error_issues.window import reporting_window + +from tests.unit.telemetry_error_issues.report_support import ( + FINGERPRINT_KEY, + LOG_SOURCE, + REPORT_DATE, + exception_record, + failed_request_record, + failures_with_and_without_a_route, + hours_after_window_noon, + message_record, + request_count, + sentinel_row, +) + + +def _group(records: list, request_counts=()) -> object: + groups = build_groups( + records, + list(request_counts), + reporting_window(REPORT_DATE), + FINGERPRINT_KEY, + ) + return groups[0] + + +def _render(group: object) -> tuple[str, str]: + window = reporting_window(REPORT_DATE) + return ( + issue_title(group), + issue_body(group, window, LOG_SOURCE), + ) + + +def _section(body: str) -> str: + start = body.index(SECTION_START) + end = body.index(SECTION_END) + len(SECTION_END) + return body[start:end] + + +def _logs_query(url: str) -> tuple[str, dict[str, str]]: + """Split a Logs Explorer URL into its decoded query and other settings.""" + parsed = urlsplit(url) + settings = dict( + segment.split("=", 1) + for segment in parsed.path.split(";")[1:] + if "=" in segment + ) + return unquote(settings.pop("query")), settings + + +def test_render_title_names_the_exception_and_its_most_frequent_route() -> None: + group = _group([exception_record()]) + + title, _ = _render(group) + + assert title == ( + "[telemetry] policyengine-api: StaleDataError on `//report` " + f"({group.fingerprint})" + ) + + +def test_render_title_for_a_logged_error() -> None: + route_group = _group([message_record()]) + operation_group = _group( + [message_record(route=None, operation_name="simulation.dispatch")], + ) + + assert issue_title(route_group).startswith( + "[telemetry] policyengine-api: logged error on `//report` (" + ) + assert issue_title(operation_group).startswith( + "[telemetry] policyengine-api: logged error in `simulation.dispatch` (" + ) + + +def test_render_title_for_a_failed_request_without_an_exception() -> None: + group = _group([failed_request_record(status_code=502)]) + + assert issue_title(group) == ( + "[telemetry] policyengine-api: HTTP 502 without recorded exception on " + f"`//report` ({group.fingerprint})" + ) + + +def test_render_title_without_a_route() -> None: + group = _group( + [ + exception_record( + service_name="policyengine-stage12-us-worker", + route=None, + ) + ], + ) + + assert issue_title(group) == ( + "[telemetry] policyengine-stage12-us-worker: StaleDataError " + f"({group.fingerprint})" + ) + + +def test_render_body_marked_section_holds_the_measures_table() -> None: + records = [ + exception_record(request_key="r1", observability_key="o1"), + exception_record(request_key="r2", timestamp=hours_after_window_noon(2)), + exception_record( + request_key="s1", + environment_name="staging", + timestamp=hours_after_window_noon(1), + ), + ] + group = _group( + records, + [request_count("//report", 400)], + ) + + _, body = _render(group) + section = _section(body) + + assert body.index(SECTION_START) > 0 + assert f"`{group.fingerprint}`" in section + assert "2026-10-20 00:00 EDT to 2026-10-21 00:00 EDT (24 hours)" in section + assert "| Measure | Production | Staging |" in section + assert "| Occurrences | 2 | 1 |" in section + assert "| Affected requests | 2 | 1 |" in section + assert "| Observability IDs | 1 | 0 |" in section + assert ( + "| Error rate on affected routes | 0.50% (2 of 400 requests) | n/a |" in section + ) + assert "| Occurrences per hour | 0.08 | 0.04 |" in section + assert "| Occurrences per hour, previous 7 days | 0.00 | 0.00 |" in section + assert ( + "| First seen in the last 8 days " + "| 2026-10-20 12:00 EDT | 2026-10-20 13:00 EDT |" in section + ) + assert "| Last seen | 2026-10-20 14:00 EDT | 2026-10-20 13:00 EDT |" in section + assert "Related fingerprints: none" in section + assert "https://console.cloud.google.com/logs/query;" in section + + +def test_render_error_rate_names_only_failures_on_routes() -> None: + group = _group( + failures_with_and_without_a_route(), + [request_count("//calculate", 1)], + ) + + _, body = _render(group) + + assert "| Error rate on affected routes | 100.00% (1 of 1 requests) | n/a |" in body + + +def test_render_body_lists_related_fingerprints() -> None: + records = [ + exception_record(trace_key="t1"), + message_record(service_name="policyengine-simulation-entry", trace_key="t1"), + ] + groups = build_groups(records, [], reporting_window(REPORT_DATE), FINGERPRINT_KEY) + exception_group = next(group for group in groups if group.kind == "exception") + other = exception_group.related_fingerprints[0] + + _, body = _render(exception_group) + + assert f"Related fingerprints (sharing traces): `{other}`" in body + + +def test_render_body_state_comment_records_the_group_and_window() -> None: + group = _group([exception_record()]) + + _, body = _render(group) + + assert parse_state(body) == { + "algorithm": 1, + "fingerprint": group.fingerprint, + "kind": "exception", + "window_end": "2026-10-21T04:00:00+00:00", + } + + +def test_render_body_stays_under_the_github_limit() -> None: + # The longest route that still passes validation (195 characters). + long_route = "/" + "/".join([""] * 13) + records = [ + exception_record( + error_type="E" * 120, + route=long_route, + trace_key=f"t{index}", + request_key=f"r{index}", + ) + for index in range(4) + ] + records += [ + message_record( + request_key=f"m{index}", + trace_key=f"t{index}", + message_template=f"template {index}", + service_name="policyengine-simulation-entry", + ) + for index in range(4) + ] + groups = build_groups(records, [], reporting_window(REPORT_DATE), FINGERPRINT_KEY) + + for group in groups: + title, body = _render(group) + assert len(body) < 65_536 + assert len(title) <= 256 + + +def test_render_logs_link_filters_exceptions_by_service_and_type() -> None: + group = _group([exception_record()]) + url = logs_explorer_url(group, reporting_window(REPORT_DATE), LOG_SOURCE) + + query, settings = _logs_query(url) + + assert urlparse(url).query == "project=policyengine-observability" + assert settings["storageScope"] == quote( + "storage,projects/policyengine-observability/locations/global/" + "buckets/policyengine-observability/views/_AllLogs", + safe="", + ) + assert query.splitlines() == [ + 'jsonPayload.schema_version="policyengine.observability.v2"', + 'jsonPayload."service.name"="policyengine-api"', + "severity>=ERROR", + 'jsonPayload."error.type"="StaleDataError"', + 'timestamp>="2026-10-20T04:00:00Z"', + 'timestamp<"2026-10-21T04:00:00Z"', + ] + for character in (" ", "\n", '"'): + assert character not in url + + +def test_render_logs_link_names_the_configured_bucket_location_and_view() -> None: + group = _group([exception_record()]) + log_source = LOG_SOURCE._replace( + project_id="other-project", + bucket="central-logs", + bucket_location="us-central1", + view="_Requests", + ) + + url = logs_explorer_url(group, reporting_window(REPORT_DATE), log_source) + + _, settings = _logs_query(url) + assert urlparse(url).query == "project=other-project" + assert settings["storageScope"] == quote( + "storage,projects/other-project/locations/us-central1/" + "buckets/central-logs/views/_Requests", + safe="", + ) + + +@pytest.mark.parametrize( + "unsafe_names", + [ + {"project_id": "SENTINEL project"}, + {"bucket": "logs/views/other"}, + {"bucket_location": "global/../other"}, + {"view": "_AllLogs?project=other"}, + ], +) +def test_render_logs_link_refuses_names_that_are_not_identifiers( + unsafe_names: dict[str, str], +) -> None: + group = _group([exception_record()]) + + with pytest.raises(ValueError, match="is not a valid name"): + logs_explorer_url( + group, reporting_window(REPORT_DATE), LOG_SOURCE._replace(**unsafe_names) + ) + + +def test_render_logs_link_filters_failed_requests_by_route_and_status() -> None: + group = _group( + [ + failed_request_record( + service_name="policyengine-simulation-py1-2-3", + route="/jobs/{job_id}", + status_code=504, + ) + ], + ) + + query, _ = _logs_query( + logs_explorer_url(group, reporting_window(REPORT_DATE), LOG_SOURCE) + ) + + assert query.splitlines()[1:5] == [ + 'jsonPayload."service.name"=~"^policyengine-simulation-py[0-9]+-[0-9]+-[0-9]+$"', + 'jsonPayload."event.name"="request.completed"', + 'jsonPayload."http.route"="/jobs/{job_id}"', + 'jsonPayload."http.response.status_code"=504', + ] + + +def test_render_logs_link_filters_logged_errors_by_route_or_operation() -> None: + window = reporting_window(REPORT_DATE) + route_group = _group([message_record()]) + operation_group = _group( + [message_record(route=None, operation_name="simulation.dispatch")], + ) + + route_query, _ = _logs_query(logs_explorer_url(route_group, window, LOG_SOURCE)) + operation_query, _ = _logs_query( + logs_explorer_url(operation_group, window, LOG_SOURCE) + ) + + assert 'jsonPayload."http.route"="//report"' in route_query + assert "severity>=ERROR" in route_query + assert 'jsonPayload."operation.name"="simulation.dispatch"' in operation_query + + +@pytest.mark.parametrize( + "overrides", + [ + {}, + {"error_type": None}, + {"error_type": None, "event_name": "request.completed"}, + {"route": None}, + ], +) +def test_render_never_includes_values_from_records( + overrides: dict[str, object], +) -> None: + record = error_record_from_row(sentinel_row(**overrides)) + group = _group([record]) + window = reporting_window(REPORT_DATE) + + title, body = _render(group) + url = logs_explorer_url(group, window, LOG_SOURCE) + + for output in (title, body, url, unquote(url)): + assert "SENTINEL" not in output diff --git a/tests/unit/telemetry_error_issues/test_sync.py b/tests/unit/telemetry_error_issues/test_sync.py new file mode 100644 index 000000000..67cbb2448 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_sync.py @@ -0,0 +1,338 @@ +"""Tests for telemetry_error_issues.sync: opening, updating and reopening issues.""" + +from __future__ import annotations + +import json +from datetime import timedelta +from urllib.parse import urlsplit + +import pytest +from telemetry_error_issues.github_client import GitHubError +from telemetry_error_issues.rendering import ( + SECTION_END, + issue_body, + issue_section, + issue_title, +) +from telemetry_error_issues.window import reporting_window + +from tests.unit.telemetry_error_issues.github_support import ( + FakeGitHub, + bot_issue, + failures, + groups_for_report_date, + sync, +) +from tests.unit.telemetry_error_issues.report_support import ( + LOG_SOURCE, + REPORT_DATE, + WINDOW_NOON, + exception_record, + hours_after_window_noon, +) + + +def _existing_issue(group: object, number: int = 7, **fields: object) -> dict: + """Return the issue the job opened for ``group`` on an earlier day.""" + earlier = reporting_window(REPORT_DATE - timedelta(days=1)) + issue = bot_issue( + number, + title=issue_title(group), + body=issue_body(group, earlier, LOG_SOURCE), + ) + issue.update(fields) + return issue + + +def test_sync_opens_an_issue_at_three_production_occurrences_in_two_requests() -> None: + eligible = failures(3) + # Three failures that all affected the request with observability ID o1. + one_request = [ + exception_record( + error_type="KeyError", + request_key=None, + trace_key=None, + observability_key="o1", + timestamp=hours_after_window_noon(index), + ) + for index in range(3) + ] + too_few = failures(2, request_key_prefix="few", error_type="TypeError") + groups = groups_for_report_date(eligible + one_request + too_few) + github = FakeGitHub() + + result = sync(groups, github) + + created = list(github.issues.values()) + eligible_group = next( + group for group in groups if group.error_type == "StaleDataError" + ) + assert [issue["title"] for issue in created] == [issue_title(eligible_group)] + assert created[0]["labels"] == [ + {"name": "telemetry-error"}, + {"name": "service: policyengine-api"}, + ] + actions = {action.fingerprint: action.action for action in result.actions} + assert sorted(actions.values()) == ["below-threshold", "below-threshold", "created"] + + +def test_sync_never_opens_an_issue_for_a_staging_only_group() -> None: + groups = groups_for_report_date(failures(10, environment_name="staging")) + github = FakeGitHub() + + result = sync(groups, github) + + assert github.issues == {} + assert result.actions[0].action == "below-threshold" + + +def test_sync_opens_at_most_five_issues_in_order_of_production_volume() -> None: + records = [] + for index in range(7): + records += failures( + 3 + index, + request_key_prefix=f"g{index}-", + error_type=f"Error{index}", + ) + groups = groups_for_report_date(records) + github = FakeGitHub() + + result = sync(groups, github) + + titles = [issue["title"] for issue in github.issues.values()] + assert [title.split(": ")[1].split(" ")[0] for title in titles] == [ + "Error6", + "Error5", + "Error4", + "Error3", + "Error2", + ] + assert result.deferred == 2 + assert [action.action for action in result.actions].count("deferred") == 2 + + +def test_sync_replaces_only_the_marked_section_and_never_the_title() -> None: + group = groups_for_report_date(failures(3))[0] + issue = _existing_issue(group, title="Renamed by a person") + issue["body"] = f"Notes before.\n\n{issue['body']}\nNotes after.\n" + github = FakeGitHub([issue]) + + result = sync([group], github) + + patch = json.loads(github.writes[0][1].body) + expected_section = issue_section(group, reporting_window(REPORT_DATE), LOG_SOURCE) + assert list(patch) == ["body"] + assert patch["body"].startswith("Notes before.\n\n") + assert patch["body"].endswith("\nNotes after.\n") + assert expected_section in patch["body"] + assert github.issues[7]["title"] == "Renamed by a person" + assert result.actions[0].action == "updated" + assert result.actions[0].issue_number == 7 + + +def test_sync_does_not_write_when_the_section_is_unchanged() -> None: + group = groups_for_report_date(failures(3))[0] + github = FakeGitHub([_existing_issue(group)]) + + sync([group], github) + first_run_writes = len(github.writes) + second = sync([group], github) + + assert first_run_writes == 1 + assert len(github.writes) == 1 + assert second.actions[0].action == "unchanged" + + +def test_sync_adds_a_deleted_section_back_at_the_end() -> None: + group = groups_for_report_date(failures(3))[0] + issue = _existing_issue(group, body="Only human notes now.") + github = FakeGitHub([issue]) + + sync([group], github) + + body = github.issues[7]["body"] + assert body.startswith("Only human notes now.\n\n") + assert body.rstrip().endswith(SECTION_END) + assert len(github.issues) == 1 + + +def test_sync_ignores_issues_opened_by_other_authors() -> None: + group = groups_for_report_date(failures(3))[0] + person_issue = _existing_issue(group, user={"login": "someone"}) + github = FakeGitHub([person_issue]) + + sync([group], github) + + assert github.issues[7]["body"] == person_issue["body"] + assert len(github.issues) == 2 + + +def test_sync_reopens_a_completed_issue_when_production_failures_return() -> None: + group = groups_for_report_date(failures(4))[0] + closed_at = (WINDOW_NOON - timedelta(days=2)).isoformat().replace("+00:00", "Z") + issue = _existing_issue( + group, + state="closed", + state_reason="completed", + closed_at=closed_at, + ) + github = FakeGitHub([issue], labels=("telemetry-error",)) + + result = sync([group], github) + + reopened = github.issues[7] + assert reopened["state"] == "open" + assert reopened["state_reason"] == "reopened" + assert {"name": "regression"} in reopened["labels"] + assert "regression" in github.labels + assert github.comments[7] == [ + "Production failures returned after this issue was closed: 4 occurrences " + "in 4 requests since 2026-10-19 12:00 EDT, 24 hours after it was closed." + ] + assert result.actions[0].action == "reopened" + + +def test_sync_treats_an_issue_closed_without_a_reason_as_completed() -> None: + group = groups_for_report_date(failures(3))[0] + closed_at = (WINDOW_NOON - timedelta(days=2)).isoformat() + issue = _existing_issue( + group, state="closed", state_reason=None, closed_at=closed_at + ) + github = FakeGitHub([issue]) + + result = sync([group], github) + + assert github.issues[7]["state"] == "open" + assert result.actions[0].action == "reopened" + + +def test_sync_keeps_a_completed_issue_closed_within_24_hours_of_closing() -> None: + group = groups_for_report_date(failures(4))[0] + closed_at = (WINDOW_NOON - timedelta(hours=12)).isoformat() + issue = _existing_issue( + group, + state="closed", + state_reason="completed", + closed_at=closed_at, + ) + github = FakeGitHub([issue]) + + result = sync([group], github) + + assert github.issues[7]["state"] == "closed" + assert 7 not in github.comments + assert ( + issue_section(group, reporting_window(REPORT_DATE), LOG_SOURCE) + in github.issues[7]["body"] + ) + assert result.actions[0].action == "updated" + + +@pytest.mark.parametrize("reason", ["not_planned", "duplicate"]) +def test_sync_updates_but_never_reopens_not_planned_or_duplicate_issues( + reason: str, +) -> None: + group = groups_for_report_date(failures(10))[0] + closed_at = (WINDOW_NOON - timedelta(days=5)).isoformat() + issue = _existing_issue( + group, state="closed", state_reason=reason, closed_at=closed_at + ) + github = FakeGitHub([issue]) + + result = sync([group], github) + + assert github.issues[7]["state"] == "closed" + assert result.actions[0].action == "updated" + assert len(github.issues) == 1 + + +def test_sync_keeps_a_newer_report_instead_of_overwriting_it() -> None: + group = groups_for_report_date(failures(3))[0] + later = reporting_window(REPORT_DATE + timedelta(days=1)) + issue = bot_issue( + 7, + title=issue_title(group), + body=issue_body(group, later, LOG_SOURCE), + ) + github = FakeGitHub([issue]) + + result = sync([group], github) + + assert github.writes == [] + assert result.actions[0].action == "kept-newer" + + +def test_sync_creates_missing_labels_before_opening_issues() -> None: + groups = groups_for_report_date(failures(3)) + github = FakeGitHub(labels=("telemetry-error",)) + + sync(groups, github) + + created_labels = [ + json.loads(request.body)["name"] + for _, request in github.writes + if urlsplit(request.url).path.endswith("/labels") + ] + assert created_labels == ["service: policyengine-api"] + + +def test_sync_uses_existing_labels_whatever_their_case() -> None: + groups = groups_for_report_date(failures(3)) + github = FakeGitHub(labels=("Telemetry-Error", "Service: Policyengine-Api")) + + sync(groups, github) + + created_labels = [ + request + for _, request in github.writes + if urlsplit(request.url).path.endswith("/labels") + ] + assert created_labels == [] + assert github.issues[1]["labels"] == [ + {"name": "Telemetry-Error"}, + {"name": "Service: Policyengine-Api"}, + ] + + +def test_sync_reopens_with_an_existing_regression_label_of_another_case() -> None: + group = groups_for_report_date(failures(4))[0] + closed_at = (WINDOW_NOON - timedelta(days=2)).isoformat() + issue = _existing_issue( + group, + state="closed", + state_reason="completed", + closed_at=closed_at, + ) + github = FakeGitHub([issue], labels=("Regression",)) + + result = sync([group], github) + + assert result.actions[0].action == "reopened" + assert {"name": "Regression"} in github.issues[7]["labels"] + assert github.labels == {"Regression"} + + +def test_sync_adds_labels_a_create_response_is_missing() -> None: + groups = groups_for_report_date(failures(3)) + github = FakeGitHub(drop_created_labels=True) + + with pytest.raises(GitHubError, match="labels"): + sync(groups, github) + + added = [ + request + for _, request in github.writes + if urlsplit(request.url).path.endswith("/issues/1/labels") + ] + assert len(added) == 1 + + +def test_sync_dry_run_makes_no_write_calls() -> None: + eligible = groups_for_report_date(failures(3))[0] + existing = groups_for_report_date(failures(3, error_type="KeyError"))[0] + github = FakeGitHub([_existing_issue(existing)]) + + result = sync([eligible, existing], github, dry_run=True) + + assert github.writes == [] + assert sorted(action.action for action in result.actions) == ["created", "updated"] diff --git a/tests/unit/telemetry_error_issues/test_window.py b/tests/unit/telemetry_error_issues/test_window.py new file mode 100644 index 000000000..3deea8625 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_window.py @@ -0,0 +1,58 @@ +"""Tests for telemetry_error_issues.window: the reporting window.""" + +from __future__ import annotations + +from datetime import UTC, date, datetime, timedelta + +import pytest +from telemetry_error_issues.window import default_report_date, reporting_window + +from tests.unit.telemetry_error_issues.report_support import EASTERN + + +@pytest.mark.parametrize( + "run_time", + [ + datetime(2026, 10, 21, 0, 45, tzinfo=EASTERN), + datetime(2026, 10, 21, 3, 30, tzinfo=EASTERN), + datetime(2026, 10, 21, 4, 45, tzinfo=UTC), + ], +) +def test_window_scheduled_run_reports_the_previous_eastern_day( + run_time: datetime, +) -> None: + window = reporting_window(default_report_date(run_time)) + + assert window.start == datetime(2026, 10, 20, 0, 0, tzinfo=EASTERN) + assert window.end == datetime(2026, 10, 21, 0, 0, tzinfo=EASTERN) + assert window.hours == 24 + + +def test_window_default_date_follows_eastern_time_not_utc() -> None: + # 02:00 UTC on 21 October is still 22:00 on 20 October in New York. + run_time = datetime(2026, 10, 21, 2, 0, tzinfo=UTC) + + assert default_report_date(run_time) == date(2026, 10, 19) + + +def test_window_spring_forward_day_lasts_23_hours() -> None: + window = reporting_window(date(2026, 3, 8)) + + assert window.start.utcoffset() == timedelta(hours=-5) + assert window.end.utcoffset() == timedelta(hours=-4) + assert window.hours == 23 + + +def test_window_fall_back_day_lasts_25_hours() -> None: + window = reporting_window(date(2026, 11, 1)) + + assert window.start.utcoffset() == timedelta(hours=-4) + assert window.end.utcoffset() == timedelta(hours=-5) + assert window.hours == 25 + + +def test_window_bounds_convert_to_utc_for_queries() -> None: + window = reporting_window(date(2026, 11, 1)) + + assert window.start.astimezone(UTC) == datetime(2026, 11, 1, 4, 0, tzinfo=UTC) + assert window.end.astimezone(UTC) == datetime(2026, 11, 2, 5, 0, tzinfo=UTC) diff --git a/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py b/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py new file mode 100644 index 000000000..b85ca91e2 --- /dev/null +++ b/tests/unit/telemetry_error_issues/test_workflow_and_wrapper.py @@ -0,0 +1,256 @@ +"""Tests for the workflow, its shell wrapper and the script they run.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tomllib +from pathlib import Path + +import pytest +import yaml + +from tests.unit.telemetry_error_issues.github_support import REPOSITORY +from tests.unit.telemetry_error_issues.report_support import ( + LOG_SOURCE_ENVIRONMENT, + PROJECT_ID, +) + +REPO = Path(__file__).resolve().parents[3] +SCRIPT_PATH = REPO / ".github" / "scripts" / "sync_telemetry_error_issues.py" +WORKFLOW_PATH = REPO / ".github" / "workflows" / "telemetry-error-issues.yml" +WRAPPER_PATH = REPO / ".github" / "scripts" / "sync_telemetry_error_issues.sh" +DEPENDENCY_GROUP = "telemetry-error-issues" +# The wrapper runs the script in an environment of only its locked group. +UV_RUN_ARGUMENTS = [ + "run", + "--frozen", + "--only-group", + DEPENDENCY_GROUP, + "--project", + str(REPO), + "python", + str(SCRIPT_PATH), +] +WRAPPER_ENVIRONMENT = { + "GOOGLE_OAUTH_ACCESS_TOKEN": "fake-access-token", + "GITHUB_TOKEN": "fake-github-token", + "GITHUB_REPOSITORY": REPOSITORY, + "TELEMETRY_ERROR_FINGERPRINT_KEY": "f" * 64, + "OBSERVABILITY_PROJECT_ID": PROJECT_ID, + **LOG_SOURCE_ENVIRONMENT, +} + + +def _workflow() -> dict: + return yaml.safe_load(WORKFLOW_PATH.read_text(encoding="utf-8")) + + +def _sync_job() -> dict: + return _workflow()["jobs"]["sync"] + + +def _step(name: str) -> dict: + return next(step for step in _sync_job()["steps"] if step["name"] == name) + + +def test_workflow_runs_at_00_45_eastern_and_on_request() -> None: + # PyYAML reads the bare key "on" as the boolean True. + triggers = _workflow()[True] + + assert triggers["schedule"] == [ + {"cron": "45 0 * * *", "timezone": "America/New_York"} + ] + inputs = triggers["workflow_dispatch"]["inputs"] + assert inputs["dry_run"]["type"] == "boolean" + assert inputs["dry_run"]["default"] is True + assert inputs["report_date"]["type"] == "string" + assert inputs["report_date"]["default"] == "" + + +def test_workflow_job_is_limited_to_master_in_its_environment() -> None: + workflow = _workflow() + job = _sync_job() + + assert workflow["concurrency"] == { + "group": "telemetry-error-issues", + "cancel-in-progress": False, + } + assert job["if"] == "github.repository == 'PolicyEngine/policyengine-api'" + assert job["environment"] == "telemetry-error-issues" + assert job["timeout-minutes"] == 15 + assert job["permissions"] == { + "contents": "read", + "issues": "write", + "id-token": "write", + } + assert job["steps"][0] == { + "name": "Require master branch", + "if": "github.ref != 'refs/heads/master'", + "run": "exit 1", + } + + +def test_workflow_runs_the_report_with_the_pinned_uv_and_python() -> None: + assert _sync_job()["env"] == {"UV_PYTHON": "3.12"} + assert _step("Set up uv") == { + "name": "Set up uv", + "uses": "astral-sh/setup-uv@v6", + "with": {"version": "0.13.4"}, + } + + +def test_wrapper_dependency_group_holds_the_bigquery_client_library() -> None: + pyproject = tomllib.loads((REPO / "pyproject.toml").read_text(encoding="utf-8")) + + group = pyproject["dependency-groups"][DEPENDENCY_GROUP] + + assert [requirement.split(">")[0] for requirement in group] == [ + "google-cloud-bigquery" + ] + + +def test_workflow_authenticates_as_the_report_service_account() -> None: + auth = _step("Authenticate to Google Cloud") + + assert auth["id"] == "auth" + assert auth["uses"] == "google-github-actions/auth@v2" + assert auth["with"] == { + "workload_identity_provider": "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}", + "service_account": "${{ vars.TELEMETRY_ERROR_ISSUES_SERVICE_ACCOUNT }}", + "token_format": "access_token", + "create_credentials_file": False, + } + assert _step("Set up Python")["with"] == {"python-version": "3.12"} + + +def test_workflow_passes_settings_to_the_wrapper() -> None: + sync = _step("Sync telemetry error issues") + + assert sync["run"] == "bash .github/scripts/sync_telemetry_error_issues.sh" + assert sync["env"] == { + "GOOGLE_OAUTH_ACCESS_TOKEN": "${{ steps.auth.outputs.access_token }}", + "GITHUB_TOKEN": "${{ github.token }}", + "TELEMETRY_ERROR_FINGERPRINT_KEY": ( + "${{ secrets.TELEMETRY_ERROR_FINGERPRINT_KEY }}" + ), + "OBSERVABILITY_PROJECT_ID": "${{ vars.OBSERVABILITY_PROJECT_ID }}", + "OBSERVABILITY_LOG_BUCKET": "${{ vars.OBSERVABILITY_LOG_BUCKET }}", + "OBSERVABILITY_LOG_BUCKET_LOCATION": ( + "${{ vars.OBSERVABILITY_LOG_BUCKET_LOCATION }}" + ), + "OBSERVABILITY_LOG_DATASET": "${{ vars.OBSERVABILITY_LOG_DATASET }}", + "OBSERVABILITY_LOG_VIEW": "${{ vars.OBSERVABILITY_LOG_VIEW }}", + "OBSERVABILITY_SERVICE_NAMESPACE": ( + "${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}" + ), + # A scheduled run has no inputs, so it is never a dry run. + "DRY_RUN": "${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}", + "REPORT_DATE": "${{ inputs.report_date }}", + } + + +def _run_wrapper( + tmp_path: Path, environment: dict[str, str] +) -> tuple[subprocess.CompletedProcess[str], list[str] | None]: + """Run the wrapper with a fake uv that records its arguments.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + calls = tmp_path / "uv-arguments.txt" + fake_uv = bin_dir / "uv" + fake_uv.write_text(f'#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "{calls}"\n') + fake_uv.chmod(0o755) + result = subprocess.run( + ["bash", str(WRAPPER_PATH)], + cwd=REPO, + env={"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", **environment}, + text=True, + capture_output=True, + check=False, + ) + arguments = calls.read_text().splitlines() if calls.exists() else None + return result, arguments + + +def test_wrapper_passes_bash_syntax_checking() -> None: + result = subprocess.run( + ["bash", "-n", str(WRAPPER_PATH)], capture_output=True, text=True, check=False + ) + + assert result.returncode == 0, result.stderr + + +def test_wrapper_runs_the_report_for_the_project(tmp_path: Path) -> None: + result, arguments = _run_wrapper(tmp_path, WRAPPER_ENVIRONMENT) + + assert result.returncode == 0, result.stderr + assert arguments == [*UV_RUN_ARGUMENTS, "--project-id", PROJECT_ID] + + +def test_wrapper_masks_the_access_token_and_fingerprint_key(tmp_path: Path) -> None: + result, _ = _run_wrapper(tmp_path, WRAPPER_ENVIRONMENT) + + assert result.stdout.splitlines() == [ + "::add-mask::fake-access-token", + f"::add-mask::{'f' * 64}", + ] + + +def test_wrapper_adds_dry_run_and_report_date_when_set(tmp_path: Path) -> None: + environment = { + **WRAPPER_ENVIRONMENT, + "DRY_RUN": "true", + "REPORT_DATE": "2026-10-15", + } + + result, arguments = _run_wrapper(tmp_path, environment) + + assert result.returncode == 0, result.stderr + assert arguments == [ + *UV_RUN_ARGUMENTS, + "--project-id", + PROJECT_ID, + "--dry-run", + "--report-date", + "2026-10-15", + ] + + +def test_wrapper_runs_live_when_dry_run_is_false(tmp_path: Path) -> None: + environment = {**WRAPPER_ENVIRONMENT, "DRY_RUN": "false", "REPORT_DATE": ""} + + _, arguments = _run_wrapper(tmp_path, environment) + + assert arguments == [*UV_RUN_ARGUMENTS, "--project-id", PROJECT_ID] + + +@pytest.mark.parametrize("missing", sorted(WRAPPER_ENVIRONMENT)) +def test_wrapper_fails_before_running_when_a_variable_is_missing( + tmp_path: Path, missing: str +) -> None: + environment = { + name: value for name, value in WRAPPER_ENVIRONMENT.items() if name != missing + } + + result, arguments = _run_wrapper(tmp_path, environment) + + assert result.returncode != 0 + assert f"{missing} is required" in result.stderr + assert arguments is None + assert result.stdout == "" + + +def test_script_imports_its_package_from_another_working_directory( + tmp_path: Path, +) -> None: + result = subprocess.run( + [sys.executable, str(SCRIPT_PATH), "--help"], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "Report each day's telemetry error groups as GitHub issues." in result.stdout diff --git a/uv.lock b/uv.lock index 84a152e4e..beb814a87 100644 --- a/uv.lock +++ b/uv.lock @@ -1033,6 +1033,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/76/d241a5c927433420507215df6cac1b1fa4ac0ba7a794df42a84326c68da8/google_auth-2.49.2-py3-none-any.whl", hash = "sha256:c2720924dfc82dedb962c9f52cabb2ab16714fd0a6a707e40561d217574ed6d5", size = 240638, upload-time = "2026-04-10T00:41:14.501Z" }, ] +[package.optional-dependencies] +pyopenssl = [ + { name = "pyopenssl" }, +] + [[package]] name = "google-cloud-appengine-logging" version = "1.9.0" @@ -1062,6 +1067,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/40/79fa535b6e3321d5e07b2a9ab4bb63860d3fea12230c765837881348003c/google_cloud_audit_log-0.5.0-py3-none-any.whl", hash = "sha256:3f4632f25bf67446fa9085c52868f3cb42fb1afbab9489ba8978e30991afc79f", size = 44862, upload-time = "2026-03-30T22:47:57.533Z" }, ] +[[package]] +name = "google-cloud-bigquery" +version = "3.46.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth", extra = ["pyopenssl"] }, + { name = "google-cloud-core" }, + { name = "google-resumable-media" }, + { name = "packaging" }, + { name = "python-dateutil" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/df/1b6f6fa4c119babcbbd002826a0a24ac287066808fbc873e7733b5ecc2bf/google_cloud_bigquery-3.46.2.tar.gz", hash = "sha256:a87e2e8026133e709deab9ebf537a6a2eee3fbcf4befa334a2f9f6164435b331", size = 530538, upload-time = "2026-10-15T17:49:37.245Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/ba/e89552c32a1cd5483261483c303da560e6504365d7314ecd88216ff9e119/google_cloud_bigquery-3.46.2-py3-none-any.whl", hash = "sha256:5e9a4e15f778cc00ee88167de9934a4c1bbb7fc20058e8f573f2bfdaf0d99052", size = 269195, upload-time = "2026-10-15T17:49:06.152Z" }, +] + [[package]] name = "google-cloud-core" version = "2.5.1" @@ -1113,6 +1136,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/4b/6dd1e2efd9a2e73aa847fd455a1ce375d8d3cba1a2c4f7fd69f9bf0b9dce/google_cloud_secret_manager-2.27.0-py3-none-any.whl", hash = "sha256:e5540bece65a3ad720146f3b438973faf9315109b3ffa012a58711843047a3dc", size = 225577, upload-time = "2026-03-26T22:15:19.622Z" }, ] +[[package]] +name = "google-crc32c" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/25/9cb0c1c31c45b893eb8f11ae70b3f4309432d59b5acaebca5dbe791729a4/google_crc32c-1.9.0.tar.gz", hash = "sha256:7b8c84c3d159ab6817fe3f74e6e6cef099c3f95dcec3abc0d8afb1404642efbe", size = 14857, upload-time = "2026-09-24T21:39:32.067Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/55/a2f07f15e624f0de79359b1a6c1deb59ec5061bd3b38744b3b2849400662/google_crc32c-1.9.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:457d0d9a4718fd52b1494eac5c200ad25beeadbdc91843d550a003910838589f", size = 31958, upload-time = "2026-09-24T21:19:00.994Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b3/923743597b774bbcf12a7c3e00e48d745e15fd616ad7489a40a63fff8f2f/google_crc32c-1.9.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:ccfe40021fd6afe23361175cf7551e3cef5fd34dc1ebe319f14993a83579e0eb", size = 31805, upload-time = "2026-09-24T21:22:25.019Z" }, + { url = "https://files.pythonhosted.org/packages/df/a6/4d0352fe889663e0d81cea7fc664ec9158727384de4a44ab10e9967a7682/google_crc32c-1.9.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fbef61a3794e011c65fb4396a196cf123a7f474fe5a443db8e5dd7d751b9e6d4", size = 38119, upload-time = "2026-09-24T21:38:06.634Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e3/26685384e4b66ff0928d9566ef6110a7df76029175a1842329d7e3515f10/google_crc32c-1.9.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86764b99e7a607830d93cb5b75e0ec3ff6cb06d3c274624418473cee701900d4", size = 36677, upload-time = "2026-09-24T21:38:08.082Z" }, + { url = "https://files.pythonhosted.org/packages/cb/ce/4e90102e84880e97d3cf935f2672ecd29191bdeacf57f01740f92debda00/google_crc32c-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:43a2dc26f9be213fbe0b4fc4a1088c5d45cbfcb3247420ccc820f0fc3edeea86", size = 35096, upload-time = "2026-09-24T21:39:28.201Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5d/0730e1b3a14d054d1466f2fec88dadf978509c749a3d96d8b069cc56d38a/google_crc32c-1.9.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:53fdafef58e230d0c946ab5f8446d123d9f548230a73b29c8b41c9546f268bc1", size = 31961, upload-time = "2026-09-24T21:19:01.724Z" }, + { url = "https://files.pythonhosted.org/packages/dd/32/d085abaf2fd907121975b92245bb3480fb8be40c37d03f9d6c41857f84c3/google_crc32c-1.9.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:8b91f41645b15a720357183fa5716682ada441873e3c462c15f9714be36f146b", size = 31804, upload-time = "2026-09-24T21:22:25.81Z" }, + { url = "https://files.pythonhosted.org/packages/94/78/dd1935432337e5da7af391a6fc9f161c1c8e9b9002a402b9190135fe1b59/google_crc32c-1.9.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:16865b477d7941712cb0e0aad8ad4815e984fb5fc16d3fdaef7d986e26e53c95", size = 37775, upload-time = "2026-09-24T21:38:09.249Z" }, + { url = "https://files.pythonhosted.org/packages/9e/43/9db03635bb10188d93dcbab9baa2a8670a0da4e868b4370cdbd98d65fed8/google_crc32c-1.9.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3abb18297d9ef0ab120531838be0e6d68c9fa876570e11c229c48f2edac23ce7", size = 36347, upload-time = "2026-09-24T21:38:10.141Z" }, + { url = "https://files.pythonhosted.org/packages/cf/eb/94dee516c846bd9382c3f566d8f8e5fb9e90599e45afeb697f9fc2533528/google_crc32c-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:fb63a8d7fa2e95dcff1ca16af2f4d88b526fa5ff72d1696285884ac2d49b6963", size = 35097, upload-time = "2026-09-24T21:39:28.934Z" }, + { url = "https://files.pythonhosted.org/packages/3f/34/cb484e8b6174f130f8c6dc79c733a9dd8869b410ad6511fb6104c46b973a/google_crc32c-1.9.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:f1dc17d987ddcc5eba12a7ce48f0eb93141dea236b170c1101151396edf2f0cf", size = 31960, upload-time = "2026-09-24T21:19:02.454Z" }, + { url = "https://files.pythonhosted.org/packages/af/25/3e8e567bd48448e225ea27318ccf2b94e05124e7b8b97b13eaec9e127199/google_crc32c-1.9.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:f894a2877650b56201d26a012a257b76d54a68834dc3913a93830ca8a047b075", size = 31803, upload-time = "2026-09-24T21:22:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/f0/18/bee0dd59ae622482dc6463636c79e4bde7c954d061c859c9256362c9931a/google_crc32c-1.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4488f1553a9ab7e86cdedc833374a7e904031803b995dc0bd0be48c271fa6556", size = 37776, upload-time = "2026-09-24T21:38:11.056Z" }, + { url = "https://files.pythonhosted.org/packages/fd/b6/e76e80fed5f2558273c7839e622f98095c9b36c719c7147e38e3c055cb70/google_crc32c-1.9.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0568b17ed90ac596f29400d99e243fd0cc6276766183def888d1bf8d1dc13827", size = 36350, upload-time = "2026-09-24T21:38:12.138Z" }, + { url = "https://files.pythonhosted.org/packages/87/34/165542bfa99dfef91a76471cc48cce74b8ff4e295722896087ab2b8e8611/google_crc32c-1.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:8583ec21d56b565d68ab2963cc7e21b3b271247c29b04286068255ef65f221bd", size = 35090, upload-time = "2026-09-24T21:39:29.764Z" }, + { url = "https://files.pythonhosted.org/packages/8f/eb/43ea41f4061a1cad87b2b6559c98e960e45bf551fe66f83d833b98aaf0c9/google_crc32c-1.9.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:6a3b2c8a343c570ed8100a7627c20badfd92c6caa2067093a86be45af27f5b1b", size = 31963, upload-time = "2026-09-24T21:19:03.208Z" }, + { url = "https://files.pythonhosted.org/packages/45/d2/a968c0c29ccd2b0c980ff4f9e3f7035cee28c23a1c57541825cc8221858c/google_crc32c-1.9.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:13179f7e3282617923e957b8e54b8f9c3968030f48640a9f47fd7c5c38c4a215", size = 31805, upload-time = "2026-09-24T21:22:27.917Z" }, + { url = "https://files.pythonhosted.org/packages/03/73/388e493d6c3e252e37165d22efe5a1361f872a24425391b999822861b23a/google_crc32c-1.9.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:265233aff33d835f5b909584fe36ab29647b598c271b661a300001099109e53e", size = 37925, upload-time = "2026-09-24T21:38:13.32Z" }, + { url = "https://files.pythonhosted.org/packages/98/36/190d32caa363ef25d685f422ed1bbf93ff1140fb22fd4d90f24cec209977/google_crc32c-1.9.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dee799544cae42a42b17a88e38b59cf2c271051dc001da2117a8ff240ffa0548", size = 36495, upload-time = "2026-09-24T21:38:14.211Z" }, + { url = "https://files.pythonhosted.org/packages/d3/fd/81cefea6adae7bd92abb23d4567d199f6485a20ec0a305ca5fa04c52b9c5/google_crc32c-1.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:af73200fa9791ccd380f3598235dba8d82b8af0905df045b3dc60b59836e8ddd", size = 35535, upload-time = "2026-09-24T21:39:30.52Z" }, + { url = "https://files.pythonhosted.org/packages/c5/18/19d4f17f3f33f8fdffcb3e1e69219d6f7ec2c359c160867b04dac1d0a64d/google_crc32c-1.9.0-cp315-cp315-macosx_12_0_arm64.whl", hash = "sha256:e6e8be8a94436079cb5340f6d495d9d7ba30124d8b952703994c739c7c06e236", size = 31977, upload-time = "2026-09-24T21:19:03.976Z" }, + { url = "https://files.pythonhosted.org/packages/81/b4/8010372c4b46f2ee2352dfdb630c397570cd85522a315df024ad2f9459aa/google_crc32c-1.9.0-cp315-cp315-macosx_12_0_x86_64.whl", hash = "sha256:f2b64641bca27497b986b9d87883014035aa904cb4fa333407c6752b3afee9ba", size = 31809, upload-time = "2026-09-24T21:22:29.1Z" }, + { url = "https://files.pythonhosted.org/packages/c5/f8/7e33845d6b90ce1cf37cfabf25cb859277c7d3533ef1b6b1e1ca58581549/google_crc32c-1.9.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f97c3806dcea41c29c04965347b0e12481561b75e0045dc7a4f69d75dec5d9b1", size = 38359, upload-time = "2026-09-24T21:38:14.983Z" }, + { url = "https://files.pythonhosted.org/packages/36/ff/556b2423f449a7515af6b8222a4d7833cbe09ff3e8d2f0b80471f5f6d02e/google_crc32c-1.9.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0abe7e202c25909869c35672ab0f2fe748a7acf276eb78577332a7c38999740f", size = 36858, upload-time = "2026-09-24T21:38:15.799Z" }, + { url = "https://files.pythonhosted.org/packages/40/71/4733f1b7c921d04a2bb9b9916cf66498bf7ad0860a06289413830da83192/google_crc32c-1.9.0-cp315-cp315-win_amd64.whl", hash = "sha256:5695c8b9327e040b2aba12c6659b0acb5995314ef0af0192da66e662e011103b", size = 35534, upload-time = "2026-09-24T21:39:31.337Z" }, +] + +[[package]] +name = "google-resumable-media" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-crc32c" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cf/64/df6a482d5aa39d7f7be186d892377d605cae6c88525fd851d456e8bbe9c9/google_resumable_media-2.11.0.tar.gz", hash = "sha256:febd83686752799661b4de575f0b993c5c25c349a5362556fc4d7be164056a37", size = 2164914, upload-time = "2026-09-29T19:26:13.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/2e/4f0a152f2e576e496f31ba1c3c62ed174a8878d06008916a7edc58b1bb28/google_resumable_media-2.11.0-py3-none-any.whl", hash = "sha256:f43d15e6a7f818f762eaead0f369c551f8275a4179c9d6225d0d259f49b87b5d", size = 81515, upload-time = "2026-09-29T19:25:47.31Z" }, +] + [[package]] name = "googleapis-common-protos" version = "1.74.0" @@ -2833,6 +2901,11 @@ dev = [ { name = "towncrier" }, ] +[package.dev-dependencies] +telemetry-error-issues = [ + { name = "google-cloud-bigquery" }, +] + [package.metadata] requires-dist = [ { name = "a2wsgi", specifier = ">=1.10,<2" }, @@ -2883,6 +2956,9 @@ requires-dist = [ ] provides-extras = ["dev"] +[package.metadata.requires-dev] +telemetry-error-issues = [{ name = "google-cloud-bigquery", specifier = ">=3.30,<4" }] + [[package]] name = "policyengine-canada" version = "0.96.3" @@ -3513,6 +3589,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl", hash = "sha256:e6b1d89711dd51f8f74b1631fe08f039e7d76cf67a42a323d3178f0f25762ed9", size = 45300, upload-time = "2025-08-24T12:55:53.394Z" }, ] +[[package]] +name = "pyopenssl" +version = "26.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/51/27a5ad5f939d08f690a326ef9582cda7140555180db71695f6fb747d6a36/pyopenssl-26.2.0.tar.gz", hash = "sha256:8c6fcecd1183a7fc897548dfe388b0cdb7f37e018200d8409cf33959dbe35387", size = 182195, upload-time = "2026-05-04T23:06:09.72Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/b8/a0e2790ae249d6f38c9f66de7a211621a7ab2650217bcd04e1262f578a56/pyopenssl-26.2.0-py3-none-any.whl", hash = "sha256:4f9d971bc5298b8bc1fab282803da04bf000c755d4ad9d99b52de2569ca19a70", size = 55823, upload-time = "2026-05-04T23:06:08.395Z" }, +] + [[package]] name = "pyproject-hooks" version = "1.2.0"