From 58ccd3475d57c5f6465e1e748ea285f7f86fbab2 Mon Sep 17 00:00:00 2001 From: Grace Qi Date: Wed, 18 Nov 2026 04:05:07 +0000 Subject: [PATCH 1/2] fix(reporting): stream BigQuery rows and isolate reporting errors in EvalService BigQuery enforces a hard quota of 1,500 load jobs per table per day (load_job_per_table.long). BigQueryReporter.store() previously created a separate load_table_from_dataframe() load job for each of the four EvalBench tables (configs, results, scores, summary) on every evaluation, exhausting the per-table token bucket under high-concurrency CI suites. In addition, EvalService._process_results() called reporter.store() without isolating per-reporter exceptions, so a BigQuery failure aborted CSV/artifact reporters, skipped temp-file cleanup, and failed the entire completed Eval() RPC. - Switch BigQueryReporter.store() from load_table_from_dataframe() to streaming inserts via client.insert_rows_json() (0 load jobs consumed), creating missing tables on NotFound, patching new columns via update_table() to preserve schema-addition behavior, coercing DataFrame values to target BigQuery SchemaField types, splitting requests below BigQuery's 10 MB insertAll limit, and retrying transient/schema-propagation errors. - Extract _store_reports() in eval_service.py (matching evalbench.py) so a reporter failure logs the error and records summary["reporting_errors"] without discarding completed evaluation scores or leaking /tmp files. - Add unit tests in bqstore_test.py and eval_service_reporting_test.py. --- evalbench/eval_service.py | 97 ++-- evalbench/reporting/bqstore.py | 518 ++++++++++++++---- evalbench/test/bqstore_test.py | 234 ++++++++ evalbench/test/eval_service_reporting_test.py | 105 ++++ 4 files changed, 822 insertions(+), 132 deletions(-) create mode 100644 evalbench/test/bqstore_test.py create mode 100644 evalbench/test/eval_service_reporting_test.py diff --git a/evalbench/eval_service.py b/evalbench/eval_service.py index 97d974e8d..98aa5fc2f 100644 --- a/evalbench/eval_service.py +++ b/evalbench/eval_service.py @@ -746,29 +746,69 @@ def _summary_count(value) -> float | int: return int(count) if count.is_integer() else round(count, 2) +def _store_reports( + reporters, config_df, results_df, scores_df, summary_scores_df +) -> list[dict[str, str]]: + """Writes evaluation outputs to each configured reporter. + + Isolates per-reporter failures so a reporting error (such as a BigQuery + permission or quota failure) neither blocks other reporters (CSV, GCS + artifacts) nor fails a completed evaluation RPC. + + Returns: + A list of {"reporter": , "error": } dicts for any + reporters that failed. + """ + errors: list[dict[str, str]] = [] + for reporter in reporters: + name = type(reporter).__name__ + try: + reporter.store(config_df, report.STORETYPE.CONFIGS) + reporter.store(results_df, report.STORETYPE.EVALS) + reporter.store(scores_df, report.STORETYPE.SCORES) + reporter.store(summary_scores_df, report.STORETYPE.SUMMARY) + except Exception as exc: + logging.exception( + "Reporter %s failed while storing evaluation results: %s", + name, + exc, + ) + errors.append({"reporter": name, "error": str(exc)}) + return errors + + def _process_results( reporters, job_id, run_time, results_tf, scores_tf, multi_trial_scores_tf, config, model_config, db_configs ): - config_df = config_to_df( - job_id, - run_time, - config, - model_config, - db_configs, - ) - results = load_json(results_tf) - results_df = report.get_dataframe(results) - if results_df.empty: - raise EmptyEvalResultError( - "No matching evals were produced for this run. Check that the dataset, " - "dialect filters, and database configuration line up." + try: + config_df = config_to_df( + job_id, + run_time, + config, + model_config, + db_configs, ) - report.quick_summary(results_df) - scores = load_json(scores_tf) - if multi_trial_scores_tf: - multi_trial_scores = load_json(multi_trial_scores_tf) - if multi_trial_scores: - scores.extend(multi_trial_scores) + results = load_json(results_tf) + results_df = report.get_dataframe(results) + if results_df.empty: + raise EmptyEvalResultError( + "No matching evals were produced for this run. Check that the dataset, " + "dialect filters, and database configuration line up." + ) + report.quick_summary(results_df) + scores = load_json(scores_tf) + if multi_trial_scores_tf: + multi_trial_scores = load_json(multi_trial_scores_tf) + if multi_trial_scores: + scores.extend(multi_trial_scores) + finally: + # k8s emptyDir /tmp does not auto cleanup, so always unlink temp files. + if results_tf: + pathlib.Path(results_tf).unlink(missing_ok=True) + if scores_tf: + pathlib.Path(scores_tf).unlink(missing_ok=True) + if multi_trial_scores_tf: + pathlib.Path(multi_trial_scores_tf).unlink(missing_ok=True) num_prompts = len(set(r.get("prompt_id") for r in results if r.get("prompt_id"))) @@ -779,18 +819,10 @@ def _process_results( summary_scores_df["job_id"] = job_id summary_scores_df["run_time"] = run_time - # Store the reports in specified outputs - for reporter in reporters: - reporter.store(config_df, report.STORETYPE.CONFIGS) - reporter.store(results_df, report.STORETYPE.EVALS) - reporter.store(scores_df, report.STORETYPE.SCORES) - reporter.store(summary_scores_df, report.STORETYPE.SUMMARY) - - # k8s emptyDir /tmp does not auto cleanup, so we explicitly delete - pathlib.Path(results_tf).unlink() - pathlib.Path(scores_tf).unlink() - if multi_trial_scores_tf: - pathlib.Path(multi_trial_scores_tf).unlink() + # Store the reports in specified outputs without aborting completed evals + reporting_errors = _store_reports( + reporters, config_df, results_df, scores_df, summary_scores_df + ) # Build summary dict from summary_scores_df summary = {"total": 0, "scores": {}} @@ -810,4 +842,7 @@ def _process_results( "p90": round(latencies.quantile(0.9), 2), } + if reporting_errors: + summary["reporting_errors"] = reporting_errors + return summary diff --git a/evalbench/reporting/bqstore.py b/evalbench/reporting/bqstore.py index bbc2e1233..50a22750d 100644 --- a/evalbench/reporting/bqstore.py +++ b/evalbench/reporting/bqstore.py @@ -1,11 +1,24 @@ -from google.cloud import bigquery -from google.api_core.exceptions import TooManyRequests, ServiceUnavailable +from collections.abc import Iterator, Sequence +import datetime +import json import logging +import math +import time +from typing import Any +import urllib.parse + +from google.api_core.exceptions import ( + Conflict, + InternalServerError, + NotFound, + PreconditionFailed, + ServiceUnavailable, + TooManyRequests, +) +from google.cloud import bigquery import pandas as pd from reporting.report import Reporter, STORETYPE from util.gcp import get_gcp_project -import urllib.parse -import time try: import google.colab # type: ignore @@ -16,6 +29,21 @@ _IN_COLAB = False _DEFAULT_CHUNK_SIZE = 500 +# BigQuery tabledata.insertAll caps each HTTP request at 10 MB; flush below 8 MiB +# so large multi-turn conversation_history rows never hit 413/BadRequest. +_MAX_STREAMING_REQUEST_BYTES = 8 * 1024 * 1024 +_SUMMARY_AND_SCORES_INT_COLS = frozenset({ + "metric_score", + "totalLatencyMs", + "total_results_count", + "correct_results_count", +}) +_TRANSIENT_ROW_ERROR_REASONS = frozenset({ + "backendError", + "internalError", + "rateLimitExceeded", + "timeout", +}) _REPORT_QUERY = """ WITH all_runs_with_set_tag AS ( @@ -62,24 +90,265 @@ """ -def _split_dataframe(df, chunk_size): - """Splits a pandas DataFrame into chunks of a specified size. - - Args: - df: The DataFrame to split. - chunk_size: The desired size of each chunk. - - Yields: - A generator that yields each chunk of the DataFrame. - """ +def _split_dataframe(df: pd.DataFrame, chunk_size: int) -> Iterator[pd.DataFrame]: + """Splits a pandas DataFrame into chunks of a specified size.""" num_chunks = len(df) // chunk_size + (len(df) % chunk_size > 0) for i in range(num_chunks): start = i * chunk_size - # Py/Pandas slicing handles not going out of bound end = (i + 1) * chunk_size yield df[start:end] +def _is_missing(val: Any) -> bool: + """Returns True for None, NaN, Inf, pd.NA, or pd.NaT without failing on sequences.""" + if val is None or val is pd.NA or val is pd.NaT: + return True + if isinstance(val, float): + return not math.isfinite(val) + if isinstance(val, (list, tuple, dict, set)): + return False + try: + na_check = pd.isna(val) + return bool(na_check) if isinstance(na_check, bool) else False + except (TypeError, ValueError): + return False + + +def _infer_scalar_bq_type(val: Any) -> str: + """Infers a BigQuery scalar column type from a non-null Python/pandas value.""" + if isinstance(val, pd.Timestamp): + return "TIMESTAMP" if val.tzinfo is not None else "DATETIME" + if isinstance(val, datetime.datetime): + return "TIMESTAMP" if val.tzinfo is not None else "DATETIME" + if isinstance(val, datetime.date): + return "DATE" + if isinstance(val, bool): + return "BOOLEAN" + if isinstance(val, int): + return "INTEGER" + if isinstance(val, float): + return "FLOAT" + return "STRING" + + +def _infer_schema( + df: pd.DataFrame, store_type: STORETYPE +) -> list[bigquery.SchemaField]: + """Infers BigQuery SchemaFields for new tables or newly added columns.""" + fields: list[bigquery.SchemaField] = [] + for col in df.columns: + col_name = str(col) + series = df[col] + if ( + store_type in (STORETYPE.SUMMARY, STORETYPE.SCORES) + and col_name in _SUMMARY_AND_SCORES_INT_COLS + ): + fields.append(bigquery.SchemaField(col_name, "INTEGER")) + continue + + non_null = [v for v in series.tolist() if not _is_missing(v)] + if non_null and isinstance(non_null[0], (list, tuple)): + elem_type = "STRING" + for seq in non_null: + if isinstance(seq, (list, tuple)): + elems = [e for e in seq if not _is_missing(e)] + if elems: + elem_type = _infer_scalar_bq_type(elems[0]) + break + fields.append( + bigquery.SchemaField(col_name, elem_type, mode="REPEATED") + ) + continue + + if pd.api.types.is_datetime64_any_dtype(series.dtype): + tz = getattr(series.dtype, "tz", None) + fields.append( + bigquery.SchemaField( + col_name, "TIMESTAMP" if tz is not None else "DATETIME" + ) + ) + elif pd.api.types.is_bool_dtype(series.dtype): + fields.append(bigquery.SchemaField(col_name, "BOOLEAN")) + elif pd.api.types.is_integer_dtype(series.dtype): + fields.append(bigquery.SchemaField(col_name, "INTEGER")) + elif pd.api.types.is_float_dtype(series.dtype): + bq_type = ( + "STRING" + if store_type in (STORETYPE.SUMMARY, STORETYPE.SCORES) + else "FLOAT" + ) + fields.append(bigquery.SchemaField(col_name, bq_type)) + elif non_null: + inferred = _infer_scalar_bq_type(non_null[0]) + if ( + inferred == "FLOAT" + and store_type in (STORETYPE.SUMMARY, STORETYPE.SCORES) + ): + inferred = "STRING" + fields.append(bigquery.SchemaField(col_name, inferred)) + else: + fields.append(bigquery.SchemaField(col_name, "STRING")) + return fields + + +def _coerce_scalar(val: Any, field_type: str) -> Any: + """Coerces a non-missing scalar value to match a BigQuery SchemaField type.""" + if hasattr(val, "item") and callable(val.item) and not isinstance( + val, (pd.Timestamp, datetime.date, bytes, str) + ): + try: + val = val.item() + except ValueError: + pass + if _is_missing(val): + return None + + upper = field_type.upper() + if upper in ("INTEGER", "INT64"): + try: + return int(round(float(val))) + except (TypeError, ValueError, OverflowError): + return None + if upper in ("FLOAT", "FLOAT64", "NUMERIC", "BIGNUMERIC"): + try: + num = float(val) + return num if math.isfinite(num) else None + except (TypeError, ValueError): + return None + if upper in ("BOOLEAN", "BOOL"): + if isinstance(val, bool): + return val + if isinstance(val, str): + lowered = val.strip().lower() + if lowered in ("true", "1", "t", "yes"): + return True + if lowered in ("false", "0", "f", "no", ""): + return False + return None + return bool(val) + if upper == "DATETIME": + if isinstance(val, pd.Timestamp): + val = val.to_pydatetime() + if isinstance(val, datetime.datetime): + if val.tzinfo is not None: + val = val.astimezone(datetime.timezone.utc).replace(tzinfo=None) + return val.isoformat(timespec="microseconds") + text = str(val).strip() + if text.endswith("Z"): + return text[:-1] + if text.endswith("+00:00"): + return text[:-6] + return text + if upper == "TIMESTAMP": + if isinstance(val, pd.Timestamp): + val = val.to_pydatetime() + if isinstance(val, datetime.datetime): + if val.tzinfo is None: + val = val.replace(tzinfo=datetime.timezone.utc) + return val.isoformat() + return str(val) + if upper == "DATE": + if isinstance(val, pd.Timestamp): + val = val.to_pydatetime() + if isinstance(val, datetime.datetime): + return val.date().isoformat() + if isinstance(val, datetime.date): + return val.isoformat() + return str(val) + if upper == "JSON": + return val if isinstance(val, str) else json.dumps(val, default=str) + + # Default STRING serialization: strip trailing .0 on integral floats to + # match legacy bqstore behavior, and JSON-encode nested dicts/lists. + if isinstance(val, float): + return str(int(val)) if val.is_integer() else str(val) + if isinstance(val, (dict, list, tuple)): + return json.dumps(val, default=str) + if isinstance(val, (datetime.datetime, datetime.date, pd.Timestamp)): + return val.isoformat() + return str(val) + + +def _dataframe_to_json_rows( + chunk: pd.DataFrame, schema: Sequence[bigquery.SchemaField] +) -> list[dict[str, Any]]: + """Converts a DataFrame chunk into JSON-serializable dicts matching `schema`.""" + schema_by_lower = {field.name.lower(): field for field in schema} + rows: list[dict[str, Any]] = [] + for record in chunk.to_dict(orient="records"): + json_row: dict[str, Any] = {} + for col, raw_val in record.items(): + field = schema_by_lower.get(str(col).lower()) + target_col = field.name if field is not None else str(col) + field_type = field.field_type if field is not None else "STRING" + if field is not None and field.mode == "REPEATED": + if _is_missing(raw_val): + json_row[target_col] = [] + elif isinstance(raw_val, (list, tuple)): + coerced_items = [ + _coerce_scalar(elem, field_type) + for elem in raw_val + if not _is_missing(elem) + ] + json_row[target_col] = [ + item for item in coerced_items if item is not None + ] + else: + coerced_single = _coerce_scalar(raw_val, field_type) + json_row[target_col] = ( + [coerced_single] if coerced_single is not None else [] + ) + elif _is_missing(raw_val): + json_row[target_col] = None + else: + json_row[target_col] = _coerce_scalar(raw_val, field_type) + rows.append(json_row) + return rows + + +def _iter_streaming_batches( + rows: Sequence[dict[str, Any]], + max_rows: int, + max_bytes: int = _MAX_STREAMING_REQUEST_BYTES, +) -> Iterator[list[dict[str, Any]]]: + """Splits JSON rows by both count and serialized byte size for insertAll.""" + batch: list[dict[str, Any]] = [] + batch_bytes = 0 + for row in rows: + row_bytes = len(json.dumps(row).encode("utf-8")) + 2 + if batch and ( + len(batch) >= max_rows or batch_bytes + row_bytes > max_bytes + ): + yield batch + batch = [] + batch_bytes = 0 + batch.append(row) + batch_bytes += row_bytes + if batch: + yield batch + + +def _is_transient_insert_error(errors: Sequence[dict[str, Any]]) -> bool: + """Returns True when insert_rows_json row errors are transient or schema-lag.""" + if not errors: + return False + for row_err in errors: + item_errors = row_err.get("errors") or [] + if not item_errors: + return False + for err in item_errors: + reason = str(err.get("reason") or "") + message = str(err.get("message") or "").lower() + if reason in _TRANSIENT_ROW_ERROR_REASONS: + continue + # tabledata.insertAll caches schemas briefly after create_table or + # update_table and can report "no such field" for a few seconds. + if "no such field" in message: + continue + return False + return True + + class BigQueryReporter(Reporter): def __init__(self, reporting_config, job_id, run_time): super().__init__(reporting_config, job_id, run_time) @@ -98,100 +367,147 @@ def __init__(self, reporting_config, job_id, run_time): # Make chunk size configurable, defaulting to 500 self.chunk_size = int(reporting_config.get( "chunk_size") or _DEFAULT_CHUNK_SIZE) + self._dataset_ensured = False + self._tables: dict[str, bigquery.Table] = {} - def store(self, results, type: STORETYPE): - if results is None or results.empty: - logging.info(f"No results to store for {type}") + def _ensure_dataset(self) -> None: + if self._dataset_ensured: return - dataset = bigquery.Dataset(self.dataset_path) dataset.location = self.location - dataset = self.client.create_dataset( - dataset, exists_ok=True, timeout=30) - logging.info( - "Created dataset {}.{} for {}".format( - self.client.project, dataset.dataset_id, type - ) - ) - job_config = bigquery.LoadJobConfig() - job_config.autodetect = True - job_config.allow_quoted_newlines = True - job_config.schema_update_options = [ - bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION, - bigquery.SchemaUpdateOption.ALLOW_FIELD_RELAXATION, + self.client.create_dataset(dataset, exists_ok=True, timeout=30) + self._dataset_ensured = True + + def _ensure_table_and_schema( + self, table_id: str, chunk: pd.DataFrame, store_type: STORETYPE + ) -> bigquery.Table: + """Ensures `table_id` exists and includes any new columns in `chunk`.""" + table_obj = self._tables.get(table_id) + if table_obj is None: + try: + table_obj = self.client.get_table(table_id) + except NotFound: + schema = _infer_schema(chunk, store_type) + table_obj = self.client.create_table( + bigquery.Table(table_id, schema=schema), exists_ok=True + ) + if not table_obj.schema: + table_obj = self.client.get_table(table_id) + + existing_lower = {field.name.lower() for field in table_obj.schema} + missing_cols = [ + col for col in chunk.columns if str(col).lower() not in existing_lower ] - if type == STORETYPE.CONFIGS: - table = self.configs_table - elif type == STORETYPE.EVALS: - table = self.results_table - elif type == STORETYPE.SCORES: - table = self.scores_table - elif type == STORETYPE.SUMMARY: - table = self.summary_table - - # Chunk this to avoid BQ OOM - job_config.write_disposition = ( - bigquery.job.WriteDisposition.WRITE_APPEND # type: ignore - ) - for chunk in _split_dataframe(results, self.chunk_size): - # Workaround for pyarrow truncation error when inserting floats - # into INT64 columns. This happens if BQ autodetected a column as - # INT64 in a previous run. Casting to string avoids the client-side - # pyarrow crash and lets BQ handle the conversion. - if type in [STORETYPE.SUMMARY, STORETYPE.SCORES]: - chunk = chunk.copy() - # 1. Identify columns that BQ might have already locked - # as INT64 common ones are 'metric_score', etc. - int_cols = [ - "metric_score", "totalLatencyMs", - "total_results_count", "correct_results_count" - ] - for col in int_cols: - if col in chunk.columns: - # Round and cast to Int64 (nullable integer type) - chunk[col] = pd.to_numeric( - chunk[col], errors="coerce" - ).round().astype("Int64") - - # 2. For any remaining float columns that aren't in our - # "must-be-int" list, we cast to string to avoid pyarrow's - # strict truncation check if BQ thinks it's an int. - float_cols = chunk.select_dtypes( - include=["float64", "float32"] - ).columns - for col in float_cols: - if col not in int_cols: - # Convert to string and strip .0 if effectively int - chunk[col] = ( - chunk[col] - .astype(str) - .str.replace(r"\.0$", "", regex=True) - .replace("nan", None) + if missing_cols: + # Refresh once before patching in case another worker added the field. + table_obj = self.client.get_table(table_id) + existing_lower = {field.name.lower() for field in table_obj.schema} + missing_cols = [ + col for col in chunk.columns + if str(col).lower() not in existing_lower + ] + if missing_cols: + added_fields = _infer_schema(chunk[missing_cols], store_type) + table_obj.schema = list(table_obj.schema) + added_fields + try: + table_obj = self.client.update_table(table_obj, ["schema"]) + except (Conflict, PreconditionFailed): + table_obj = self.client.get_table(table_id) + existing_lower = { + field.name.lower() for field in table_obj.schema + } + still_missing = [ + col for col in chunk.columns + if str(col).lower() not in existing_lower + ] + if still_missing: + table_obj.schema = list(table_obj.schema) + _infer_schema( + chunk[still_missing], store_type + ) + table_obj = self.client.update_table( + table_obj, ["schema"] ) - # Retry loop for BQ load job to handle transient rate limits - max_retries = 5 - initial_delay = 2 - backoff_factor = 2 + self._tables[table_id] = table_obj + return table_obj - for attempt in range(max_retries): - try: - job = self.client.load_table_from_dataframe( - chunk, table, job_config=job_config - ) - job.result() # Wait for the job to complete. - break # Success, exit retry loop - except (TooManyRequests, ServiceUnavailable) as e: - if attempt == max_retries - 1: - logging.error( - f"BQ write failed after {max_retries} attempts: {e}") - raise - delay = initial_delay * (backoff_factor ** attempt) - logging.warning( - f"BQ write failed with transient error: {e}. " - f"Retrying in {delay}s... (Attempt {attempt + 1}/{max_retries})" - ) - time.sleep(delay) + def _resolve_table_id(self, store_type: STORETYPE) -> str: + if store_type == STORETYPE.CONFIGS: + return self.configs_table + if store_type == STORETYPE.EVALS: + return self.results_table + if store_type == STORETYPE.SCORES: + return self.scores_table + if store_type == STORETYPE.SUMMARY: + return self.summary_table + raise ValueError(f"Unsupported store type: {store_type}") + + def store(self, results, type: STORETYPE): + if results is None or results.empty: + logging.info(f"No results to store for {type}") + return + + self._ensure_dataset() + table = self._resolve_table_id(type) + + # Use BigQuery streaming inserts (tabledata.insertAll) instead of per-eval + # load jobs so high-concurrency CI suites never exhaust BigQuery's hard + # 1,500 load_job_per_table.long daily limit. + max_retries = 5 + initial_delay = 2 + backoff_factor = 2 + + for chunk in _split_dataframe(results, self.chunk_size): + table_obj = self._ensure_table_and_schema(table, chunk, type) + json_rows = _dataframe_to_json_rows(chunk, table_obj.schema) + + for batch in _iter_streaming_batches(json_rows, self.chunk_size): + for attempt in range(max_retries): + try: + errors = self.client.insert_rows_json(table, batch) + if not errors: + break + if ( + attempt < max_retries - 1 + and _is_transient_insert_error(errors) + ): + self._tables.pop(table, None) + table_obj = self._ensure_table_and_schema( + table, chunk, type + ) + delay = initial_delay * (backoff_factor ** attempt) + logging.warning( + "BQ streaming write had transient/schema-lag " + f"errors: {errors[:2]}. Retrying in {delay}s... " + f"(Attempt {attempt + 1}/{max_retries})" + ) + time.sleep(delay) + continue + raise RuntimeError( + f"BigQuery streaming insert into {table} failed: " + f"{errors[:3]}" + ) + except ( + TooManyRequests, + ServiceUnavailable, + InternalServerError, + NotFound, + ) as e: + if attempt == max_retries - 1: + logging.error( + f"BQ write failed after {max_retries} attempts: {e}" + ) + raise + if isinstance(e, NotFound): + self._tables.pop(table, None) + self._ensure_table_and_schema(table, chunk, type) + delay = initial_delay * (backoff_factor ** attempt) + logging.warning( + f"BQ write failed with transient error: {e}. " + f"Retrying in {delay}s... " + f"(Attempt {attempt + 1}/{max_retries})" + ) + time.sleep(delay) def print_dashboard_links(self): report_date = self.run_time.strftime("%Y-%m-%d") diff --git a/evalbench/test/bqstore_test.py b/evalbench/test/bqstore_test.py new file mode 100644 index 000000000..c2220f308 --- /dev/null +++ b/evalbench/test/bqstore_test.py @@ -0,0 +1,234 @@ +import datetime +import math +import unittest +from unittest.mock import MagicMock, patch + +from google.api_core.exceptions import Forbidden, NotFound, ServiceUnavailable +from google.cloud import bigquery +import pandas as pd +from reporting.bqstore import BigQueryReporter +from reporting.report import STORETYPE + + +class TestBigQueryReporterStreaming(unittest.TestCase): + + def setUp(self): + self.run_time = datetime.datetime(2026, 11, 18, 3, 15, 0) + self.mock_client_patcher = patch("reporting.bqstore.bigquery.Client") + self.mock_bq_client_cls = self.mock_client_patcher.start() + self.mock_client = MagicMock() + self.mock_bq_client_cls.return_value = self.mock_client + + self.reporter = BigQueryReporter( + {"gcp_project_id": "test-project", "dataset_id": "evalbench"}, + job_id="job-123", + run_time=self.run_time, + ) + + def tearDown(self): + self.mock_client_patcher.stop() + + def test_store_uses_streaming_insert_rows_json_and_zero_load_jobs(self): + configs_schema = [ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("run_time", "DATETIME"), + bigquery.SchemaField("config", "STRING"), + bigquery.SchemaField("value", "STRING"), + ] + table_obj = bigquery.Table( + "test-project.evalbench.configs", schema=configs_schema + ) + self.mock_client.get_table.return_value = table_obj + self.mock_client.insert_rows_json.return_value = [] + + df = pd.DataFrame([ + { + "job_id": "job-123", + "run_time": self.run_time, + "config": "model_config.generator", + "value": "agy_cli", + }, + { + "job_id": "job-123", + "run_time": self.run_time, + "config": "experiment_config.empty_val", + "value": pd.NA, + }, + ]) + + self.reporter.store(df, STORETYPE.CONFIGS) + + self.mock_client.load_table_from_dataframe.assert_not_called() + self.mock_client.insert_rows_json.assert_called_once() + target_table, rows = self.mock_client.insert_rows_json.call_args[0] + self.assertEqual(target_table, "test-project.evalbench.configs") + self.assertEqual( + rows, + [ + { + "job_id": "job-123", + "run_time": "2026-11-18T03:15:00.000000", + "config": "model_config.generator", + "value": "agy_cli", + }, + { + "job_id": "job-123", + "run_time": "2026-11-18T03:15:00.000000", + "config": "experiment_config.empty_val", + "value": None, + }, + ], + ) + + def test_store_creates_missing_table_and_rounds_summary_integers(self): + self.mock_client.get_table.side_effect = NotFound("Table not found") + + def _create_table(table_arg, exists_ok=False): + return table_arg + + self.mock_client.create_table.side_effect = _create_table + self.mock_client.insert_rows_json.return_value = [] + + summary_df = pd.DataFrame([ + { + "metric_name": "goal_completion", + "metric_score": 100.0, + "correct_results_count": 1.67, + "total_results_count": 2, + "job_id": "job-123", + "run_time": self.run_time, + } + ]) + + self.reporter.store(summary_df, STORETYPE.SUMMARY) + + self.mock_client.create_table.assert_called_once() + created_table = self.mock_client.create_table.call_args[0][0] + schema_types = {f.name: f.field_type for f in created_table.schema} + self.assertEqual(schema_types["metric_score"], "INTEGER") + self.assertEqual(schema_types["correct_results_count"], "INTEGER") + self.assertEqual(schema_types["total_results_count"], "INTEGER") + self.assertEqual(schema_types["run_time"], "DATETIME") + + _, rows = self.mock_client.insert_rows_json.call_args[0] + self.assertEqual(rows[0]["correct_results_count"], 2) + self.assertEqual(rows[0]["metric_score"], 100) + self.assertEqual(rows[0]["total_results_count"], 2) + + def test_store_adds_missing_columns_to_existing_table_schema(self): + existing_table = bigquery.Table( + "test-project.evalbench.results", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("eval_id", "STRING"), + ], + ) + updated_table = bigquery.Table( + "test-project.evalbench.results", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("eval_id", "STRING"), + bigquery.SchemaField("accumulated_skills", "STRING"), + ], + ) + self.mock_client.get_table.side_effect = [existing_table, existing_table] + self.mock_client.update_table.return_value = updated_table + self.mock_client.insert_rows_json.return_value = [] + + results_df = pd.DataFrame([ + { + "job_id": "job-123", + "eval_id": "scenario-1", + "accumulated_skills": "['bigquery']", + } + ]) + + self.reporter.store(results_df, STORETYPE.EVALS) + + self.mock_client.update_table.assert_called_once() + patched_table = self.mock_client.update_table.call_args[0][0] + self.assertEqual( + [f.name for f in patched_table.schema], + ["job_id", "eval_id", "accumulated_skills"], + ) + _, rows = self.mock_client.insert_rows_json.call_args[0] + self.assertEqual(rows[0]["accumulated_skills"], "['bigquery']") + + def test_store_handles_repeated_fields_and_float_nan_safely(self): + scores_table = bigquery.Table( + "test-project.evalbench.scores", + schema=[ + bigquery.SchemaField("id", "STRING"), + bigquery.SchemaField("comparator", "STRING"), + bigquery.SchemaField("score", "FLOAT"), + bigquery.SchemaField("dialects", "STRING", mode="REPEATED"), + bigquery.SchemaField("job_id", "STRING"), + ], + ) + self.mock_client.get_table.return_value = scores_table + self.mock_client.insert_rows_json.return_value = [] + + scores_df = pd.DataFrame([ + { + "id": "s1", + "comparator": "goal_completion", + "score": 95.5, + "dialects": ["bigquery"], + "job_id": "job-123", + }, + { + "id": "s2", + "comparator": "goal_completion", + "score": math.nan, + "dialects": None, + "job_id": "job-123", + }, + ]) + + self.reporter.store(scores_df, STORETYPE.SCORES) + + _, rows = self.mock_client.insert_rows_json.call_args[0] + self.assertEqual(rows[0]["score"], 95.5) + self.assertEqual(rows[0]["dialects"], ["bigquery"]) + self.assertIsNone(rows[1]["score"]) + self.assertEqual(rows[1]["dialects"], []) + + @patch("reporting.bqstore.time.sleep") + def test_store_retries_transient_errors_and_schema_lag_but_not_forbidden( + self, mock_sleep + ): + table_obj = bigquery.Table( + "test-project.evalbench.configs", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("config", "STRING"), + ], + ) + self.mock_client.get_table.return_value = table_obj + self.mock_client.insert_rows_json.side_effect = [ + ServiceUnavailable("503 backend transient"), + [{ + "index": 0, + "errors": [{"reason": "invalid", "message": "no such field: config."}], + }], + [], + ] + + df = pd.DataFrame([{"job_id": "job-123", "config": "k"}]) + self.reporter.store(df, STORETYPE.CONFIGS) + self.assertEqual(self.mock_client.insert_rows_json.call_count, 3) + self.assertEqual(mock_sleep.call_count, 2) + + # Permanent 403 Forbidden (e.g. IAM deny or hard quotaExceeded) fails + # immediately without sleeping or retrying. + mock_sleep.reset_mock() + self.mock_client.insert_rows_json.side_effect = Forbidden( + "403 Access Denied" + ) + with self.assertRaises(Forbidden): + self.reporter.store(df, STORETYPE.CONFIGS) + mock_sleep.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/evalbench/test/eval_service_reporting_test.py b/evalbench/test/eval_service_reporting_test.py new file mode 100644 index 000000000..39be1514f --- /dev/null +++ b/evalbench/test/eval_service_reporting_test.py @@ -0,0 +1,105 @@ +import datetime +import json +import os +import tempfile +import unittest +from unittest.mock import MagicMock + +from eval_service import _process_results, EmptyEvalResultError + + +class BigQueryReporter(MagicMock): + """Fake BigQuery reporter named for _store_reports diagnostics.""" + + +class CsvReporter(MagicMock): + """Fake CSV reporter named for _store_reports diagnostics.""" + + +class TestEvalServiceProcessResultsReporting(unittest.TestCase): + + def _write_json_tempfile(self, payload) -> str: + fd, path = tempfile.mkstemp(suffix=".json") + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(payload, handle) + return path + + def test_reporter_failure_does_not_discard_eval_summary_or_block_other_reporters( + self, + ): + results_tf = self._write_json_tempfile([ + { + "id": "scenario-1", + "prompt_id": "scenario-1", + "generated_sql": "", + "generated_error": None, + } + ]) + scores_tf = self._write_json_tempfile([ + { + "id": "scenario-1", + "prompt_id": "scenario-1", + "comparator": "goal_completion", + "score": 100, + "generated_sql": "", + "generated_error": None, + } + ]) + + bq_reporter = BigQueryReporter() + bq_reporter.store.side_effect = RuntimeError( + "403 Quota exceeded: load_job_per_table.long" + ) + csv_reporter = CsvReporter() + + summary = _process_results( + reporters=[bq_reporter, csv_reporter], + job_id="job-abc", + run_time=datetime.datetime(2026, 11, 18, 3, 20, 0), + results_tf=results_tf, + scores_tf=scores_tf, + multi_trial_scores_tf=None, + config={ + "orchestrator": "geminicli", + "scorers": {"goal_completion": {}}, + }, + model_config={"generator": "agy_cli"}, + db_configs=[], + ) + + self.assertEqual(summary["total"], 1) + self.assertEqual(summary["scores"]["goal_completion"], 1) + self.assertEqual(csv_reporter.store.call_count, 4) + self.assertEqual(len(summary["reporting_errors"]), 1) + self.assertEqual( + summary["reporting_errors"][0]["reporter"], "BigQueryReporter" + ) + self.assertIn( + "load_job_per_table.long", summary["reporting_errors"][0]["error"] + ) + self.assertFalse(os.path.exists(results_tf)) + self.assertFalse(os.path.exists(scores_tf)) + + def test_empty_results_still_unlinks_tempfiles_and_raises(self): + results_tf = self._write_json_tempfile([]) + scores_tf = self._write_json_tempfile([]) + + with self.assertRaises(EmptyEvalResultError): + _process_results( + reporters=[], + job_id="job-empty", + run_time=datetime.datetime(2026, 11, 18, 3, 20, 0), + results_tf=results_tf, + scores_tf=scores_tf, + multi_trial_scores_tf=None, + config={"scorers": {}}, + model_config={}, + db_configs=[], + ) + + self.assertFalse(os.path.exists(results_tf)) + self.assertFalse(os.path.exists(scores_tf)) + + +if __name__ == "__main__": + unittest.main() From 8ac1f84f7e1b11f0238d39b5a362c5e764d49ab1 Mon Sep 17 00:00:00 2001 From: Grace Qi Date: Sat, 21 Nov 2026 02:05:51 +0000 Subject: [PATCH 2/2] fix(reporting): harden BigQuery streaming retries, schema inference, coercion, and reporter init isolation Address self-review findings on PR #729: - Allow sibling 'stopped' row errors when checking retryable BigQuery insertErrors while still failing fast on permanent row errors. - Pass stable per-batch row_ids to insert_rows_json() and retry only failed indices on partial retryable errors to prevent duplicate rows. - Infer schema across the full DataFrame (all non-null values/elements, unwrapping NumPy scalars) before chunking, and retry update_table() up to 5 times on concurrent Conflict/PreconditionFailed. - Only refresh cached table schemas on NotFound or 'no such field' errors, and re-coerce pending rows when the refreshed schema changed. - Fix _coerce_scalar() / _dataframe_to_json_rows() edge cases: exact INT64 precision above 2^53, empty/offset DATETIME/TIMESTAMP/DATE strings (Python 3.10-safe), NaN/Inf sanitization in JSON, and recursive RECORD/STRUCT coercion. - Move reporter construction inside _process_results() via _build_reporters() so reporter init failures are isolated and /tmp files are always unlinked; raise in Eval() when summary_in_response is falsy and every configured reporter failed. - Expand unit tests in bqstore_test.py and eval_service_reporting_test.py. --- evalbench/eval_service.py | 75 +++- evalbench/reporting/bqstore.py | 388 +++++++++++++----- evalbench/test/bqstore_test.py | 290 ++++++++++++- evalbench/test/eval_service_reporting_test.py | 102 ++++- 4 files changed, 733 insertions(+), 122 deletions(-) diff --git a/evalbench/eval_service.py b/evalbench/eval_service.py index 98aa5fc2f..b3d3d534e 100644 --- a/evalbench/eval_service.py +++ b/evalbench/eval_service.py @@ -260,18 +260,14 @@ async def Eval( await executor.run(ctx.run, evaluator.evaluate, dataset) job_id, run_time, results_tf, scores_tf, multi_trial_scores_tf = evaluator.process() - # Fallback to empty dict if reporting is present but null in YAML - reporters = get_reporters( - config.get("reporting") or {}, job_id, run_time - ) - # Offload blocking results processing to a thread pool + # Offload blocking results processing and reporter setup to a thread pool logging.info("Offloading results processing to thread pool...") ctx = contextvars.copy_context() summary = await executor.run( ctx.run, _process_results, - reporters, + None, job_id, run_time, results_tf, @@ -280,6 +276,7 @@ async def Eval( config, with_agent_version(model_config, evaluator), db_configs, + not config.get("summary_in_response"), ) logging.info( @@ -469,8 +466,6 @@ async def read_from_client(): # Process final scoring and reporting job_id, run_time, results_tf, scores_tf = orchestrator.process() - reporters = get_reporters(config.get( - "reporting") or {}, job_id, run_time) logging.info( "Offloading interactive results processing to thread pool...") @@ -478,7 +473,7 @@ async def read_from_client(): summary = await executor.run( ctx.run, _process_results, - reporters, + None, job_id, run_time, results_tf, @@ -602,14 +597,11 @@ async def run_eval_and_process(): scores_tf, multi_trial_scores_tf, ) = orchestrator.process() - reporters = get_reporters( - config.get("reporting") or {}, job_id, run_time - ) logging.info("Processing agent evaluation results...") summary = await executor.run( ctx.run, _process_results, - reporters, + None, job_id, run_time, results_tf, @@ -746,6 +738,27 @@ def _summary_count(value) -> float | int: return int(count) if count.is_integer() else round(count, 2) +def _build_reporters( + reporting_config, job_id, run_time +) -> tuple[list, list[dict[str, str]]]: + """Constructs each configured reporter with per-reporter error isolation.""" + reporters = [] + errors: list[dict[str, str]] = [] + if not isinstance(reporting_config, dict): + return reporters, errors + for key, cfg in reporting_config.items(): + try: + reporters.extend(get_reporters({key: cfg}, job_id, run_time)) + except Exception as exc: + logging.exception( + "Reporter %s failed to initialize: %s", + key, + exc, + ) + errors.append({"reporter": str(key), "error": str(exc)}) + return reporters, errors + + def _store_reports( reporters, config_df, results_df, scores_df, summary_scores_df ) -> list[dict[str, str]]: @@ -778,7 +791,16 @@ def _store_reports( def _process_results( - reporters, job_id, run_time, results_tf, scores_tf, multi_trial_scores_tf, config, model_config, db_configs + reporters, + job_id, + run_time, + results_tf, + scores_tf, + multi_trial_scores_tf, + config, + model_config, + db_configs, + fail_if_all_reporters_failed=False, ): try: config_df = config_to_df( @@ -819,11 +841,32 @@ def _process_results( summary_scores_df["job_id"] = job_id summary_scores_df["run_time"] = run_time + if reporters is None: + reporters, reporting_errors = _build_reporters( + config.get("reporting") or {}, job_id, run_time + ) + else: + reporting_errors = [] + + total_reporters = len(reporters) + len(reporting_errors) + # Store the reports in specified outputs without aborting completed evals - reporting_errors = _store_reports( - reporters, config_df, results_df, scores_df, summary_scores_df + reporting_errors.extend( + _store_reports( + reporters, config_df, results_df, scores_df, summary_scores_df + ) ) + if ( + fail_if_all_reporters_failed + and total_reporters > 0 + and len(reporting_errors) >= total_reporters + ): + raise RuntimeError( + f"All configured reporters failed for job {job_id}: " + f"{reporting_errors}" + ) + # Build summary dict from summary_scores_df summary = {"total": 0, "scores": {}} for _, row in summary_scores_df.iterrows(): diff --git a/evalbench/reporting/bqstore.py b/evalbench/reporting/bqstore.py index 50a22750d..8b3ee771d 100644 --- a/evalbench/reporting/bqstore.py +++ b/evalbench/reporting/bqstore.py @@ -6,6 +6,7 @@ import time from typing import Any import urllib.parse +import uuid from google.api_core.exceptions import ( Conflict, @@ -32,6 +33,7 @@ # BigQuery tabledata.insertAll caps each HTTP request at 10 MB; flush below 8 MiB # so large multi-turn conversation_history rows never hit 413/BadRequest. _MAX_STREAMING_REQUEST_BYTES = 8 * 1024 * 1024 +_MAX_SCHEMA_UPDATE_RETRIES = 5 _SUMMARY_AND_SCORES_INT_COLS = frozenset({ "metric_score", "totalLatencyMs", @@ -99,23 +101,59 @@ def _split_dataframe(df: pd.DataFrame, chunk_size: int) -> Iterator[pd.DataFrame yield df[start:end] +def _unwrap_numpy_scalar(val: Any) -> Any: + """Converts a 0-d NumPy scalar to its native Python counterpart.""" + if hasattr(val, "item") and callable(val.item) and not isinstance( + val, (pd.Timestamp, datetime.date, bytes, str) + ): + try: + return val.item() + except ValueError: + pass + return val + + def _is_missing(val: Any) -> bool: """Returns True for None, NaN, Inf, pd.NA, or pd.NaT without failing on sequences.""" if val is None or val is pd.NA or val is pd.NaT: return True - if isinstance(val, float): - return not math.isfinite(val) if isinstance(val, (list, tuple, dict, set)): return False + val = _unwrap_numpy_scalar(val) + if val is None or val is pd.NA or val is pd.NaT: + return True + if isinstance(val, float): + return not math.isfinite(val) try: - na_check = pd.isna(val) + na_check = _unwrap_numpy_scalar(pd.isna(val)) return bool(na_check) if isinstance(na_check, bool) else False except (TypeError, ValueError): return False +def _sanitize_for_json(val: Any) -> Any: + """Recursively converts nested structures into strict RFC-8259 JSON values.""" + if _is_missing(val): + return None + val = _unwrap_numpy_scalar(val) + if isinstance(val, dict): + return {str(k): _sanitize_for_json(v) for k, v in val.items()} + if isinstance(val, (list, tuple)): + return [_sanitize_for_json(v) for v in val] + if ( + hasattr(val, "tolist") + and callable(val.tolist) + and not isinstance(val, (str, bytes)) + ): + return _sanitize_for_json(val.tolist()) + if isinstance(val, (datetime.datetime, datetime.date, pd.Timestamp)): + return val.isoformat() + return val + + def _infer_scalar_bq_type(val: Any) -> str: """Infers a BigQuery scalar column type from a non-null Python/pandas value.""" + val = _unwrap_numpy_scalar(val) if isinstance(val, pd.Timestamp): return "TIMESTAMP" if val.tzinfo is not None else "DATETIME" if isinstance(val, datetime.datetime): @@ -131,6 +169,29 @@ def _infer_scalar_bq_type(val: Any) -> str: return "STRING" +def _merge_bq_types( + types: set[str], store_type: STORETYPE | None +) -> str: + """Merges inferred BigQuery scalar types across all non-null values.""" + if not types: + return "STRING" + if len(types) == 1: + inferred = next(iter(types)) + elif types == {"INTEGER", "FLOAT"}: + inferred = "FLOAT" + elif types == {"DATETIME", "TIMESTAMP"}: + inferred = "TIMESTAMP" + else: + inferred = "STRING" + + if ( + inferred == "FLOAT" + and store_type in (STORETYPE.SUMMARY, STORETYPE.SCORES) + ): + return "STRING" + return inferred + + def _infer_schema( df: pd.DataFrame, store_type: STORETYPE ) -> list[bigquery.SchemaField]: @@ -146,20 +207,6 @@ def _infer_schema( fields.append(bigquery.SchemaField(col_name, "INTEGER")) continue - non_null = [v for v in series.tolist() if not _is_missing(v)] - if non_null and isinstance(non_null[0], (list, tuple)): - elem_type = "STRING" - for seq in non_null: - if isinstance(seq, (list, tuple)): - elems = [e for e in seq if not _is_missing(e)] - if elems: - elem_type = _infer_scalar_bq_type(elems[0]) - break - fields.append( - bigquery.SchemaField(col_name, elem_type, mode="REPEATED") - ) - continue - if pd.api.types.is_datetime64_any_dtype(series.dtype): tz = getattr(series.dtype, "tz", None) fields.append( @@ -178,35 +225,51 @@ def _infer_schema( else "FLOAT" ) fields.append(bigquery.SchemaField(col_name, bq_type)) - elif non_null: - inferred = _infer_scalar_bq_type(non_null[0]) - if ( - inferred == "FLOAT" - and store_type in (STORETYPE.SUMMARY, STORETYPE.SCORES) - ): - inferred = "STRING" - fields.append(bigquery.SchemaField(col_name, inferred)) else: - fields.append(bigquery.SchemaField(col_name, "STRING")) + non_null = [v for v in series.tolist() if not _is_missing(v)] + if not non_null: + fields.append(bigquery.SchemaField(col_name, "STRING")) + elif all(isinstance(v, (list, tuple)) for v in non_null): + elem_types = { + _infer_scalar_bq_type(e) + for seq in non_null + for e in seq + if not _is_missing(e) + } + elem_type = _merge_bq_types(elem_types, store_type=None) + fields.append( + bigquery.SchemaField(col_name, elem_type, mode="REPEATED") + ) + else: + scalar_types = {_infer_scalar_bq_type(v) for v in non_null} + inferred = _merge_bq_types(scalar_types, store_type) + fields.append(bigquery.SchemaField(col_name, inferred)) return fields def _coerce_scalar(val: Any, field_type: str) -> Any: """Coerces a non-missing scalar value to match a BigQuery SchemaField type.""" - if hasattr(val, "item") and callable(val.item) and not isinstance( - val, (pd.Timestamp, datetime.date, bytes, str) - ): - try: - val = val.item() - except ValueError: - pass + val = _unwrap_numpy_scalar(val) if _is_missing(val): return None upper = field_type.upper() if upper in ("INTEGER", "INT64"): + if isinstance(val, bool): + return int(val) + if isinstance(val, int): + return val + if isinstance(val, str): + text = val.strip() + if not text: + return None + try: + return int(text) + except ValueError: + pass try: - return int(round(float(val))) + num = float(val) + return int(round(num)) if math.isfinite(num) else None except (TypeError, ValueError, OverflowError): return None if upper in ("FLOAT", "FLOAT64", "NUMERIC", "BIGNUMERIC"): @@ -234,11 +297,24 @@ def _coerce_scalar(val: Any, field_type: str) -> Any: val = val.astimezone(datetime.timezone.utc).replace(tzinfo=None) return val.isoformat(timespec="microseconds") text = str(val).strip() - if text.endswith("Z"): - return text[:-1] - if text.endswith("+00:00"): - return text[:-6] - return text + if not text: + return None + iso_text = ( + f"{text[:-1]}+00:00" if text.endswith(("Z", "z")) else text + ) + try: + parsed = datetime.datetime.fromisoformat(iso_text) + except ValueError: + if text.endswith(("Z", "z")): + return text[:-1] + if text.endswith("+00:00"): + return text[:-6] + return text + if parsed.tzinfo is not None: + parsed = parsed.astimezone(datetime.timezone.utc).replace( + tzinfo=None + ) + return parsed.isoformat(timespec="microseconds") if upper == "TIMESTAMP": if isinstance(val, pd.Timestamp): val = val.to_pydatetime() @@ -246,7 +322,8 @@ def _coerce_scalar(val: Any, field_type: str) -> Any: if val.tzinfo is None: val = val.replace(tzinfo=datetime.timezone.utc) return val.isoformat() - return str(val) + text = str(val).strip() + return text or None if upper == "DATE": if isinstance(val, pd.Timestamp): val = val.to_pydatetime() @@ -254,21 +331,84 @@ def _coerce_scalar(val: Any, field_type: str) -> Any: return val.date().isoformat() if isinstance(val, datetime.date): return val.isoformat() - return str(val) + text = str(val).strip() + return text or None if upper == "JSON": - return val if isinstance(val, str) else json.dumps(val, default=str) + if isinstance(val, str): + text = val.strip() + return text or None + return json.dumps(_sanitize_for_json(val), default=str, allow_nan=False) # Default STRING serialization: strip trailing .0 on integral floats to # match legacy bqstore behavior, and JSON-encode nested dicts/lists. if isinstance(val, float): return str(int(val)) if val.is_integer() else str(val) if isinstance(val, (dict, list, tuple)): - return json.dumps(val, default=str) + return json.dumps(_sanitize_for_json(val), default=str, allow_nan=False) if isinstance(val, (datetime.datetime, datetime.date, pd.Timestamp)): return val.isoformat() return str(val) +def _coerce_single_value( + val: Any, field: bigquery.SchemaField | None +) -> Any: + """Coerces a single non-repeated value (scalar or RECORD/STRUCT).""" + field_type = field.field_type if field is not None else "STRING" + if field_type.upper() in ("RECORD", "STRUCT"): + val = _unwrap_numpy_scalar(val) + if _is_missing(val): + return None + if isinstance(val, str): + text = val.strip() + if not text: + return None + try: + parsed = json.loads(text) + except ValueError: + return None + if not isinstance(parsed, dict): + return None + val = parsed + if not isinstance(val, dict): + return None + if field is not None and field.fields: + sub_by_lower = {f.name.lower(): f for f in field.fields} + coerced_record: dict[str, Any] = {} + for sub_key, sub_val in val.items(): + sub_field = sub_by_lower.get(str(sub_key).lower()) + target_key = ( + sub_field.name if sub_field is not None else str(sub_key) + ) + coerced_record[target_key] = _coerce_field_value( + sub_val, sub_field + ) + return coerced_record + return _sanitize_for_json(val) + return _coerce_scalar(val, field_type) + + +def _coerce_field_value( + raw_val: Any, field: bigquery.SchemaField | None +) -> Any: + """Coerces a field value according to its BigQuery SchemaField mode and type.""" + if field is not None and field.mode == "REPEATED": + if _is_missing(raw_val): + return [] + if isinstance(raw_val, (list, tuple)): + coerced_items = [ + _coerce_single_value(elem, field) + for elem in raw_val + if not _is_missing(elem) + ] + return [item for item in coerced_items if item is not None] + coerced_single = _coerce_single_value(raw_val, field) + return [coerced_single] if coerced_single is not None else [] + if _is_missing(raw_val): + return None + return _coerce_single_value(raw_val, field) + + def _dataframe_to_json_rows( chunk: pd.DataFrame, schema: Sequence[bigquery.SchemaField] ) -> list[dict[str, Any]]: @@ -280,28 +420,7 @@ def _dataframe_to_json_rows( for col, raw_val in record.items(): field = schema_by_lower.get(str(col).lower()) target_col = field.name if field is not None else str(col) - field_type = field.field_type if field is not None else "STRING" - if field is not None and field.mode == "REPEATED": - if _is_missing(raw_val): - json_row[target_col] = [] - elif isinstance(raw_val, (list, tuple)): - coerced_items = [ - _coerce_scalar(elem, field_type) - for elem in raw_val - if not _is_missing(elem) - ] - json_row[target_col] = [ - item for item in coerced_items if item is not None - ] - else: - coerced_single = _coerce_scalar(raw_val, field_type) - json_row[target_col] = ( - [coerced_single] if coerced_single is not None else [] - ) - elif _is_missing(raw_val): - json_row[target_col] = None - else: - json_row[target_col] = _coerce_scalar(raw_val, field_type) + json_row[target_col] = _coerce_field_value(raw_val, field) rows.append(json_row) return rows @@ -315,7 +434,7 @@ def _iter_streaming_batches( batch: list[dict[str, Any]] = [] batch_bytes = 0 for row in rows: - row_bytes = len(json.dumps(row).encode("utf-8")) + 2 + row_bytes = len(json.dumps(row, allow_nan=False).encode("utf-8")) + 2 if batch and ( len(batch) >= max_rows or batch_bytes + row_bytes > max_bytes ): @@ -328,10 +447,21 @@ def _iter_streaming_batches( yield batch +def _has_schema_lag_error(errors: Sequence[dict[str, Any]]) -> bool: + """Returns True when any insert_rows_json row error indicates schema lag.""" + for row_err in errors: + for err in row_err.get("errors") or []: + message = str(err.get("message") or "").lower() + if "no such field" in message: + return True + return False + + def _is_transient_insert_error(errors: Sequence[dict[str, Any]]) -> bool: """Returns True when insert_rows_json row errors are transient or schema-lag.""" if not errors: return False + saw_retryable = False for row_err in errors: item_errors = row_err.get("errors") or [] if not item_errors: @@ -339,14 +469,20 @@ def _is_transient_insert_error(errors: Sequence[dict[str, Any]]) -> bool: for err in item_errors: reason = str(err.get("reason") or "") message = str(err.get("message") or "").lower() + if reason == "stopped": + # BigQuery aborts sibling rows with reason="stopped" whenever + # skip_invalid_rows=False and another row in the batch fails. + continue if reason in _TRANSIENT_ROW_ERROR_REASONS: + saw_retryable = True continue # tabledata.insertAll caches schemas briefly after create_table or # update_table and can report "no such field" for a few seconds. if "no such field" in message: + saw_retryable = True continue return False - return True + return saw_retryable class BigQueryReporter(Reporter): @@ -379,15 +515,15 @@ def _ensure_dataset(self) -> None: self._dataset_ensured = True def _ensure_table_and_schema( - self, table_id: str, chunk: pd.DataFrame, store_type: STORETYPE + self, table_id: str, df: pd.DataFrame, store_type: STORETYPE ) -> bigquery.Table: - """Ensures `table_id` exists and includes any new columns in `chunk`.""" + """Ensures `table_id` exists and includes any new columns in `df`.""" table_obj = self._tables.get(table_id) if table_obj is None: try: table_obj = self.client.get_table(table_id) except NotFound: - schema = _infer_schema(chunk, store_type) + schema = _infer_schema(df, store_type) table_obj = self.client.create_table( bigquery.Table(table_id, schema=schema), exists_ok=True ) @@ -396,37 +532,30 @@ def _ensure_table_and_schema( existing_lower = {field.name.lower() for field in table_obj.schema} missing_cols = [ - col for col in chunk.columns if str(col).lower() not in existing_lower + col for col in df.columns if str(col).lower() not in existing_lower ] if missing_cols: - # Refresh once before patching in case another worker added the field. - table_obj = self.client.get_table(table_id) - existing_lower = {field.name.lower() for field in table_obj.schema} - missing_cols = [ - col for col in chunk.columns - if str(col).lower() not in existing_lower - ] - if missing_cols: - added_fields = _infer_schema(chunk[missing_cols], store_type) + for attempt in range(_MAX_SCHEMA_UPDATE_RETRIES): + # Refresh before patching in case another worker added the field. + table_obj = self.client.get_table(table_id) + existing_lower = { + field.name.lower() for field in table_obj.schema + } + missing_cols = [ + col for col in df.columns + if str(col).lower() not in existing_lower + ] + if not missing_cols: + break + added_fields = _infer_schema(df[missing_cols], store_type) table_obj.schema = list(table_obj.schema) + added_fields try: table_obj = self.client.update_table(table_obj, ["schema"]) + break except (Conflict, PreconditionFailed): - table_obj = self.client.get_table(table_id) - existing_lower = { - field.name.lower() for field in table_obj.schema - } - still_missing = [ - col for col in chunk.columns - if str(col).lower() not in existing_lower - ] - if still_missing: - table_obj.schema = list(table_obj.schema) + _infer_schema( - chunk[still_missing], store_type - ) - table_obj = self.client.update_table( - table_obj, ["schema"] - ) + if attempt == _MAX_SCHEMA_UPDATE_RETRIES - 1: + raise + time.sleep(0.5 * (2 ** attempt)) self._tables[table_id] = table_obj return table_obj @@ -457,24 +586,68 @@ def store(self, results, type: STORETYPE): initial_delay = 2 backoff_factor = 2 + table_obj = self._ensure_table_and_schema(table, results, type) + current_schema = list(table_obj.schema) + for chunk in _split_dataframe(results, self.chunk_size): - table_obj = self._ensure_table_and_schema(table, chunk, type) - json_rows = _dataframe_to_json_rows(chunk, table_obj.schema) + chunk_schema = current_schema + json_rows = _dataframe_to_json_rows(chunk, chunk_schema) + chunk_offset = 0 for batch in _iter_streaming_batches(json_rows, self.chunk_size): + batch_len = len(batch) + pending_positions = list( + range(chunk_offset, chunk_offset + batch_len) + ) + chunk_offset += batch_len + if current_schema != chunk_schema: + pending_rows = _dataframe_to_json_rows( + chunk.iloc[pending_positions], current_schema + ) + else: + pending_rows = batch + pending_row_ids = [str(uuid.uuid4()) for _ in pending_rows] + for attempt in range(max_retries): try: - errors = self.client.insert_rows_json(table, batch) + errors = self.client.insert_rows_json( + table, pending_rows, row_ids=pending_row_ids + ) if not errors: break if ( attempt < max_retries - 1 and _is_transient_insert_error(errors) ): - self._tables.pop(table, None) - table_obj = self._ensure_table_and_schema( - table, chunk, type - ) + failed_indices = sorted({ + err["index"] + for err in errors + if isinstance(err.get("index"), int) + and 0 <= err["index"] < len(pending_rows) + }) + if failed_indices: + pending_positions = [ + pending_positions[i] + for i in failed_indices + ] + pending_rows = [ + pending_rows[i] for i in failed_indices + ] + pending_row_ids = [ + pending_row_ids[i] for i in failed_indices + ] + if _has_schema_lag_error(errors): + self._tables.pop(table, None) + refreshed = self._ensure_table_and_schema( + table, results, type + ) + refreshed_schema = list(refreshed.schema) + if refreshed_schema != current_schema: + current_schema = refreshed_schema + pending_rows = _dataframe_to_json_rows( + chunk.iloc[pending_positions], + current_schema, + ) delay = initial_delay * (backoff_factor ** attempt) logging.warning( "BQ streaming write had transient/schema-lag " @@ -500,7 +673,16 @@ def store(self, results, type: STORETYPE): raise if isinstance(e, NotFound): self._tables.pop(table, None) - self._ensure_table_and_schema(table, chunk, type) + refreshed = self._ensure_table_and_schema( + table, results, type + ) + refreshed_schema = list(refreshed.schema) + if refreshed_schema != current_schema: + current_schema = refreshed_schema + pending_rows = _dataframe_to_json_rows( + chunk.iloc[pending_positions], + current_schema, + ) delay = initial_delay * (backoff_factor ** attempt) logging.warning( f"BQ write failed with transient error: {e}. " diff --git a/evalbench/test/bqstore_test.py b/evalbench/test/bqstore_test.py index c2220f308..b0a3847ee 100644 --- a/evalbench/test/bqstore_test.py +++ b/evalbench/test/bqstore_test.py @@ -1,12 +1,26 @@ import datetime +import json import math import unittest from unittest.mock import MagicMock, patch -from google.api_core.exceptions import Forbidden, NotFound, ServiceUnavailable +from google.api_core.exceptions import ( + Conflict, + Forbidden, + NotFound, + PreconditionFailed, + ServiceUnavailable, +) from google.cloud import bigquery +import numpy as np import pandas as pd -from reporting.bqstore import BigQueryReporter +from reporting.bqstore import ( + BigQueryReporter, + _coerce_scalar, + _dataframe_to_json_rows, + _infer_schema, + _iter_streaming_batches, +) from reporting.report import STORETYPE @@ -229,6 +243,278 @@ def test_store_retries_transient_errors_and_schema_lag_but_not_forbidden( self.reporter.store(df, STORETYPE.CONFIGS) mock_sleep.assert_not_called() + @patch("reporting.bqstore.time.sleep") + def test_store_retries_multi_row_stopped_siblings_with_stable_row_ids( + self, mock_sleep + ): + table_obj = bigquery.Table( + "test-project.evalbench.configs", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("config", "STRING"), + ], + ) + self.mock_client.get_table.return_value = table_obj + self.mock_client.insert_rows_json.side_effect = [ + [ + { + "index": 0, + "errors": [{ + "reason": "invalid", + "message": "no such field: config.", + }], + }, + { + "index": 1, + "errors": [{"reason": "stopped", "message": ""}], + }, + ], + [], + ] + + df = pd.DataFrame([ + {"job_id": "job-123", "config": "k1"}, + {"job_id": "job-123", "config": "k2"}, + ]) + self.reporter.store(df, STORETYPE.CONFIGS) + + self.assertEqual(self.mock_client.insert_rows_json.call_count, 2) + self.assertEqual(mock_sleep.call_count, 1) + first_call = self.mock_client.insert_rows_json.call_args_list[0] + second_call = self.mock_client.insert_rows_json.call_args_list[1] + self.assertEqual(len(first_call.kwargs["row_ids"]), 2) + self.assertEqual( + first_call.kwargs["row_ids"], second_call.kwargs["row_ids"] + ) + + @patch("reporting.bqstore.time.sleep") + def test_store_partial_transient_error_retries_only_failed_rows_without_get_table( + self, mock_sleep + ): + table_obj = bigquery.Table( + "test-project.evalbench.configs", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("config", "STRING"), + ], + ) + self.mock_client.get_table.return_value = table_obj + self.mock_client.insert_rows_json.side_effect = [ + [{ + "index": 1, + "errors": [{"reason": "backendError", "message": "try again"}], + }], + [], + ] + + df = pd.DataFrame([ + {"job_id": "job-123", "config": "k0"}, + {"job_id": "job-123", "config": "k1"}, + {"job_id": "job-123", "config": "k2"}, + ]) + self.reporter.store(df, STORETYPE.CONFIGS) + + self.assertEqual(self.mock_client.insert_rows_json.call_count, 2) + # Pure transient backendError must not trigger extra get_table RPCs. + self.assertEqual(self.mock_client.get_table.call_count, 1) + first_call = self.mock_client.insert_rows_json.call_args_list[0] + second_call = self.mock_client.insert_rows_json.call_args_list[1] + self.assertEqual(len(second_call.args[1]), 1) + self.assertEqual(second_call.args[1][0]["config"], "k1") + self.assertEqual( + second_call.kwargs["row_ids"], [first_call.kwargs["row_ids"][1]] + ) + + @patch("reporting.bqstore.time.sleep") + def test_store_permanent_row_error_with_stopped_sibling_fails_fast( + self, mock_sleep + ): + table_obj = bigquery.Table( + "test-project.evalbench.configs", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("config", "STRING"), + ], + ) + self.mock_client.get_table.return_value = table_obj + self.mock_client.insert_rows_json.return_value = [ + { + "index": 0, + "errors": [{"reason": "invalid", "message": "bad integer"}], + }, + { + "index": 1, + "errors": [{"reason": "stopped", "message": ""}], + }, + ] + + df = pd.DataFrame([ + {"job_id": "job-123", "config": "k0"}, + {"job_id": "job-123", "config": "k1"}, + ]) + with self.assertRaises(RuntimeError): + self.reporter.store(df, STORETYPE.CONFIGS) + self.assertEqual(self.mock_client.insert_rows_json.call_count, 1) + mock_sleep.assert_not_called() + + @patch("reporting.bqstore.time.sleep") + def test_store_re_coerces_pending_rows_when_schema_changes_on_not_found( + self, mock_sleep + ): + old_table = bigquery.Table( + "test-project.evalbench.scores", + schema=[ + bigquery.SchemaField("id", "STRING"), + bigquery.SchemaField("score", "STRING"), + ], + ) + recreated_table = bigquery.Table( + "test-project.evalbench.scores", + schema=[ + bigquery.SchemaField("id", "STRING"), + bigquery.SchemaField("score", "INTEGER"), + ], + ) + self.mock_client.get_table.side_effect = [old_table, recreated_table] + self.mock_client.insert_rows_json.side_effect = [ + NotFound("Table was recreated"), + [], + ] + + df = pd.DataFrame([{"id": "s1", "score": 95.6}]) + self.reporter.store(df, STORETYPE.SCORES) + + self.assertEqual(self.mock_client.insert_rows_json.call_count, 2) + first_rows = self.mock_client.insert_rows_json.call_args_list[0].args[1] + second_rows = self.mock_client.insert_rows_json.call_args_list[1].args[1] + self.assertEqual(first_rows[0]["score"], "95.6") + self.assertEqual(second_rows[0]["score"], 96) + + @patch("reporting.bqstore.time.sleep") + def test_store_raises_not_found_after_exhausting_retries(self, mock_sleep): + table_obj = bigquery.Table( + "test-project.evalbench.configs", + schema=[bigquery.SchemaField("job_id", "STRING")], + ) + self.mock_client.get_table.return_value = table_obj + self.mock_client.insert_rows_json.side_effect = NotFound("missing") + + df = pd.DataFrame([{"job_id": "job-123"}]) + with self.assertRaises(NotFound): + self.reporter.store(df, STORETYPE.CONFIGS) + self.assertEqual(self.mock_client.insert_rows_json.call_count, 5) + self.assertEqual(mock_sleep.call_count, 4) + + @patch("reporting.bqstore.time.sleep") + def test_update_table_retries_multiple_concurrent_conflicts(self, mock_sleep): + final_table = bigquery.Table( + "test-project.evalbench.results", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("new_col", "STRING"), + ], + ) + # Return a fresh Table per get_table() call, matching BigQuery client + # behavior (update_table mutates table_obj.schema on the local object). + self.mock_client.get_table.side_effect = lambda _: bigquery.Table( + "test-project.evalbench.results", + schema=[bigquery.SchemaField("job_id", "STRING")], + ) + self.mock_client.update_table.side_effect = [ + PreconditionFailed("etag mismatch 1"), + Conflict("etag mismatch 2"), + final_table, + ] + self.mock_client.insert_rows_json.return_value = [] + + df = pd.DataFrame([{"job_id": "job-123", "new_col": "v"}]) + self.reporter.store(df, STORETYPE.EVALS) + + self.assertEqual(self.mock_client.update_table.call_count, 3) + self.assertEqual(mock_sleep.call_count, 2) + + def test_infer_schema_scans_all_values_chunks_and_numpy_scalars(self): + # Mixed object and repeated columns widen instead of locking on row 0; + # NumPy scalars are unwrapped to native BigQuery types. + df = pd.DataFrame({ + "mixed_obj": pd.Series([100, "N/A"], dtype="object"), + "int_and_float": pd.Series([1, 2.5], dtype="object"), + "mixed_repeated": pd.Series([[1], ["foo"]], dtype="object"), + "numpy_int": pd.Series([np.int64(7), None], dtype="object"), + }) + fields = {f.name: f for f in _infer_schema(df, STORETYPE.EVALS)} + self.assertEqual(fields["mixed_obj"].field_type, "STRING") + self.assertEqual(fields["int_and_float"].field_type, "FLOAT") + self.assertEqual(fields["mixed_repeated"].mode, "REPEATED") + self.assertEqual(fields["mixed_repeated"].field_type, "STRING") + self.assertEqual(fields["numpy_int"].field_type, "INTEGER") + + # All-null in chunk 0 but populated as REPEATED in chunk 1 must infer + # REPEATED STRING across the whole DataFrame before chunking. + self.reporter.chunk_size = 1 + self.mock_client.get_table.side_effect = NotFound("missing") + self.mock_client.create_table.side_effect = ( + lambda table_arg, exists_ok=False: table_arg + ) + self.mock_client.insert_rows_json.return_value = [] + multi_chunk_df = pd.DataFrame({ + "id": ["s1", "s2"], + "dialects": pd.Series([None, ["bigquery"]], dtype="object"), + }) + self.reporter.store(multi_chunk_df, STORETYPE.SCORES) + created_table = self.mock_client.create_table.call_args[0][0] + created_fields = {f.name: f for f in created_table.schema} + self.assertEqual(created_fields["dialects"].mode, "REPEATED") + self.assertEqual(created_fields["dialects"].field_type, "STRING") + + def test_coerce_scalar_and_record_edge_cases(self): + big_int = 9007199254740993 # 2**53 + 1 + self.assertEqual(_coerce_scalar(big_int, "INT64"), big_int) + self.assertEqual(_coerce_scalar(str(big_int), "INT64"), big_int) + self.assertEqual(_coerce_scalar(True, "INT64"), 1) + self.assertIsNone(_coerce_scalar("", "INT64")) + + self.assertIsNone(_coerce_scalar("", "DATETIME")) + self.assertIsNone(_coerce_scalar(" ", "TIMESTAMP")) + self.assertIsNone(_coerce_scalar("", "DATE")) + self.assertEqual( + _coerce_scalar("2026-11-18T03:15:00-07:00", "DATETIME"), + "2026-11-18T10:15:00.000000", + ) + self.assertEqual( + _coerce_scalar("2026-11-18T03:15:00Z", "DATETIME"), + "2026-11-18T03:15:00.000000", + ) + + json_str = _coerce_scalar({"a": float("nan"), "b": 1}, "JSON") + self.assertEqual(json.loads(json_str), {"a": None, "b": 1}) + + record_schema = [ + bigquery.SchemaField( + "meta", + "RECORD", + fields=[ + bigquery.SchemaField("count", "INT64"), + bigquery.SchemaField("tag", "STRING"), + ], + ) + ] + rows = _dataframe_to_json_rows( + pd.DataFrame([{"meta": {"count": 42, "tag": "ok"}}]), + record_schema, + ) + self.assertEqual(rows, [{"meta": {"count": 42, "tag": "ok"}}]) + + def test_iter_streaming_batches_splits_by_byte_size(self): + rows = [ + {"id": "1", "payload": "a" * 80}, + {"id": "2", "payload": "b" * 80}, + {"id": "3", "payload": "c" * 80}, + ] + batches = list(_iter_streaming_batches(rows, max_rows=10, max_bytes=150)) + self.assertEqual(len(batches), 3) + self.assertEqual([b[0]["id"] for b in batches], ["1", "2", "3"]) + if __name__ == "__main__": unittest.main() diff --git a/evalbench/test/eval_service_reporting_test.py b/evalbench/test/eval_service_reporting_test.py index 39be1514f..7e00300ed 100644 --- a/evalbench/test/eval_service_reporting_test.py +++ b/evalbench/test/eval_service_reporting_test.py @@ -3,7 +3,7 @@ import os import tempfile import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from eval_service import _process_results, EmptyEvalResultError @@ -100,6 +100,106 @@ def test_empty_results_still_unlinks_tempfiles_and_raises(self): self.assertFalse(os.path.exists(results_tf)) self.assertFalse(os.path.exists(scores_tf)) + @patch("eval_service.get_reporters") + def test_reporter_init_failure_is_isolated_and_unlinks_tempfiles( + self, mock_get_reporters + ): + results_tf = self._write_json_tempfile([ + { + "id": "scenario-1", + "prompt_id": "scenario-1", + "generated_sql": "", + "generated_error": None, + } + ]) + scores_tf = self._write_json_tempfile([ + { + "id": "scenario-1", + "prompt_id": "scenario-1", + "comparator": "goal_completion", + "score": 100, + "generated_sql": "", + "generated_error": None, + } + ]) + + csv_reporter = CsvReporter() + + def _get_reporters_side_effect(reporting_cfg, job_id, run_time): + if "bigquery" in reporting_cfg: + raise ValueError("GCP project ID not set") + return [csv_reporter] + + mock_get_reporters.side_effect = _get_reporters_side_effect + + summary = _process_results( + reporters=None, + job_id="job-init-err", + run_time=datetime.datetime(2026, 11, 18, 3, 20, 0), + results_tf=results_tf, + scores_tf=scores_tf, + multi_trial_scores_tf=None, + config={ + "orchestrator": "geminicli", + "scorers": {"goal_completion": {}}, + "reporting": {"bigquery": {}, "csv": {"output_directory": "x"}}, + }, + model_config={"generator": "agy_cli"}, + db_configs=[], + fail_if_all_reporters_failed=True, + ) + + self.assertEqual(summary["total"], 1) + self.assertEqual(csv_reporter.store.call_count, 4) + self.assertEqual(len(summary["reporting_errors"]), 1) + self.assertEqual(summary["reporting_errors"][0]["reporter"], "bigquery") + self.assertIn("GCP project ID", summary["reporting_errors"][0]["error"]) + self.assertFalse(os.path.exists(results_tf)) + self.assertFalse(os.path.exists(scores_tf)) + + def test_all_reporters_failing_raises_when_caller_has_no_summary(self): + results_tf = self._write_json_tempfile([ + { + "id": "scenario-1", + "prompt_id": "scenario-1", + "generated_sql": "", + "generated_error": None, + } + ]) + scores_tf = self._write_json_tempfile([ + { + "id": "scenario-1", + "prompt_id": "scenario-1", + "comparator": "goal_completion", + "score": 100, + "generated_sql": "", + "generated_error": None, + } + ]) + + bq_reporter = BigQueryReporter() + bq_reporter.store.side_effect = RuntimeError("403 Forbidden") + + with self.assertRaises(RuntimeError): + _process_results( + reporters=[bq_reporter], + job_id="job-all-failed", + run_time=datetime.datetime(2026, 11, 18, 3, 20, 0), + results_tf=results_tf, + scores_tf=scores_tf, + multi_trial_scores_tf=None, + config={ + "orchestrator": "geminicli", + "scorers": {"goal_completion": {}}, + }, + model_config={"generator": "agy_cli"}, + db_configs=[], + fail_if_all_reporters_failed=True, + ) + + self.assertFalse(os.path.exists(results_tf)) + self.assertFalse(os.path.exists(scores_tf)) + if __name__ == "__main__": unittest.main()