diff --git a/evalbench/eval_service.py b/evalbench/eval_service.py index 97d974e8..b3d3d534 100644 --- a/evalbench/eval_service.py +++ b/evalbench/eval_service.py @@ -260,18 +260,14 @@ async def Eval( await executor.run(ctx.run, evaluator.evaluate, dataset) job_id, run_time, results_tf, scores_tf, multi_trial_scores_tf = evaluator.process() - # Fallback to empty dict if reporting is present but null in YAML - reporters = get_reporters( - config.get("reporting") or {}, job_id, run_time - ) - # Offload blocking results processing to a thread pool + # Offload blocking results processing and reporter setup to a thread pool logging.info("Offloading results processing to thread pool...") ctx = contextvars.copy_context() summary = await executor.run( ctx.run, _process_results, - reporters, + None, job_id, run_time, results_tf, @@ -280,6 +276,7 @@ async def Eval( config, with_agent_version(model_config, evaluator), db_configs, + not config.get("summary_in_response"), ) logging.info( @@ -469,8 +466,6 @@ async def read_from_client(): # Process final scoring and reporting job_id, run_time, results_tf, scores_tf = orchestrator.process() - reporters = get_reporters(config.get( - "reporting") or {}, job_id, run_time) logging.info( "Offloading interactive results processing to thread pool...") @@ -478,7 +473,7 @@ async def read_from_client(): summary = await executor.run( ctx.run, _process_results, - reporters, + None, job_id, run_time, results_tf, @@ -602,14 +597,11 @@ async def run_eval_and_process(): scores_tf, multi_trial_scores_tf, ) = orchestrator.process() - reporters = get_reporters( - config.get("reporting") or {}, job_id, run_time - ) logging.info("Processing agent evaluation results...") summary = await executor.run( ctx.run, _process_results, - reporters, + None, job_id, run_time, results_tf, @@ -746,29 +738,99 @@ def _summary_count(value) -> float | int: return int(count) if count.is_integer() else round(count, 2) +def _build_reporters( + reporting_config, job_id, run_time +) -> tuple[list, list[dict[str, str]]]: + """Constructs each configured reporter with per-reporter error isolation.""" + reporters = [] + errors: list[dict[str, str]] = [] + if not isinstance(reporting_config, dict): + return reporters, errors + for key, cfg in reporting_config.items(): + try: + reporters.extend(get_reporters({key: cfg}, job_id, run_time)) + except Exception as exc: + logging.exception( + "Reporter %s failed to initialize: %s", + key, + exc, + ) + errors.append({"reporter": str(key), "error": str(exc)}) + return reporters, errors + + +def _store_reports( + reporters, config_df, results_df, scores_df, summary_scores_df +) -> list[dict[str, str]]: + """Writes evaluation outputs to each configured reporter. + + Isolates per-reporter failures so a reporting error (such as a BigQuery + permission or quota failure) neither blocks other reporters (CSV, GCS + artifacts) nor fails a completed evaluation RPC. + + Returns: + A list of {"reporter": , "error": } dicts for any + reporters that failed. + """ + errors: list[dict[str, str]] = [] + for reporter in reporters: + name = type(reporter).__name__ + try: + reporter.store(config_df, report.STORETYPE.CONFIGS) + reporter.store(results_df, report.STORETYPE.EVALS) + reporter.store(scores_df, report.STORETYPE.SCORES) + reporter.store(summary_scores_df, report.STORETYPE.SUMMARY) + except Exception as exc: + logging.exception( + "Reporter %s failed while storing evaluation results: %s", + name, + exc, + ) + errors.append({"reporter": name, "error": str(exc)}) + return errors + + def _process_results( - reporters, job_id, run_time, results_tf, scores_tf, multi_trial_scores_tf, config, model_config, db_configs + reporters, + job_id, + run_time, + results_tf, + scores_tf, + multi_trial_scores_tf, + config, + model_config, + db_configs, + fail_if_all_reporters_failed=False, ): - config_df = config_to_df( - job_id, - run_time, - config, - model_config, - db_configs, - ) - results = load_json(results_tf) - results_df = report.get_dataframe(results) - if results_df.empty: - raise EmptyEvalResultError( - "No matching evals were produced for this run. Check that the dataset, " - "dialect filters, and database configuration line up." + try: + config_df = config_to_df( + job_id, + run_time, + config, + model_config, + db_configs, ) - report.quick_summary(results_df) - scores = load_json(scores_tf) - if multi_trial_scores_tf: - multi_trial_scores = load_json(multi_trial_scores_tf) - if multi_trial_scores: - scores.extend(multi_trial_scores) + results = load_json(results_tf) + results_df = report.get_dataframe(results) + if results_df.empty: + raise EmptyEvalResultError( + "No matching evals were produced for this run. Check that the dataset, " + "dialect filters, and database configuration line up." + ) + report.quick_summary(results_df) + scores = load_json(scores_tf) + if multi_trial_scores_tf: + multi_trial_scores = load_json(multi_trial_scores_tf) + if multi_trial_scores: + scores.extend(multi_trial_scores) + finally: + # k8s emptyDir /tmp does not auto cleanup, so always unlink temp files. + if results_tf: + pathlib.Path(results_tf).unlink(missing_ok=True) + if scores_tf: + pathlib.Path(scores_tf).unlink(missing_ok=True) + if multi_trial_scores_tf: + pathlib.Path(multi_trial_scores_tf).unlink(missing_ok=True) num_prompts = len(set(r.get("prompt_id") for r in results if r.get("prompt_id"))) @@ -779,18 +841,31 @@ def _process_results( summary_scores_df["job_id"] = job_id summary_scores_df["run_time"] = run_time - # Store the reports in specified outputs - for reporter in reporters: - reporter.store(config_df, report.STORETYPE.CONFIGS) - reporter.store(results_df, report.STORETYPE.EVALS) - reporter.store(scores_df, report.STORETYPE.SCORES) - reporter.store(summary_scores_df, report.STORETYPE.SUMMARY) + if reporters is None: + reporters, reporting_errors = _build_reporters( + config.get("reporting") or {}, job_id, run_time + ) + else: + reporting_errors = [] - # k8s emptyDir /tmp does not auto cleanup, so we explicitly delete - pathlib.Path(results_tf).unlink() - pathlib.Path(scores_tf).unlink() - if multi_trial_scores_tf: - pathlib.Path(multi_trial_scores_tf).unlink() + total_reporters = len(reporters) + len(reporting_errors) + + # Store the reports in specified outputs without aborting completed evals + reporting_errors.extend( + _store_reports( + reporters, config_df, results_df, scores_df, summary_scores_df + ) + ) + + if ( + fail_if_all_reporters_failed + and total_reporters > 0 + and len(reporting_errors) >= total_reporters + ): + raise RuntimeError( + f"All configured reporters failed for job {job_id}: " + f"{reporting_errors}" + ) # Build summary dict from summary_scores_df summary = {"total": 0, "scores": {}} @@ -810,4 +885,7 @@ def _process_results( "p90": round(latencies.quantile(0.9), 2), } + if reporting_errors: + summary["reporting_errors"] = reporting_errors + return summary diff --git a/evalbench/reporting/bqstore.py b/evalbench/reporting/bqstore.py index bbc2e123..8b3ee771 100644 --- a/evalbench/reporting/bqstore.py +++ b/evalbench/reporting/bqstore.py @@ -1,11 +1,25 @@ -from google.cloud import bigquery -from google.api_core.exceptions import TooManyRequests, ServiceUnavailable +from collections.abc import Iterator, Sequence +import datetime +import json import logging +import math +import time +from typing import Any +import urllib.parse +import uuid + +from google.api_core.exceptions import ( + Conflict, + InternalServerError, + NotFound, + PreconditionFailed, + ServiceUnavailable, + TooManyRequests, +) +from google.cloud import bigquery import pandas as pd from reporting.report import Reporter, STORETYPE from util.gcp import get_gcp_project -import urllib.parse -import time try: import google.colab # type: ignore @@ -16,6 +30,22 @@ _IN_COLAB = False _DEFAULT_CHUNK_SIZE = 500 +# BigQuery tabledata.insertAll caps each HTTP request at 10 MB; flush below 8 MiB +# so large multi-turn conversation_history rows never hit 413/BadRequest. +_MAX_STREAMING_REQUEST_BYTES = 8 * 1024 * 1024 +_MAX_SCHEMA_UPDATE_RETRIES = 5 +_SUMMARY_AND_SCORES_INT_COLS = frozenset({ + "metric_score", + "totalLatencyMs", + "total_results_count", + "correct_results_count", +}) +_TRANSIENT_ROW_ERROR_REASONS = frozenset({ + "backendError", + "internalError", + "rateLimitExceeded", + "timeout", +}) _REPORT_QUERY = """ WITH all_runs_with_set_tag AS ( @@ -62,24 +92,399 @@ """ -def _split_dataframe(df, chunk_size): - """Splits a pandas DataFrame into chunks of a specified size. - - Args: - df: The DataFrame to split. - chunk_size: The desired size of each chunk. - - Yields: - A generator that yields each chunk of the DataFrame. - """ +def _split_dataframe(df: pd.DataFrame, chunk_size: int) -> Iterator[pd.DataFrame]: + """Splits a pandas DataFrame into chunks of a specified size.""" num_chunks = len(df) // chunk_size + (len(df) % chunk_size > 0) for i in range(num_chunks): start = i * chunk_size - # Py/Pandas slicing handles not going out of bound end = (i + 1) * chunk_size yield df[start:end] +def _unwrap_numpy_scalar(val: Any) -> Any: + """Converts a 0-d NumPy scalar to its native Python counterpart.""" + if hasattr(val, "item") and callable(val.item) and not isinstance( + val, (pd.Timestamp, datetime.date, bytes, str) + ): + try: + return val.item() + except ValueError: + pass + return val + + +def _is_missing(val: Any) -> bool: + """Returns True for None, NaN, Inf, pd.NA, or pd.NaT without failing on sequences.""" + if val is None or val is pd.NA or val is pd.NaT: + return True + if isinstance(val, (list, tuple, dict, set)): + return False + val = _unwrap_numpy_scalar(val) + if val is None or val is pd.NA or val is pd.NaT: + return True + if isinstance(val, float): + return not math.isfinite(val) + try: + na_check = _unwrap_numpy_scalar(pd.isna(val)) + return bool(na_check) if isinstance(na_check, bool) else False + except (TypeError, ValueError): + return False + + +def _sanitize_for_json(val: Any) -> Any: + """Recursively converts nested structures into strict RFC-8259 JSON values.""" + if _is_missing(val): + return None + val = _unwrap_numpy_scalar(val) + if isinstance(val, dict): + return {str(k): _sanitize_for_json(v) for k, v in val.items()} + if isinstance(val, (list, tuple)): + return [_sanitize_for_json(v) for v in val] + if ( + hasattr(val, "tolist") + and callable(val.tolist) + and not isinstance(val, (str, bytes)) + ): + return _sanitize_for_json(val.tolist()) + if isinstance(val, (datetime.datetime, datetime.date, pd.Timestamp)): + return val.isoformat() + return val + + +def _infer_scalar_bq_type(val: Any) -> str: + """Infers a BigQuery scalar column type from a non-null Python/pandas value.""" + val = _unwrap_numpy_scalar(val) + if isinstance(val, pd.Timestamp): + return "TIMESTAMP" if val.tzinfo is not None else "DATETIME" + if isinstance(val, datetime.datetime): + return "TIMESTAMP" if val.tzinfo is not None else "DATETIME" + if isinstance(val, datetime.date): + return "DATE" + if isinstance(val, bool): + return "BOOLEAN" + if isinstance(val, int): + return "INTEGER" + if isinstance(val, float): + return "FLOAT" + return "STRING" + + +def _merge_bq_types( + types: set[str], store_type: STORETYPE | None +) -> str: + """Merges inferred BigQuery scalar types across all non-null values.""" + if not types: + return "STRING" + if len(types) == 1: + inferred = next(iter(types)) + elif types == {"INTEGER", "FLOAT"}: + inferred = "FLOAT" + elif types == {"DATETIME", "TIMESTAMP"}: + inferred = "TIMESTAMP" + else: + inferred = "STRING" + + if ( + inferred == "FLOAT" + and store_type in (STORETYPE.SUMMARY, STORETYPE.SCORES) + ): + return "STRING" + return inferred + + +def _infer_schema( + df: pd.DataFrame, store_type: STORETYPE +) -> list[bigquery.SchemaField]: + """Infers BigQuery SchemaFields for new tables or newly added columns.""" + fields: list[bigquery.SchemaField] = [] + for col in df.columns: + col_name = str(col) + series = df[col] + if ( + store_type in (STORETYPE.SUMMARY, STORETYPE.SCORES) + and col_name in _SUMMARY_AND_SCORES_INT_COLS + ): + fields.append(bigquery.SchemaField(col_name, "INTEGER")) + continue + + if pd.api.types.is_datetime64_any_dtype(series.dtype): + tz = getattr(series.dtype, "tz", None) + fields.append( + bigquery.SchemaField( + col_name, "TIMESTAMP" if tz is not None else "DATETIME" + ) + ) + elif pd.api.types.is_bool_dtype(series.dtype): + fields.append(bigquery.SchemaField(col_name, "BOOLEAN")) + elif pd.api.types.is_integer_dtype(series.dtype): + fields.append(bigquery.SchemaField(col_name, "INTEGER")) + elif pd.api.types.is_float_dtype(series.dtype): + bq_type = ( + "STRING" + if store_type in (STORETYPE.SUMMARY, STORETYPE.SCORES) + else "FLOAT" + ) + fields.append(bigquery.SchemaField(col_name, bq_type)) + else: + non_null = [v for v in series.tolist() if not _is_missing(v)] + if not non_null: + fields.append(bigquery.SchemaField(col_name, "STRING")) + elif all(isinstance(v, (list, tuple)) for v in non_null): + elem_types = { + _infer_scalar_bq_type(e) + for seq in non_null + for e in seq + if not _is_missing(e) + } + elem_type = _merge_bq_types(elem_types, store_type=None) + fields.append( + bigquery.SchemaField(col_name, elem_type, mode="REPEATED") + ) + else: + scalar_types = {_infer_scalar_bq_type(v) for v in non_null} + inferred = _merge_bq_types(scalar_types, store_type) + fields.append(bigquery.SchemaField(col_name, inferred)) + return fields + + +def _coerce_scalar(val: Any, field_type: str) -> Any: + """Coerces a non-missing scalar value to match a BigQuery SchemaField type.""" + val = _unwrap_numpy_scalar(val) + if _is_missing(val): + return None + + upper = field_type.upper() + if upper in ("INTEGER", "INT64"): + if isinstance(val, bool): + return int(val) + if isinstance(val, int): + return val + if isinstance(val, str): + text = val.strip() + if not text: + return None + try: + return int(text) + except ValueError: + pass + try: + num = float(val) + return int(round(num)) if math.isfinite(num) else None + except (TypeError, ValueError, OverflowError): + return None + if upper in ("FLOAT", "FLOAT64", "NUMERIC", "BIGNUMERIC"): + try: + num = float(val) + return num if math.isfinite(num) else None + except (TypeError, ValueError): + return None + if upper in ("BOOLEAN", "BOOL"): + if isinstance(val, bool): + return val + if isinstance(val, str): + lowered = val.strip().lower() + if lowered in ("true", "1", "t", "yes"): + return True + if lowered in ("false", "0", "f", "no", ""): + return False + return None + return bool(val) + if upper == "DATETIME": + if isinstance(val, pd.Timestamp): + val = val.to_pydatetime() + if isinstance(val, datetime.datetime): + if val.tzinfo is not None: + val = val.astimezone(datetime.timezone.utc).replace(tzinfo=None) + return val.isoformat(timespec="microseconds") + text = str(val).strip() + if not text: + return None + iso_text = ( + f"{text[:-1]}+00:00" if text.endswith(("Z", "z")) else text + ) + try: + parsed = datetime.datetime.fromisoformat(iso_text) + except ValueError: + if text.endswith(("Z", "z")): + return text[:-1] + if text.endswith("+00:00"): + return text[:-6] + return text + if parsed.tzinfo is not None: + parsed = parsed.astimezone(datetime.timezone.utc).replace( + tzinfo=None + ) + return parsed.isoformat(timespec="microseconds") + if upper == "TIMESTAMP": + if isinstance(val, pd.Timestamp): + val = val.to_pydatetime() + if isinstance(val, datetime.datetime): + if val.tzinfo is None: + val = val.replace(tzinfo=datetime.timezone.utc) + return val.isoformat() + text = str(val).strip() + return text or None + if upper == "DATE": + if isinstance(val, pd.Timestamp): + val = val.to_pydatetime() + if isinstance(val, datetime.datetime): + return val.date().isoformat() + if isinstance(val, datetime.date): + return val.isoformat() + text = str(val).strip() + return text or None + if upper == "JSON": + if isinstance(val, str): + text = val.strip() + return text or None + return json.dumps(_sanitize_for_json(val), default=str, allow_nan=False) + + # Default STRING serialization: strip trailing .0 on integral floats to + # match legacy bqstore behavior, and JSON-encode nested dicts/lists. + if isinstance(val, float): + return str(int(val)) if val.is_integer() else str(val) + if isinstance(val, (dict, list, tuple)): + return json.dumps(_sanitize_for_json(val), default=str, allow_nan=False) + if isinstance(val, (datetime.datetime, datetime.date, pd.Timestamp)): + return val.isoformat() + return str(val) + + +def _coerce_single_value( + val: Any, field: bigquery.SchemaField | None +) -> Any: + """Coerces a single non-repeated value (scalar or RECORD/STRUCT).""" + field_type = field.field_type if field is not None else "STRING" + if field_type.upper() in ("RECORD", "STRUCT"): + val = _unwrap_numpy_scalar(val) + if _is_missing(val): + return None + if isinstance(val, str): + text = val.strip() + if not text: + return None + try: + parsed = json.loads(text) + except ValueError: + return None + if not isinstance(parsed, dict): + return None + val = parsed + if not isinstance(val, dict): + return None + if field is not None and field.fields: + sub_by_lower = {f.name.lower(): f for f in field.fields} + coerced_record: dict[str, Any] = {} + for sub_key, sub_val in val.items(): + sub_field = sub_by_lower.get(str(sub_key).lower()) + target_key = ( + sub_field.name if sub_field is not None else str(sub_key) + ) + coerced_record[target_key] = _coerce_field_value( + sub_val, sub_field + ) + return coerced_record + return _sanitize_for_json(val) + return _coerce_scalar(val, field_type) + + +def _coerce_field_value( + raw_val: Any, field: bigquery.SchemaField | None +) -> Any: + """Coerces a field value according to its BigQuery SchemaField mode and type.""" + if field is not None and field.mode == "REPEATED": + if _is_missing(raw_val): + return [] + if isinstance(raw_val, (list, tuple)): + coerced_items = [ + _coerce_single_value(elem, field) + for elem in raw_val + if not _is_missing(elem) + ] + return [item for item in coerced_items if item is not None] + coerced_single = _coerce_single_value(raw_val, field) + return [coerced_single] if coerced_single is not None else [] + if _is_missing(raw_val): + return None + return _coerce_single_value(raw_val, field) + + +def _dataframe_to_json_rows( + chunk: pd.DataFrame, schema: Sequence[bigquery.SchemaField] +) -> list[dict[str, Any]]: + """Converts a DataFrame chunk into JSON-serializable dicts matching `schema`.""" + schema_by_lower = {field.name.lower(): field for field in schema} + rows: list[dict[str, Any]] = [] + for record in chunk.to_dict(orient="records"): + json_row: dict[str, Any] = {} + for col, raw_val in record.items(): + field = schema_by_lower.get(str(col).lower()) + target_col = field.name if field is not None else str(col) + json_row[target_col] = _coerce_field_value(raw_val, field) + rows.append(json_row) + return rows + + +def _iter_streaming_batches( + rows: Sequence[dict[str, Any]], + max_rows: int, + max_bytes: int = _MAX_STREAMING_REQUEST_BYTES, +) -> Iterator[list[dict[str, Any]]]: + """Splits JSON rows by both count and serialized byte size for insertAll.""" + batch: list[dict[str, Any]] = [] + batch_bytes = 0 + for row in rows: + row_bytes = len(json.dumps(row, allow_nan=False).encode("utf-8")) + 2 + if batch and ( + len(batch) >= max_rows or batch_bytes + row_bytes > max_bytes + ): + yield batch + batch = [] + batch_bytes = 0 + batch.append(row) + batch_bytes += row_bytes + if batch: + yield batch + + +def _has_schema_lag_error(errors: Sequence[dict[str, Any]]) -> bool: + """Returns True when any insert_rows_json row error indicates schema lag.""" + for row_err in errors: + for err in row_err.get("errors") or []: + message = str(err.get("message") or "").lower() + if "no such field" in message: + return True + return False + + +def _is_transient_insert_error(errors: Sequence[dict[str, Any]]) -> bool: + """Returns True when insert_rows_json row errors are transient or schema-lag.""" + if not errors: + return False + saw_retryable = False + for row_err in errors: + item_errors = row_err.get("errors") or [] + if not item_errors: + return False + for err in item_errors: + reason = str(err.get("reason") or "") + message = str(err.get("message") or "").lower() + if reason == "stopped": + # BigQuery aborts sibling rows with reason="stopped" whenever + # skip_invalid_rows=False and another row in the batch fails. + continue + if reason in _TRANSIENT_ROW_ERROR_REASONS: + saw_retryable = True + continue + # tabledata.insertAll caches schemas briefly after create_table or + # update_table and can report "no such field" for a few seconds. + if "no such field" in message: + saw_retryable = True + continue + return False + return saw_retryable + + class BigQueryReporter(Reporter): def __init__(self, reporting_config, job_id, run_time): super().__init__(reporting_config, job_id, run_time) @@ -98,100 +503,193 @@ def __init__(self, reporting_config, job_id, run_time): # Make chunk size configurable, defaulting to 500 self.chunk_size = int(reporting_config.get( "chunk_size") or _DEFAULT_CHUNK_SIZE) + self._dataset_ensured = False + self._tables: dict[str, bigquery.Table] = {} - def store(self, results, type: STORETYPE): - if results is None or results.empty: - logging.info(f"No results to store for {type}") + def _ensure_dataset(self) -> None: + if self._dataset_ensured: return - dataset = bigquery.Dataset(self.dataset_path) dataset.location = self.location - dataset = self.client.create_dataset( - dataset, exists_ok=True, timeout=30) - logging.info( - "Created dataset {}.{} for {}".format( - self.client.project, dataset.dataset_id, type - ) - ) - job_config = bigquery.LoadJobConfig() - job_config.autodetect = True - job_config.allow_quoted_newlines = True - job_config.schema_update_options = [ - bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION, - bigquery.SchemaUpdateOption.ALLOW_FIELD_RELAXATION, - ] - if type == STORETYPE.CONFIGS: - table = self.configs_table - elif type == STORETYPE.EVALS: - table = self.results_table - elif type == STORETYPE.SCORES: - table = self.scores_table - elif type == STORETYPE.SUMMARY: - table = self.summary_table - - # Chunk this to avoid BQ OOM - job_config.write_disposition = ( - bigquery.job.WriteDisposition.WRITE_APPEND # type: ignore - ) - for chunk in _split_dataframe(results, self.chunk_size): - # Workaround for pyarrow truncation error when inserting floats - # into INT64 columns. This happens if BQ autodetected a column as - # INT64 in a previous run. Casting to string avoids the client-side - # pyarrow crash and lets BQ handle the conversion. - if type in [STORETYPE.SUMMARY, STORETYPE.SCORES]: - chunk = chunk.copy() - # 1. Identify columns that BQ might have already locked - # as INT64 common ones are 'metric_score', etc. - int_cols = [ - "metric_score", "totalLatencyMs", - "total_results_count", "correct_results_count" - ] - for col in int_cols: - if col in chunk.columns: - # Round and cast to Int64 (nullable integer type) - chunk[col] = pd.to_numeric( - chunk[col], errors="coerce" - ).round().astype("Int64") - - # 2. For any remaining float columns that aren't in our - # "must-be-int" list, we cast to string to avoid pyarrow's - # strict truncation check if BQ thinks it's an int. - float_cols = chunk.select_dtypes( - include=["float64", "float32"] - ).columns - for col in float_cols: - if col not in int_cols: - # Convert to string and strip .0 if effectively int - chunk[col] = ( - chunk[col] - .astype(str) - .str.replace(r"\.0$", "", regex=True) - .replace("nan", None) - ) + self.client.create_dataset(dataset, exists_ok=True, timeout=30) + self._dataset_ensured = True - # Retry loop for BQ load job to handle transient rate limits - max_retries = 5 - initial_delay = 2 - backoff_factor = 2 + def _ensure_table_and_schema( + self, table_id: str, df: pd.DataFrame, store_type: STORETYPE + ) -> bigquery.Table: + """Ensures `table_id` exists and includes any new columns in `df`.""" + table_obj = self._tables.get(table_id) + if table_obj is None: + try: + table_obj = self.client.get_table(table_id) + except NotFound: + schema = _infer_schema(df, store_type) + table_obj = self.client.create_table( + bigquery.Table(table_id, schema=schema), exists_ok=True + ) + if not table_obj.schema: + table_obj = self.client.get_table(table_id) - for attempt in range(max_retries): + existing_lower = {field.name.lower() for field in table_obj.schema} + missing_cols = [ + col for col in df.columns if str(col).lower() not in existing_lower + ] + if missing_cols: + for attempt in range(_MAX_SCHEMA_UPDATE_RETRIES): + # Refresh before patching in case another worker added the field. + table_obj = self.client.get_table(table_id) + existing_lower = { + field.name.lower() for field in table_obj.schema + } + missing_cols = [ + col for col in df.columns + if str(col).lower() not in existing_lower + ] + if not missing_cols: + break + added_fields = _infer_schema(df[missing_cols], store_type) + table_obj.schema = list(table_obj.schema) + added_fields try: - job = self.client.load_table_from_dataframe( - chunk, table, job_config=job_config - ) - job.result() # Wait for the job to complete. - break # Success, exit retry loop - except (TooManyRequests, ServiceUnavailable) as e: - if attempt == max_retries - 1: - logging.error( - f"BQ write failed after {max_retries} attempts: {e}") + table_obj = self.client.update_table(table_obj, ["schema"]) + break + except (Conflict, PreconditionFailed): + if attempt == _MAX_SCHEMA_UPDATE_RETRIES - 1: raise - delay = initial_delay * (backoff_factor ** attempt) - logging.warning( - f"BQ write failed with transient error: {e}. " - f"Retrying in {delay}s... (Attempt {attempt + 1}/{max_retries})" + time.sleep(0.5 * (2 ** attempt)) + + self._tables[table_id] = table_obj + return table_obj + + def _resolve_table_id(self, store_type: STORETYPE) -> str: + if store_type == STORETYPE.CONFIGS: + return self.configs_table + if store_type == STORETYPE.EVALS: + return self.results_table + if store_type == STORETYPE.SCORES: + return self.scores_table + if store_type == STORETYPE.SUMMARY: + return self.summary_table + raise ValueError(f"Unsupported store type: {store_type}") + + def store(self, results, type: STORETYPE): + if results is None or results.empty: + logging.info(f"No results to store for {type}") + return + + self._ensure_dataset() + table = self._resolve_table_id(type) + + # Use BigQuery streaming inserts (tabledata.insertAll) instead of per-eval + # load jobs so high-concurrency CI suites never exhaust BigQuery's hard + # 1,500 load_job_per_table.long daily limit. + max_retries = 5 + initial_delay = 2 + backoff_factor = 2 + + table_obj = self._ensure_table_and_schema(table, results, type) + current_schema = list(table_obj.schema) + + for chunk in _split_dataframe(results, self.chunk_size): + chunk_schema = current_schema + json_rows = _dataframe_to_json_rows(chunk, chunk_schema) + chunk_offset = 0 + + for batch in _iter_streaming_batches(json_rows, self.chunk_size): + batch_len = len(batch) + pending_positions = list( + range(chunk_offset, chunk_offset + batch_len) + ) + chunk_offset += batch_len + if current_schema != chunk_schema: + pending_rows = _dataframe_to_json_rows( + chunk.iloc[pending_positions], current_schema ) - time.sleep(delay) + else: + pending_rows = batch + pending_row_ids = [str(uuid.uuid4()) for _ in pending_rows] + + for attempt in range(max_retries): + try: + errors = self.client.insert_rows_json( + table, pending_rows, row_ids=pending_row_ids + ) + if not errors: + break + if ( + attempt < max_retries - 1 + and _is_transient_insert_error(errors) + ): + failed_indices = sorted({ + err["index"] + for err in errors + if isinstance(err.get("index"), int) + and 0 <= err["index"] < len(pending_rows) + }) + if failed_indices: + pending_positions = [ + pending_positions[i] + for i in failed_indices + ] + pending_rows = [ + pending_rows[i] for i in failed_indices + ] + pending_row_ids = [ + pending_row_ids[i] for i in failed_indices + ] + if _has_schema_lag_error(errors): + self._tables.pop(table, None) + refreshed = self._ensure_table_and_schema( + table, results, type + ) + refreshed_schema = list(refreshed.schema) + if refreshed_schema != current_schema: + current_schema = refreshed_schema + pending_rows = _dataframe_to_json_rows( + chunk.iloc[pending_positions], + current_schema, + ) + delay = initial_delay * (backoff_factor ** attempt) + logging.warning( + "BQ streaming write had transient/schema-lag " + f"errors: {errors[:2]}. Retrying in {delay}s... " + f"(Attempt {attempt + 1}/{max_retries})" + ) + time.sleep(delay) + continue + raise RuntimeError( + f"BigQuery streaming insert into {table} failed: " + f"{errors[:3]}" + ) + except ( + TooManyRequests, + ServiceUnavailable, + InternalServerError, + NotFound, + ) as e: + if attempt == max_retries - 1: + logging.error( + f"BQ write failed after {max_retries} attempts: {e}" + ) + raise + if isinstance(e, NotFound): + self._tables.pop(table, None) + refreshed = self._ensure_table_and_schema( + table, results, type + ) + refreshed_schema = list(refreshed.schema) + if refreshed_schema != current_schema: + current_schema = refreshed_schema + pending_rows = _dataframe_to_json_rows( + chunk.iloc[pending_positions], + current_schema, + ) + delay = initial_delay * (backoff_factor ** attempt) + logging.warning( + f"BQ write failed with transient error: {e}. " + f"Retrying in {delay}s... " + f"(Attempt {attempt + 1}/{max_retries})" + ) + time.sleep(delay) def print_dashboard_links(self): report_date = self.run_time.strftime("%Y-%m-%d") diff --git a/evalbench/test/bqstore_test.py b/evalbench/test/bqstore_test.py new file mode 100644 index 00000000..b0a3847e --- /dev/null +++ b/evalbench/test/bqstore_test.py @@ -0,0 +1,520 @@ +import datetime +import json +import math +import unittest +from unittest.mock import MagicMock, patch + +from google.api_core.exceptions import ( + Conflict, + Forbidden, + NotFound, + PreconditionFailed, + ServiceUnavailable, +) +from google.cloud import bigquery +import numpy as np +import pandas as pd +from reporting.bqstore import ( + BigQueryReporter, + _coerce_scalar, + _dataframe_to_json_rows, + _infer_schema, + _iter_streaming_batches, +) +from reporting.report import STORETYPE + + +class TestBigQueryReporterStreaming(unittest.TestCase): + + def setUp(self): + self.run_time = datetime.datetime(2026, 11, 18, 3, 15, 0) + self.mock_client_patcher = patch("reporting.bqstore.bigquery.Client") + self.mock_bq_client_cls = self.mock_client_patcher.start() + self.mock_client = MagicMock() + self.mock_bq_client_cls.return_value = self.mock_client + + self.reporter = BigQueryReporter( + {"gcp_project_id": "test-project", "dataset_id": "evalbench"}, + job_id="job-123", + run_time=self.run_time, + ) + + def tearDown(self): + self.mock_client_patcher.stop() + + def test_store_uses_streaming_insert_rows_json_and_zero_load_jobs(self): + configs_schema = [ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("run_time", "DATETIME"), + bigquery.SchemaField("config", "STRING"), + bigquery.SchemaField("value", "STRING"), + ] + table_obj = bigquery.Table( + "test-project.evalbench.configs", schema=configs_schema + ) + self.mock_client.get_table.return_value = table_obj + self.mock_client.insert_rows_json.return_value = [] + + df = pd.DataFrame([ + { + "job_id": "job-123", + "run_time": self.run_time, + "config": "model_config.generator", + "value": "agy_cli", + }, + { + "job_id": "job-123", + "run_time": self.run_time, + "config": "experiment_config.empty_val", + "value": pd.NA, + }, + ]) + + self.reporter.store(df, STORETYPE.CONFIGS) + + self.mock_client.load_table_from_dataframe.assert_not_called() + self.mock_client.insert_rows_json.assert_called_once() + target_table, rows = self.mock_client.insert_rows_json.call_args[0] + self.assertEqual(target_table, "test-project.evalbench.configs") + self.assertEqual( + rows, + [ + { + "job_id": "job-123", + "run_time": "2026-11-18T03:15:00.000000", + "config": "model_config.generator", + "value": "agy_cli", + }, + { + "job_id": "job-123", + "run_time": "2026-11-18T03:15:00.000000", + "config": "experiment_config.empty_val", + "value": None, + }, + ], + ) + + def test_store_creates_missing_table_and_rounds_summary_integers(self): + self.mock_client.get_table.side_effect = NotFound("Table not found") + + def _create_table(table_arg, exists_ok=False): + return table_arg + + self.mock_client.create_table.side_effect = _create_table + self.mock_client.insert_rows_json.return_value = [] + + summary_df = pd.DataFrame([ + { + "metric_name": "goal_completion", + "metric_score": 100.0, + "correct_results_count": 1.67, + "total_results_count": 2, + "job_id": "job-123", + "run_time": self.run_time, + } + ]) + + self.reporter.store(summary_df, STORETYPE.SUMMARY) + + self.mock_client.create_table.assert_called_once() + created_table = self.mock_client.create_table.call_args[0][0] + schema_types = {f.name: f.field_type for f in created_table.schema} + self.assertEqual(schema_types["metric_score"], "INTEGER") + self.assertEqual(schema_types["correct_results_count"], "INTEGER") + self.assertEqual(schema_types["total_results_count"], "INTEGER") + self.assertEqual(schema_types["run_time"], "DATETIME") + + _, rows = self.mock_client.insert_rows_json.call_args[0] + self.assertEqual(rows[0]["correct_results_count"], 2) + self.assertEqual(rows[0]["metric_score"], 100) + self.assertEqual(rows[0]["total_results_count"], 2) + + def test_store_adds_missing_columns_to_existing_table_schema(self): + existing_table = bigquery.Table( + "test-project.evalbench.results", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("eval_id", "STRING"), + ], + ) + updated_table = bigquery.Table( + "test-project.evalbench.results", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("eval_id", "STRING"), + bigquery.SchemaField("accumulated_skills", "STRING"), + ], + ) + self.mock_client.get_table.side_effect = [existing_table, existing_table] + self.mock_client.update_table.return_value = updated_table + self.mock_client.insert_rows_json.return_value = [] + + results_df = pd.DataFrame([ + { + "job_id": "job-123", + "eval_id": "scenario-1", + "accumulated_skills": "['bigquery']", + } + ]) + + self.reporter.store(results_df, STORETYPE.EVALS) + + self.mock_client.update_table.assert_called_once() + patched_table = self.mock_client.update_table.call_args[0][0] + self.assertEqual( + [f.name for f in patched_table.schema], + ["job_id", "eval_id", "accumulated_skills"], + ) + _, rows = self.mock_client.insert_rows_json.call_args[0] + self.assertEqual(rows[0]["accumulated_skills"], "['bigquery']") + + def test_store_handles_repeated_fields_and_float_nan_safely(self): + scores_table = bigquery.Table( + "test-project.evalbench.scores", + schema=[ + bigquery.SchemaField("id", "STRING"), + bigquery.SchemaField("comparator", "STRING"), + bigquery.SchemaField("score", "FLOAT"), + bigquery.SchemaField("dialects", "STRING", mode="REPEATED"), + bigquery.SchemaField("job_id", "STRING"), + ], + ) + self.mock_client.get_table.return_value = scores_table + self.mock_client.insert_rows_json.return_value = [] + + scores_df = pd.DataFrame([ + { + "id": "s1", + "comparator": "goal_completion", + "score": 95.5, + "dialects": ["bigquery"], + "job_id": "job-123", + }, + { + "id": "s2", + "comparator": "goal_completion", + "score": math.nan, + "dialects": None, + "job_id": "job-123", + }, + ]) + + self.reporter.store(scores_df, STORETYPE.SCORES) + + _, rows = self.mock_client.insert_rows_json.call_args[0] + self.assertEqual(rows[0]["score"], 95.5) + self.assertEqual(rows[0]["dialects"], ["bigquery"]) + self.assertIsNone(rows[1]["score"]) + self.assertEqual(rows[1]["dialects"], []) + + @patch("reporting.bqstore.time.sleep") + def test_store_retries_transient_errors_and_schema_lag_but_not_forbidden( + self, mock_sleep + ): + table_obj = bigquery.Table( + "test-project.evalbench.configs", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("config", "STRING"), + ], + ) + self.mock_client.get_table.return_value = table_obj + self.mock_client.insert_rows_json.side_effect = [ + ServiceUnavailable("503 backend transient"), + [{ + "index": 0, + "errors": [{"reason": "invalid", "message": "no such field: config."}], + }], + [], + ] + + df = pd.DataFrame([{"job_id": "job-123", "config": "k"}]) + self.reporter.store(df, STORETYPE.CONFIGS) + self.assertEqual(self.mock_client.insert_rows_json.call_count, 3) + self.assertEqual(mock_sleep.call_count, 2) + + # Permanent 403 Forbidden (e.g. IAM deny or hard quotaExceeded) fails + # immediately without sleeping or retrying. + mock_sleep.reset_mock() + self.mock_client.insert_rows_json.side_effect = Forbidden( + "403 Access Denied" + ) + with self.assertRaises(Forbidden): + self.reporter.store(df, STORETYPE.CONFIGS) + mock_sleep.assert_not_called() + + @patch("reporting.bqstore.time.sleep") + def test_store_retries_multi_row_stopped_siblings_with_stable_row_ids( + self, mock_sleep + ): + table_obj = bigquery.Table( + "test-project.evalbench.configs", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("config", "STRING"), + ], + ) + self.mock_client.get_table.return_value = table_obj + self.mock_client.insert_rows_json.side_effect = [ + [ + { + "index": 0, + "errors": [{ + "reason": "invalid", + "message": "no such field: config.", + }], + }, + { + "index": 1, + "errors": [{"reason": "stopped", "message": ""}], + }, + ], + [], + ] + + df = pd.DataFrame([ + {"job_id": "job-123", "config": "k1"}, + {"job_id": "job-123", "config": "k2"}, + ]) + self.reporter.store(df, STORETYPE.CONFIGS) + + self.assertEqual(self.mock_client.insert_rows_json.call_count, 2) + self.assertEqual(mock_sleep.call_count, 1) + first_call = self.mock_client.insert_rows_json.call_args_list[0] + second_call = self.mock_client.insert_rows_json.call_args_list[1] + self.assertEqual(len(first_call.kwargs["row_ids"]), 2) + self.assertEqual( + first_call.kwargs["row_ids"], second_call.kwargs["row_ids"] + ) + + @patch("reporting.bqstore.time.sleep") + def test_store_partial_transient_error_retries_only_failed_rows_without_get_table( + self, mock_sleep + ): + table_obj = bigquery.Table( + "test-project.evalbench.configs", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("config", "STRING"), + ], + ) + self.mock_client.get_table.return_value = table_obj + self.mock_client.insert_rows_json.side_effect = [ + [{ + "index": 1, + "errors": [{"reason": "backendError", "message": "try again"}], + }], + [], + ] + + df = pd.DataFrame([ + {"job_id": "job-123", "config": "k0"}, + {"job_id": "job-123", "config": "k1"}, + {"job_id": "job-123", "config": "k2"}, + ]) + self.reporter.store(df, STORETYPE.CONFIGS) + + self.assertEqual(self.mock_client.insert_rows_json.call_count, 2) + # Pure transient backendError must not trigger extra get_table RPCs. + self.assertEqual(self.mock_client.get_table.call_count, 1) + first_call = self.mock_client.insert_rows_json.call_args_list[0] + second_call = self.mock_client.insert_rows_json.call_args_list[1] + self.assertEqual(len(second_call.args[1]), 1) + self.assertEqual(second_call.args[1][0]["config"], "k1") + self.assertEqual( + second_call.kwargs["row_ids"], [first_call.kwargs["row_ids"][1]] + ) + + @patch("reporting.bqstore.time.sleep") + def test_store_permanent_row_error_with_stopped_sibling_fails_fast( + self, mock_sleep + ): + table_obj = bigquery.Table( + "test-project.evalbench.configs", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("config", "STRING"), + ], + ) + self.mock_client.get_table.return_value = table_obj + self.mock_client.insert_rows_json.return_value = [ + { + "index": 0, + "errors": [{"reason": "invalid", "message": "bad integer"}], + }, + { + "index": 1, + "errors": [{"reason": "stopped", "message": ""}], + }, + ] + + df = pd.DataFrame([ + {"job_id": "job-123", "config": "k0"}, + {"job_id": "job-123", "config": "k1"}, + ]) + with self.assertRaises(RuntimeError): + self.reporter.store(df, STORETYPE.CONFIGS) + self.assertEqual(self.mock_client.insert_rows_json.call_count, 1) + mock_sleep.assert_not_called() + + @patch("reporting.bqstore.time.sleep") + def test_store_re_coerces_pending_rows_when_schema_changes_on_not_found( + self, mock_sleep + ): + old_table = bigquery.Table( + "test-project.evalbench.scores", + schema=[ + bigquery.SchemaField("id", "STRING"), + bigquery.SchemaField("score", "STRING"), + ], + ) + recreated_table = bigquery.Table( + "test-project.evalbench.scores", + schema=[ + bigquery.SchemaField("id", "STRING"), + bigquery.SchemaField("score", "INTEGER"), + ], + ) + self.mock_client.get_table.side_effect = [old_table, recreated_table] + self.mock_client.insert_rows_json.side_effect = [ + NotFound("Table was recreated"), + [], + ] + + df = pd.DataFrame([{"id": "s1", "score": 95.6}]) + self.reporter.store(df, STORETYPE.SCORES) + + self.assertEqual(self.mock_client.insert_rows_json.call_count, 2) + first_rows = self.mock_client.insert_rows_json.call_args_list[0].args[1] + second_rows = self.mock_client.insert_rows_json.call_args_list[1].args[1] + self.assertEqual(first_rows[0]["score"], "95.6") + self.assertEqual(second_rows[0]["score"], 96) + + @patch("reporting.bqstore.time.sleep") + def test_store_raises_not_found_after_exhausting_retries(self, mock_sleep): + table_obj = bigquery.Table( + "test-project.evalbench.configs", + schema=[bigquery.SchemaField("job_id", "STRING")], + ) + self.mock_client.get_table.return_value = table_obj + self.mock_client.insert_rows_json.side_effect = NotFound("missing") + + df = pd.DataFrame([{"job_id": "job-123"}]) + with self.assertRaises(NotFound): + self.reporter.store(df, STORETYPE.CONFIGS) + self.assertEqual(self.mock_client.insert_rows_json.call_count, 5) + self.assertEqual(mock_sleep.call_count, 4) + + @patch("reporting.bqstore.time.sleep") + def test_update_table_retries_multiple_concurrent_conflicts(self, mock_sleep): + final_table = bigquery.Table( + "test-project.evalbench.results", + schema=[ + bigquery.SchemaField("job_id", "STRING"), + bigquery.SchemaField("new_col", "STRING"), + ], + ) + # Return a fresh Table per get_table() call, matching BigQuery client + # behavior (update_table mutates table_obj.schema on the local object). + self.mock_client.get_table.side_effect = lambda _: bigquery.Table( + "test-project.evalbench.results", + schema=[bigquery.SchemaField("job_id", "STRING")], + ) + self.mock_client.update_table.side_effect = [ + PreconditionFailed("etag mismatch 1"), + Conflict("etag mismatch 2"), + final_table, + ] + self.mock_client.insert_rows_json.return_value = [] + + df = pd.DataFrame([{"job_id": "job-123", "new_col": "v"}]) + self.reporter.store(df, STORETYPE.EVALS) + + self.assertEqual(self.mock_client.update_table.call_count, 3) + self.assertEqual(mock_sleep.call_count, 2) + + def test_infer_schema_scans_all_values_chunks_and_numpy_scalars(self): + # Mixed object and repeated columns widen instead of locking on row 0; + # NumPy scalars are unwrapped to native BigQuery types. + df = pd.DataFrame({ + "mixed_obj": pd.Series([100, "N/A"], dtype="object"), + "int_and_float": pd.Series([1, 2.5], dtype="object"), + "mixed_repeated": pd.Series([[1], ["foo"]], dtype="object"), + "numpy_int": pd.Series([np.int64(7), None], dtype="object"), + }) + fields = {f.name: f for f in _infer_schema(df, STORETYPE.EVALS)} + self.assertEqual(fields["mixed_obj"].field_type, "STRING") + self.assertEqual(fields["int_and_float"].field_type, "FLOAT") + self.assertEqual(fields["mixed_repeated"].mode, "REPEATED") + self.assertEqual(fields["mixed_repeated"].field_type, "STRING") + self.assertEqual(fields["numpy_int"].field_type, "INTEGER") + + # All-null in chunk 0 but populated as REPEATED in chunk 1 must infer + # REPEATED STRING across the whole DataFrame before chunking. + self.reporter.chunk_size = 1 + self.mock_client.get_table.side_effect = NotFound("missing") + self.mock_client.create_table.side_effect = ( + lambda table_arg, exists_ok=False: table_arg + ) + self.mock_client.insert_rows_json.return_value = [] + multi_chunk_df = pd.DataFrame({ + "id": ["s1", "s2"], + "dialects": pd.Series([None, ["bigquery"]], dtype="object"), + }) + self.reporter.store(multi_chunk_df, STORETYPE.SCORES) + created_table = self.mock_client.create_table.call_args[0][0] + created_fields = {f.name: f for f in created_table.schema} + self.assertEqual(created_fields["dialects"].mode, "REPEATED") + self.assertEqual(created_fields["dialects"].field_type, "STRING") + + def test_coerce_scalar_and_record_edge_cases(self): + big_int = 9007199254740993 # 2**53 + 1 + self.assertEqual(_coerce_scalar(big_int, "INT64"), big_int) + self.assertEqual(_coerce_scalar(str(big_int), "INT64"), big_int) + self.assertEqual(_coerce_scalar(True, "INT64"), 1) + self.assertIsNone(_coerce_scalar("", "INT64")) + + self.assertIsNone(_coerce_scalar("", "DATETIME")) + self.assertIsNone(_coerce_scalar(" ", "TIMESTAMP")) + self.assertIsNone(_coerce_scalar("", "DATE")) + self.assertEqual( + _coerce_scalar("2026-11-18T03:15:00-07:00", "DATETIME"), + "2026-11-18T10:15:00.000000", + ) + self.assertEqual( + _coerce_scalar("2026-11-18T03:15:00Z", "DATETIME"), + "2026-11-18T03:15:00.000000", + ) + + json_str = _coerce_scalar({"a": float("nan"), "b": 1}, "JSON") + self.assertEqual(json.loads(json_str), {"a": None, "b": 1}) + + record_schema = [ + bigquery.SchemaField( + "meta", + "RECORD", + fields=[ + bigquery.SchemaField("count", "INT64"), + bigquery.SchemaField("tag", "STRING"), + ], + ) + ] + rows = _dataframe_to_json_rows( + pd.DataFrame([{"meta": {"count": 42, "tag": "ok"}}]), + record_schema, + ) + self.assertEqual(rows, [{"meta": {"count": 42, "tag": "ok"}}]) + + def test_iter_streaming_batches_splits_by_byte_size(self): + rows = [ + {"id": "1", "payload": "a" * 80}, + {"id": "2", "payload": "b" * 80}, + {"id": "3", "payload": "c" * 80}, + ] + batches = list(_iter_streaming_batches(rows, max_rows=10, max_bytes=150)) + self.assertEqual(len(batches), 3) + self.assertEqual([b[0]["id"] for b in batches], ["1", "2", "3"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/evalbench/test/eval_service_reporting_test.py b/evalbench/test/eval_service_reporting_test.py new file mode 100644 index 00000000..7e00300e --- /dev/null +++ b/evalbench/test/eval_service_reporting_test.py @@ -0,0 +1,205 @@ +import datetime +import json +import os +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +from eval_service import _process_results, EmptyEvalResultError + + +class BigQueryReporter(MagicMock): + """Fake BigQuery reporter named for _store_reports diagnostics.""" + + +class CsvReporter(MagicMock): + """Fake CSV reporter named for _store_reports diagnostics.""" + + +class TestEvalServiceProcessResultsReporting(unittest.TestCase): + + def _write_json_tempfile(self, payload) -> str: + fd, path = tempfile.mkstemp(suffix=".json") + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(payload, handle) + return path + + def test_reporter_failure_does_not_discard_eval_summary_or_block_other_reporters( + self, + ): + results_tf = self._write_json_tempfile([ + { + "id": "scenario-1", + "prompt_id": "scenario-1", + "generated_sql": "", + "generated_error": None, + } + ]) + scores_tf = self._write_json_tempfile([ + { + "id": "scenario-1", + "prompt_id": "scenario-1", + "comparator": "goal_completion", + "score": 100, + "generated_sql": "", + "generated_error": None, + } + ]) + + bq_reporter = BigQueryReporter() + bq_reporter.store.side_effect = RuntimeError( + "403 Quota exceeded: load_job_per_table.long" + ) + csv_reporter = CsvReporter() + + summary = _process_results( + reporters=[bq_reporter, csv_reporter], + job_id="job-abc", + run_time=datetime.datetime(2026, 11, 18, 3, 20, 0), + results_tf=results_tf, + scores_tf=scores_tf, + multi_trial_scores_tf=None, + config={ + "orchestrator": "geminicli", + "scorers": {"goal_completion": {}}, + }, + model_config={"generator": "agy_cli"}, + db_configs=[], + ) + + self.assertEqual(summary["total"], 1) + self.assertEqual(summary["scores"]["goal_completion"], 1) + self.assertEqual(csv_reporter.store.call_count, 4) + self.assertEqual(len(summary["reporting_errors"]), 1) + self.assertEqual( + summary["reporting_errors"][0]["reporter"], "BigQueryReporter" + ) + self.assertIn( + "load_job_per_table.long", summary["reporting_errors"][0]["error"] + ) + self.assertFalse(os.path.exists(results_tf)) + self.assertFalse(os.path.exists(scores_tf)) + + def test_empty_results_still_unlinks_tempfiles_and_raises(self): + results_tf = self._write_json_tempfile([]) + scores_tf = self._write_json_tempfile([]) + + with self.assertRaises(EmptyEvalResultError): + _process_results( + reporters=[], + job_id="job-empty", + run_time=datetime.datetime(2026, 11, 18, 3, 20, 0), + results_tf=results_tf, + scores_tf=scores_tf, + multi_trial_scores_tf=None, + config={"scorers": {}}, + model_config={}, + db_configs=[], + ) + + self.assertFalse(os.path.exists(results_tf)) + self.assertFalse(os.path.exists(scores_tf)) + + @patch("eval_service.get_reporters") + def test_reporter_init_failure_is_isolated_and_unlinks_tempfiles( + self, mock_get_reporters + ): + results_tf = self._write_json_tempfile([ + { + "id": "scenario-1", + "prompt_id": "scenario-1", + "generated_sql": "", + "generated_error": None, + } + ]) + scores_tf = self._write_json_tempfile([ + { + "id": "scenario-1", + "prompt_id": "scenario-1", + "comparator": "goal_completion", + "score": 100, + "generated_sql": "", + "generated_error": None, + } + ]) + + csv_reporter = CsvReporter() + + def _get_reporters_side_effect(reporting_cfg, job_id, run_time): + if "bigquery" in reporting_cfg: + raise ValueError("GCP project ID not set") + return [csv_reporter] + + mock_get_reporters.side_effect = _get_reporters_side_effect + + summary = _process_results( + reporters=None, + job_id="job-init-err", + run_time=datetime.datetime(2026, 11, 18, 3, 20, 0), + results_tf=results_tf, + scores_tf=scores_tf, + multi_trial_scores_tf=None, + config={ + "orchestrator": "geminicli", + "scorers": {"goal_completion": {}}, + "reporting": {"bigquery": {}, "csv": {"output_directory": "x"}}, + }, + model_config={"generator": "agy_cli"}, + db_configs=[], + fail_if_all_reporters_failed=True, + ) + + self.assertEqual(summary["total"], 1) + self.assertEqual(csv_reporter.store.call_count, 4) + self.assertEqual(len(summary["reporting_errors"]), 1) + self.assertEqual(summary["reporting_errors"][0]["reporter"], "bigquery") + self.assertIn("GCP project ID", summary["reporting_errors"][0]["error"]) + self.assertFalse(os.path.exists(results_tf)) + self.assertFalse(os.path.exists(scores_tf)) + + def test_all_reporters_failing_raises_when_caller_has_no_summary(self): + results_tf = self._write_json_tempfile([ + { + "id": "scenario-1", + "prompt_id": "scenario-1", + "generated_sql": "", + "generated_error": None, + } + ]) + scores_tf = self._write_json_tempfile([ + { + "id": "scenario-1", + "prompt_id": "scenario-1", + "comparator": "goal_completion", + "score": 100, + "generated_sql": "", + "generated_error": None, + } + ]) + + bq_reporter = BigQueryReporter() + bq_reporter.store.side_effect = RuntimeError("403 Forbidden") + + with self.assertRaises(RuntimeError): + _process_results( + reporters=[bq_reporter], + job_id="job-all-failed", + run_time=datetime.datetime(2026, 11, 18, 3, 20, 0), + results_tf=results_tf, + scores_tf=scores_tf, + multi_trial_scores_tf=None, + config={ + "orchestrator": "geminicli", + "scorers": {"goal_completion": {}}, + }, + model_config={"generator": "agy_cli"}, + db_configs=[], + fail_if_all_reporters_failed=True, + ) + + self.assertFalse(os.path.exists(results_tf)) + self.assertFalse(os.path.exists(scores_tf)) + + +if __name__ == "__main__": + unittest.main()