From 8e008f2e6684b82a7ac6480f0a231666ede8c5a5 Mon Sep 17 00:00:00 2001 From: EnjoyBacon7 <59032058+EnjoyBacon7@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:29:40 +0000 Subject: [PATCH] feat(evaluation): test-set parsing and file_id normalisation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `parse_testset` reads the admin-uploaded CSV (`question,expected_answer,expected_file_ids`) into `EvalTestCase` rows and rejects a malformed file at upload time rather than after a run has already spent minutes indexing a corpus. `sanitize_file_id` maps a corpus filename onto an id the indexing API will accept — it allows only `[A-Za-z0-9._:-]`, so a file cannot be uploaded under a raw human filename. It lives on its own because both sides of the ground-truth comparison have to apply it: the uploader (part 9) and the metrics (part 4). A test set naming `A B.pdf` still matches the stored `A_B.pdf`. --- openrag/core/evaluation/__init__.py | 9 ++ openrag/core/evaluation/identity.py | 20 +++ openrag/core/evaluation/testset.py | 149 +++++++++++++++++++++ tests/unit/core/evaluation/test_testset.py | 93 +++++++++++++ 4 files changed, 271 insertions(+) create mode 100644 openrag/core/evaluation/__init__.py create mode 100644 openrag/core/evaluation/identity.py create mode 100644 openrag/core/evaluation/testset.py create mode 100644 tests/unit/core/evaluation/test_testset.py diff --git a/openrag/core/evaluation/__init__.py b/openrag/core/evaluation/__init__.py new file mode 100644 index 000000000..ad6dc313d --- /dev/null +++ b/openrag/core/evaluation/__init__.py @@ -0,0 +1,9 @@ +"""Pure evaluation logic: test-set parsing, promptfoo config, metric math.""" + +from core.evaluation.identity import sanitize_file_id +from core.evaluation.testset import parse_testset + +__all__ = [ + "parse_testset", + "sanitize_file_id", +] diff --git a/openrag/core/evaluation/identity.py b/openrag/core/evaluation/identity.py new file mode 100644 index 000000000..a3816eb2c --- /dev/null +++ b/openrag/core/evaluation/identity.py @@ -0,0 +1,20 @@ +"""Filename to ``file_id`` normalisation, shared by the runner and the metrics. + +The indexing API accepts only ``[A-Za-z0-9._:-]`` in a ``file_id``, while a test +set names its ground truth by real filename. Both sides go through this function +so the two still match. +""" + +from __future__ import annotations + +import re + +_DISALLOWED = re.compile(r"[^A-Za-z0-9._:-]") + + +def sanitize_file_id(filename: str) -> str: + """Map a corpus filename onto an id the indexing API accepts.""" + return _DISALLOWED.sub("_", filename) + + +__all__ = ["sanitize_file_id"] diff --git a/openrag/core/evaluation/testset.py b/openrag/core/evaluation/testset.py new file mode 100644 index 000000000..3dbd700eb --- /dev/null +++ b/openrag/core/evaluation/testset.py @@ -0,0 +1,149 @@ +"""Parsing and validation of the uploaded test-set CSV. + +The admin uploads a plain CSV; everything promptfoo needs is derived from it +so operators never have to learn promptfoo's YAML. Validation is strict and +reports the offending 1-based row numbers, because a malformed test set must +fail at upload time rather than half-way through a run that has already +indexed a corpus. +""" + +from __future__ import annotations + +import csv +import io + +from core.models.evaluation import EvalTestCase +from core.utils.exceptions import ValidationError + +QUERY_COLUMN = "question" +ANSWER_COLUMN = "expected_answer" +FILE_IDS_COLUMN = "expected_file_ids" + +REQUIRED_COLUMNS = (QUERY_COLUMN, ANSWER_COLUMN) +OPTIONAL_COLUMNS = (FILE_IDS_COLUMN,) + +#: ``expected_file_ids`` holds several ids in one cell, separated by this. +FILE_ID_SEPARATOR = ";" + +#: Row numbers are reported to the user, so cap how many we list at once. +_MAX_REPORTED_ERRORS = 10 + + +def _decode(raw: bytes) -> str: + """Decode the upload, tolerating a UTF-8 BOM from Excel exports.""" + try: + return raw.decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise ValidationError( + "Test set must be UTF-8 encoded CSV.", + code="EVAL_TESTSET_ENCODING", + status_code=400, + ) from exc + + +def _split_file_ids(cell: str) -> tuple[str, ...]: + return tuple(part.strip() for part in cell.split(FILE_ID_SEPARATOR) if part.strip()) + + +def parse_testset(raw: bytes | str, *, max_rows: int) -> list[EvalTestCase]: + """Parse the CSV upload into test cases. + + Args: + raw: Raw upload bytes, or already-decoded text. + max_rows: Reject test sets longer than this (``EVAL_MAX_TESTSET_ROWS``). + Every row costs a retrieval call plus a graded generation per run, + so the cap is a deployment concern rather than a fixed limit. + + Returns: + One :class:`EvalTestCase` per data row, in file order. + + Raises: + ValidationError: On a missing/duplicated header, an empty file, a row + with a blank required cell, or more than ``max_rows`` rows. + """ + text = _decode(raw) if isinstance(raw, bytes) else raw + reader = csv.DictReader(io.StringIO(text)) + + if reader.fieldnames is None: + raise ValidationError( + "Test set is empty — expected a CSV header row.", + code="EVAL_TESTSET_EMPTY", + status_code=400, + ) + + headers = [(name or "").strip().lower() for name in reader.fieldnames] + missing = [column for column in REQUIRED_COLUMNS if column not in headers] + if missing: + raise ValidationError( + f"Test set is missing required column(s): {', '.join(missing)}. " + f"Expected header: {','.join((*REQUIRED_COLUMNS, *OPTIONAL_COLUMNS))}", + code="EVAL_TESTSET_COLUMNS", + status_code=400, + ) + if len(set(headers)) != len(headers): + raise ValidationError( + "Test set has duplicate column names.", + code="EVAL_TESTSET_COLUMNS", + status_code=400, + ) + + # DictReader keys off the raw header spelling; normalise so " Question " + # and "question" both resolve. + key_for = {column: reader.fieldnames[headers.index(column)] for column in headers if column} + + cases: list[EvalTestCase] = [] + errors: list[str] = [] + error_count = 0 + + for offset, row in enumerate(reader): + # +2: one for the header line, one to make it 1-based like a spreadsheet. + line = offset + 2 + query = (row.get(key_for[QUERY_COLUMN]) or "").strip() + expected = (row.get(key_for[ANSWER_COLUMN]) or "").strip() + + if not query and not expected: + continue # blank trailing line + if not query or not expected: + column = QUERY_COLUMN if not query else ANSWER_COLUMN + error_count += 1 + # Only the reported ones are retained; the rest are just counted. + if len(errors) < _MAX_REPORTED_ERRORS: + errors.append(f"row {line}: '{column}' is empty") + continue + + file_ids_key = key_for.get(FILE_IDS_COLUMN) + # Reject on the row that would exceed the cap, rather than + # materialising every remaining row only to count them afterwards. + if len(cases) >= max_rows: + raise ValidationError( + f"Test set has more than {max_rows} rows.", + code="EVAL_TESTSET_TOO_LARGE", + status_code=400, + ) + file_ids = _split_file_ids(row.get(file_ids_key) or "") if file_ids_key else () + cases.append(EvalTestCase(query=query, expected_answer=expected, expected_file_ids=file_ids)) + + if errors: + suffix = f" (+{error_count - len(errors)} more)" if error_count > len(errors) else "" + raise ValidationError( + "Test set has invalid rows — " + "; ".join(errors) + suffix, + code="EVAL_TESTSET_ROWS", + status_code=400, + ) + if not cases: + raise ValidationError( + "Test set contains no usable rows.", + code="EVAL_TESTSET_EMPTY", + status_code=400, + ) + return cases + + +__all__ = [ + "ANSWER_COLUMN", + "FILE_IDS_COLUMN", + "FILE_ID_SEPARATOR", + "QUERY_COLUMN", + "REQUIRED_COLUMNS", + "parse_testset", +] diff --git a/tests/unit/core/evaluation/test_testset.py b/tests/unit/core/evaluation/test_testset.py new file mode 100644 index 000000000..5caeedad9 --- /dev/null +++ b/tests/unit/core/evaluation/test_testset.py @@ -0,0 +1,93 @@ +"""Tests for the evaluation test-set CSV parser.""" + +from __future__ import annotations + +import pytest +from core.evaluation.testset import parse_testset +from core.utils.exceptions import ValidationError + +#: The cap is deployment config (EVAL_MAX_TESTSET_ROWS); these tests pin +#: their own so they stay independent of the shipped default. +MAX_ROWS = 500 + +VALID = ( + "question,expected_answer,expected_file_ids\n" + "What is the refund window?,30 days,policy.pdf\n" + "Who approves large spend?,The CFO,finance.pdf;approvals.pdf\n" +) + + +def test_parses_rows_and_splits_file_ids(): + cases = parse_testset(VALID, max_rows=MAX_ROWS) + assert [case.query for case in cases] == [ + "What is the refund window?", + "Who approves large spend?", + ] + assert cases[0].expected_file_ids == ("policy.pdf",) + assert cases[1].expected_file_ids == ("finance.pdf", "approvals.pdf") + + +def test_expected_file_ids_column_is_optional(): + """Answer-quality-only test sets are legitimate — the ranking metrics + just report them as skipped.""" + cases = parse_testset("question,expected_answer\nWhy?,Because\n", max_rows=MAX_ROWS) + assert cases[0].expected_file_ids == () + assert cases[0].has_ground_truth_sources is False + + +def test_accepts_bytes_with_utf8_bom(): + """Excel writes a BOM; decoding with plain utf-8 would corrupt the first + header and make the required-column check fail.""" + cases = parse_testset(VALID.encode("utf-8-sig"), max_rows=MAX_ROWS) + assert len(cases) == 2 + + +def test_header_case_and_whitespace_are_normalised(): + cases = parse_testset(" Question , Expected_Answer \nWhy?,Because\n", max_rows=MAX_ROWS) + assert cases[0].query == "Why?" + assert cases[0].expected_answer == "Because" + + +def test_blank_trailing_lines_are_ignored(): + cases = parse_testset(VALID + ",\n\n", max_rows=MAX_ROWS) + assert len(cases) == 2 + + +def test_missing_required_column_is_rejected(): + with pytest.raises(ValidationError) as excinfo: + parse_testset("question,answer\nWhy?,Because\n", max_rows=MAX_ROWS) + assert "expected_answer" in str(excinfo.value) + + +def test_empty_required_cell_reports_the_spreadsheet_row_number(): + """Row 3 = second data row, counting the header as row 1.""" + with pytest.raises(ValidationError) as excinfo: + parse_testset("question,expected_answer\nWhy?,Because\n,Orphan answer\n", max_rows=MAX_ROWS) + assert "row 3" in str(excinfo.value) + + +def test_empty_file_is_rejected(): + with pytest.raises(ValidationError): + parse_testset("", max_rows=MAX_ROWS) + + +def test_header_only_file_is_rejected(): + with pytest.raises(ValidationError): + parse_testset("question,expected_answer\n", max_rows=MAX_ROWS) + + +def test_duplicate_columns_are_rejected(): + with pytest.raises(ValidationError): + parse_testset("question,question,expected_answer\na,b,c\n", max_rows=MAX_ROWS) + + +def test_row_cap_is_enforced(): + rows = "".join(f"q{i},a{i}\n" for i in range(MAX_ROWS + 1)) + with pytest.raises(ValidationError) as excinfo: + parse_testset("question,expected_answer\n" + rows, max_rows=MAX_ROWS) + assert str(MAX_ROWS) in str(excinfo.value) + + +def test_invalid_encoding_is_rejected(): + with pytest.raises(ValidationError): + parse_testset(b"\xff\xfe\x00question", max_rows=MAX_ROWS)