feat(evaluation) 2/15: test-set parsing and file_id normalisation

openrag/core/evaluation/__init__.py

@@ -0,0 +1,9 @@
+"""Pure evaluation logic: test-set parsing, promptfoo config, metric math."""
+
+from core.evaluation.identity import sanitize_file_id
+from core.evaluation.testset import parse_testset
+
+__all__ = [
+    "parse_testset",
+    "sanitize_file_id",
+]

openrag/core/evaluation/identity.py

@@ -0,0 +1,20 @@
+"""Filename to ``file_id`` normalisation, shared by the runner and the metrics.
+
+The indexing API accepts only ``[A-Za-z0-9._:-]`` in a ``file_id``, while a test
+set names its ground truth by real filename. Both sides go through this function
+so the two still match.
+"""
+
+from __future__ import annotations
+
+import re
+
+_DISALLOWED = re.compile(r"[^A-Za-z0-9._:-]")
+
+
+def sanitize_file_id(filename: str) -> str:
+    """Map a corpus filename onto an id the indexing API accepts."""
+    return _DISALLOWED.sub("_", filename)
+
+
+__all__ = ["sanitize_file_id"]

openrag/core/evaluation/testset.py

@@ -0,0 +1,149 @@
+"""Parsing and validation of the uploaded test-set CSV.
+
+The admin uploads a plain CSV; everything promptfoo needs is derived from it
+so operators never have to learn promptfoo's YAML. Validation is strict and
+reports the offending 1-based row numbers, because a malformed test set must
+fail at upload time rather than half-way through a run that has already
+indexed a corpus.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+
+from core.models.evaluation import EvalTestCase
+from core.utils.exceptions import ValidationError
+
+QUERY_COLUMN = "question"
+ANSWER_COLUMN = "expected_answer"
+FILE_IDS_COLUMN = "expected_file_ids"
+
+REQUIRED_COLUMNS = (QUERY_COLUMN, ANSWER_COLUMN)
+OPTIONAL_COLUMNS = (FILE_IDS_COLUMN,)
+
+#: ``expected_file_ids`` holds several ids in one cell, separated by this.
+FILE_ID_SEPARATOR = ";"
+
+#: Row numbers are reported to the user, so cap how many we list at once.
+_MAX_REPORTED_ERRORS = 10
+
+
+def _decode(raw: bytes) -> str:
+    """Decode the upload, tolerating a UTF-8 BOM from Excel exports."""
+    try:
+        return raw.decode("utf-8-sig")
+    except UnicodeDecodeError as exc:
+        raise ValidationError(
+            "Test set must be UTF-8 encoded CSV.",
+            code="EVAL_TESTSET_ENCODING",
+            status_code=400,
+        ) from exc
+
+
+def _split_file_ids(cell: str) -> tuple[str, ...]:
+    return tuple(part.strip() for part in cell.split(FILE_ID_SEPARATOR) if part.strip())
+
+
+def parse_testset(raw: bytes | str, *, max_rows: int) -> list[EvalTestCase]:
+    """Parse the CSV upload into test cases.
+
+    Args:
+        raw: Raw upload bytes, or already-decoded text.
+        max_rows: Reject test sets longer than this (``EVAL_MAX_TESTSET_ROWS``).
+            Every row costs a retrieval call plus a graded generation per run,
+            so the cap is a deployment concern rather than a fixed limit.
+
+    Returns:
+        One :class:`EvalTestCase` per data row, in file order.
+
+    Raises:
+        ValidationError: On a missing/duplicated header, an empty file, a row
+            with a blank required cell, or more than ``max_rows`` rows.
+    """
+    text = _decode(raw) if isinstance(raw, bytes) else raw
+    reader = csv.DictReader(io.StringIO(text))
+
+    if reader.fieldnames is None:
+        raise ValidationError(
+            "Test set is empty — expected a CSV header row.",
+            code="EVAL_TESTSET_EMPTY",
+            status_code=400,
+        )
+
+    headers = [(name or "").strip().lower() for name in reader.fieldnames]
+    missing = [column for column in REQUIRED_COLUMNS if column not in headers]
+    if missing:
+        raise ValidationError(
+            f"Test set is missing required column(s): {', '.join(missing)}. "
+            f"Expected header: {','.join((*REQUIRED_COLUMNS, *OPTIONAL_COLUMNS))}",
+            code="EVAL_TESTSET_COLUMNS",
+            status_code=400,
+        )
+    if len(set(headers)) != len(headers):
+        raise ValidationError(
+            "Test set has duplicate column names.",
+            code="EVAL_TESTSET_COLUMNS",
+            status_code=400,
+        )
+
+    # DictReader keys off the raw header spelling; normalise so " Question "
+    # and "question" both resolve.
+    key_for = {column: reader.fieldnames[headers.index(column)] for column in headers if column}
+
+    cases: list[EvalTestCase] = []
+    errors: list[str] = []
+    error_count = 0
+
+    for offset, row in enumerate(reader):
+        # +2: one for the header line, one to make it 1-based like a spreadsheet.
+        line = offset + 2
+        query = (row.get(key_for[QUERY_COLUMN]) or "").strip()
+        expected = (row.get(key_for[ANSWER_COLUMN]) or "").strip()
+
+        if not query and not expected:
+            continue  # blank trailing line
+        if not query or not expected:
+            column = QUERY_COLUMN if not query else ANSWER_COLUMN
+            error_count += 1
+            # Only the reported ones are retained; the rest are just counted.
+            if len(errors) < _MAX_REPORTED_ERRORS:
+                errors.append(f"row {line}: '{column}' is empty")
+            continue
+
+        file_ids_key = key_for.get(FILE_IDS_COLUMN)
+        # Reject on the row that would exceed the cap, rather than
+        # materialising every remaining row only to count them afterwards.
+        if len(cases) >= max_rows:
+            raise ValidationError(
+                f"Test set has more than {max_rows} rows.",
+                code="EVAL_TESTSET_TOO_LARGE",
+                status_code=400,
+            )
+        file_ids = _split_file_ids(row.get(file_ids_key) or "") if file_ids_key else ()
+        cases.append(EvalTestCase(query=query, expected_answer=expected, expected_file_ids=file_ids))
+
+    if errors:
+        suffix = f" (+{error_count - len(errors)} more)" if error_count > len(errors) else ""
+        raise ValidationError(
+            "Test set has invalid rows — " + "; ".join(errors) + suffix,
+            code="EVAL_TESTSET_ROWS",
+            status_code=400,
+        )
+    if not cases:
+        raise ValidationError(
+            "Test set contains no usable rows.",
+            code="EVAL_TESTSET_EMPTY",
+            status_code=400,
+        )
+    return cases
+
+
+__all__ = [
+    "ANSWER_COLUMN",
+    "FILE_IDS_COLUMN",
+    "FILE_ID_SEPARATOR",
+    "QUERY_COLUMN",
+    "REQUIRED_COLUMNS",
+    "parse_testset",
+]

tests/unit/core/evaluation/test_testset.py

@@ -0,0 +1,93 @@
+"""Tests for the evaluation test-set CSV parser."""
+
+from __future__ import annotations
+
+import pytest
+from core.evaluation.testset import parse_testset
+from core.utils.exceptions import ValidationError
+
+#: The cap is deployment config (EVAL_MAX_TESTSET_ROWS); these tests pin
+#: their own so they stay independent of the shipped default.
+MAX_ROWS = 500
+
+VALID = (
+    "question,expected_answer,expected_file_ids\n"
+    "What is the refund window?,30 days,policy.pdf\n"
+    "Who approves large spend?,The CFO,finance.pdf;approvals.pdf\n"
+)
+
+
+def test_parses_rows_and_splits_file_ids():
+    cases = parse_testset(VALID, max_rows=MAX_ROWS)
+    assert [case.query for case in cases] == [
+        "What is the refund window?",
+        "Who approves large spend?",
+    ]
+    assert cases[0].expected_file_ids == ("policy.pdf",)
+    assert cases[1].expected_file_ids == ("finance.pdf", "approvals.pdf")
+
+
+def test_expected_file_ids_column_is_optional():
+    """Answer-quality-only test sets are legitimate — the ranking metrics
+    just report them as skipped."""
+    cases = parse_testset("question,expected_answer\nWhy?,Because\n", max_rows=MAX_ROWS)
+    assert cases[0].expected_file_ids == ()
+    assert cases[0].has_ground_truth_sources is False
+
+
+def test_accepts_bytes_with_utf8_bom():
+    """Excel writes a BOM; decoding with plain utf-8 would corrupt the first
+    header and make the required-column check fail."""
+    cases = parse_testset(VALID.encode("utf-8-sig"), max_rows=MAX_ROWS)
+    assert len(cases) == 2
+
+
+def test_header_case_and_whitespace_are_normalised():
+    cases = parse_testset(" Question , Expected_Answer \nWhy?,Because\n", max_rows=MAX_ROWS)
+    assert cases[0].query == "Why?"
+    assert cases[0].expected_answer == "Because"
+
+
+def test_blank_trailing_lines_are_ignored():
+    cases = parse_testset(VALID + ",\n\n", max_rows=MAX_ROWS)
+    assert len(cases) == 2
+
+
+def test_missing_required_column_is_rejected():
+    with pytest.raises(ValidationError) as excinfo:
+        parse_testset("question,answer\nWhy?,Because\n", max_rows=MAX_ROWS)
+    assert "expected_answer" in str(excinfo.value)
+
+
+def test_empty_required_cell_reports_the_spreadsheet_row_number():
+    """Row 3 = second data row, counting the header as row 1."""
+    with pytest.raises(ValidationError) as excinfo:
+        parse_testset("question,expected_answer\nWhy?,Because\n,Orphan answer\n", max_rows=MAX_ROWS)
+    assert "row 3" in str(excinfo.value)
+
+
+def test_empty_file_is_rejected():
+    with pytest.raises(ValidationError):
+        parse_testset("", max_rows=MAX_ROWS)
+
+
+def test_header_only_file_is_rejected():
+    with pytest.raises(ValidationError):
+        parse_testset("question,expected_answer\n", max_rows=MAX_ROWS)
+
+
+def test_duplicate_columns_are_rejected():
+    with pytest.raises(ValidationError):
+        parse_testset("question,question,expected_answer\na,b,c\n", max_rows=MAX_ROWS)
+
+
+def test_row_cap_is_enforced():
+    rows = "".join(f"q{i},a{i}\n" for i in range(MAX_ROWS + 1))
+    with pytest.raises(ValidationError) as excinfo:
+        parse_testset("question,expected_answer\n" + rows, max_rows=MAX_ROWS)
+    assert str(MAX_ROWS) in str(excinfo.value)
+
+
+def test_invalid_encoding_is_rejected():
+    with pytest.raises(ValidationError):
+        parse_testset(b"\xff\xfe\x00question", max_rows=MAX_ROWS)