diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80acc29..33fde7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,8 +30,14 @@ jobs: - name: Install locked dependencies run: uv sync --frozen - - name: Run tests - run: uv run python -m unittest discover -s tests -v + - name: Run tests with coverage + run: | + uv run --with coverage==7.13.4 coverage run -m unittest discover -s tests -v + uv run --with coverage==7.13.4 coverage report + + - name: Static type check + if: matrix.python-version == '3.11' + run: uv run --with mypy==1.18.2 mypy - name: Run Ruff if: matrix.python-version == '3.11' @@ -39,8 +45,33 @@ jobs: uvx --from ruff==0.16.1 ruff check . uvx --from ruff==0.16.1 ruff format --check . + - name: Audit runtime dependencies + if: matrix.python-version == '3.11' + run: | + uv export --frozen --no-dev --format requirements-txt \ + --no-emit-project -o /tmp/runtime-requirements.txt + uvx --from pip-audit==2.10.1 pip-audit \ + -r /tmp/runtime-requirements.txt \ + --progress-spinner off \ + --ignore-vuln CVE-2026-45829 + - name: Build and smoke-test package if: matrix.python-version == '3.11' run: | uv build + python - <<'PY' + import glob + import zipfile + + wheel, = glob.glob("dist/local_ai_agent-*.whl") + with zipfile.ZipFile(wheel) as archive: + names = set(archive.namelist()) + required = { + "local_ai_agent/data/rag_cases.json", + "local_ai_agent/data/realistic_restaurant_reviews.csv", + } + missing = required - names + if missing: + raise SystemExit(f"wheel is missing packaged assets: {sorted(missing)}") + PY uvx --from dist/local_ai_agent-*.whl local-ai-agent --help diff --git a/.gitignore b/.gitignore index e64a7f2..05b09e8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ build/ dist/ wheels/ *.egg-info +.coverage # Virtual environments .venv diff --git a/README.md b/README.md index 921b77e..73177a5 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,16 @@ By default, runtime review data, embeddings, and prompts are processed by embedd - **Safe offline state:** analytics still load when Ollama is unavailable, while the app shows exact setup commands instead of crashing. - **Adaptive CSV upload:** automatically detect common headers, manually map unfamiliar names, and isolate every dataset in content-addressed Chroma storage. - **Reconciled indexing:** content-derived IDs survive reordering; additions, changed records, and deletions are synchronized safely. -- **Measured RAG:** a four-case evaluation set reports retrieval recall, citation correctness, reference-grounded faithfulness, and abstention accuracy. +- **Measured RAG:** a versioned 30-case evaluation across nine answerable domains plus abstention compares semantic retrieval with a deterministic BM25 keyword baseline, then measures citation validity, a transparent reference-term support proxy, expected-action accuracy, answer success, and abstention recall. - **Installable CLI:** the packaged `local-ai-agent` command exposes status, ask, chat, and evaluate workflows. - **Local execution by default:** Ollama handles embeddings and answer generation unless a remote host is explicitly configured. See the [architecture diagram and boundary notes](docs/architecture.md). +## Live workflow demo + +[Watch the silent 48-second dashboard demo](docs/assets/local-ai-agent-v0.2-demo.mp4). It shows the real local question trigger, indexing state, generated answer, validated citations, and an expanded source record using the bundled dataset. + ## Requirements - [Python 3.11 or newer](https://www.python.org/downloads/) @@ -127,6 +131,21 @@ Run the measured RAG evaluation against the configured local models: uv run local-ai-agent evaluate ``` +Write a reproducible machine-readable report and a Markdown summary: + +```bash +uv run local-ai-agent evaluate --report-dir evaluation/results/my-run +``` + +The versioned case manifest is tied to the dataset SHA-256 and uses immutable +content-derived source IDs for gold relevance. The report records model tags +and immutable Ollama digests, dataset and case-set hashes, retrieval limit, +runtime versions, per-case RAG and BM25 rankings, and aggregate metrics. +Retrieval quality is reported as recall@k, hit rate@k, and MRR@k for both +semantic search and the BM25 baseline. Relevance judgments are known-positive, +not exhaustive. The generated report is evidence for this fixed benchmark +configuration, not a claim about general RAG performance. + The original source-tree command remains available for development: ```bash @@ -174,18 +193,32 @@ The suite covers: - adaptive rating, date, and categorical filtering; - Ollama health states; - evidence-alias-to-source citation validation and model abstention; -- all four RAG evaluation metrics; +- the 30-case evaluation set, BM25 baseline, retrieval metrics, and decomposed + answer, citation, support-proxy, and abstention metrics; +- evaluation-report serialization and credential-safe provenance; - deterministic upload storage; - Streamlit rendering without Ollama. -Run lint and formatting checks: +Run the same quality gates used by CI: ```bash +uv run --with coverage==7.13.4 coverage run -m unittest discover -s tests -v +uv run --with coverage==7.13.4 coverage report +uv run --with mypy==1.18.2 mypy uvx --from ruff==0.16.1 ruff check . uvx --from ruff==0.16.1 ruff format --check . +uv export --frozen --no-dev --format requirements-txt \ + --no-emit-project -o /tmp/runtime-requirements.txt +uvx --from pip-audit==2.10.1 pip-audit \ + -r /tmp/runtime-requirements.txt --progress-spinner off \ + --ignore-vuln CVE-2026-45829 ``` -GitHub Actions runs the tests on Python 3.11 and Python 3.14. +GitHub Actions tests Python 3.11 and Python 3.14 and reports coverage for all +six core Python modules, including `evaluation.py`. On Python 3.11 it also runs +static typing, Ruff, formatting, the runtime-dependency audit, package build, +and an installed-CLI smoke test. Coverage remains visible without an arbitrary +pass threshold. ## Project layout @@ -202,6 +235,8 @@ GitHub Actions runs the tests on Python 3.11 and Python 3.14. │ └── data/rag_cases.json # Curated RAG evaluation set ├── docs/architecture.md # Architecture and privacy boundaries ├── docs/architecture.svg # Editable architecture diagram +├── evaluation/results/ # Versioned JSON and Markdown benchmark reports +├── SECURITY.md # Reporting policy and scoped risk acceptance ├── tests/ # Unit, integration, evaluation, and dashboard tests ├── pyproject.toml └── uv.lock @@ -209,7 +244,7 @@ GitHub Actions runs the tests on Python 3.11 and Python 3.14. ## Dependency audit -ChromaDB is declared directly at the newest PyPI release verified during this update (`1.5.9`). `pip-audit` still reports `PYSEC-2026-311` / `CVE-2026-45829`, and the advisory currently lists no fixed release. It concerns an unauthenticated Chroma HTTP server endpoint that accepts `trust_remote_code`; this application uses embedded Chroma and does not start that server. The finding is documented rather than misrepresented as resolved. +ChromaDB is declared directly at the newest PyPI release verified during this update (`1.5.9`). `pip-audit` still reports `PYSEC-2026-311` / `CVE-2026-45829`, and the advisory currently lists no fixed release. It concerns an unauthenticated Chroma HTTP server endpoint that accepts `trust_remote_code`; this application uses embedded Chroma and does not start that server. CI ignores only this finding and fails on every other known vulnerability. The scope and removal conditions are documented in [SECURITY.md](SECURITY.md). ## Current limitations @@ -218,7 +253,7 @@ ChromaDB is declared directly at the newest PyPI release verified during this up - Rating, date, sentiment, restaurant, and country filters are applied after semantic ranking. This is suitable for small local datasets but not optimized for very large collections. - Automatic mapping is conservative. Unfamiliar or ambiguous headers require confirmation in the dashboard. - Sentiment labels are displayed and filtered as supplied. The application does not infer sentiment when the dataset lacks a sentiment column. -- The faithfulness score is a transparent reference-term check against cited source text, not an LLM judge or proof that every possible claim is correct. +- The reference-term support proxy checks expected terms against answers and cited source text; it is not an LLM judge or proof that every claim is correct. - Topic modeling, hybrid keyword retrieval, and reranking are not implemented. - The application does not provide authentication or multi-user isolation and should not be exposed directly as a shared public service. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md new file mode 100644 index 0000000..f7cc51c --- /dev/null +++ b/RELEASE_NOTES.md @@ -0,0 +1,69 @@ +# Local AI Agent v0.2.0 + +## What changed + +- Expanded the bundled RAG evaluation from 4 to 30 cases: 25 answerable cases across nine domains and 5 abstention cases. +- Bound evaluation relevance judgments to immutable content-derived source IDs and validated the dataset hash and row count. +- Added a deterministic BM25 keyword baseline alongside semantic retrieval. +- Added versioned JSON and Markdown reports with exact model digests, runtime versions, dataset/case hashes, per-case outcomes, rankings, and latency. +- Split end-to-end behavior into explicit metrics: expected-action accuracy, answer success rate, abstention recall, citation validity, and a lexical reference-term support proxy. +- Added stricter manifest/schema validation, duplicate-observation rejection, CLI/report tests, typing, coverage reporting, dependency auditing, package-asset checks, and installed-wheel smoke testing. +- Added a scoped security policy for the embedded-Chroma deployment boundary. +- Added a silent live-dashboard demo showing the real question trigger, answer, validated citations, and expanded evidence. + +## Verified local benchmark + +Runtime: + +- Ollama `0.32.5` +- `llama3.2:latest` digest `a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72` +- `mxbai-embed-large:latest` digest `468836162de7f81e041c43663fedbbba921dcea9b9fefea135685a39b2d83dd8` +- 123 bundled restaurant reviews +- 30 evaluation cases; retrieval limit 5 + +Retrieval results on the 25 answerable cases: + +| Metric | Semantic retrieval | BM25 baseline | +| --- | ---: | ---: | +| Recall@5 | 0.913 | 0.770 | +| Hit rate@5 | 1.000 | 0.880 | +| MRR@5 | 0.960 | 0.753 | + +Model-dependent RAG results: + +| Metric | Result | +| --- | ---: | +| Citation validity | 0.560 | +| Reference-term support proxy | 0.520 | +| Answer success rate | 0.560 | +| Abstention recall | 1.000 | +| Expected-action accuracy | 0.633 | +| Mean latency | 4.359 s | +| Median latency | 2.006 s | + +Outcomes: 14 answered, 13 model abstentions, and 3 citation-validation rejections. + +Authoritative artifacts: + +- `evaluation/results/v0.2.0-ollama-0.32.5/evaluation-report.json` +- `evaluation/results/v0.2.0-ollama-0.32.5/README.md` +- `evaluation/results/v0.2.0-ollama-0.32.5/run.log` +- `docs/assets/local-ai-agent-v0.2-demo.mp4` + +Report SHA-256: `5700748c2abe1b5446a850c08291c41e8bbab6aff663cd5ac783513880d02718` + +## Verification + +- 61 tests passed. +- Ruff check and format check passed. +- Mypy passed for six core modules. +- Coverage reports all six core modules, including `evaluation.py`: 84% overall. No arbitrary pass threshold is imposed. +- Wheel and source distribution passed Twine checks. +- Wheel contains both bundled evaluation/data assets and the installed CLI smoke test passed. +- Runtime dependency audit reported no known vulnerabilities other than one explicitly ignored advisory: `CVE-2026-45829` / `PYSEC-2026-311`. + +## Evidence boundaries + +These results apply only to the checked-in dataset, case manifest, model digests, runtime, retrieval limit, and machine recorded in the report. Known-positive relevance judgments are not exhaustive, so the semantic-versus-BM25 comparison is not evidence of general retrieval superiority. Citation validity checks whether citations resolve to retrieved evidence; the reference-term metric is a lexical support proxy, not claim-level factual-faithfulness evaluation. + +The Chroma advisory exception is accepted only for embedded, process-local Chroma. It must be removed if a fixed compatible release becomes available or the application exposes/uses a remote Chroma server. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..b5d5981 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,41 @@ +# Security policy + +## Supported version + +Security fixes target the latest release and the `main` branch. + +## Reporting a vulnerability + +Do not open a public issue for a suspected vulnerability. Use GitHub's private +security-advisory flow for this repository so the report can be investigated +before details are disclosed. + +Do not include API keys, model credentials, private review data, or other +secrets in a report. + +## Runtime boundary + +Local AI Agent runs Chroma in embedded, process-local mode. It does not start +or expose a Chroma HTTP server. Ollama is expected to bind to localhost unless +the operator deliberately configures a different endpoint. + +Treat imported CSV files and model output as untrusted. The application parses +CSV data, validates its schema, escapes it into structured documents, validates +model citations against retrieved source IDs, and does not execute model or CSV +content as code. + +## Accepted dependency advisory + +CI ignores **CVE-2026-45829 / PYSEC-2026-311** for `chromadb` because the +advisory applies to Chroma's unauthenticated HTTP collection endpoint with +`trust_remote_code=true`. This application uses embedded Chroma and never +exposes that endpoint. As of 2026-07-31, pip-audit reports no fixed release. + +This is a scoped risk acceptance, not a claim that the dependency is generally +safe. The exception must be removed when either: + +1. a fixed compatible Chroma release becomes available; or +2. the application begins exposing a Chroma server or accepting remote Chroma + endpoints. + +The dependency audit still fails CI for every other known vulnerability. \ No newline at end of file diff --git a/docs/assets/local-ai-agent-v0.2-demo.mp4 b/docs/assets/local-ai-agent-v0.2-demo.mp4 new file mode 100644 index 0000000..fe43edb Binary files /dev/null and b/docs/assets/local-ai-agent-v0.2-demo.mp4 differ diff --git a/evaluation.py b/evaluation.py index 5470d79..3fec482 100644 --- a/evaluation.py +++ b/evaluation.py @@ -1,8 +1,16 @@ +import csv import json -from collections.abc import Iterable +import math +import re +import unicodedata +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass +from datetime import UTC, datetime from importlib.resources import files from pathlib import Path +from statistics import median +from time import perf_counter from typing import Any from agent import answer_question @@ -25,6 +33,10 @@ class EvaluationCase: relevant_titles: tuple[str, ...] reference_facts: tuple[ReferenceFact, ...] should_abstain: bool = False + category: str = "uncategorized" + gold_source_ids: tuple[str, ...] = () + split: str = "test" + difficulty: str = "unspecified" @dataclass(frozen=True) @@ -36,44 +48,266 @@ class EvaluationObservation: cited_text: str answer: str abstained: bool + outcome: str = "unknown" + latency_ms: float | None = None @dataclass(frozen=True) class EvaluationMetrics: retrieval_recall: float - citation_correctness: float - answer_faithfulness: float - abstention_accuracy: float + citation_validity: float + reference_term_support_proxy: float + expected_action_accuracy: float case_count: int + answer_success_rate: float = 0.0 + abstention_recall: float = 0.0 + answerable_case_count: int = 0 + abstention_case_count: int = 0 + + @property + def citation_correctness(self) -> float: + """Backward-compatible alias; this metric checks citation validity only.""" + return self.citation_validity + + @property + def answer_faithfulness(self) -> float: + """Compatibility alias for the earlier, less precise metric name.""" + return self.reference_term_support_proxy + + @property + def abstention_accuracy(self) -> float: + """Compatibility alias for the earlier aggregate metric name.""" + return self.expected_action_accuracy def as_dict(self) -> dict[str, float | int]: return { "retrieval_recall": self.retrieval_recall, - "citation_correctness": self.citation_correctness, - "answer_faithfulness": self.answer_faithfulness, - "abstention_accuracy": self.abstention_accuracy, + "citation_validity": self.citation_validity, + "reference_term_support_proxy": self.reference_term_support_proxy, + "expected_action_accuracy": self.expected_action_accuracy, + "answer_success_rate": self.answer_success_rate, + "abstention_recall": self.abstention_recall, "case_count": self.case_count, + "answerable_case_count": self.answerable_case_count, + "abstention_case_count": self.abstention_case_count, + } + + +@dataclass(frozen=True) +class RetrievalMetrics: + recall_at_k: float + hit_rate_at_k: float + mrr_at_k: float + evaluated_case_count: int + limit: int + + def as_dict(self) -> dict[str, float | int]: + return { + "recall_at_k": self.recall_at_k, + "hit_rate_at_k": self.hit_rate_at_k, + "mrr_at_k": self.mrr_at_k, + "evaluated_case_count": self.evaluated_case_count, + "limit": self.limit, } -def load_evaluation_cases(path: str | Path) -> tuple[EvaluationCase, ...]: +_TOKEN_PATTERN = re.compile(r"(?u)\b[^\W_]{2,}\b") + + +def _tokens(text: str) -> tuple[str, ...]: + normalized = unicodedata.normalize("NFKC", text).casefold() + return tuple(_TOKEN_PATTERN.findall(normalized)) + + +class BM25Retriever: + """Small deterministic BM25 keyword baseline for retrieval comparison.""" + + def __init__( + self, + documents: Sequence[Any], + *, + k1: float = 1.5, + b: float = 0.75, + ) -> None: + if not documents: + raise ValueError("BM25 requires at least one document") + self._k1 = k1 + self._b = b + self._source_ids = tuple(_source_id(document) for document in documents) + if any(not source_id for source_id in self._source_ids): + raise ValueError("every BM25 document requires a source ID") + if len(set(self._source_ids)) != len(self._source_ids): + raise ValueError("BM25 document source IDs must be unique") + self._term_frequencies = tuple( + Counter(_tokens(str(document.page_content))) for document in documents + ) + self._document_lengths = tuple( + sum(frequencies.values()) for frequencies in self._term_frequencies + ) + self._average_document_length = sum(self._document_lengths) / len( + self._document_lengths + ) + self._document_frequencies = Counter( + term for frequencies in self._term_frequencies for term in frequencies + ) + + def search(self, query: str, *, limit: int = 5) -> tuple[str, ...]: + if limit < 1: + raise ValueError("limit must be at least 1") + query_terms = set(_tokens(query)) + document_count = len(self._source_ids) + scored: list[tuple[float, str]] = [] + for source_id, frequencies, document_length in zip( + self._source_ids, + self._term_frequencies, + self._document_lengths, + strict=True, + ): + score = 0.0 + for term in query_terms: + frequency = frequencies.get(term, 0) + if not frequency: + continue + document_frequency = self._document_frequencies[term] + inverse_document_frequency = math.log( + 1 + + (document_count - document_frequency + 0.5) + / (document_frequency + 0.5) + ) + length_normalization = self._k1 * ( + 1 + - self._b + + self._b + * document_length + / max(self._average_document_length, 1.0) + ) + score += inverse_document_frequency * ( + frequency * (self._k1 + 1) / (frequency + length_normalization) + ) + if score > 0: + scored.append((score, source_id)) + scored.sort(key=lambda item: (-item[0], item[1])) + return tuple(source_id for _, source_id in scored[:limit]) + + +def load_evaluation_cases( + path: str | Path, + *, + dataset_path: str | Path | None = None, +) -> tuple[EvaluationCase, ...]: payload = json.loads(Path(path).read_text(encoding="utf-8")) - return tuple( - EvaluationCase( - case_id=item["id"], - question=item["question"], - relevant_titles=tuple(item.get("relevant_titles", ())), - reference_facts=tuple( + if ( + not isinstance(payload, dict) + or payload.get("schema_version") != "rag-eval-cases/2.0" + ): + raise ValueError("evaluation set must use schema rag-eval-cases/2.0") + items = payload.get("cases") + if not isinstance(items, list) or not items: + raise ValueError("evaluation set requires at least one case") + dataset = payload.get("dataset") + if not isinstance(dataset, dict): + raise TypeError("evaluation set dataset provenance must be an object") + declared_hash = dataset.get("sha256") + declared_row_count = dataset.get("row_count") + if not isinstance(declared_hash, str) or not re.fullmatch( + r"[0-9a-f]{64}", declared_hash + ): + raise ValueError("evaluation dataset requires a lowercase SHA-256") + if not isinstance(declared_row_count, int) or declared_row_count < 1: + raise ValueError("evaluation dataset requires a positive integer row_count") + if dataset_path is not None: + from hashlib import sha256 + + resolved_dataset = Path(dataset_path) + actual_hash = sha256(resolved_dataset.read_bytes()).hexdigest() + if declared_hash != actual_hash: + raise ValueError("evaluation dataset SHA-256 does not match the manifest") + with resolved_dataset.open(encoding="utf-8", newline="") as handle: + actual_row_count = sum(1 for _ in csv.DictReader(handle)) + if declared_row_count != actual_row_count: + raise ValueError("evaluation dataset row_count does not match the manifest") + + def parse_strings(value: Any, *, field: str) -> tuple[str, ...]: + if not isinstance(value, list): + raise TypeError(f"{field} must be an array of strings") + if any(not isinstance(item, str) or not item.strip() for item in value): + raise ValueError(f"{field} must contain only non-empty strings") + return tuple(item.strip() for item in value) + + def parse_case(item: Any) -> EvaluationCase: + if not isinstance(item, dict): + raise TypeError("every evaluation case must be an object") + action = item.get("expected_action") + if action not in {"answer", "abstain"}: + raise ValueError("expected_action must be answer or abstain") + gold_ids = parse_strings(item.get("gold_source_ids"), field="gold_source_ids") + source_labels = parse_strings(item.get("source_labels"), field="source_labels") + raw_facts = item.get("reference_facts") + if not isinstance(raw_facts, list): + raise TypeError("reference_facts must be an array") + facts_list: list[ReferenceFact] = [] + for fact in raw_facts: + if not isinstance(fact, dict): + raise TypeError("every reference fact must be an object") + facts_list.append( ReferenceFact( - answer_terms=tuple(fact["answer_terms"]), - source_terms=tuple(fact["source_terms"]), + answer_terms=parse_strings( + fact.get("answer_terms"), field="answer_terms" + ), + source_terms=parse_strings( + fact.get("source_terms"), field="source_terms" + ), ) - for fact in item.get("reference_facts", ()) - ), - should_abstain=bool(item.get("should_abstain", False)), + ) + facts = tuple(facts_list) + case = EvaluationCase( + case_id=str(item.get("id") or "").strip(), + question=str(item.get("question") or "").strip(), + relevant_titles=source_labels, + reference_facts=facts, + should_abstain=action == "abstain", + category=str(item.get("category") or "").strip(), + gold_source_ids=gold_ids, + split=str(item.get("split") or "").strip(), + difficulty=str(item.get("difficulty") or "").strip(), ) - for item in payload - ) + if not case.case_id or not case.question: + raise ValueError("evaluation cases require non-empty IDs and questions") + if not case.category or not case.split or not case.difficulty: + raise ValueError("evaluation cases require category, split, and difficulty") + if len(set(case.gold_source_ids)) != len(case.gold_source_ids): + raise ValueError( + f"evaluation case {case.case_id} has duplicate gold source IDs" + ) + if case.should_abstain and ( + case.gold_source_ids or case.relevant_titles or case.reference_facts + ): + raise ValueError( + "abstention cases cannot declare gold sources, source labels, " + "or reference facts" + ) + if len(case.gold_source_ids) != len(case.relevant_titles): + raise ValueError( + f"evaluation case {case.case_id} must label every gold source" + ) + if any( + not fact.answer_terms or not fact.source_terms + for fact in case.reference_facts + ): + raise ValueError( + f"evaluation case {case.case_id} has an empty reference fact" + ) + if not case.should_abstain and not case.gold_source_ids: + raise ValueError("answer cases require at least one gold source ID") + if not case.should_abstain and not case.reference_facts: + raise ValueError("answer cases require at least one reference fact") + return case + + cases = tuple(parse_case(item) for item in items) + case_ids = [case.case_id for case in cases] + if len(set(case_ids)) != len(case_ids): + raise ValueError("evaluation case IDs must be unique") + return cases def _source_id(document: Any) -> str: @@ -82,7 +316,52 @@ def _source_id(document: Any) -> str: def _mean(values: Iterable[float]) -> float: materialized = list(values) - return sum(materialized) / len(materialized) if materialized else 1.0 + return sum(materialized) / len(materialized) if materialized else 0.0 + + +def retrieval_metrics_from_observations( + cases: tuple[EvaluationCase, ...], + observations: tuple[EvaluationObservation, ...], + *, + limit: int, +) -> RetrievalMetrics: + if limit < 1: + raise ValueError("limit must be at least 1") + by_id = {observation.case_id: observation for observation in observations} + if len(observations) != len(by_id) or len(observations) != len(cases): + raise ValueError("observations must contain each evaluation case exactly once") + if set(by_id) != {case.case_id for case in cases}: + raise ValueError("observations must match every evaluation case exactly") + + recall_scores: list[float] = [] + hit_scores: list[float] = [] + reciprocal_ranks: list[float] = [] + for case in cases: + observation = by_id[case.case_id] + relevant = observation.relevant_source_ids + if not relevant: + continue + retrieved = observation.retrieved_source_ids[:limit] + retrieved_set = set(retrieved) + recall_scores.append(len(relevant & retrieved_set) / len(relevant)) + hit_scores.append(float(bool(relevant & retrieved_set))) + reciprocal_ranks.append( + next( + ( + 1.0 / rank + for rank, source_id in enumerate(retrieved, start=1) + if source_id in relevant + ), + 0.0, + ) + ) + return RetrievalMetrics( + recall_at_k=_mean(recall_scores), + hit_rate_at_k=_mean(hit_scores), + mrr_at_k=_mean(reciprocal_ranks), + evaluated_case_count=len(recall_scores), + limit=limit, + ) def _faithfulness(case: EvaluationCase, observation: EvaluationObservation) -> float: @@ -107,13 +386,19 @@ def score_evaluation( cases: tuple[EvaluationCase, ...], observations: tuple[EvaluationObservation, ...], ) -> EvaluationMetrics: + if not cases: + raise ValueError("evaluation requires at least one case") by_id = {observation.case_id: observation for observation in observations} + if len(observations) != len(by_id) or len(observations) != len(cases): + raise ValueError("observations must contain each evaluation case exactly once") if set(by_id) != {case.case_id for case in cases}: raise ValueError("observations must match every evaluation case exactly") retrieval_scores: list[float] = [] citation_scores: list[float] = [] - faithfulness_scores: list[float] = [] + support_scores: list[float] = [] + expected_action_scores: list[float] = [] + answer_success_scores: list[float] = [] abstention_scores: list[float] = [] for case in cases: observation = by_id[case.case_id] @@ -124,21 +409,31 @@ def score_evaluation( cited = set(observation.cited_source_ids) retrieved = set(observation.retrieved_source_ids) - citation_scores.append( - 1.0 - if (case.should_abstain and not cited) - or (not case.should_abstain and bool(cited) and cited <= retrieved) - else 0.0 + answer_succeeded = observation.outcome == "answered" or ( + observation.outcome == "unknown" + and not observation.abstained + and bool(cited) ) - faithfulness_scores.append(_faithfulness(case, observation)) - abstention_scores.append(float(observation.abstained == case.should_abstain)) + if case.should_abstain: + abstention_succeeded = observation.abstained and not cited + abstention_scores.append(float(abstention_succeeded)) + expected_action_scores.append(float(abstention_succeeded)) + else: + citation_scores.append(float(bool(cited) and cited <= retrieved)) + support_scores.append(_faithfulness(case, observation)) + answer_success_scores.append(float(answer_succeeded)) + expected_action_scores.append(float(answer_succeeded)) return EvaluationMetrics( retrieval_recall=_mean(retrieval_scores), - citation_correctness=_mean(citation_scores), - answer_faithfulness=_mean(faithfulness_scores), - abstention_accuracy=_mean(abstention_scores), + citation_validity=_mean(citation_scores), + reference_term_support_proxy=_mean(support_scores), + expected_action_accuracy=_mean(expected_action_scores), case_count=len(cases), + answer_success_rate=_mean(answer_success_scores), + abstention_recall=_mean(abstention_scores), + answerable_case_count=len(answer_success_scores), + abstention_case_count=len(abstention_scores), ) @@ -150,6 +445,7 @@ def run_rag_evaluation( limit: int = 5, ) -> tuple[EvaluationMetrics, tuple[EvaluationObservation, ...]]: collection = vector_store.get(include=["metadatas"]) + available_source_ids = {str(source_id) for source_id in collection["ids"]} title_to_ids: dict[str, set[str]] = {} for source_id, metadata in zip( collection["ids"], collection["metadatas"], strict=True @@ -160,25 +456,27 @@ def run_rag_evaluation( observations: list[EvaluationObservation] = [] for case in cases: - missing_titles = [ - title for title in case.relevant_titles if title not in title_to_ids - ] - if missing_titles: - raise ValueError( - f"evaluation case {case.case_id} references missing titles: " - + ", ".join(missing_titles) - ) - relevant_ids = frozenset( - source_id - for title in case.relevant_titles - for source_id in title_to_ids[title] + relevant_ids = _relevant_ids( + case, + title_to_ids, + available_source_ids=available_source_ids, ) + started = perf_counter() result = answer_question( case.question, vector_store=vector_store, model=model, limit=limit, ) + latency_ms = (perf_counter() - started) * 1000 + if not result.retrieved_source_ids: + outcome = "empty_retrieval" + elif result.abstained: + outcome = "model_abstention" + elif not result.sources: + outcome = "citation_validation_rejection" + else: + outcome = "answered" observations.append( EvaluationObservation( case_id=case.case_id, @@ -192,8 +490,278 @@ def run_rag_evaluation( ), answer=result.answer, abstained=result.abstained, + outcome=outcome, + latency_ms=latency_ms, ) ) materialized = tuple(observations) return score_evaluation(cases, materialized), materialized + + +def run_bm25_baseline( + cases: tuple[EvaluationCase, ...], + *, + vector_store: Any, + limit: int = 5, +) -> tuple[RetrievalMetrics, tuple[EvaluationObservation, ...]]: + collection = vector_store.get(include=["metadatas", "documents"]) + documents = tuple( + _document_from_collection(source_id, metadata, content) + for source_id, metadata, content in zip( + collection["ids"], + collection["metadatas"], + collection["documents"], + strict=True, + ) + ) + title_to_ids = _title_to_source_ids(documents) + available_source_ids = {_source_id(document) for document in documents} + retriever = BM25Retriever(documents) + observations = tuple( + EvaluationObservation( + case_id=case.case_id, + relevant_source_ids=_relevant_ids( + case, + title_to_ids, + available_source_ids=available_source_ids, + ), + retrieved_source_ids=retriever.search(case.question, limit=limit), + cited_source_ids=(), + cited_text="", + answer="", + abstained=False, + outcome="retrieval_only", + ) + for case in cases + ) + return ( + retrieval_metrics_from_observations(cases, observations, limit=limit), + observations, + ) + + +def _document_from_collection( + source_id: object, + metadata: object, + content: object, +) -> Any: + from langchain_core.documents import Document + + if metadata is None: + resolved_metadata = {} + elif isinstance(metadata, Mapping): + resolved_metadata = dict(metadata) + else: + raise TypeError("collection metadata must be a mapping") + resolved_metadata.setdefault("source_id", str(source_id)) + return Document( + id=str(source_id), + page_content=str(content or ""), + metadata=resolved_metadata, + ) + + +def _title_to_source_ids(documents: Sequence[Any]) -> dict[str, set[str]]: + title_to_ids: dict[str, set[str]] = {} + for document in documents: + title = str(document.metadata.get("title") or "") + if title: + title_to_ids.setdefault(title, set()).add(_source_id(document)) + return title_to_ids + + +def _relevant_ids( + case: EvaluationCase, + title_to_ids: Mapping[str, set[str]], + *, + available_source_ids: set[str], +) -> frozenset[str]: + if case.gold_source_ids: + missing_ids = sorted(set(case.gold_source_ids) - available_source_ids) + if missing_ids: + raise ValueError( + f"evaluation case {case.case_id} references missing source IDs: " + + ", ".join(missing_ids) + ) + return frozenset(case.gold_source_ids) + missing_titles = [ + title for title in case.relevant_titles if title not in title_to_ids + ] + if missing_titles: + raise ValueError( + f"evaluation case {case.case_id} references missing titles: " + + ", ".join(missing_titles) + ) + return frozenset( + source_id for title in case.relevant_titles for source_id in title_to_ids[title] + ) + + +def _as_mapping(value: Mapping[str, Any] | Any) -> dict[str, Any]: + if isinstance(value, Mapping): + return dict(value) + converted = value.as_dict() + if not isinstance(converted, Mapping): + raise TypeError("metric as_dict() must return a mapping") + return dict(converted) + + +def build_evaluation_report( + *, + cases: tuple[EvaluationCase, ...], + rag_metrics: Mapping[str, Any] | EvaluationMetrics, + semantic_metrics: Mapping[str, Any] | RetrievalMetrics, + baseline_metrics: Mapping[str, Any] | RetrievalMetrics, + observations: tuple[EvaluationObservation, ...], + configuration: Mapping[str, Any], + provenance: Mapping[str, Any], + baseline_observations: tuple[EvaluationObservation, ...] = (), + generated_at: str | None = None, +) -> dict[str, Any]: + timestamp = generated_at or datetime.now(UTC).isoformat().replace("+00:00", "Z") + categories = Counter(case.category for case in cases) + latencies = [ + observation.latency_ms + for observation in observations + if observation.latency_ms is not None + ] + return { + "schema_version": 2, + "generated_at": timestamp, + "configuration": dict(configuration), + "provenance": dict(provenance), + "evaluation_set": { + "case_count": len(cases), + "answerable_case_count": sum(not case.should_abstain for case in cases), + "abstention_case_count": sum(case.should_abstain for case in cases), + "categories": dict(sorted(categories.items())), + }, + "results": { + "semantic_retrieval": _as_mapping(semantic_metrics), + "bm25_baseline": _as_mapping(baseline_metrics), + "rag": _as_mapping(rag_metrics), + }, + "timing": { + "rag_total_latency_ms": round(sum(latencies), 3), + "rag_mean_latency_ms": round(_mean(latencies), 3), + "rag_median_latency_ms": round(median(latencies), 3) if latencies else 0.0, + }, + "observations": { + "rag": [_observation_as_dict(observation) for observation in observations], + "bm25_baseline": [ + _observation_as_dict(observation) + for observation in baseline_observations + ], + }, + } + + +def _observation_as_dict(observation: EvaluationObservation) -> dict[str, Any]: + return { + "case_id": observation.case_id, + "relevant_source_ids": sorted(observation.relevant_source_ids), + "retrieved_source_ids": list(observation.retrieved_source_ids), + "cited_source_ids": list(observation.cited_source_ids), + "answer": observation.answer, + "abstained": observation.abstained, + "outcome": observation.outcome, + "latency_ms": observation.latency_ms, + } + + +def _metric(value: object) -> str: + if not isinstance(value, (str, int, float)): + raise TypeError("metric value must be numeric") + return f"{float(value):.3f}" + + +def _report_markdown(report: Mapping[str, Any]) -> str: + results = report["results"] + semantic = results["semantic_retrieval"] + baseline = results["bm25_baseline"] + rag = results["rag"] + evaluation_set = report["evaluation_set"] + configuration = report["configuration"] + provenance = report["provenance"] + timing = report["timing"] + return "\n".join( + ( + "# Evaluation report", + "", + f"Generated: `{report['generated_at']}`", + "", + "## Scope", + "", + ( + f"- Cases: **{evaluation_set['case_count']}** " + f"({evaluation_set['answerable_case_count']} answerable, " + f"{evaluation_set['abstention_case_count']} abstention)" + ), + f"- Chat model: `{configuration.get('chat_model', 'unknown')}`", + f"- Embedding model: `{configuration.get('embedding_model', 'unknown')}`", + f"- Ollama runtime: `{configuration.get('ollama_version', 'unknown')}`", + f"- Evidence limit: **{semantic['limit']}**", + f"- Dataset SHA-256: `{provenance.get('dataset_sha256', 'unknown')}`", + f"- Cases SHA-256: `{provenance.get('cases_sha256', 'unknown')}`", + "", + "## Retrieval comparison", + "", + "| Retriever | Recall@k | Hit rate@k | MRR@k |", + "| --- | ---: | ---: | ---: |", + ( + f"| Semantic | {_metric(semantic['recall_at_k'])} | " + f"{_metric(semantic['hit_rate_at_k'])} | " + f"{_metric(semantic['mrr_at_k'])} |" + ), + ( + f"| BM25 keyword baseline | {_metric(baseline['recall_at_k'])} | " + f"{_metric(baseline['hit_rate_at_k'])} | " + f"{_metric(baseline['mrr_at_k'])} |" + ), + "", + "## Model-dependent results", + "", + "| Metric | Score |", + "| --- | ---: |", + f"| Citation validity | {_metric(rag['citation_validity'])} |", + ( + "| Reference-term support proxy | " + f"{_metric(rag['reference_term_support_proxy'])} |" + ), + f"| Expected-action accuracy | {_metric(rag['expected_action_accuracy'])} |", + f"| Answer success (answerable cases) | {_metric(rag['answer_success_rate'])} |", + f"| Abstention recall (abstention cases) | {_metric(rag['abstention_recall'])} |", + "", + "## Timing", + "", + f"- Total RAG latency: **{timing['rag_total_latency_ms'] / 1000:.2f}s**", + f"- Mean per case: **{timing['rag_mean_latency_ms']:.1f}ms**", + f"- Median per case: **{timing['rag_median_latency_ms']:.1f}ms**", + "", + ( + "These scores describe this fixed dataset, case set, retrieval limit, " + "and local model configuration. Relevance judgments are known-positive, " + "not exhaustive. The reference-term support score is a transparent " + "heuristic, not an LLM judge or a general factuality guarantee." + ), + "", + ) + ) + + +def write_evaluation_report( + report: Mapping[str, Any], + *, + json_path: str | Path, + markdown_path: str | Path, +) -> None: + resolved_json = Path(json_path) + resolved_markdown = Path(markdown_path) + resolved_json.parent.mkdir(parents=True, exist_ok=True) + resolved_markdown.parent.mkdir(parents=True, exist_ok=True) + resolved_json.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + resolved_markdown.write_text(_report_markdown(report), encoding="utf-8") diff --git a/evaluation/results/v0.2.0-ollama-0.32.5/README.md b/evaluation/results/v0.2.0-ollama-0.32.5/README.md new file mode 100644 index 0000000..aca97e2 --- /dev/null +++ b/evaluation/results/v0.2.0-ollama-0.32.5/README.md @@ -0,0 +1,38 @@ +# Evaluation report + +Generated: `2026-07-31T16:42:39.160738Z` + +## Scope + +- Cases: **30** (25 answerable, 5 abstention) +- Chat model: `llama3.2` +- Embedding model: `mxbai-embed-large` +- Ollama runtime: `0.32.5` +- Evidence limit: **5** +- Dataset SHA-256: `8ace5c1cb728c3c8a355d90a414849ebd645e021aae8a5b32303ef310917254e` +- Cases SHA-256: `b46d008043ad5a815fc6f9563362901f2a4368a408739c811ae80a614556a141` + +## Retrieval comparison + +| Retriever | Recall@k | Hit rate@k | MRR@k | +| --- | ---: | ---: | ---: | +| Semantic | 0.913 | 1.000 | 0.960 | +| BM25 keyword baseline | 0.770 | 0.880 | 0.753 | + +## Model-dependent results + +| Metric | Score | +| --- | ---: | +| Citation validity | 0.560 | +| Reference-term support proxy | 0.520 | +| Expected-action accuracy | 0.633 | +| Answer success (answerable cases) | 0.560 | +| Abstention recall (abstention cases) | 1.000 | + +## Timing + +- Total RAG latency: **130.76s** +- Mean per case: **4358.7ms** +- Median per case: **2006.2ms** + +These scores describe this fixed dataset, case set, retrieval limit, and local model configuration. Relevance judgments are known-positive, not exhaustive. The reference-term support score is a transparent heuristic, not an LLM judge or a general factuality guarantee. diff --git a/evaluation/results/v0.2.0-ollama-0.32.5/evaluation-report.json b/evaluation/results/v0.2.0-ollama-0.32.5/evaluation-report.json new file mode 100644 index 0000000..44645c0 --- /dev/null +++ b/evaluation/results/v0.2.0-ollama-0.32.5/evaluation-report.json @@ -0,0 +1,1283 @@ +{ + "configuration": { + "chat_model": "llama3.2", + "embedding_model": "mxbai-embed-large", + "evidence_limit": 5, + "models": { + "llama3.2": { + "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72", + "modified_at": "2026-07-31 16:14:24.118841+00:00", + "resolved_name": "llama3.2:latest", + "size": 2019393189 + }, + "mxbai-embed-large": { + "digest": "468836162de7f81e041c43663fedbbba921dcea9b9fefea135685a39b2d83dd8", + "modified_at": "2026-07-31 16:14:28.495867+00:00", + "resolved_name": "mxbai-embed-large:latest", + "size": 669615493 + } + }, + "ollama_host": "http://127.0.0.1:11434", + "ollama_version": "0.32.5" + }, + "evaluation_set": { + "abstention_case_count": 5, + "answerable_case_count": 25, + "case_count": 30, + "categories": { + "abstention": 5, + "beverages": 1, + "dietary": 2, + "ingredients": 2, + "menu": 6, + "operations": 4, + "quality-defect": 3, + "style": 5, + "texture": 1, + "value": 1 + } + }, + "generated_at": "2026-07-31T16:42:39.160738Z", + "observations": { + "bm25_baseline": [ + { + "abstained": false, + "answer": "", + "case_id": "crispy-crust", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_13f6c41bf74292e7087a1a7ccf100c6d", + "review_a23134f6b63ffdb4c6fab9baa2cce095", + "review_cf6397aa45938d26e42aca081d576a93" + ], + "retrieved_source_ids": [ + "review_e82c59e5e8a0069ab75de732490c39d1", + "review_0cee7aec6b30d32a93b194f3047ef5b4", + "review_13f6c41bf74292e7087a1a7ccf100c6d", + "review_8b58fbd7743e24c7f9332a65fb828318", + "review_b9521c53f19812b44e478bb616d381e3" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "vegan-cheese", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_08222ec4bad55a761d23c3dddf0dae72", + "review_20c52415ea843838a45b29821f7ba112" + ], + "retrieved_source_ids": [ + "review_08222ec4bad55a761d23c3dddf0dae72", + "review_20c52415ea843838a45b29821f7ba112", + "review_4803ec6912573da6b2b09ee8d11efca4", + "review_231bf1e7bc7a4983da6e8971d932733c", + "review_4ced72c09a43162a44694cc2fdb4171a" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "dessert-pizza", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_f624da623120634ef3ae535a9501688c" + ], + "retrieved_source_ids": [ + "review_08222ec4bad55a761d23c3dddf0dae72", + "review_58f10e007784450cfe4c44e239a3bed7", + "review_9671a6061820222dd8af1a72592a0a76", + "review_f624da623120634ef3ae535a9501688c", + "review_182d10267c2e4a1654aae9786af494d5" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "gluten-free-crust", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_6464cf45ebbd480454abc07065075c3b" + ], + "retrieved_source_ids": [ + "review_6464cf45ebbd480454abc07065075c3b", + "review_1135e8f866f2f79d951325072190f564", + "review_49f2ddf7f8434772a008801d3f7dfd95", + "review_4803ec6912573da6b2b09ee8d11efca4", + "review_2f293fad89162c9ceb164d9812bc3d83" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "margherita-components", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_13809cb33172b3c40040348b296245df", + "review_5117c8c073d626838c7798930f15b8e1" + ], + "retrieved_source_ids": [ + "review_637d1fbaecd307a046cc5b8d9f87614c", + "review_35bbb51a785b5d72e21b309d41d0db80", + "review_49f2ddf7f8434772a008801d3f7dfd95", + "review_5117c8c073d626838c7798930f15b8e1", + "review_13809cb33172b3c40040348b296245df" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "hawaiian-pizza", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_211ffae3b56d5e65fc7edc3f92143b12" + ], + "retrieved_source_ids": [ + "review_211ffae3b56d5e65fc7edc3f92143b12", + "review_e0cf78dbbb255982fbfb1288297e52ea", + "review_d9ca96b76c7ab69c0f900f804d4a62f2", + "review_a92b57aa28366732db18740deb7d39c7", + "review_efa9ed9172b37cc0f45b45301c7ed9c3" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "white-pizza", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_1615431adc386cbc8063f8a3e7983f10", + "review_a6014a29ee06cc625b8b71a2bf39759b" + ], + "retrieved_source_ids": [ + "review_1615431adc386cbc8063f8a3e7983f10", + "review_a6014a29ee06cc625b8b71a2bf39759b", + "review_5666239ad9f3340465ef839c7b49fb11", + "review_08709a30ce1fa58227877b0f647f43a3", + "review_637d1fbaecd307a046cc5b8d9f87614c" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "vodka-sauce", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_bfeabf92afa1ff6c1644bee468624ca2" + ], + "retrieved_source_ids": [ + "review_bfeabf92afa1ff6c1644bee468624ca2", + "review_12574e71aaf0b76482d6fc6a57d34502", + "review_a92b57aa28366732db18740deb7d39c7", + "review_4803ec6912573da6b2b09ee8d11efca4", + "review_442067c8d75bbb31443c4aa64f1a1a45" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "buffalo-chicken", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_2ab5eb185f34f7f6233d315242b13034" + ], + "retrieved_source_ids": [ + "review_2ab5eb185f34f7f6233d315242b13034", + "review_49f2ddf7f8434772a008801d3f7dfd95", + "review_13809cb33172b3c40040348b296245df", + "review_30d736e417c06e347851e710514d1046", + "review_0689a695ad22db80c369549a9ae071b2" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "breakfast-pizza", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_413d502bdf96213e0bd1eb04378cd88e" + ], + "retrieved_source_ids": [ + "review_413d502bdf96213e0bd1eb04378cd88e", + "review_a74544dcca71945035e4da08cbefb263", + "review_08709a30ce1fa58227877b0f647f43a3", + "review_46deb903486e7bdbf280ccf75a8053b0", + "review_5df87053cbf1f73cd631288f814a9208" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "roman-style", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_d9ca96b76c7ab69c0f900f804d4a62f2" + ], + "retrieved_source_ids": [ + "review_d9ca96b76c7ab69c0f900f804d4a62f2", + "review_0e5472a8c24c61a7e4de614412461918", + "review_1615431adc386cbc8063f8a3e7983f10", + "review_dad04092fb528c74b9d80ce442930ee4", + "review_43cc47cc7c997e68800e2faa043b18fc" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "detroit-style", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_dad04092fb528c74b9d80ce442930ee4" + ], + "retrieved_source_ids": [ + "review_dad04092fb528c74b9d80ce442930ee4", + "review_b9521c53f19812b44e478bb616d381e3", + "review_d9ca96b76c7ab69c0f900f804d4a62f2", + "review_efa9ed9172b37cc0f45b45301c7ed9c3", + "review_08222ec4bad55a761d23c3dddf0dae72" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "chicago-deep-dish", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_442067c8d75bbb31443c4aa64f1a1a45" + ], + "retrieved_source_ids": [ + "review_442067c8d75bbb31443c4aa64f1a1a45", + "review_0689a695ad22db80c369549a9ae071b2", + "review_dad04092fb528c74b9d80ce442930ee4", + "review_d9ca96b76c7ab69c0f900f804d4a62f2", + "review_0e5472a8c24c61a7e4de614412461918" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "new-york-style", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_0689a695ad22db80c369549a9ae071b2", + "review_0e5472a8c24c61a7e4de614412461918" + ], + "retrieved_source_ids": [ + "review_0689a695ad22db80c369549a9ae071b2", + "review_0e5472a8c24c61a7e4de614412461918", + "review_b9e259e601836dfac086841092fe5c3b", + "review_dad04092fb528c74b9d80ce442930ee4", + "review_d9ca96b76c7ab69c0f900f804d4a62f2" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "delivery-problems", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_46deb903486e7bdbf280ccf75a8053b0", + "review_50429ddac7fcf92eb87d761a7b7e4d11", + "review_c79555a5cf3345966342cfb9207d2405", + "review_fb4a5b16a84ab35efc47e281e01894ce" + ], + "retrieved_source_ids": [ + "review_01089957f487f383fe7854b0f2a4f945", + "review_c79555a5cf3345966342cfb9207d2405", + "review_9671a6061820222dd8af1a72592a0a76", + "review_50429ddac7fcf92eb87d761a7b7e4d11", + "review_8b58fbd7743e24c7f9332a65fb828318" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "long-waits", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_4306c4305c5959dd3587f8e870437a4a", + "review_442067c8d75bbb31443c4aa64f1a1a45", + "review_dad04092fb528c74b9d80ce442930ee4" + ], + "retrieved_source_ids": [ + "review_4306c4305c5959dd3587f8e870437a4a", + "review_dad04092fb528c74b9d80ce442930ee4", + "review_1a28353790d8c4bc342444fd14f8be78", + "review_fcf71fc37903247f060ccea2cf271710", + "review_f20617ff46e7cc6abbc2073fca2f43cd" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "large-groups", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_9d84ed9fd7582f0a9f352fcdcc03483e", + "review_d8101a592a128df0fb622215f018391c" + ], + "retrieved_source_ids": [ + "review_a92b57aa28366732db18740deb7d39c7", + "review_d8101a592a128df0fb622215f018391c", + "review_41461655b861f0c1a884f500840e0fa0", + "review_9d84ed9fd7582f0a9f352fcdcc03483e", + "review_425a2cada3d6ceebbccbd0b303368183" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "children-and-families", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_42e9a66d38512d7ecd0a86e52f12b2d2", + "review_d57690690aac0645faada4223d374551", + "review_f20617ff46e7cc6abbc2073fca2f43cd" + ], + "retrieved_source_ids": [ + "review_f624da623120634ef3ae535a9501688c", + "review_08222ec4bad55a761d23c3dddf0dae72", + "review_8fcebe00f6c14a995fcc2c90235da9fc", + "review_38424247e3760fb18b79d5ce3d40c4a2", + "review_a74544dcca71945035e4da08cbefb263" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "drink-pairings", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_2f293fad89162c9ceb164d9812bc3d83", + "review_3618241908457302b051385404a70ab0" + ], + "retrieved_source_ids": [ + "review_3618241908457302b051385404a70ab0", + "review_13f6c41bf74292e7087a1a7ccf100c6d", + "review_0bcb5c336ae1650ea59a3fe5d4ebad60", + "review_4ced72c09a43162a44694cc2fdb4171a", + "review_9d84ed9fd7582f0a9f352fcdcc03483e" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "price-and-value", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_49f2ddf7f8434772a008801d3f7dfd95", + "review_80b54150ee2f71f582c9798263dd92fe", + "review_b9e259e601836dfac086841092fe5c3b", + "review_bb1e3ccbdbad8d3f35e41770cc6e6b61" + ], + "retrieved_source_ids": [ + "review_b9e259e601836dfac086841092fe5c3b", + "review_49f2ddf7f8434772a008801d3f7dfd95", + "review_a92b57aa28366732db18740deb7d39c7", + "review_08709a30ce1fa58227877b0f647f43a3", + "review_0689a695ad22db80c369549a9ae071b2" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "undercooked-centers", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_1a28353790d8c4bc342444fd14f8be78", + "review_28360738a639784c3ea351484aef20d2", + "review_43cc47cc7c997e68800e2faa043b18fc", + "review_e8050c6a11acaab43c264cebcdc532a4" + ], + "retrieved_source_ids": [ + "review_43cc47cc7c997e68800e2faa043b18fc", + "review_28360738a639784c3ea351484aef20d2", + "review_9671a6061820222dd8af1a72592a0a76", + "review_e8050c6a11acaab43c264cebcdc532a4", + "review_49f2ddf7f8434772a008801d3f7dfd95" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "excessive-grease", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_182d10267c2e4a1654aae9786af494d5", + "review_9671a6061820222dd8af1a72592a0a76", + "review_e58826d3c2728e96a0f7b4f055d4e6b1" + ], + "retrieved_source_ids": [ + "review_9671a6061820222dd8af1a72592a0a76", + "review_cf6397aa45938d26e42aca081d576a93", + "review_43cc47cc7c997e68800e2faa043b18fc", + "review_e58826d3c2728e96a0f7b4f055d4e6b1", + "review_182d10267c2e4a1654aae9786af494d5" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "negative-sauce-quality", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_12574e71aaf0b76482d6fc6a57d34502", + "review_823c8cf17bd94ada444a824145796489", + "review_a92b57aa28366732db18740deb7d39c7", + "review_bb161bcd8bbe609cc1be19e232fef089" + ], + "retrieved_source_ids": [ + "review_08222ec4bad55a761d23c3dddf0dae72", + "review_9ade82029a7fb5aa8e00ace4a188e0ae", + "review_42e9a66d38512d7ecd0a86e52f12b2d2", + "review_aa7e1ac88a6d0b562f97f4f00767e253", + "review_413d502bdf96213e0bd1eb04378cd88e" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "fresh-ingredients", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_924b8212f2c0052946059cb4c867d038", + "review_dc2ae1a58c8bbd89048c1c0b569af1af", + "review_e0cf78dbbb255982fbfb1288297e52ea" + ], + "retrieved_source_ids": [ + "review_49f2ddf7f8434772a008801d3f7dfd95", + "review_a92b57aa28366732db18740deb7d39c7", + "review_bb1e3ccbdbad8d3f35e41770cc6e6b61", + "review_41461655b861f0c1a884f500840e0fa0", + "review_9671a6061820222dd8af1a72592a0a76" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "seasonal-specials", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [ + "review_4a9fb2adb5ce77a1c6cbddd7bb4be043", + "review_5666239ad9f3340465ef839c7b49fb11" + ], + "retrieved_source_ids": [ + "review_5666239ad9f3340465ef839c7b49fb11", + "review_4a9fb2adb5ce77a1c6cbddd7bb4be043", + "review_58f10e007784450cfe4c44e239a3bed7", + "review_dc2ae1a58c8bbd89048c1c0b569af1af", + "review_08709a30ce1fa58227877b0f647f43a3" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "parking-abstention", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [], + "retrieved_source_ids": [ + "review_9a960b06110fc713601c0d788e7b47e2", + "review_9065fb1d99a02d0c960161b729015f39", + "review_efa9ed9172b37cc0f45b45301c7ed9c3", + "review_01089957f487f383fe7854b0f2a4f945", + "review_bb1e3ccbdbad8d3f35e41770cc6e6b61" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "wheelchair-abstention", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [], + "retrieved_source_ids": [ + "review_0ab44afae543aa0a4193a55fd4158e2d", + "review_2f293fad89162c9ceb164d9812bc3d83", + "review_2ab5eb185f34f7f6233d315242b13034", + "review_3618241908457302b051385404a70ab0", + "review_5df87053cbf1f73cd631288f814a9208" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "reservations-abstention", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [], + "retrieved_source_ids": [ + "review_30d736e417c06e347851e710514d1046", + "review_f239cd569a160545f30b51fc79098f0b", + "review_425a2cada3d6ceebbccbd0b303368183", + "review_50429ddac7fcf92eb87d761a7b7e4d11", + "review_01089957f487f383fe7854b0f2a4f945" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "wifi-abstention", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [], + "retrieved_source_ids": [ + "review_9a960b06110fc713601c0d788e7b47e2", + "review_6464cf45ebbd480454abc07065075c3b", + "review_50429ddac7fcf92eb87d761a7b7e4d11", + "review_08222ec4bad55a761d23c3dddf0dae72", + "review_46deb903486e7bdbf280ccf75a8053b0" + ] + }, + { + "abstained": false, + "answer": "", + "case_id": "delivery-fee-abstention", + "cited_source_ids": [], + "latency_ms": null, + "outcome": "retrieval_only", + "relevant_source_ids": [], + "retrieved_source_ids": [ + "review_50429ddac7fcf92eb87d761a7b7e4d11", + "review_46deb903486e7bdbf280ccf75a8053b0", + "review_260dae3b81d315fbd9ee924b7f63c49a", + "review_01089957f487f383fe7854b0f2a4f945", + "review_9065fb1d99a02d0c960161b729015f39" + ] + } + ], + "rag": [ + { + "abstained": false, + "answer": "Insufficient Evidence.\n\nThe reviews do not provide clear descriptions of the pizza crust's texture, particularly in relation to being crispy while still having some chew. While some reviewers mention the crust's texture (e.g., [1], [2]), others describe it as tough or gritty (e.g., [3], [4]). There is no consistent description that matches the question.", + "case_id": "crispy-crust", + "cited_source_ids": [ + "review_cf6397aa45938d26e42aca081d576a93", + "review_f239cd569a160545f30b51fc79098f0b", + "review_27843d67520f3d3e98ad5dfd65b75bf1", + "review_66ff650d6d920ffa6d1f7797f1e67d11" + ], + "latency_ms": 5595.177419018, + "outcome": "answered", + "relevant_source_ids": [ + "review_13f6c41bf74292e7087a1a7ccf100c6d", + "review_a23134f6b63ffdb4c6fab9baa2cce095", + "review_cf6397aa45938d26e42aca081d576a93" + ], + "retrieved_source_ids": [ + "review_cf6397aa45938d26e42aca081d576a93", + "review_f239cd569a160545f30b51fc79098f0b", + "review_27843d67520f3d3e98ad5dfd65b75bf1", + "review_66ff650d6d920ffa6d1f7797f1e67d11", + "review_8fe21bed523d1395f51ec2a0141d9f7f" + ] + }, + { + "abstained": false, + "answer": "Based on the provided reviews, reviewers report contrasting experiences about the vegan cheese. Some reviewers, such as [1] and [2], express disappointment with the product's taste, texture, and aftertaste, citing issues like a grainy texture, strong coconut flavor, and unpleasant chemical tastes.\n\nOn the other hand, reviewer [3] reports a positive experience with the vegan pizza, praising the meltability of their cashew cheese and the freshness of their vegetable toppings. Reviewer [4] also mentions that their non-vegan friends enjoy the product, suggesting that it may be well-received by some consumers.\n\nHowever, there is no clear consensus on the overall quality or characteristics of the vegan cheese across all reviews.", + "case_id": "vegan-cheese", + "cited_source_ids": [ + "review_08222ec4bad55a761d23c3dddf0dae72", + "review_8c0489c3ad5568aac51608d88d7a8bc6", + "review_20c52415ea843838a45b29821f7ba112", + "review_231bf1e7bc7a4983da6e8971d932733c" + ], + "latency_ms": 10427.238863077946, + "outcome": "answered", + "relevant_source_ids": [ + "review_08222ec4bad55a761d23c3dddf0dae72", + "review_20c52415ea843838a45b29821f7ba112" + ], + "retrieved_source_ids": [ + "review_08222ec4bad55a761d23c3dddf0dae72", + "review_20c52415ea843838a45b29821f7ba112", + "review_231bf1e7bc7a4983da6e8971d932733c", + "review_bb161bcd8bbe609cc1be19e232fef089", + "review_8c0489c3ad5568aac51608d88d7a8bc6" + ] + }, + { + "abstained": false, + "answer": "Yes, the reviews mention dessert pizzas.\n\nThe recommended flavor of dessert pizza is not explicitly stated in any review. However, one review mentions that they have dessert pizzas that are \"surprisingly good\" and recommends trying the Nutella and banana combination (review [1]).", + "case_id": "dessert-pizza", + "cited_source_ids": [ + "review_f624da623120634ef3ae535a9501688c" + ], + "latency_ms": 4179.776731994934, + "outcome": "answered", + "relevant_source_ids": [ + "review_f624da623120634ef3ae535a9501688c" + ], + "retrieved_source_ids": [ + "review_46cb53200f8521feb75e7642d41ab245", + "review_f624da623120634ef3ae535a9501688c", + "review_1d3902b9c73a6a4fd453e618e8b731dc", + "review_3618241908457302b051385404a70ab0", + "review_42e9a66d38512d7ecd0a86e52f12b2d2" + ] + }, + { + "abstained": false, + "answer": "I could not produce an answer with citations that match the retrieved reviews.", + "case_id": "gluten-free-crust", + "cited_source_ids": [], + "latency_ms": 3111.7266649380326, + "outcome": "citation_validation_rejection", + "relevant_source_ids": [ + "review_6464cf45ebbd480454abc07065075c3b" + ], + "retrieved_source_ids": [ + "review_6464cf45ebbd480454abc07065075c3b", + "review_13f6c41bf74292e7087a1a7ccf100c6d", + "review_e8050c6a11acaab43c264cebcdc532a4", + "review_cf6397aa45938d26e42aca081d576a93", + "review_8fe21bed523d1395f51ec2a0141d9f7f" + ] + }, + { + "abstained": false, + "answer": "The praised ingredients in Margherita pizzas are:\n\n* Tomato sauce (mentioned in reviews [1], [2])\n* Fresh mozzarella (mentioned in reviews [1], [3], [2])\n* Basil (mentioned in reviews [1], [3])\n\nThere is no clear evidence of other ingredients being praised, but the cooking qualities mentioned include:\n\n* Perfectly chewy-crisp crust (review [1])\n* Perfect char (review [3])\n* Cooking time perfectly (review [4])\n* Expertise and attention to detail (review [4])", + "case_id": "margherita-components", + "cited_source_ids": [ + "review_13809cb33172b3c40040348b296245df", + "review_dc2ae1a58c8bbd89048c1c0b569af1af", + "review_5117c8c073d626838c7798930f15b8e1", + "review_35bbb51a785b5d72e21b309d41d0db80" + ], + "latency_ms": 8270.35483205691, + "outcome": "answered", + "relevant_source_ids": [ + "review_13809cb33172b3c40040348b296245df", + "review_5117c8c073d626838c7798930f15b8e1" + ], + "retrieved_source_ids": [ + "review_13809cb33172b3c40040348b296245df", + "review_5117c8c073d626838c7798930f15b8e1", + "review_35bbb51a785b5d72e21b309d41d0db80", + "review_dc2ae1a58c8bbd89048c1c0b569af1af", + "review_ca3130944bc8520c4936ece2e143cb12" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "hawaiian-pizza", + "cited_source_ids": [], + "latency_ms": 875.66630798392, + "outcome": "model_abstention", + "relevant_source_ids": [ + "review_211ffae3b56d5e65fc7edc3f92143b12" + ], + "retrieved_source_ids": [ + "review_211ffae3b56d5e65fc7edc3f92143b12", + "review_f624da623120634ef3ae535a9501688c", + "review_8ae0a84472fa7202da17945f5f037092", + "review_46cb53200f8521feb75e7642d41ab245", + "review_35bbb51a785b5d72e21b309d41d0db80" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "white-pizza", + "cited_source_ids": [], + "latency_ms": 887.166297994554, + "outcome": "model_abstention", + "relevant_source_ids": [ + "review_1615431adc386cbc8063f8a3e7983f10", + "review_a6014a29ee06cc625b8b71a2bf39759b" + ], + "retrieved_source_ids": [ + "review_1615431adc386cbc8063f8a3e7983f10", + "review_a6014a29ee06cc625b8b71a2bf39759b", + "review_ca3130944bc8520c4936ece2e143cb12", + "review_4a9fb2adb5ce77a1c6cbddd7bb4be043", + "review_e0cf78dbbb255982fbfb1288297e52ea" + ] + }, + { + "abstained": false, + "answer": "The description of the vodka-sauce pizza is that it has a creamy, slightly spicy sauce as its base. The optional topping suggested by one reviewer is prosciutto for an extra flavor dimension [1]. However, there is no mention of other toppings or specific descriptions of the sauce beyond being \"suspiciously sweet\" and lacking complexity [2], or having a \"nice texture\" but lacking personality in terms of herbs, garlic, or salt [3].", + "case_id": "vodka-sauce", + "cited_source_ids": [ + "review_bfeabf92afa1ff6c1644bee468624ca2", + "review_a92b57aa28366732db18740deb7d39c7", + "review_823c8cf17bd94ada444a824145796489" + ], + "latency_ms": 6786.548349075019, + "outcome": "answered", + "relevant_source_ids": [ + "review_bfeabf92afa1ff6c1644bee468624ca2" + ], + "retrieved_source_ids": [ + "review_bfeabf92afa1ff6c1644bee468624ca2", + "review_a92b57aa28366732db18740deb7d39c7", + "review_823c8cf17bd94ada444a824145796489", + "review_4803ec6912573da6b2b09ee8d11efca4", + "review_3477ac06147ab1bfaa40bf4f96bbca82" + ] + }, + { + "abstained": false, + "answer": "Based on the provided reviews, it appears that the balance and ingredient quality of the buffalo chicken pizza are praised. \n\nReview [1] mentions that the buffalo chicken pizza balances spicy and creamy perfectly, with high-quality and juicy chicken. Review [2] praises the application of buffalo mozzarella, stating it maintains its creamy integrity after being applied to the pizza.\n\nThere is no direct mention of specific ingredients or their quality in reviews [3], [4], and [5]. However, review [3] does mention a \"perfect balance of flavors\" from the 'Mediterranean' pizza, which may imply that other pizzas also have well-balanced flavors.", + "case_id": "buffalo-chicken", + "cited_source_ids": [ + "review_2ab5eb185f34f7f6233d315242b13034", + "review_30d736e417c06e347851e710514d1046", + "review_ca3130944bc8520c4936ece2e143cb12", + "review_8b58fbd7743e24c7f9332a65fb828318", + "review_0ab44afae543aa0a4193a55fd4158e2d" + ], + "latency_ms": 9527.296634041704, + "outcome": "answered", + "relevant_source_ids": [ + "review_2ab5eb185f34f7f6233d315242b13034" + ], + "retrieved_source_ids": [ + "review_2ab5eb185f34f7f6233d315242b13034", + "review_ca3130944bc8520c4936ece2e143cb12", + "review_30d736e417c06e347851e710514d1046", + "review_8b58fbd7743e24c7f9332a65fb828318", + "review_0ab44afae543aa0a4193a55fd4158e2d" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "breakfast-pizza", + "cited_source_ids": [], + "latency_ms": 900.0685749342665, + "outcome": "model_abstention", + "relevant_source_ids": [ + "review_413d502bdf96213e0bd1eb04378cd88e" + ], + "retrieved_source_ids": [ + "review_413d502bdf96213e0bd1eb04378cd88e", + "review_46cb53200f8521feb75e7642d41ab245", + "review_a6014a29ee06cc625b8b71a2bf39759b", + "review_4a9fb2adb5ce77a1c6cbddd7bb4be043", + "review_f624da623120634ef3ae535a9501688c" + ] + }, + { + "abstained": false, + "answer": "Based on the provided reviews, it appears that Roman-style pizza differs from a usual round pie in terms of its crust shape and texture.\n\nSome reviewers mention that the crust is rectangular or has a focaccia-like base (Review [1]), which is distinct from the traditional round pies. However, there is no consistent description of the crust's shape across all reviews.\n\nRegarding the crust's texture, some reviewers express disappointment with its toughness or lack of chewiness (Reviews [2] and [3]). One reviewer mentions that the crust was \"leathery\" and had to be torn apart (Review [2]), while another notes that it was still doughy and pale (Review [3]).\n\nIn contrast, Review [4] describes a crust with an excessive edge, making up almost half of each slice, which is not typical of Roman-style pizza. Review [5] mentions that the crust was burnt, but this is not a characteristic commonly associated with Roman-style pizza.\n\nOverall, while there are some differences in texture and shape, the reviews do not provide consistent evidence to support a clear distinction between Roman-style pizza and usual round pies.", + "case_id": "roman-style", + "cited_source_ids": [ + "review_d9ca96b76c7ab69c0f900f804d4a62f2", + "review_27843d67520f3d3e98ad5dfd65b75bf1", + "review_e8050c6a11acaab43c264cebcdc532a4", + "review_432fa44d59e772d65107db19fcf2ed20", + "review_44f1b7b944f624aad2586e2dffa0ea12" + ], + "latency_ms": 16624.680222012103, + "outcome": "answered", + "relevant_source_ids": [ + "review_d9ca96b76c7ab69c0f900f804d4a62f2" + ], + "retrieved_source_ids": [ + "review_d9ca96b76c7ab69c0f900f804d4a62f2", + "review_432fa44d59e772d65107db19fcf2ed20", + "review_44f1b7b944f624aad2586e2dffa0ea12", + "review_27843d67520f3d3e98ad5dfd65b75bf1", + "review_e8050c6a11acaab43c264cebcdc532a4" + ] + }, + { + "abstained": false, + "answer": "Based on the provided reviews, it appears that the structural details praised in the Detroit-style pizza are:\n\n* Crispy cheese edges [1]\n* Fluffy interior [1]\n* Corner pieces [1]\n\nThere is no mention of a specific crust type or texture being praised. Review [2] mentions \"perfect crispness\" but does not specify what type of crust it refers to.\n\nInsufficient evidence exists for the structural details of the crust itself, such as its thickness, flakiness, or layering.", + "case_id": "detroit-style", + "cited_source_ids": [ + "review_dad04092fb528c74b9d80ce442930ee4", + "review_f239cd569a160545f30b51fc79098f0b" + ], + "latency_ms": 7712.886944995262, + "outcome": "answered", + "relevant_source_ids": [ + "review_dad04092fb528c74b9d80ce442930ee4" + ], + "retrieved_source_ids": [ + "review_dad04092fb528c74b9d80ce442930ee4", + "review_442067c8d75bbb31443c4aa64f1a1a45", + "review_f239cd569a160545f30b51fc79098f0b", + "review_8b58fbd7743e24c7f9332a65fb828318", + "review_9b3c07f09c27b1ad9d58b3e4b4dbcc83" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "chicago-deep-dish", + "cited_source_ids": [], + "latency_ms": 884.7970690112561, + "outcome": "model_abstention", + "relevant_source_ids": [ + "review_442067c8d75bbb31443c4aa64f1a1a45" + ], + "retrieved_source_ids": [ + "review_442067c8d75bbb31443c4aa64f1a1a45", + "review_dad04092fb528c74b9d80ce442930ee4", + "review_0e5472a8c24c61a7e4de614412461918", + "review_8b58fbd7743e24c7f9332a65fb828318", + "review_a23134f6b63ffdb4c6fab9baa2cce095" + ] + }, + { + "abstained": false, + "answer": "Insufficient Evidence.\n\nThe reviews do not provide consistent or clear characterization of the New York-style slices. While some reviewers mention specific aspects such as the foldability [1] and taste [2], others criticize the presentation [3] and quality of the crust [4]. There is no unified view on what constitutes a \"New York-style slice\" across all reviews.", + "case_id": "new-york-style", + "cited_source_ids": [ + "review_0689a695ad22db80c369549a9ae071b2", + "review_0e5472a8c24c61a7e4de614412461918", + "review_202af3d22a12a639d21c216037e81d49", + "review_e8050c6a11acaab43c264cebcdc532a4" + ], + "latency_ms": 5413.699205964804, + "outcome": "answered", + "relevant_source_ids": [ + "review_0689a695ad22db80c369549a9ae071b2", + "review_0e5472a8c24c61a7e4de614412461918" + ], + "retrieved_source_ids": [ + "review_0e5472a8c24c61a7e4de614412461918", + "review_0689a695ad22db80c369549a9ae071b2", + "review_202af3d22a12a639d21c216037e81d49", + "review_e8050c6a11acaab43c264cebcdc532a4", + "review_efa9ed9172b37cc0f45b45301c7ed9c3" + ] + }, + { + "abstained": false, + "answer": "I could not produce an answer with citations that match the retrieved reviews.", + "case_id": "delivery-problems", + "cited_source_ids": [], + "latency_ms": 739.1056179767475, + "outcome": "citation_validation_rejection", + "relevant_source_ids": [ + "review_46deb903486e7bdbf280ccf75a8053b0", + "review_50429ddac7fcf92eb87d761a7b7e4d11", + "review_c79555a5cf3345966342cfb9207d2405", + "review_fb4a5b16a84ab35efc47e281e01894ce" + ], + "retrieved_source_ids": [ + "review_c79555a5cf3345966342cfb9207d2405", + "review_50429ddac7fcf92eb87d761a7b7e4d11", + "review_fb4a5b16a84ab35efc47e281e01894ce", + "review_9671a6061820222dd8af1a72592a0a76", + "review_f3684b15c85f4e974d3bf5606c7129aa" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "long-waits", + "cited_source_ids": [], + "latency_ms": 900.727892993018, + "outcome": "model_abstention", + "relevant_source_ids": [ + "review_4306c4305c5959dd3587f8e870437a4a", + "review_442067c8d75bbb31443c4aa64f1a1a45", + "review_dad04092fb528c74b9d80ce442930ee4" + ], + "retrieved_source_ids": [ + "review_fb4a5b16a84ab35efc47e281e01894ce", + "review_4306c4305c5959dd3587f8e870437a4a", + "review_35bbb51a785b5d72e21b309d41d0db80", + "review_c79555a5cf3345966342cfb9207d2405", + "review_50429ddac7fcf92eb87d761a7b7e4d11" + ] + }, + { + "abstained": false, + "answer": "The evidence suggests that the restaurant can handle large groups or parties because reviews [1], [2], and [3] mention accommodating for big groups, party packages, and hosting kids' parties with various activities. Review [4] also mentions a \"party package\" but does not specifically state handling large groups.\n\nInsufficient Evidence", + "case_id": "large-groups", + "cited_source_ids": [ + "review_9d84ed9fd7582f0a9f352fcdcc03483e", + "review_d8101a592a128df0fb622215f018391c", + "review_d57690690aac0645faada4223d374551", + "review_ad6e457de9e0de9ddca62dd81012d94a" + ], + "latency_ms": 5021.321017993614, + "outcome": "answered", + "relevant_source_ids": [ + "review_9d84ed9fd7582f0a9f352fcdcc03483e", + "review_d8101a592a128df0fb622215f018391c" + ], + "retrieved_source_ids": [ + "review_9d84ed9fd7582f0a9f352fcdcc03483e", + "review_d8101a592a128df0fb622215f018391c", + "review_f624da623120634ef3ae535a9501688c", + "review_d57690690aac0645faada4223d374551", + "review_ad6e457de9e0de9ddca62dd81012d94a" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "children-and-families", + "cited_source_ids": [], + "latency_ms": 880.2069170633331, + "outcome": "model_abstention", + "relevant_source_ids": [ + "review_42e9a66d38512d7ecd0a86e52f12b2d2", + "review_d57690690aac0645faada4223d374551", + "review_f20617ff46e7cc6abbc2073fca2f43cd" + ], + "retrieved_source_ids": [ + "review_f20617ff46e7cc6abbc2073fca2f43cd", + "review_d57690690aac0645faada4223d374551", + "review_f624da623120634ef3ae535a9501688c", + "review_42e9a66d38512d7ecd0a86e52f12b2d2", + "review_10c14c4f4777cd0e92889dd3f2c8e96b" + ] + }, + { + "abstained": false, + "answer": "I could not produce an answer with citations that match the retrieved reviews.", + "case_id": "drink-pairings", + "cited_source_ids": [], + "latency_ms": 722.0424159895629, + "outcome": "citation_validation_rejection", + "relevant_source_ids": [ + "review_2f293fad89162c9ceb164d9812bc3d83", + "review_3618241908457302b051385404a70ab0" + ], + "retrieved_source_ids": [ + "review_3618241908457302b051385404a70ab0", + "review_2f293fad89162c9ceb164d9812bc3d83", + "review_2787e993842fe9bd113a2d1d458e3295", + "review_9b9578da35871932c39da7a0fdc46c1b", + "review_58f10e007784450cfe4c44e239a3bed7" + ] + }, + { + "abstained": false, + "answer": "The specific price or portion complaints that make reviewers question the value are:\n\n* Review [1]: The reviewer feels that the 8-slice large pizza has slices so small that two bites are gone, making the portion size \"simply unacceptable\" for the price charged.\n* Review [2]: The reviewer expects a larger portion size for $15 and is disappointed when they receive a 'personal' pizza that's only the size of a CD.\n* Review [3]: The reviewer finds the prices too high, especially for basic large cheese pizzas ($30-$40), considering the quality doesn't justify these premium prices.\n\nThese complaints suggest that reviewers are questioning the value due to perceived overpricing and/or small portion sizes.", + "case_id": "price-and-value", + "cited_source_ids": [ + "review_b9e259e601836dfac086841092fe5c3b", + "review_80b54150ee2f71f582c9798263dd92fe", + "review_bb1e3ccbdbad8d3f35e41770cc6e6b61" + ], + "latency_ms": 10290.357508929446, + "outcome": "answered", + "relevant_source_ids": [ + "review_49f2ddf7f8434772a008801d3f7dfd95", + "review_80b54150ee2f71f582c9798263dd92fe", + "review_b9e259e601836dfac086841092fe5c3b", + "review_bb1e3ccbdbad8d3f35e41770cc6e6b61" + ], + "retrieved_source_ids": [ + "review_b9e259e601836dfac086841092fe5c3b", + "review_49f2ddf7f8434772a008801d3f7dfd95", + "review_80b54150ee2f71f582c9798263dd92fe", + "review_bb1e3ccbdbad8d3f35e41770cc6e6b61", + "review_9001d44a210559a0a3a9033e63f2fed4" + ] + }, + { + "abstained": false, + "answer": "Based on the provided reviews, signs of an undercooked center reported include:\n\n* Unmelted cheese (review [1])\n* Cold center with undermelted cheese (review [2])\n* Doughy and undercooked middle (reviews [3] and [4])\n\nNote that review [5] mentions a strange metallic aftertaste, but it is not directly related to the center being undercooked.", + "case_id": "undercooked-centers", + "cited_source_ids": [ + "review_1a28353790d8c4bc342444fd14f8be78", + "review_28360738a639784c3ea351484aef20d2", + "review_43cc47cc7c997e68800e2faa043b18fc", + "review_e8050c6a11acaab43c264cebcdc532a4", + "review_8c0489c3ad5568aac51608d88d7a8bc6" + ], + "latency_ms": 6167.138125980273, + "outcome": "answered", + "relevant_source_ids": [ + "review_1a28353790d8c4bc342444fd14f8be78", + "review_28360738a639784c3ea351484aef20d2", + "review_43cc47cc7c997e68800e2faa043b18fc", + "review_e8050c6a11acaab43c264cebcdc532a4" + ], + "retrieved_source_ids": [ + "review_1a28353790d8c4bc342444fd14f8be78", + "review_28360738a639784c3ea351484aef20d2", + "review_43cc47cc7c997e68800e2faa043b18fc", + "review_e8050c6a11acaab43c264cebcdc532a4", + "review_8c0489c3ad5568aac51608d88d7a8bc6" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "excessive-grease", + "cited_source_ids": [], + "latency_ms": 872.8065741015598, + "outcome": "model_abstention", + "relevant_source_ids": [ + "review_182d10267c2e4a1654aae9786af494d5", + "review_9671a6061820222dd8af1a72592a0a76", + "review_e58826d3c2728e96a0f7b4f055d4e6b1" + ], + "retrieved_source_ids": [ + "review_9671a6061820222dd8af1a72592a0a76", + "review_e58826d3c2728e96a0f7b4f055d4e6b1", + "review_182d10267c2e4a1654aae9786af494d5", + "review_9065fb1d99a02d0c960161b729015f39", + "review_e8050c6a11acaab43c264cebcdc532a4" + ] + }, + { + "abstained": false, + "answer": "Based on the provided reviews, the following sauce defects are described by dissatisfied reviewers:\n\n* Unpleasant chemical aftertaste (review [1])\n* Artificial flavor (review [1])\n* Suspiciously sweet and one-dimensional taste, reminiscent of ketchup (review [2])\n* Rancid notes from spoiled tomatoes (review [3])\n\nNote that the reviews do not specifically mention other sauce defects such as greasiness or sloppy presentation, which are mentioned in reviews [4] and [5], respectively.", + "case_id": "negative-sauce-quality", + "cited_source_ids": [ + "review_bb161bcd8bbe609cc1be19e232fef089", + "review_a92b57aa28366732db18740deb7d39c7", + "review_12574e71aaf0b76482d6fc6a57d34502", + "review_9671a6061820222dd8af1a72592a0a76", + "review_202af3d22a12a639d21c216037e81d49" + ], + "latency_ms": 7478.698421968147, + "outcome": "answered", + "relevant_source_ids": [ + "review_12574e71aaf0b76482d6fc6a57d34502", + "review_823c8cf17bd94ada444a824145796489", + "review_a92b57aa28366732db18740deb7d39c7", + "review_bb161bcd8bbe609cc1be19e232fef089" + ], + "retrieved_source_ids": [ + "review_bb161bcd8bbe609cc1be19e232fef089", + "review_a92b57aa28366732db18740deb7d39c7", + "review_12574e71aaf0b76482d6fc6a57d34502", + "review_9671a6061820222dd8af1a72592a0a76", + "review_202af3d22a12a639d21c216037e81d49" + ] + }, + { + "abstained": false, + "answer": "The evidence for fresh or premium ingredients in positive reviews comes from:\n\n* Review [1]: Fresh tomatoes are used instead of canned.\n* Review [2]: High-quality olive oil is added just before serving, which enhances flavors and adds a wonderful aromatic element.\n* Review [3]: No mention of specific ingredients being used, but the reviewer praises the \"perfect combination\" of toppings and distribution.\n* Review [4]: No mention of specific ingredients being used, but the reviewer praises the cooking time and attention to detail.\n* Review [5]: San Marzano tomatoes and in-house mozzarella production are mentioned as evidence of a commitment to quality ingredients.\n\nNote that while these reviews provide some evidence for fresh or premium ingredients, they do not provide direct evidence for all types of premium ingredients.", + "case_id": "fresh-ingredients", + "cited_source_ids": [ + "review_e0cf78dbbb255982fbfb1288297e52ea", + "review_2787e993842fe9bd113a2d1d458e3295", + "review_9b9578da35871932c39da7a0fdc46c1b", + "review_35bbb51a785b5d72e21b309d41d0db80", + "review_dc2ae1a58c8bbd89048c1c0b569af1af" + ], + "latency_ms": 11251.433104975149, + "outcome": "answered", + "relevant_source_ids": [ + "review_924b8212f2c0052946059cb4c867d038", + "review_dc2ae1a58c8bbd89048c1c0b569af1af", + "review_e0cf78dbbb255982fbfb1288297e52ea" + ], + "retrieved_source_ids": [ + "review_e0cf78dbbb255982fbfb1288297e52ea", + "review_2787e993842fe9bd113a2d1d458e3295", + "review_9b9578da35871932c39da7a0fdc46c1b", + "review_35bbb51a785b5d72e21b309d41d0db80", + "review_dc2ae1a58c8bbd89048c1c0b569af1af" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "seasonal-specials", + "cited_source_ids": [], + "latency_ms": 884.2669880250469, + "outcome": "model_abstention", + "relevant_source_ids": [ + "review_4a9fb2adb5ce77a1c6cbddd7bb4be043", + "review_5666239ad9f3340465ef839c7b49fb11" + ], + "retrieved_source_ids": [ + "review_4a9fb2adb5ce77a1c6cbddd7bb4be043", + "review_5666239ad9f3340465ef839c7b49fb11", + "review_46cb53200f8521feb75e7642d41ab245", + "review_ca3130944bc8520c4936ece2e143cb12", + "review_e0cf78dbbb255982fbfb1288297e52ea" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "parking-abstention", + "cited_source_ids": [], + "latency_ms": 875.8160950383171, + "outcome": "model_abstention", + "relevant_source_ids": [], + "retrieved_source_ids": [ + "review_ad6e457de9e0de9ddca62dd81012d94a", + "review_9a960b06110fc713601c0d788e7b47e2", + "review_9065fb1d99a02d0c960161b729015f39", + "review_01089957f487f383fe7854b0f2a4f945", + "review_4306c4305c5959dd3587f8e870437a4a" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "wheelchair-abstention", + "cited_source_ids": [], + "latency_ms": 870.9037119988352, + "outcome": "model_abstention", + "relevant_source_ids": [], + "retrieved_source_ids": [ + "review_ad6e457de9e0de9ddca62dd81012d94a", + "review_f624da623120634ef3ae535a9501688c", + "review_10c14c4f4777cd0e92889dd3f2c8e96b", + "review_d57690690aac0645faada4223d374551", + "review_9065fb1d99a02d0c960161b729015f39" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "reservations-abstention", + "cited_source_ids": [], + "latency_ms": 868.6721380800009, + "outcome": "model_abstention", + "relevant_source_ids": [], + "retrieved_source_ids": [ + "review_ad6e457de9e0de9ddca62dd81012d94a", + "review_f624da623120634ef3ae535a9501688c", + "review_9a960b06110fc713601c0d788e7b47e2", + "review_01089957f487f383fe7854b0f2a4f945", + "review_44993d1fac543a96025bdbf0aad40207" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "wifi-abstention", + "cited_source_ids": [], + "latency_ms": 877.0650050137192, + "outcome": "model_abstention", + "relevant_source_ids": [], + "retrieved_source_ids": [ + "review_ad6e457de9e0de9ddca62dd81012d94a", + "review_9a960b06110fc713601c0d788e7b47e2", + "review_44993d1fac543a96025bdbf0aad40207", + "review_9065fb1d99a02d0c960161b729015f39", + "review_5df87053cbf1f73cd631288f814a9208" + ] + }, + { + "abstained": true, + "answer": "I could not find any reviews matching the current question and filters.", + "case_id": "delivery-fee-abstention", + "cited_source_ids": [], + "latency_ms": 862.8429740201682, + "outcome": "model_abstention", + "relevant_source_ids": [], + "retrieved_source_ids": [ + "review_49f2ddf7f8434772a008801d3f7dfd95", + "review_50429ddac7fcf92eb87d761a7b7e4d11", + "review_fb4a5b16a84ab35efc47e281e01894ce", + "review_b9e259e601836dfac086841092fe5c3b", + "review_4306c4305c5959dd3587f8e870437a4a" + ] + } + ] + }, + "provenance": { + "application_version": "0.2.0", + "cases_file": "rag_cases.json", + "cases_sha256": "b46d008043ad5a815fc6f9563362901f2a4368a408739c811ae80a614556a141", + "dataset_file": "realistic_restaurant_reviews.csv", + "dataset_sha256": "8ace5c1cb728c3c8a355d90a414849ebd645e021aae8a5b32303ef310917254e", + "dependency_versions": { + "chromadb": "1.5.9", + "langchain-ollama": "1.1.0", + "pandas": "3.0.2" + }, + "git_commit": "b9800bd2c1440c424b5407b25c7093488c83eb00", + "git_dirty": true, + "platform": "Linux-7.0.0-1010-azure-x86_64-with-glibc2.43", + "python_version": "3.11.15", + "review_count": 123 + }, + "results": { + "bm25_baseline": { + "evaluated_case_count": 25, + "hit_rate_at_k": 0.88, + "limit": 5, + "mrr_at_k": 0.7533333333333333, + "recall_at_k": 0.77 + }, + "rag": { + "abstention_case_count": 5, + "abstention_recall": 1.0, + "answer_success_rate": 0.56, + "answerable_case_count": 25, + "case_count": 30, + "citation_validity": 0.56, + "expected_action_accuracy": 0.6333333333333333, + "reference_term_support_proxy": 0.52, + "retrieval_recall": 0.9133333333333333 + }, + "semantic_retrieval": { + "evaluated_case_count": 25, + "hit_rate_at_k": 1.0, + "limit": 5, + "mrr_at_k": 0.96, + "recall_at_k": 0.9133333333333333 + } + }, + "schema_version": 2, + "timing": { + "rag_mean_latency_ms": 4358.683, + "rag_median_latency_ms": 2006.227, + "rag_total_latency_ms": 130760.489 + } +} diff --git a/evaluation/results/v0.2.0-ollama-0.32.5/run.log b/evaluation/results/v0.2.0-ollama-0.32.5/run.log new file mode 100644 index 0000000..b885e92 --- /dev/null +++ b/evaluation/results/v0.2.0-ollama-0.32.5/run.log @@ -0,0 +1,43 @@ +{ + "abstention_case_count": 5, + "abstention_recall": 1.0, + "answer_success_rate": 0.56, + "answerable_case_count": 25, + "case_count": 30, + "citation_validity": 0.56, + "expected_action_accuracy": 0.6333333333333333, + "reference_term_support_proxy": 0.52, + "retrieval_recall": 0.9133333333333333 +} +crispy-crust: retrieved=5 cited=4 abstained=False +vegan-cheese: retrieved=5 cited=4 abstained=False +dessert-pizza: retrieved=5 cited=1 abstained=False +gluten-free-crust: retrieved=5 cited=0 abstained=False +margherita-components: retrieved=5 cited=4 abstained=False +hawaiian-pizza: retrieved=5 cited=0 abstained=True +white-pizza: retrieved=5 cited=0 abstained=True +vodka-sauce: retrieved=5 cited=3 abstained=False +buffalo-chicken: retrieved=5 cited=5 abstained=False +breakfast-pizza: retrieved=5 cited=0 abstained=True +roman-style: retrieved=5 cited=5 abstained=False +detroit-style: retrieved=5 cited=2 abstained=False +chicago-deep-dish: retrieved=5 cited=0 abstained=True +new-york-style: retrieved=5 cited=4 abstained=False +delivery-problems: retrieved=5 cited=0 abstained=False +long-waits: retrieved=5 cited=0 abstained=True +large-groups: retrieved=5 cited=4 abstained=False +children-and-families: retrieved=5 cited=0 abstained=True +drink-pairings: retrieved=5 cited=0 abstained=False +price-and-value: retrieved=5 cited=3 abstained=False +undercooked-centers: retrieved=5 cited=5 abstained=False +excessive-grease: retrieved=5 cited=0 abstained=True +negative-sauce-quality: retrieved=5 cited=5 abstained=False +fresh-ingredients: retrieved=5 cited=5 abstained=False +seasonal-specials: retrieved=5 cited=0 abstained=True +parking-abstention: retrieved=5 cited=0 abstained=True +wheelchair-abstention: retrieved=5 cited=0 abstained=True +reservations-abstention: retrieved=5 cited=0 abstained=True +wifi-abstention: retrieved=5 cited=0 abstained=True +delivery-fee-abstention: retrieved=5 cited=0 abstained=True +Wrote evaluation/results/v0.2.0-ollama-0.32.5-isolated/evaluation-report.json +Wrote evaluation/results/v0.2.0-ollama-0.32.5-isolated/README.md diff --git a/local_ai_agent/data/rag_cases.json b/local_ai_agent/data/rag_cases.json index e128940..33c0805 100644 --- a/local_ai_agent/data/rag_cases.json +++ b/local_ai_agent/data/rag_cases.json @@ -1,45 +1,887 @@ -[ - { - "id": "crispy-crust", - "question": "Do guests describe the pizza crust as crispy?", - "relevant_titles": ["Best pizza in town"], - "reference_facts": [ - { - "answer_terms": ["crispy", "crisp"], - "source_terms": ["perfectly crispy"] - } - ] +{ + "schema_version": "rag-eval-cases/2.0", + "evaluation_set_id": "bundled-restaurant-reviews-v2.0", + "dataset": { + "file": "realistic_restaurant_reviews.csv", + "sha256": "8ace5c1cb728c3c8a355d90a414849ebd645e021aae8a5b32303ef310917254e", + "row_count": 123, + "relevance_judgments": "known_positives_only" }, - { - "id": "vegan-cheese", - "question": "What do reviewers say about the vegan cheese options?", - "relevant_titles": [ - "Hidden gem for vegans", - "Tasteless cheese substitute" - ], - "reference_facts": [ - { - "answer_terms": ["vegan cheese", "cashew cheese"], - "source_terms": ["cashew cheese", "vegan cheese option"] - } - ] - }, - { - "id": "dessert-pizza", - "question": "Do the reviews mention dessert pizza?", - "relevant_titles": ["Outstanding variety of options"], - "reference_facts": [ - { - "answer_terms": ["dessert", "Nutella"], - "source_terms": ["dessert pizzas", "Nutella and banana"] - } - ] - }, - { - "id": "parking-abstention", - "question": "Is parking available at the restaurant?", - "relevant_titles": [], - "reference_facts": [], - "should_abstain": true - } -] + "cases": [ + { + "id": "crispy-crust", + "split": "test", + "category": "texture", + "difficulty": "hard", + "question": "Do guests describe the pizza crust as crispy while still having some chew?", + "expected_action": "answer", + "gold_source_ids": [ + "review_a23134f6b63ffdb4c6fab9baa2cce095", + "review_13f6c41bf74292e7087a1a7ccf100c6d", + "review_cf6397aa45938d26e42aca081d576a93" + ], + "source_labels": [ + "Best pizza in town", + "Phenomenal crust", + "Crispy yet chewy perfection" + ], + "reference_facts": [ + { + "answer_terms": [ + "crispy", + "crisp" + ], + "source_terms": [ + "perfectly crispy", + "combination of crispy and chewy", + "crispy on the outside and chewy inside" + ] + }, + { + "answer_terms": [ + "chewy", + "chew" + ], + "source_terms": [ + "chewy inside", + "slight sourdough tang", + "simultaneously crispy on the outside and chewy inside" + ] + } + ] + }, + { + "id": "vegan-cheese", + "split": "test", + "category": "dietary", + "difficulty": "medium", + "question": "What contrasting experiences do reviewers report about the vegan cheese?", + "expected_action": "answer", + "gold_source_ids": [ + "review_20c52415ea843838a45b29821f7ba112", + "review_08222ec4bad55a761d23c3dddf0dae72" + ], + "source_labels": [ + "Hidden gem for vegans", + "Tasteless cheese substitute" + ], + "reference_facts": [ + { + "answer_terms": [ + "cashew", + "melts", + "meltability" + ], + "source_terms": [ + "cashew cheese", + "no meltability" + ] + }, + { + "answer_terms": [ + "grainy", + "coconut", + "negative" + ], + "source_terms": [ + "grainy texture", + "tasted strongly of coconut" + ] + } + ] + }, + { + "id": "dessert-pizza", + "split": "test", + "category": "menu", + "difficulty": "easy", + "question": "Do the reviews mention a dessert pizza, and which flavor is recommended?", + "expected_action": "answer", + "gold_source_ids": [ + "review_f624da623120634ef3ae535a9501688c" + ], + "source_labels": [ + "Outstanding variety of options" + ], + "reference_facts": [ + { + "answer_terms": [ + "dessert", + "Nutella" + ], + "source_terms": [ + "dessert pizzas", + "Nutella and banana" + ] + } + ] + }, + { + "id": "gluten-free-crust", + "split": "test", + "category": "dietary", + "difficulty": "easy", + "question": "What does the reviewer with celiac disease say about the gluten-free crust?", + "expected_action": "answer", + "gold_source_ids": [ + "review_6464cf45ebbd480454abc07065075c3b" + ], + "source_labels": [ + "Great gluten-free option" + ], + "reference_facts": [ + { + "answer_terms": [ + "cauliflower", + "doesn't fall apart", + "good" + ], + "source_terms": [ + "celiac disease", + "cauliflower crust", + "doesn't fall apart" + ] + } + ] + }, + { + "id": "margherita-components", + "split": "test", + "category": "style", + "difficulty": "medium", + "question": "Which ingredients and cooking qualities are praised in the Margherita pizzas?", + "expected_action": "answer", + "gold_source_ids": [ + "review_5117c8c073d626838c7798930f15b8e1", + "review_13809cb33172b3c40040348b296245df" + ], + "source_labels": [ + "Authentic Italian experience", + "Excellence in simplicity" + ], + "reference_facts": [ + { + "answer_terms": [ + "tomato", + "mozzarella", + "basil" + ], + "source_terms": [ + "buffalo mozzarella", + "fresh mozzarella", + "fragrant basil" + ] + }, + { + "answer_terms": [ + "char", + "crust" + ], + "source_terms": [ + "perfect char", + "chewy-crisp crust" + ] + } + ] + }, + { + "id": "hawaiian-pizza", + "split": "test", + "category": "menu", + "difficulty": "easy", + "question": "What makes the recommended Hawaiian pizza distinctive?", + "expected_action": "answer", + "gold_source_ids": [ + "review_211ffae3b56d5e65fc7edc3f92143b12" + ], + "source_labels": [ + "Best Hawaiian in the city" + ], + "reference_facts": [ + { + "answer_terms": [ + "pineapple", + "ham" + ], + "source_terms": [ + "fresh pineapple", + "house-cured ham", + "sweet and salty balance" + ] + } + ] + }, + { + "id": "white-pizza", + "split": "test", + "category": "menu", + "difficulty": "medium", + "question": "What ingredients appear in the praised spinach and ricotta white pizzas?", + "expected_action": "answer", + "gold_source_ids": [ + "review_a6014a29ee06cc625b8b71a2bf39759b", + "review_1615431adc386cbc8063f8a3e7983f10" + ], + "source_labels": [ + "Best white pizza ever", + "Exceptional spinach ricotta pie" + ], + "reference_facts": [ + { + "answer_terms": [ + "ricotta", + "spinach", + "garlic" + ], + "source_terms": [ + "ricotta, mozzarella, garlic, and spinach", + "spinach and ricotta", + "garlic-infused olive oil" + ] + } + ] + }, + { + "id": "vodka-sauce", + "split": "test", + "category": "menu", + "difficulty": "easy", + "question": "How is the vodka-sauce pizza described, and what optional topping is suggested?", + "expected_action": "answer", + "gold_source_ids": [ + "review_bfeabf92afa1ff6c1644bee468624ca2" + ], + "source_labels": [ + "Incredible vodka sauce base" + ], + "reference_facts": [ + { + "answer_terms": [ + "creamy", + "spicy", + "prosciutto" + ], + "source_terms": [ + "creamy, slightly spicy sauce", + "Add prosciutto" + ] + } + ] + }, + { + "id": "buffalo-chicken", + "split": "test", + "category": "menu", + "difficulty": "easy", + "question": "What balance and ingredient quality are praised in the buffalo chicken pizza?", + "expected_action": "answer", + "gold_source_ids": [ + "review_2ab5eb185f34f7f6233d315242b13034" + ], + "source_labels": [ + "Brilliant buffalo chicken pizza" + ], + "reference_facts": [ + { + "answer_terms": [ + "spicy", + "creamy", + "juicy", + "blue cheese" + ], + "source_terms": [ + "balances spicy and creamy", + "high-quality and juicy", + "blue cheese" + ] + } + ] + }, + { + "id": "breakfast-pizza", + "split": "test", + "category": "menu", + "difficulty": "easy", + "question": "What comes on the breakfast pizza, and when is it offered?", + "expected_action": "answer", + "gold_source_ids": [ + "review_413d502bdf96213e0bd1eb04378cd88e" + ], + "source_labels": [ + "Outstanding breakfast pizza" + ], + "reference_facts": [ + { + "answer_terms": [ + "bacon", + "eggs", + "hollandaise", + "brunch" + ], + "source_terms": [ + "bacon, eggs, and hollandaise sauce", + "weekend brunch" + ] + } + ] + }, + { + "id": "roman-style", + "split": "test", + "category": "style", + "difficulty": "easy", + "question": "How does the Roman-style pizza differ from a usual round pie?", + "expected_action": "answer", + "gold_source_ids": [ + "review_d9ca96b76c7ab69c0f900f804d4a62f2" + ], + "source_labels": [ + "Perfect niche for Roman-style pizza" + ], + "reference_facts": [ + { + "answer_terms": [ + "rectangular", + "focaccia", + "airy" + ], + "source_terms": [ + "pizza al taglio", + "rectangular", + "focaccia-like base", + "airy, light crust" + ] + } + ] + }, + { + "id": "detroit-style", + "split": "test", + "category": "style", + "difficulty": "easy", + "question": "Which structural details are praised in the Detroit-style pizza?", + "expected_action": "answer", + "gold_source_ids": [ + "review_dad04092fb528c74b9d80ce442930ee4" + ], + "source_labels": [ + "Amazing Detroit-style" + ], + "reference_facts": [ + { + "answer_terms": [ + "square", + "crispy", + "fluffy", + "corner" + ], + "source_terms": [ + "square pizza", + "crispy cheese edges", + "fluffy interior", + "corner pieces" + ] + } + ] + }, + { + "id": "chicago-deep-dish", + "split": "test", + "category": "style", + "difficulty": "easy", + "question": "What identifies the praised Chicago-style deep-dish pizza?", + "expected_action": "answer", + "gold_source_ids": [ + "review_442067c8d75bbb31443c4aa64f1a1a45" + ], + "source_labels": [ + "Excellent Chicago deep dish" + ], + "reference_facts": [ + { + "answer_terms": [ + "buttery", + "flaky", + "cheese", + "sauce" + ], + "source_terms": [ + "buttery, flaky crust", + "cheese under the sauce", + "chunky tomato topping" + ] + } + ] + }, + { + "id": "new-york-style", + "split": "test", + "category": "style", + "difficulty": "medium", + "question": "How do reviewers characterize the New York-style slices?", + "expected_action": "answer", + "gold_source_ids": [ + "review_0e5472a8c24c61a7e4de614412461918", + "review_0689a695ad22db80c369549a9ae071b2" + ], + "source_labels": [ + "Perfect NY-style slices", + "Expert-level pizza folding" + ], + "reference_facts": [ + { + "answer_terms": [ + "thin", + "foldable", + "chew" + ], + "source_terms": [ + "thin, foldable slices", + "fold perfectly New York-style", + "balance of crisp and chew" + ] + } + ] + }, + { + "id": "delivery-problems", + "split": "test", + "category": "operations", + "difficulty": "hard", + "question": "Which delivery and takeout failures appear across the negative reviews?", + "expected_action": "answer", + "gold_source_ids": [ + "review_fb4a5b16a84ab35efc47e281e01894ce", + "review_50429ddac7fcf92eb87d761a7b7e4d11", + "review_c79555a5cf3345966342cfb9207d2405", + "review_46deb903486e7bdbf280ccf75a8053b0" + ], + "source_labels": [ + "Disappointed with service", + "Terrible delivery experience", + "Pizza barely survived the drive home", + "Pizza arrived upside-down" + ], + "reference_facts": [ + { + "answer_terms": [ + "late", + "cold", + "two hours", + "2 hours" + ], + "source_terms": [ + "over an hour for delivery", + "2 hours to arrive", + "cold" + ] + }, + { + "answer_terms": [ + "soggy", + "upside-down", + "upside down" + ], + "source_terms": [ + "soggy mess", + "completely flipped over" + ] + } + ] + }, + { + "id": "long-waits", + "split": "test", + "category": "operations", + "difficulty": "hard", + "question": "What long wait or cooking times are explicitly reported?", + "expected_action": "answer", + "gold_source_ids": [ + "review_4306c4305c5959dd3587f8e870437a4a", + "review_dad04092fb528c74b9d80ce442930ee4", + "review_442067c8d75bbb31443c4aa64f1a1a45" + ], + "source_labels": [ + "Not worth the wait", + "Amazing Detroit-style", + "Excellent Chicago deep dish" + ], + "reference_facts": [ + { + "answer_terms": [ + "90", + "30", + "45", + "minute" + ], + "source_terms": [ + "90-minute wait", + "30-minute wait", + "45-minute cook time" + ] + } + ] + }, + { + "id": "large-groups", + "split": "test", + "category": "operations", + "difficulty": "medium", + "question": "What evidence suggests the restaurant can handle large groups or parties?", + "expected_action": "answer", + "gold_source_ids": [ + "review_9d84ed9fd7582f0a9f352fcdcc03483e", + "review_d8101a592a128df0fb622215f018391c" + ], + "source_labels": [ + "Perfect for big groups", + "Perfect for large parties" + ], + "reference_facts": [ + { + "answer_terms": [ + "12", + "20", + "group", + "party" + ], + "source_terms": [ + "party of 12", + "group of 20", + "shareable pizzas", + "party package" + ] + } + ] + }, + { + "id": "children-and-families", + "split": "test", + "category": "operations", + "difficulty": "hard", + "question": "Which activities or menu options are described for children and families?", + "expected_action": "answer", + "gold_source_ids": [ + "review_f20617ff46e7cc6abbc2073fca2f43cd", + "review_d57690690aac0645faada4223d374551", + "review_42e9a66d38512d7ecd0a86e52f12b2d2" + ], + "source_labels": [ + "Family night favorite", + "Perfect for kids' parties", + "Perfect for picky eaters" + ], + "reference_facts": [ + { + "answer_terms": [ + "dough", + "mini", + "kid", + "quadrant" + ], + "source_terms": [ + "dough for them to play with", + "kid-sized pizzas", + "make their own mini pizzas", + "divided into quadrants" + ] + } + ] + }, + { + "id": "drink-pairings", + "split": "test", + "category": "beverages", + "difficulty": "medium", + "question": "What drink-pairing help do reviewers praise?", + "expected_action": "answer", + "gold_source_ids": [ + "review_3618241908457302b051385404a70ab0", + "review_2f293fad89162c9ceb164d9812bc3d83" + ], + "source_labels": [ + "Excellent beer pairing suggestions", + "Excellent wine pairings" + ], + "reference_facts": [ + { + "answer_terms": [ + "beer", + "wine", + "pairing" + ], + "source_terms": [ + "craft beer selection", + "perfect pairing", + "curated wine list", + "house red" + ] + } + ] + }, + { + "id": "price-and-value", + "split": "test", + "category": "value", + "difficulty": "hard", + "question": "What specific price or portion complaints make reviewers question the value?", + "expected_action": "answer", + "gold_source_ids": [ + "review_49f2ddf7f8434772a008801d3f7dfd95", + "review_bb1e3ccbdbad8d3f35e41770cc6e6b61", + "review_80b54150ee2f71f582c9798263dd92fe", + "review_b9e259e601836dfac086841092fe5c3b" + ], + "source_labels": [ + "Overpriced for what you get", + "Too expensive for pizza", + "Tiny portion sizes", + "Absurdly small slices" + ], + "reference_facts": [ + { + "answer_terms": [ + "$24", + "$30", + "$15", + "expensive", + "overpriced" + ], + "source_terms": [ + "$24 for a medium", + "$30 for a basic large", + "For $15", + "portion size" + ] + } + ] + }, + { + "id": "undercooked-centers", + "split": "test", + "category": "quality-defect", + "difficulty": "hard", + "question": "What signs of an undercooked center are reported?", + "expected_action": "answer", + "gold_source_ids": [ + "review_43cc47cc7c997e68800e2faa043b18fc", + "review_e8050c6a11acaab43c264cebcdc532a4", + "review_28360738a639784c3ea351484aef20d2", + "review_1a28353790d8c4bc342444fd14f8be78" + ], + "source_labels": [ + "Undercooked in the middle", + "Doughy and undercooked", + "Cold center burnt edges", + "Unmelted cheese in the center" + ], + "reference_facts": [ + { + "answer_terms": [ + "doughy", + "cold", + "unmelted", + "sliding" + ], + "source_terms": [ + "middle was doughy and undercooked", + "toppings were sliding off", + "center was barely warm", + "cold, unmelted cheese" + ] + } + ] + }, + { + "id": "excessive-grease", + "split": "test", + "category": "quality-defect", + "difficulty": "hard", + "question": "How severe is the grease problem in the negative reviews?", + "expected_action": "answer", + "gold_source_ids": [ + "review_9671a6061820222dd8af1a72592a0a76", + "review_182d10267c2e4a1654aae9786af494d5", + "review_e58826d3c2728e96a0f7b4f055d4e6b1" + ], + "source_labels": [ + "Greasy and disappointing", + "Too greasy to enjoy", + "Excessive grease pooling" + ], + "reference_facts": [ + { + "answer_terms": [ + "swimming", + "blot", + "pool", + "napkin" + ], + "source_terms": [ + "swimming in grease", + "blot my pizza with napkins", + "pools of orange grease" + ] + } + ] + }, + { + "id": "negative-sauce-quality", + "split": "test", + "category": "quality-defect", + "difficulty": "hard", + "question": "Which different sauce defects are described by dissatisfied reviewers?", + "expected_action": "answer", + "gold_source_ids": [ + "review_823c8cf17bd94ada444a824145796489", + "review_a92b57aa28366732db18740deb7d39c7", + "review_bb161bcd8bbe609cc1be19e232fef089", + "review_12574e71aaf0b76482d6fc6a57d34502" + ], + "source_labels": [ + "Bland sauce needs work", + "Sauce like ketchup", + "Sauce had an odd chemical taste", + "Tomato sauce tasted rancid" + ], + "reference_facts": [ + { + "answer_terms": [ + "bland", + "ketchup", + "chemical", + "rancid" + ], + "source_terms": [ + "lacked any personality", + "reminiscent of ketchup", + "artificial flavor", + "rancid notes" + ] + } + ] + }, + { + "id": "fresh-ingredients", + "split": "test", + "category": "ingredients", + "difficulty": "hard", + "question": "What evidence do positive reviews give for fresh or premium ingredients?", + "expected_action": "answer", + "gold_source_ids": [ + "review_e0cf78dbbb255982fbfb1288297e52ea", + "review_dc2ae1a58c8bbd89048c1c0b569af1af", + "review_924b8212f2c0052946059cb4c867d038" + ], + "source_labels": [ + "Fresh ingredients make the difference", + "Superior ingredients make the difference", + "Exceptional sauce-making technique" + ], + "reference_facts": [ + { + "answer_terms": [ + "fresh", + "Italy", + "San Marzano", + "house", + "hand-crushed" + ], + "source_terms": [ + "fresh tomatoes", + "import their flour from Italy", + "San Marzano tomatoes", + "mozzarella in-house", + "hand-crushed tomatoes" + ] + } + ] + }, + { + "id": "seasonal-specials", + "split": "test", + "category": "ingredients", + "difficulty": "medium", + "question": "Which seasonal ingredients appear in the spring pizza specials?", + "expected_action": "answer", + "gold_source_ids": [ + "review_5666239ad9f3340465ef839c7b49fb11", + "review_4a9fb2adb5ce77a1c6cbddd7bb4be043" + ], + "source_labels": [ + "Creative seasonal specials", + "Brilliant seasonal rotation" + ], + "reference_facts": [ + { + "answer_terms": [ + "asparagus", + "ricotta", + "prosciutto", + "peas", + "garlic" + ], + "source_terms": [ + "asparagus, lemon ricotta, and prosciutto", + "asparagus, green garlic, and fresh peas" + ] + } + ] + }, + { + "id": "parking-abstention", + "split": "test", + "category": "abstention", + "difficulty": "medium", + "question": "Is customer parking available at the restaurant?", + "expected_action": "abstain", + "gold_source_ids": [], + "source_labels": [], + "reference_facts": [] + }, + { + "id": "wheelchair-abstention", + "split": "test", + "category": "abstention", + "difficulty": "medium", + "question": "Is the entrance and dining room wheelchair accessible?", + "expected_action": "abstain", + "gold_source_ids": [], + "source_labels": [], + "reference_facts": [] + }, + { + "id": "reservations-abstention", + "split": "test", + "category": "abstention", + "difficulty": "medium", + "question": "Does the restaurant accept reservations through its website?", + "expected_action": "abstain", + "gold_source_ids": [], + "source_labels": [], + "reference_facts": [] + }, + { + "id": "wifi-abstention", + "split": "test", + "category": "abstention", + "difficulty": "medium", + "question": "Does the restaurant provide free Wi-Fi to customers?", + "expected_action": "abstain", + "gold_source_ids": [], + "source_labels": [], + "reference_facts": [] + }, + { + "id": "delivery-fee-abstention", + "split": "test", + "category": "abstention", + "difficulty": "medium", + "question": "How much does the restaurant charge as a delivery fee?", + "expected_action": "abstain", + "gold_source_ids": [], + "source_labels": [], + "reference_facts": [] + } + ] +} diff --git a/main.py b/main.py index e5d9af0..9648d3d 100644 --- a/main.py +++ b/main.py @@ -1,21 +1,36 @@ import argparse +import hashlib import json +import platform +import subprocess import sys from collections.abc import Sequence from datetime import date +from importlib.metadata import PackageNotFoundError, version from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +from httpx import HTTPError +from ollama import ResponseError from agent import answer_question, create_chat_model from evaluation import ( DEFAULT_EVALUATION_PATH, + build_evaluation_report, load_evaluation_cases, + retrieval_metrics_from_observations, + run_bm25_baseline, run_rag_evaluation, + write_evaluation_report, ) +from local_ai_agent import __version__ from ollama_health import ( DEFAULT_CHAT_MODEL, DEFAULT_EMBEDDING_MODEL, DEFAULT_OLLAMA_HOST, check_ollama, + model_metadata, + ollama_version, ) from vector import ( DEFAULT_DATA_PATH, @@ -78,6 +93,11 @@ def build_parser() -> argparse.ArgumentParser: ) evaluate_parser.add_argument("--cases", type=Path, default=DEFAULT_EVALUATION_PATH) evaluate_parser.add_argument("--limit", type=int, default=5) + evaluate_parser.add_argument( + "--report-dir", + type=Path, + help="write evaluation-report.json and README.md to this directory", + ) _add_runtime_arguments(evaluate_parser) chat_parser = subparsers.add_parser("chat", help="Start the interactive terminal") @@ -120,6 +140,76 @@ def _print_answer(result) -> None: print(source.document.page_content.replace("\n", " | ")) +def _safe_endpoint(value: str) -> str: + """Return an endpoint suitable for a committed report without credentials.""" + parsed = urlsplit(value) + if not parsed.scheme or not parsed.hostname: + return value.split("?", 1)[0].split("#", 1)[0] + hostname = parsed.hostname + if ":" in hostname and not hostname.startswith("["): + hostname = f"[{hostname}]" + netloc = hostname + if parsed.port is not None: + netloc = f"{netloc}:{parsed.port}" + return urlunsplit((parsed.scheme, netloc, parsed.path, "", "")) + + +def _ollama_report_metadata(arguments: argparse.Namespace): + try: + runtime_version = ollama_version(arguments.ollama_host) + models = model_metadata( + (arguments.chat_model, arguments.embedding_model), + host=arguments.ollama_host, + ) + except (HTTPError, OSError, ResponseError) as error: + raise ValueError( + f"could not collect Ollama report metadata: {error}" + ) from error + return runtime_version, models + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _git_provenance() -> dict[str, str | bool | None]: + root = Path(__file__).resolve().parent + try: + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + dirty = bool( + subprocess.run( + ["git", "status", "--porcelain"], + cwd=root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + ) + except (OSError, subprocess.CalledProcessError): + return {"git_commit": None, "git_dirty": None} + return {"git_commit": commit, "git_dirty": dirty} + + +def _dependency_versions() -> dict[str, str]: + resolved: dict[str, str] = {} + for package in ("chromadb", "langchain-ollama", "pandas"): + try: + resolved[package] = version(package) + except PackageNotFoundError: + resolved[package] = "not-installed" + return resolved + + def _create_runtime(arguments: argparse.Namespace): vector_store = create_vector_store( arguments.data, @@ -168,13 +258,26 @@ def run(arguments: argparse.Namespace) -> int: if arguments.command == "evaluate": vector_store, model = _create_runtime(arguments) - cases = load_evaluation_cases(arguments.cases) + cases = load_evaluation_cases( + arguments.cases, + dataset_path=arguments.data, + ) metrics, observations = run_rag_evaluation( cases, vector_store=vector_store, model=model, limit=arguments.limit, ) + semantic_metrics = retrieval_metrics_from_observations( + cases, + observations, + limit=arguments.limit, + ) + baseline_metrics, baseline_observations = run_bm25_baseline( + cases, + vector_store=vector_store, + limit=arguments.limit, + ) print(json.dumps(metrics.as_dict(), indent=2, sort_keys=True)) for observation in observations: print( @@ -182,6 +285,45 @@ def run(arguments: argparse.Namespace) -> int: f"cited={len(observation.cited_source_ids)} " f"abstained={observation.abstained}" ) + if arguments.report_dir is not None: + runtime_version, models = _ollama_report_metadata(arguments) + report = build_evaluation_report( + cases=cases, + rag_metrics=metrics, + semantic_metrics=semantic_metrics, + baseline_metrics=baseline_metrics, + observations=observations, + baseline_observations=baseline_observations, + configuration={ + "chat_model": arguments.chat_model, + "embedding_model": arguments.embedding_model, + "ollama_version": runtime_version, + "ollama_host": _safe_endpoint(arguments.ollama_host), + "evidence_limit": arguments.limit, + "models": models, + }, + provenance={ + "application_version": __version__, + "python_version": platform.python_version(), + "platform": platform.platform(), + "dataset_file": arguments.data.name, + "dataset_sha256": _file_sha256(arguments.data), + "review_count": len(dataframe), + "cases_file": arguments.cases.name, + "cases_sha256": _file_sha256(arguments.cases), + "dependency_versions": _dependency_versions(), + **_git_provenance(), + }, + ) + json_path = arguments.report_dir / "evaluation-report.json" + markdown_path = arguments.report_dir / "README.md" + write_evaluation_report( + report, + json_path=json_path, + markdown_path=markdown_path, + ) + print(f"Wrote {json_path}") + print(f"Wrote {markdown_path}") return 0 if arguments.command == "ask": @@ -211,7 +353,7 @@ def main(argv: Sequence[str] | None = None) -> int: arguments = parser.parse_args(supplied_arguments) try: return run(arguments) - except (ReviewDataError, ValueError) as error: + except (ReviewDataError, TypeError, ValueError) as error: parser.error(str(error)) return 2 diff --git a/ollama_health.py b/ollama_health.py index 76f92e1..be5c7db 100644 --- a/ollama_health.py +++ b/ollama_health.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from typing import Any +import httpx from httpx import HTTPError from ollama import Client, ResponseError @@ -37,6 +38,21 @@ def create_ollama_client(host: str = DEFAULT_OLLAMA_HOST) -> Client: return Client(host=host) +def ollama_version( + host: str = DEFAULT_OLLAMA_HOST, + *, + request: Any | None = None, +) -> str: + """Return the server-reported Ollama runtime version.""" + getter = request or httpx.get + response = getter(f"{host.rstrip('/')}/api/version", timeout=5.0) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict) or not str(payload.get("version") or "").strip(): + raise ValueError("Ollama version response is missing a version") + return str(payload["version"]) + + def _available_model_names(response: Any) -> tuple[str, ...]: models = ( response.get("models", []) if isinstance(response, dict) else response.models @@ -60,6 +76,56 @@ def _model_is_available(required: str, available: tuple[str, ...]) -> bool: ) +def _model_value(model: Any, key: str) -> Any: + if isinstance(model, dict): + return model.get(key) + return getattr(model, key, None) + + +def model_metadata( + required_models: tuple[str, ...], + *, + host: str = DEFAULT_OLLAMA_HOST, + client: Any | None = None, +) -> dict[str, dict[str, str | int | None]]: + """Resolve configured model tags to the immutable digests Ollama reports.""" + ollama_client = client or create_ollama_client(host) + response = ollama_client.list() + models = ( + response.get("models", []) if isinstance(response, dict) else response.models + ) + resolved: dict[str, dict[str, str | int | None]] = {} + for required in required_models: + candidates = [] + for model in models: + name = _model_value(model, "model") or _model_value(model, "name") + if name and _model_is_available(required, (str(name),)): + candidates.append(model) + if not candidates: + continue + selected = min( + candidates, + key=lambda model: ( + str(_model_value(model, "model") or _model_value(model, "name")) + != required, + not str( + _model_value(model, "model") or _model_value(model, "name") + ).endswith(":latest"), + str(_model_value(model, "model") or _model_value(model, "name")), + ), + ) + size = _model_value(selected, "size") + resolved[required] = { + "resolved_name": str( + _model_value(selected, "model") or _model_value(selected, "name") + ), + "digest": str(_model_value(selected, "digest") or "") or None, + "size": int(size) if size is not None else None, + "modified_at": str(_model_value(selected, "modified_at") or "") or None, + } + return resolved + + def check_ollama( *, required_models: tuple[str, ...] = ( diff --git a/pyproject.toml b/pyproject.toml index 6e880c8..c067885 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,3 +37,29 @@ include-package-data = true [tool.setuptools.package-data] "local_ai_agent.data" = ["*.csv", "*.json"] + +[tool.coverage.run] +source = ["."] + +[tool.coverage.report] +include = [ + "agent.py", + "dashboard_support.py", + "evaluation.py", + "main.py", + "ollama_health.py", + "vector.py", +] +show_missing = true + +[tool.mypy] +files = [ + "agent.py", + "dashboard_support.py", + "evaluation.py", + "main.py", + "ollama_health.py", + "vector.py", +] +check_untyped_defs = true +ignore_missing_imports = true diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py new file mode 100644 index 0000000..42e17a6 --- /dev/null +++ b/tests/test_benchmark.py @@ -0,0 +1,317 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from langchain_core.documents import Document + +from evaluation import ( + BM25Retriever, + EvaluationCase, + EvaluationObservation, + build_evaluation_report, + load_evaluation_cases, + retrieval_metrics_from_observations, + score_evaluation, + write_evaluation_report, +) + + +class BenchmarkTest(unittest.TestCase): + def test_bm25_ranks_exact_keyword_evidence_above_unrelated_text(self) -> None: + documents = ( + Document( + id="vegan", + page_content="Title: Vegan option\nReview: House-made cashew cheese melts well.", + metadata={"source_id": "vegan", "title": "Vegan option"}, + ), + Document( + id="delivery", + page_content="Title: Slow delivery\nReview: The pizza arrived cold after two hours.", + metadata={"source_id": "delivery", "title": "Slow delivery"}, + ), + ) + + retriever = BM25Retriever(documents) + ranked = retriever.search("Does the vegan pizza use cashew cheese?", limit=2) + + self.assertEqual(ranked[0], "vegan") + self.assertEqual(set(ranked), {"vegan", "delivery"}) + + def test_bm25_normalizes_unicode_and_does_not_pad_zero_overlap(self) -> None: + documents = ( + Document( + id="crispy", + page_content="ＣＲＩＳＰＹ crust", + metadata={"source_id": "crispy"}, + ), + Document( + id="other", + page_content="quiet dining room", + metadata={"source_id": "other"}, + ), + ) + retriever = BM25Retriever(documents) + + self.assertEqual(retriever.search("crispy", limit=5), ("crispy",)) + self.assertEqual(retriever.search("wheelchair parking", limit=5), ()) + + def test_empty_evaluation_cannot_report_perfect_scores(self) -> None: + with self.assertRaisesRegex(ValueError, "at least one case"): + score_evaluation((), ()) + + def test_retrieval_metrics_report_recall_hit_rate_and_reciprocal_rank(self) -> None: + cases = ( + EvaluationCase( + case_id="first", + question="first", + relevant_titles=("A",), + reference_facts=(), + ), + EvaluationCase( + case_id="second", + question="second", + relevant_titles=("B", "C"), + reference_facts=(), + ), + EvaluationCase( + case_id="abstain", + question="unknown", + relevant_titles=(), + reference_facts=(), + should_abstain=True, + ), + ) + observations = ( + EvaluationObservation( + case_id="first", + relevant_source_ids=frozenset({"a"}), + retrieved_source_ids=("x", "a"), + cited_source_ids=(), + cited_text="", + answer="", + abstained=False, + ), + EvaluationObservation( + case_id="second", + relevant_source_ids=frozenset({"b", "c"}), + retrieved_source_ids=("b", "x"), + cited_source_ids=(), + cited_text="", + answer="", + abstained=False, + ), + EvaluationObservation( + case_id="abstain", + relevant_source_ids=frozenset(), + retrieved_source_ids=("x",), + cited_source_ids=(), + cited_text="", + answer="", + abstained=True, + ), + ) + + metrics = retrieval_metrics_from_observations(cases, observations, limit=2) + + self.assertEqual(metrics.evaluated_case_count, 2) + self.assertEqual(metrics.limit, 2) + self.assertAlmostEqual(metrics.recall_at_k, 0.75) + self.assertAlmostEqual(metrics.hit_rate_at_k, 1.0) + self.assertAlmostEqual(metrics.mrr_at_k, 0.75) + + def test_retrieval_metrics_reject_duplicate_observations(self) -> None: + case = EvaluationCase( + case_id="one", + question="one", + relevant_titles=("One",), + reference_facts=(), + ) + observation = EvaluationObservation( + case_id="one", + relevant_source_ids=frozenset({"one"}), + retrieved_source_ids=("one",), + cited_source_ids=(), + cited_text="", + answer="", + abstained=False, + ) + + with self.assertRaisesRegex(ValueError, "exactly once"): + retrieval_metrics_from_observations( + (case,), (observation, observation), limit=1 + ) + + def test_report_is_machine_readable_and_writes_matching_markdown(self) -> None: + cases = ( + EvaluationCase( + case_id="answer", + question="Is it crisp?", + relevant_titles=("Crisp",), + reference_facts=(), + category="quality", + ), + EvaluationCase( + case_id="abstain", + question="Is there parking?", + relevant_titles=(), + reference_facts=(), + should_abstain=True, + category="abstention", + ), + ) + report = build_evaluation_report( + cases=cases, + rag_metrics={ + "retrieval_recall": 1.0, + "citation_validity": 1.0, + "reference_term_support_proxy": 1.0, + "expected_action_accuracy": 1.0, + "answer_success_rate": 1.0, + "abstention_recall": 1.0, + "case_count": 2, + }, + semantic_metrics={ + "recall_at_k": 1.0, + "hit_rate_at_k": 1.0, + "mrr_at_k": 1.0, + "evaluated_case_count": 1, + "limit": 5, + }, + baseline_metrics={ + "recall_at_k": 0.0, + "hit_rate_at_k": 0.0, + "mrr_at_k": 0.0, + "evaluated_case_count": 1, + "limit": 5, + }, + observations=(), + configuration={"chat_model": "llama3.2", "embedding_model": "mxbai"}, + provenance={"dataset_sha256": "abc", "cases_sha256": "def"}, + generated_at="2026-07-31T00:00:00Z", + ) + + self.assertEqual(report["schema_version"], 2) + self.assertEqual(report["evaluation_set"]["case_count"], 2) + self.assertEqual(report["evaluation_set"]["abstention_case_count"], 1) + self.assertEqual(report["results"]["bm25_baseline"]["recall_at_k"], 0.0) + + with tempfile.TemporaryDirectory() as directory: + json_path = Path(directory) / "report.json" + markdown_path = Path(directory) / "report.md" + write_evaluation_report( + report, + json_path=json_path, + markdown_path=markdown_path, + ) + + written = json.loads(json_path.read_text(encoding="utf-8")) + markdown = markdown_path.read_text(encoding="utf-8") + + self.assertEqual(written, report) + self.assertIn("BM25 keyword baseline", markdown) + self.assertIn("Model-dependent results", markdown) + self.assertIn("2026-07-31T00:00:00Z", markdown) + + +class EvaluationSetQualityTest(unittest.TestCase): + def test_bundled_set_has_broad_unique_coverage(self) -> None: + from evaluation import DEFAULT_EVALUATION_PATH + + cases = load_evaluation_cases(DEFAULT_EVALUATION_PATH) + + self.assertGreaterEqual(len(cases), 30) + self.assertEqual(len({case.case_id for case in cases}), len(cases)) + self.assertGreaterEqual(sum(case.should_abstain for case in cases), 5) + self.assertGreaterEqual(len({case.category for case in cases}), 6) + self.assertTrue(all(case.category for case in cases)) + + def test_manifest_uses_immutable_gold_ids(self) -> None: + from evaluation import DEFAULT_EVALUATION_PATH + + cases = load_evaluation_cases(DEFAULT_EVALUATION_PATH) + answerable = [case for case in cases if not case.should_abstain] + + self.assertTrue(answerable) + self.assertTrue( + all( + source_id.startswith("review_") + for case in answerable + for source_id in case.gold_source_ids + ) + ) + + def test_loader_rejects_empty_case_sets(self) -> None: + from evaluation import DEFAULT_EVALUATION_PATH + + payload = json.loads(DEFAULT_EVALUATION_PATH.read_text(encoding="utf-8")) + payload["cases"] = [] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "cases.json" + path.write_text(json.dumps(payload), encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "at least one case"): + load_evaluation_cases(path) + + def test_loader_rejects_contradictory_abstention_cases(self) -> None: + from evaluation import DEFAULT_EVALUATION_PATH + + payload = json.loads(DEFAULT_EVALUATION_PATH.read_text(encoding="utf-8")) + payload["cases"][0]["expected_action"] = "abstain" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "cases.json" + path.write_text(json.dumps(payload), encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "abstention cases"): + load_evaluation_cases(path) + + def test_loader_rejects_abstention_cases_with_source_labels(self) -> None: + from evaluation import DEFAULT_EVALUATION_PATH + + payload = json.loads(DEFAULT_EVALUATION_PATH.read_text(encoding="utf-8")) + case = payload["cases"][0] + case["expected_action"] = "abstain" + case["gold_source_ids"] = [] + case["source_labels"] = ["Unexpected label"] + case["reference_facts"] = [] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "cases.json" + path.write_text(json.dumps(payload), encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "abstention cases"): + load_evaluation_cases(path) + + def test_loader_rejects_string_term_lists_and_empty_terms(self) -> None: + from evaluation import DEFAULT_EVALUATION_PATH + + payload = json.loads(DEFAULT_EVALUATION_PATH.read_text(encoding="utf-8")) + payload["cases"][0]["reference_facts"][0]["answer_terms"] = "crispy" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "cases.json" + path.write_text(json.dumps(payload), encoding="utf-8") + with self.assertRaisesRegex(TypeError, "array of strings"): + load_evaluation_cases(path) + + payload = json.loads(DEFAULT_EVALUATION_PATH.read_text(encoding="utf-8")) + payload["cases"][0]["reference_facts"][0]["source_terms"] = [""] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "cases.json" + path.write_text(json.dumps(payload), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "non-empty strings"): + load_evaluation_cases(path) + + def test_loader_verifies_dataset_row_count(self) -> None: + from evaluation import DEFAULT_EVALUATION_PATH + from vector import DEFAULT_DATA_PATH + + payload = json.loads(DEFAULT_EVALUATION_PATH.read_text(encoding="utf-8")) + payload["dataset"]["row_count"] += 1 + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "cases.json" + path.write_text(json.dumps(payload), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "row_count"): + load_evaluation_cases(path, dataset_path=DEFAULT_DATA_PATH) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..e81ceee --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,186 @@ +import json +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from io import StringIO +from pathlib import Path +from unittest.mock import patch + +from httpx import HTTPError + +from evaluation import ( + EvaluationCase, + EvaluationMetrics, + EvaluationObservation, + RetrievalMetrics, +) +from main import _safe_endpoint, build_parser, main, run +from ollama_health import OllamaHealth + + +class EvaluationCLIReportTest(unittest.TestCase): + def test_evaluate_parser_accepts_report_directory(self) -> None: + arguments = build_parser().parse_args( + ["evaluate", "--report-dir", "docs/evaluation"] + ) + + self.assertEqual(arguments.report_dir, Path("docs/evaluation")) + + def test_safe_endpoint_removes_credentials_query_and_fragment(self) -> None: + endpoint = _safe_endpoint( + "https://user:secret@example.com:11434/api?token=x#part" + ) + + self.assertEqual(endpoint, "https://example.com:11434/api") + + def test_evaluate_writes_json_and_markdown_reports(self) -> None: + case = EvaluationCase( + case_id="answer", + question="Is it crisp?", + relevant_titles=("Crisp",), + reference_facts=(), + category="quality", + ) + observation = EvaluationObservation( + case_id="answer", + relevant_source_ids=frozenset({"a"}), + retrieved_source_ids=("a",), + cited_source_ids=("a",), + cited_text="crisp", + answer="It is crisp [1].", + abstained=False, + ) + rag_metrics = EvaluationMetrics(1.0, 1.0, 1.0, 1.0, 1) + retrieval_metrics = RetrievalMetrics(1.0, 1.0, 1.0, 1, 5) + health = OllamaHealth(True, ("chat:latest", "embed:latest"), ()) + + with tempfile.TemporaryDirectory() as directory: + report_dir = Path(directory) / "evaluation" + arguments = build_parser().parse_args( + [ + "evaluate", + "--chat-model", + "chat", + "--embedding-model", + "embed", + "--report-dir", + str(report_dir), + ] + ) + with ( + patch("main.load_reviews", return_value=[object()]), + patch("main._health_for_arguments", return_value=health), + patch("main._create_runtime", return_value=(object(), object())), + patch("main.load_evaluation_cases", return_value=(case,)), + patch( + "main.run_rag_evaluation", + return_value=(rag_metrics, (observation,)), + ), + patch( + "main.run_bm25_baseline", + return_value=(retrieval_metrics, (observation,)), + ), + patch( + "main.model_metadata", + return_value={ + "chat": { + "resolved_name": "chat:latest", + "digest": "sha256:chat", + "size": 1, + "modified_at": None, + }, + "embed": { + "resolved_name": "embed:latest", + "digest": "sha256:embed", + "size": 2, + "modified_at": None, + }, + }, + ), + patch("main.ollama_version", return_value="0.32.5"), + ): + exit_code = run(arguments) + + payload = json.loads( + (report_dir / "evaluation-report.json").read_text(encoding="utf-8") + ) + markdown = (report_dir / "README.md").read_text(encoding="utf-8") + + self.assertEqual(exit_code, 0) + self.assertEqual( + payload["configuration"]["models"]["chat"]["digest"], "sha256:chat" + ) + self.assertEqual(payload["configuration"]["ollama_version"], "0.32.5") + self.assertEqual(payload["results"]["semantic_retrieval"]["mrr_at_k"], 1.0) + self.assertIn("Retrieval comparison", markdown) + + def test_evaluate_reports_late_ollama_provenance_failures(self) -> None: + case = EvaluationCase( + case_id="answer", + question="Is it crisp?", + relevant_titles=("Crisp",), + reference_facts=(), + category="quality", + ) + observation = EvaluationObservation( + case_id="answer", + relevant_source_ids=frozenset({"a"}), + retrieved_source_ids=("a",), + cited_source_ids=("a",), + cited_text="crisp", + answer="It is crisp [1].", + abstained=False, + ) + rag_metrics = EvaluationMetrics(1.0, 1.0, 1.0, 1.0, 1) + retrieval_metrics = RetrievalMetrics(1.0, 1.0, 1.0, 1, 5) + health = OllamaHealth(True, ("chat:latest", "embed:latest"), ()) + failures = ( + ("ollama_version", HTTPError("Ollama stopped")), + ("model_metadata", OSError("Ollama stopped")), + ) + + for target, error in failures: + with ( + self.subTest(target=target), + tempfile.TemporaryDirectory() as directory, + ): + report_dir = Path(directory) / "evaluation" + stderr = StringIO() + stdout = StringIO() + patches = { + "ollama_version": patch( + "main.ollama_version", return_value="0.32.5" + ), + "model_metadata": patch("main.model_metadata", return_value={}), + } + patches[target] = patch(f"main.{target}", side_effect=error) + with ( + patch("main.load_reviews", return_value=[object()]), + patch("main._health_for_arguments", return_value=health), + patch("main._create_runtime", return_value=(object(), object())), + patch("main.load_evaluation_cases", return_value=(case,)), + patch( + "main.run_rag_evaluation", + return_value=(rag_metrics, (observation,)), + ), + patch( + "main.run_bm25_baseline", + return_value=(retrieval_metrics, (observation,)), + ), + patches["ollama_version"], + patches["model_metadata"], + redirect_stderr(stderr), + redirect_stdout(stdout), + self.assertRaises(SystemExit) as raised, + ): + main(["evaluate", "--report-dir", str(report_dir)]) + + self.assertEqual(raised.exception.code, 2) + self.assertIn( + "could not collect Ollama report metadata", stderr.getvalue() + ) + self.assertNotIn("Traceback", stderr.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 2e3ebe3..5ac801d 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -57,8 +57,9 @@ class RAGEvaluationTest(unittest.TestCase): def test_loads_curated_evaluation_set(self) -> None: cases = load_evaluation_cases(DEFAULT_EVALUATION_PATH) - self.assertEqual(len(cases), 4) - self.assertTrue(any(case.should_abstain for case in cases)) + self.assertEqual(len(cases), 30) + self.assertEqual(sum(case.should_abstain for case in cases), 5) + self.assertTrue(all(case.category for case in cases)) self.assertTrue(any(case.reference_facts for case in cases)) def test_scores_retrieval_citations_faithfulness_and_abstention(self) -> None: @@ -186,6 +187,55 @@ def test_penalizes_missing_retrieval_invalid_citation_and_false_answer( self.assertEqual(metrics.answer_faithfulness, 0.0) self.assertEqual(metrics.abstention_accuracy, 0.0) + def test_rejection_is_not_a_successful_answer_or_support_score(self) -> None: + answer_case = EvaluationCase( + case_id="answer", + question="Is the crust crisp?", + relevant_titles=("Best pizza",), + reference_facts=( + ReferenceFact( + answer_terms=("crispy",), + source_terms=("crispy",), + ), + ), + ) + abstain_case = EvaluationCase( + case_id="abstain", + question="Is parking available?", + relevant_titles=(), + reference_facts=(), + should_abstain=True, + ) + rejected = EvaluationObservation( + case_id="answer", + relevant_source_ids=frozenset({"review-a"}), + retrieved_source_ids=("review-a",), + cited_source_ids=(), + cited_text="", + answer="I could not produce an answer with valid citations.", + abstained=False, + outcome="citation_validation_rejection", + ) + abstained = EvaluationObservation( + case_id="abstain", + relevant_source_ids=frozenset(), + retrieved_source_ids=("review-a",), + cited_source_ids=(), + cited_text="", + answer="I could not find matching evidence.", + abstained=True, + outcome="model_abstention", + ) + + metrics = score_evaluation((answer_case, abstain_case), (rejected, abstained)) + + self.assertEqual(metrics.expected_action_accuracy, 0.5) + self.assertEqual(metrics.answer_success_rate, 0.0) + self.assertEqual(metrics.abstention_recall, 1.0) + self.assertEqual(metrics.reference_term_support_proxy, 0.0) + with self.assertRaisesRegex(ValueError, "exactly once"): + score_evaluation((answer_case,), (rejected, rejected)) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ollama_health.py b/tests/test_ollama_health.py index 89f6289..8703d42 100644 --- a/tests/test_ollama_health.py +++ b/tests/test_ollama_health.py @@ -1,6 +1,6 @@ import unittest -from ollama_health import check_ollama +from ollama_health import check_ollama, model_metadata, ollama_version class FakeClient: @@ -17,6 +17,51 @@ def list(self) -> dict[str, list[dict[str, str]]]: class OllamaHealthTest(unittest.TestCase): + def test_reports_server_version(self) -> None: + class Response: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, str]: + return {"version": "0.32.5"} + + def request(url: str, *, timeout: float): + self.assertEqual(url, "http://localhost:11434/api/version") + self.assertEqual(timeout, 5.0) + return Response() + + self.assertEqual( + ollama_version("http://localhost:11434/", request=request), + "0.32.5", + ) + + def test_model_metadata_resolves_tags_and_preserves_digests(self) -> None: + class MetadataClient: + def list(self): + return { + "models": [ + { + "model": "llama3.2:latest", + "digest": "sha256:chat", + "size": 123, + }, + { + "model": "mxbai-embed-large:latest", + "digest": "sha256:embed", + "size": 456, + }, + ] + } + + metadata = model_metadata( + ("llama3.2", "mxbai-embed-large"), + client=MetadataClient(), + ) + + self.assertEqual(metadata["llama3.2"]["resolved_name"], "llama3.2:latest") + self.assertEqual(metadata["llama3.2"]["digest"], "sha256:chat") + self.assertEqual(metadata["mxbai-embed-large"]["size"], 456) + def test_reports_available_models(self) -> None: health = check_ollama( required_models=("llama3.2", "mxbai-embed-large"), diff --git a/vector.py b/vector.py index 125ab42..ee7035f 100644 --- a/vector.py +++ b/vector.py @@ -6,7 +6,7 @@ from hashlib import sha256 from importlib.resources import files from pathlib import Path -from typing import IO, Any, cast +from typing import IO, Any, TypeAlias, cast import pandas as pd from langchain_chroma import Chroma @@ -79,7 +79,7 @@ "country": "Country", } -ReviewSource = str | Path | IO[str] | IO[bytes] | pd.DataFrame +ReviewSource: TypeAlias = str | Path | IO[str] | IO[bytes] | pd.DataFrame class ReviewDataError(ValueError): @@ -441,10 +441,10 @@ def _documents_and_ids(dataframe: pd.DataFrame) -> tuple[list[Document], list[st if base_key == "extra_": continue safe_key = base_key - suffix = 2 + collision_suffix = 2 while safe_key in metadata: - safe_key = f"{base_key}_{suffix}" - suffix += 1 + safe_key = f"{base_key}_{collision_suffix}" + collision_suffix += 1 metadata[safe_key] = str(raw_value) content_lines.append(f"{source_column}: {raw_value}") content_lines.append(f"Review: {row['Review']}")