From 8b344eb784bd6693d62f5159520262a1ddb10e72 Mon Sep 17 00:00:00 2001 From: Connor Shorten Date: Tue, 16 Jun 2026 06:47:41 -0400 Subject: [PATCH 1/5] add FilteredCars, bump engram version --- pyproject.toml | 2 +- .../adapters/agents/engram_dspy_agent.py | 28 +++++++++-- .../adapters/database/database_config.py | 1 + .../adapters/database/database_registry.py | 17 +++++++ .../adapters/dataset/huggingface_loader.py | 46 +++++++++++++++++++ .../internal/adapters/dataset/registry.py | 2 + .../internal/config/benchmark-config.yml | 2 +- .../internal/config/config.py | 1 + .../config/database_loader_config.yml | 2 +- .../internal/core/domain/metrics_config.py | 5 ++ tests/functional/test_database_loading.py | 1 + tests/functional/test_database_weaviate.py | 1 + tests/functional/test_dataset_loading.py | 1 + 13 files changed, 101 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d64c9fc..f2923d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "wheel>=0.45.1", "twine>=6.2.0", "PyMuPDF>=1.24.0", - "weaviate-engram>=0.6.0", + "weaviate-engram>=1.0.0", "python-dotenv>=1.1.1", "fastapi>=0.135.3", "uvicorn>=0.44.0", diff --git a/query_agent_benchmarking/internal/adapters/agents/engram_dspy_agent.py b/query_agent_benchmarking/internal/adapters/agents/engram_dspy_agent.py index 29cc5e8..4884e76 100644 --- a/query_agent_benchmarking/internal/adapters/agents/engram_dspy_agent.py +++ b/query_agent_benchmarking/internal/adapters/agents/engram_dspy_agent.py @@ -13,7 +13,20 @@ import dspy from pydantic import BaseModel -from engram import EngramClient, RetrievalConfig +from engram import ( + EngramClient, + BM25Retrieval, + FetchRetrieval, + HybridRetrieval, + VectorRetrieval, +) + +_RETRIEVAL_CLASSES = { + "hybrid": HybridRetrieval, + "bm25": BM25Retrieval, + "vector": VectorRetrieval, + "fetch": FetchRetrieval, +} # --------------------------------------------------------------------------- @@ -148,14 +161,19 @@ def run( user_id = f"{self.user_id_prefix}{tenant_id}" + try: + retrieval_cls = _RETRIEVAL_CLASSES[self.retrieval_type] + except KeyError: + raise ValueError( + f"Unsupported retrieval_type '{self.retrieval_type}'. " + f"Supported: {sorted(_RETRIEVAL_CLASSES)}" + ) + memories = self.engram_client.memories.search( query=query, user_id=user_id, group=self.engram_group, - retrieval_config=RetrievalConfig( - retrieval_type=self.retrieval_type, - limit=self.retrieval_limit, - ), + retrieval_config=retrieval_cls(limit=self.retrieval_limit), ) retrieved = [ diff --git a/query_agent_benchmarking/internal/adapters/database/database_config.py b/query_agent_benchmarking/internal/adapters/database/database_config.py index 3771caf..a795d52 100644 --- a/query_agent_benchmarking/internal/adapters/database/database_config.py +++ b/query_agent_benchmarking/internal/adapters/database/database_config.py @@ -15,6 +15,7 @@ "bright/psychology", "bright/robotics", "enron", + "filtered-cars", "freshstack-angular", "freshstack-godot", "freshstack-langchain", diff --git a/query_agent_benchmarking/internal/adapters/database/database_registry.py b/query_agent_benchmarking/internal/adapters/database/database_registry.py index 7df6cd7..7226aba 100644 --- a/query_agent_benchmarking/internal/adapters/database/database_registry.py +++ b/query_agent_benchmarking/internal/adapters/database/database_registry.py @@ -63,6 +63,23 @@ .with_text2vec() .build()), + (DatasetSpecBuilder("filtered-cars") + .with_static_name("FilteredCars") + .with_text_property("make") + .with_text_property("model") + .with_int_property("year") + .with_text_property("fuel_type") + .with_text_property("body_type") + .with_int_property("mileage") + .with_int_property("price") + .with_text_property("color") + .with_text_property("color_family") + .with_text_property("transmission") + .with_text_property("drivetrain") + .with_dataset_id() + .with_text2vec() + .build()), + (DatasetSpecBuilder("multihoprag") .with_static_name("MultiHopRAG") .with_text_property("content", source_field="body") diff --git a/query_agent_benchmarking/internal/adapters/dataset/huggingface_loader.py b/query_agent_benchmarking/internal/adapters/dataset/huggingface_loader.py index 2867a69..93c49e3 100644 --- a/query_agent_benchmarking/internal/adapters/dataset/huggingface_loader.py +++ b/query_agent_benchmarking/internal/adapters/dataset/huggingface_loader.py @@ -152,6 +152,52 @@ def load_irpapers(): return docs, questions +def load_filtered_cars(): + """Load the FilteredCars structured-schema benchmark. + + The corpus is a list of car listings with categorical and numeric + columns (make, model, year, fuel_type, body_type, mileage, price, + color, color_family, transmission, drivetrain). Queries are natural + language and must be translated by the Query Agent into structured + filters over these columns. + + The dataset's HuggingFace config refers to a CSV path that does not + exist in the repository, so the JSON files are fetched directly via + hf_hub_download instead of `load_dataset`. + """ + from huggingface_hub import hf_hub_download + import json as _json + + print("Loading FilteredCars dataset...") + + docs_path = hf_hub_download( + "weaviate/FilteredCars", "FilteredCars.json", repo_type="dataset" + ) + with open(docs_path) as fh: + docs = _json.load(fh) + for doc in docs: + doc["dataset_id"] = str(doc["id"]) + + queries_path = hf_hub_download( + "weaviate/FilteredCars", "filtered_cars_queries.json", repo_type="dataset" + ) + with open(queries_path) as fh: + raw_queries = _json.load(fh) + + questions: list[InMemoryQuery] = [] + for i, item in enumerate(raw_queries): + questions.append( + InMemoryQuery( + question=item["query"], + query_id=str(i), + dataset_ids=[str(d) for d in item["dataset_ids"]], + ) + ) + + print(f"Loaded {len(docs)} documents and {len(questions)} queries") + return docs, questions + + def load_vidore(): print("Loading ViDoRe dataset...") corpus = load_dataset("vidore/vidore_v3_hr", "corpus")["test"] diff --git a/query_agent_benchmarking/internal/adapters/dataset/registry.py b/query_agent_benchmarking/internal/adapters/dataset/registry.py index 46e5330..5390778 100644 --- a/query_agent_benchmarking/internal/adapters/dataset/registry.py +++ b/query_agent_benchmarking/internal/adapters/dataset/registry.py @@ -62,6 +62,8 @@ def _dispatch_search_loader(dataset_name: str): return huggingface_loader.load_wixqa() if dataset_name in ("irpapers", "irpapers-text-only"): return huggingface_loader.load_irpapers() + if dataset_name == "filtered-cars": + return huggingface_loader.load_filtered_cars() if dataset_name == "multihoprag": return huggingface_loader.load_multihoprag() if dataset_name == "vidore_v3_hr": diff --git a/query_agent_benchmarking/internal/config/benchmark-config.yml b/query_agent_benchmarking/internal/config/benchmark-config.yml index 532e685..1d682d7 100644 --- a/query_agent_benchmarking/internal/config/benchmark-config.yml +++ b/query_agent_benchmarking/internal/config/benchmark-config.yml @@ -2,7 +2,7 @@ # Search Mode Settings # ============================================================================= search_agent_name: "external_service" -search_dataset: bright/biology +search_dataset: filtered-cars # See `named_vectors.md` for supported target vectors. search_target: "text_content_weaviate" diff --git a/query_agent_benchmarking/internal/config/config.py b/query_agent_benchmarking/internal/config/config.py index e040122..f46331f 100644 --- a/query_agent_benchmarking/internal/config/config.py +++ b/query_agent_benchmarking/internal/config/config.py @@ -15,6 +15,7 @@ "bright/psychology", "bright/robotics", "enron", + "filtered-cars", "freshstack-angular", "freshstack-godot", "freshstack-langchain", diff --git a/query_agent_benchmarking/internal/config/database_loader_config.yml b/query_agent_benchmarking/internal/config/database_loader_config.yml index d2b08d0..fa95302 100644 --- a/query_agent_benchmarking/internal/config/database_loader_config.yml +++ b/query_agent_benchmarking/internal/config/database_loader_config.yml @@ -1,4 +1,4 @@ -dataset_name: longmemeval-s +dataset_name: filtered-cars # Multiple datasets — used by populate_db_multi dataset_names: diff --git a/query_agent_benchmarking/internal/core/domain/metrics_config.py b/query_agent_benchmarking/internal/core/domain/metrics_config.py index c4a9056..3b0d7e9 100644 --- a/query_agent_benchmarking/internal/core/domain/metrics_config.py +++ b/query_agent_benchmarking/internal/core/domain/metrics_config.py @@ -82,6 +82,11 @@ class MetricsProfile: MetricSpec("recall", {"k": 5}), MetricSpec("recall", {"k": 20}), ), primary_metric="recall_at_5"), + MetricsProfile("filtered-cars", ( + MetricSpec("recall", {"k": 1}), + MetricSpec("recall", {"k": 5}), + MetricSpec("recall", {"k": 20}), + ), primary_metric="recall_at_5"), MetricsProfile("vidore_v3_hr", ( MetricSpec("recall", {"k": 1}), MetricSpec("recall", {"k": 5}), diff --git a/tests/functional/test_database_loading.py b/tests/functional/test_database_loading.py index 70c5cb5..0a86b54 100644 --- a/tests/functional/test_database_loading.py +++ b/tests/functional/test_database_loading.py @@ -32,6 +32,7 @@ "wixqa", "irpapers", "irpapers-text-only", + "filtered-cars", "multihoprag", "longmemeval-s", "lotte/lifestyle/test/search", diff --git a/tests/functional/test_database_weaviate.py b/tests/functional/test_database_weaviate.py index f55f819..f5c9928 100644 --- a/tests/functional/test_database_weaviate.py +++ b/tests/functional/test_database_weaviate.py @@ -42,6 +42,7 @@ "freshstack-langchain", "wixqa", "irpapers-text-only", + "filtered-cars", "multihoprag", "lotte/lifestyle/test/search", ] diff --git a/tests/functional/test_dataset_loading.py b/tests/functional/test_dataset_loading.py index 7ef2a12..8b3b663 100644 --- a/tests/functional/test_dataset_loading.py +++ b/tests/functional/test_dataset_loading.py @@ -37,6 +37,7 @@ "enron", "wixqa", "irpapers", + "filtered-cars", "multihoprag", "longmemeval-s", # officeqa and vidore_v3_hr are excluded: From a706b83f98a97360fa45767dc00c57903466280d Mon Sep 17 00:00:00 2001 From: Connor Shorten Date: Tue, 16 Jun 2026 07:45:04 -0400 Subject: [PATCH 2/5] add filtering parameter to search mode --- .../internal/adapters/agents/factory.py | 6 +++++ .../adapters/agents/weaviate_query_agent.py | 26 ++++++++++++++++--- .../adapters/database/database_loader.py | 26 ++++++++++++++++++- .../core/services/search_benchmark.py | 17 ++++++++++++ 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/query_agent_benchmarking/internal/adapters/agents/factory.py b/query_agent_benchmarking/internal/adapters/agents/factory.py index 0a6100d..110e795 100644 --- a/query_agent_benchmarking/internal/adapters/agents/factory.py +++ b/query_agent_benchmarking/internal/adapters/agents/factory.py @@ -33,6 +33,7 @@ def create_search_agent( image_embedding_model: Optional[str] = None, embedding_providers: Optional[Sequence[str]] = None, external_service_host: Optional[str] = None, + filtering: Optional[str] = None, ): """Create a SearchAgent adapter based on agent_name. @@ -49,6 +50,10 @@ def create_search_agent( image_embedding_model: Image embedding model. embedding_providers: Explicit provider list for header injection. external_service_host: Host URL for external service mode. + filtering: Query Agent search filtering strategy, "recall" (default, + generate multiple Weaviate queries) or "precision" (generate a + single query for the most likely interpretation). Only applies to + "query-agent-search-mode". Returns: An object implementing the SearchAgent protocol. @@ -67,6 +72,7 @@ def create_search_agent( dataset_name=dataset_name, docs_collection=docs_collection, agents_host=agents_host, + filtering=filtering, **_embedding_kwargs, ) diff --git a/query_agent_benchmarking/internal/adapters/agents/weaviate_query_agent.py b/query_agent_benchmarking/internal/adapters/agents/weaviate_query_agent.py index 03791f0..bf556d0 100644 --- a/query_agent_benchmarking/internal/adapters/agents/weaviate_query_agent.py +++ b/query_agent_benchmarking/internal/adapters/agents/weaviate_query_agent.py @@ -5,7 +5,7 @@ """ import os -from typing import Any, Optional, Sequence +from typing import Any, Literal, Optional, Sequence import weaviate from weaviate.auth import Auth @@ -19,6 +19,24 @@ resolve_headers_for_models, ) +Filtering = Literal["recall", "precision"] + + +def _validate_filtering(filtering: Optional[str]) -> Filtering: + """Normalize and validate the search filtering strategy. + + "recall" (default) generates multiple Weaviate queries spanning different + filters/interpretations; "precision" generates a single query targeting the + most likely interpretation. Defaults to "recall" when unset. + """ + if filtering is None: + return "recall" + if filtering not in ("recall", "precision"): + raise ValueError( + f"filtering must be 'recall' or 'precision'; got {filtering!r}." + ) + return filtering + class WeaviateQueryAgentSearch: """SearchAgent adapter wrapping Weaviate QueryAgent in search-only mode.""" @@ -32,11 +50,13 @@ def __init__( text_embedding_model: Optional[str] = None, image_embedding_model: Optional[str] = None, embedding_providers: Optional[Sequence[str]] = None, + filtering: Optional[str] = None, ): info = resolve_collection_info(dataset_name, docs_collection) self.collection = info["collection"] self.id_property = info["id_property"] self.agents_host = agents_host or "https://api.agents.weaviate.io" + self.filtering: Filtering = _validate_filtering(filtering) self.headers = resolve_headers_for_models( embedding_model=embedding_model, text_embedding_model=text_embedding_model, @@ -83,14 +103,14 @@ async def initialize_async(self): def run(self, query: str, tenant: Optional[str] = None) -> list[ObjectID]: if self._agent is None: self.initialize_sync() - response = self._agent.search(query, limit=20) + response = self._agent.search(query, limit=20, filtering=self.filtering) return [ ObjectID(object_id=obj.properties[self.id_property]) for obj in response.search_results.objects ] async def run_async(self, query: str, tenant: Optional[str] = None) -> list[ObjectID]: - response = await self._agent.search(query, limit=20) + response = await self._agent.search(query, limit=20, filtering=self.filtering) return [ ObjectID(object_id=obj.properties[self.id_property]) for obj in response.search_results.objects diff --git a/query_agent_benchmarking/internal/adapters/database/database_loader.py b/query_agent_benchmarking/internal/adapters/database/database_loader.py index 0e00140..afceffa 100644 --- a/query_agent_benchmarking/internal/adapters/database/database_loader.py +++ b/query_agent_benchmarking/internal/adapters/database/database_loader.py @@ -126,6 +126,30 @@ def database_loader(recreate: bool = True, tag: str = "Default") -> None: populate_db(recreate=recreate, tag=tag) +# Private helper: force an explicit vector index type onto named-vector configs. +def _ensure_explicit_vector_index(vector_config: Any) -> Any: + """Attach a default HNSW index config to any vector lacking one. + + The new named-vector ``Configure.Vectors.*`` API omits ``vectorIndexType`` + from the create payload when the server reports >= 1.37.5, trusting the + server to fill in the HNSW default. Some servers (incl. certain Weaviate + Cloud builds) report >= 1.37.5 but do *not* apply that default; they read + the missing field as the ``"none"`` sentinel and reject the create with a + 422 ("cannot create a new class with vectorIndexType 'none'"). Setting an + explicit ``vector_index_config`` makes the client always emit + ``vectorIndexType: "hnsw"``, sidestepping the handshake gap. + + Only vectors that don't already declare an index config are touched, so + multivector image configs (which carry their own multivector-enabled HNSW + config) are left untouched. + """ + configs = vector_config if isinstance(vector_config, list) else [vector_config] + for cfg in configs: + if getattr(cfg, "vectorIndexConfig", "unset") is None: + cfg.vectorIndexConfig = wvcc.Configure.VectorIndex.hnsw() + return vector_config + + # Private helper: low-level create/delete wrapper used internally to keep public loaders concise. def _drop_and_create_collection( client: weaviate.WeaviateClient, @@ -141,7 +165,7 @@ def _drop_and_create_collection( if not client.collections.exists(name): create_kwargs: dict[str, Any] = { "name": name, - "vector_config": vector_config, + "vector_config": _ensure_explicit_vector_index(vector_config), "properties": list(properties), } if multi_tenancy_config is not None: diff --git a/query_agent_benchmarking/internal/core/services/search_benchmark.py b/query_agent_benchmarking/internal/core/services/search_benchmark.py index d3a2e7f..0b4465a 100644 --- a/query_agent_benchmarking/internal/core/services/search_benchmark.py +++ b/query_agent_benchmarking/internal/core/services/search_benchmark.py @@ -230,6 +230,10 @@ async def _run_search_eval(config: dict[str, Any]) -> dict[str, Any]: agent_cfg = _load_agent_config(agent_name) resolved_agents_host = agent_cfg.get("agents_host", agents_host) resolved_external_service_host = agent_cfg.get("external_service_host", config.get("external_service_host")) + # Query Agent search filtering strategy ("recall" | "precision"). + # Eval-level config takes precedence, then the agent default, then "recall". + resolved_filtering = config.get("filtering") or agent_cfg.get("filtering") or "recall" + config["filtering"] = resolved_filtering query_agent = create_search_agent( agent_name, @@ -241,6 +245,7 @@ async def _run_search_eval(config: dict[str, Any]) -> dict[str, Any]: image_embedding_model=image_embedding_model, embedding_providers=embedding_providers, external_service_host=resolved_external_service_host, + filtering=resolved_filtering, ) num_trials = config.get("num_trials", 1) @@ -321,6 +326,7 @@ def run_search_eval( embedding_providers: Optional[Union[str, list[str]]] = None, search_target: Optional[str] = None, search_target_vector: Optional[str] = None, + filtering: Optional[str] = None, **kwargs ) -> dict[str, Any]: """ @@ -356,6 +362,10 @@ def run_search_eval( search_target: Canonical named vector(s), e.g. "text_content_weaviate", "image_content_weaviate", or "text_content_weaviate+image_content_weaviate". search_target_vector: Legacy vector-name key. Prefer search_target. + filtering: Query Agent search filtering strategy: "recall" (default — + generate multiple Weaviate queries spanning different filters and + interpretations) or "precision" (generate a single query targeting the + most likely interpretation). Only applies to "query-agent-search-mode". **kwargs: Additional config overrides. Returns: @@ -388,6 +398,7 @@ def run_search_eval( "embedding_providers": embedding_providers, "search_target": search_target, "search_target_vector": search_target_vector, + "filtering": filtering, **kwargs } @@ -418,6 +429,7 @@ def compare_search_agents( embedding_providers: Optional[Union[str, list[str]]] = None, search_target: Optional[str] = None, search_target_vector: Optional[str] = None, + filtering: Optional[str] = None, **kwargs ) -> dict[str, dict[str, Any]]: """Run search benchmark for multiple query agents and compare results.""" @@ -452,6 +464,7 @@ def compare_search_agents( "embedding_providers": embedding_providers, "search_target": search_target, "search_target_vector": search_target_vector, + "filtering": filtering, **kwargs } @@ -494,6 +507,7 @@ def run_search_evals( embedding_providers: Optional[Union[str, list[str]]] = None, search_target: Optional[str] = None, search_target_vector: Optional[str] = None, + filtering: Optional[str] = None, **kwargs ) -> dict[str, dict[str, Any]]: """ @@ -523,6 +537,8 @@ def run_search_evals( embedding_providers: Provider list for header injection. search_target: Canonical named vector(s). search_target_vector: Legacy vector-name key. + filtering: Query Agent search filtering strategy, "recall" (default) or + "precision". Only applies to "query-agent-search-mode". **kwargs: Additional config overrides. Returns: @@ -560,6 +576,7 @@ def run_search_evals( "embedding_providers": embedding_providers, "search_target": search_target, "search_target_vector": search_target_vector, + "filtering": filtering, **kwargs } From 7125213dbab2edeb572f876027e5c295be60b697 Mon Sep 17 00:00:00 2001 From: Connor Shorten Date: Tue, 16 Jun 2026 09:59:46 -0400 Subject: [PATCH 3/5] improve structured retrieval visualization and result logging --- console/app/api/compare/queries/route.ts | 33 + console/app/components/SearchPlan.tsx | 171 ++++ console/app/layout.tsx | 2 +- console/app/results/compare/page.tsx | 847 +++++++++++++++++- .../[id]/trial/[trialNum]/page.tsx | 111 ++- console/lib/results.ts | 279 ++++++ query_agent_benchmarking/__init__.py | 4 + .../adapters/agents/external_service.py | 38 +- .../adapters/agents/weaviate_query_agent.py | 71 +- .../internal/adapters/metrics/ir_metrics.py | 42 + .../adapters/metrics/ir_metrics_calculator.py | 4 +- .../adapters/results/serialization.py | 35 +- .../internal/core/domain/metrics_config.py | 5 +- .../internal/core/domain/models.py | 48 +- .../internal/core/domain/query_execution.py | 27 +- .../internal/core/ports/search_agent.py | 28 +- tests/adapters/test_agents.py | 14 +- tests/adapters/test_json_repository.py | 45 + 18 files changed, 1733 insertions(+), 71 deletions(-) create mode 100644 console/app/api/compare/queries/route.ts create mode 100644 console/app/components/SearchPlan.tsx diff --git a/console/app/api/compare/queries/route.ts b/console/app/api/compare/queries/route.ts new file mode 100644 index 0000000..57c8970 --- /dev/null +++ b/console/app/api/compare/queries/route.ts @@ -0,0 +1,33 @@ +import { NextRequest, NextResponse } from "next/server"; +import { buildQueryComparison } from "@/lib/results"; + +export const dynamic = "force-dynamic"; + +/** + * GET /api/compare/queries?ids=id1,id2,... + * + * Returns a per-query comparison across the requested experiments, aligning + * queries by question text and classifying each as all-correct / all-wrong / + * mixed (disagreement). Loaded on demand by the compare view. + */ +export function GET(request: NextRequest) { + const idsParam = request.nextUrl.searchParams.get("ids"); + if (!idsParam) { + return NextResponse.json({ error: "Missing ids parameter" }, { status: 400 }); + } + + const requestedIds = idsParam.split(",").map((s) => s.trim()).filter(Boolean); + if (requestedIds.length < 2) { + return NextResponse.json({ error: "Need at least 2 experiment ids" }, { status: 400 }); + } + + const comparison = buildQueryComparison(requestedIds); + if (!comparison) { + return NextResponse.json( + { error: "Could not find at least 2 of the requested experiments" }, + { status: 404 } + ); + } + + return NextResponse.json(comparison); +} diff --git a/console/app/components/SearchPlan.tsx b/console/app/components/SearchPlan.tsx new file mode 100644 index 0000000..4cd6ea8 --- /dev/null +++ b/console/app/components/SearchPlan.tsx @@ -0,0 +1,171 @@ +"use client"; + +import type { AgentSearch, FilterNode, FilterGroup, QuerySort } from "@/lib/results"; + +/* ── Search-plan rendering ────────────────────────────────────────────────── */ +// Shared between the trial detail view and the per-query comparison view. + +// Map raw Query Agent operators to readable symbols/words. +const OPERATOR_LABELS: Record = { + "=": "=", + "!=": "≠", + "<": "<", + ">": ">", + "<=": "≤", + ">=": "≥", + LIKE: "like", + contains_any: "contains any", + contains_all: "contains all", +}; + +function isFilterGroup(node: FilterNode): node is FilterGroup { + return (node as FilterGroup).combine !== undefined; +} + +function formatFilterValue(value: unknown): string { + if (value === null || value === undefined) return "∅"; + if (Array.isArray(value)) return value.map((v) => formatFilterValue(v)).join(", "); + if (typeof value === "object") { + const v = value as Record; + // Date filter value shapes (exact / from / to / between) + if ("exact_timestamp" in v) return String(v.exact_timestamp); + if ("date_from" in v && "date_to" in v) return `${v.date_from} → ${v.date_to}`; + if ("date_from" in v) return `from ${v.date_from}`; + if ("date_to" in v) return `to ${v.date_to}`; + return JSON.stringify(v); + } + return String(value); +} + +/** A single leaf filter rendered as a monospace chip. */ +function FilterChip({ node }: { node: Exclude }) { + let body: string; + if (node.filter_type === "is_null") { + body = `${node.property_name} is ${node.is_null ? "null" : "not null"}`; + } else if (node.filter_type === "geo") { + body = `${node.property_name} within ${node.max_distance_meters}m of (${node.latitude}, ${node.longitude})`; + } else { + const op = node.operator ? OPERATOR_LABELS[node.operator] ?? node.operator : ""; + body = `${node.property_name ?? "?"} ${op} ${formatFilterValue(node.value)}`.trim(); + } + return ( + + {body} + + ); +} + +/** Recursively render a filter tree: leaves as chips, groups joined by AND/OR. */ +export function FilterTree({ node }: { node: FilterNode }) { + if (!isFilterGroup(node)) { + return ; + } + return ( + + ( + {node.filters.map((child, i) => ( + + {i > 0 && ( + + {node.combine} + + )} + + + ))} + ) + + ); +} + +function formatSort(sort: QuerySort): string { + const arrow = sort.order === "descending" ? "↓" : "↑"; + let label = `${sort.property_name} ${arrow}`; + if (sort.tie_break) label += `, then ${formatSort(sort.tie_break)}`; + return label; +} + +/** The agent's search plan for one benchmark query: a list of sub-searches. */ +export function SearchPlan({ searches }: { searches: AgentSearch[] }) { + if (searches.length === 0) { + return ( +

+ No search plan recorded for this agent. +

+ ); + } + return ( +
+ {searches.map((s, i) => ( +
+
+ Search {i + 1} + + {s.collection} + + {s.uuid_value && ( + + uuid: {s.uuid_value.slice(0, 8)}… + + )} +
+ +
+ query + + {s.query ? ( + “{s.query}” + ) : ( + none (filter / lookup only) + )} + + + filters + + {s.filters ? ( + + ) : ( + none + )} + + + {s.sort_property && ( + <> + sort + + {formatSort(s.sort_property)} + + + )} +
+
+ ))} +
+ ); +} diff --git a/console/app/layout.tsx b/console/app/layout.tsx index 3b3f516..6440427 100644 --- a/console/app/layout.tsx +++ b/console/app/layout.tsx @@ -48,7 +48,7 @@ export default function RootLayout({
Weaviate; + wins: number[]; +}; + +/** Escape pipe characters so user labels can't break markdown table rows. */ +function escPipe(s: string): string { + return s.replace(/\|/g, "\\|"); +} + +function formatMetricValue(key: string, val: number | null): string { + if (val === null) return "—"; + return isTimeMetric(key) ? `${val.toFixed(2)}s` : `${(val * 100).toFixed(2)}%`; +} + +/** Short column header for an experiment in the report (label preferred). */ +function reportColLabel(exp: { label: string; agent_name: string }, i: number): string { + return `[${i + 1}] ${escPipe(exp.label || exp.agent_name)}`; +} + +/** + * Build a detailed markdown comparison report from the aggregated compare data, + * the derived analysis, and (optionally) the per-query comparison. Always + * surfaces each experiment's user-assigned label prominently. + */ +function buildMarkdownReport( + data: CompareData, + analysis: CompareAnalysis | null, + queries: QueryComparison | null, + generatedAt: string, +): string { + const { metricKeys, experiments } = data; + const lines: string[] = []; + const L = (s = "") => lines.push(s); + + L(`# Experiment Comparison Report`); + L(); + L(`_Generated ${generatedAt} · comparing ${experiments.length} experiments_`); + L(); + + // ── Experiments ────────────────────────────────────────────────────────── + L(`## Experiments`); + L(); + experiments.forEach((exp, i) => { + L(`### [${i + 1}] ${exp.label || `${exp.dataset} / ${exp.agent_name}`}`); + L(); + L(`- **Label:** ${exp.label ? exp.label : "_(none assigned)_"}`); + L(`- **Dataset:** ${exp.dataset}`); + L(`- **Agent:** \`${exp.agent_name}\``); + L(`- **Mode:** ${exp.mode}`); + L(`- **Trials:** ${exp.num_trials}`); + L(`- **Timestamp:** ${exp.timestamp ? new Date(exp.timestamp).toLocaleString() : "—"}`); + L(`- **ID:** \`${decodeURIComponent(exp.id)}\``); + L(); + }); + + // ── Metrics table ──────────────────────────────────────────────────────── + L(`## Metrics`); + L(); + if (metricKeys.length === 0) { + L(`_No aggregated metrics available._`); + L(); + } else { + const twoWay = experiments.length === 2; + const header = ["Metric", ...experiments.map((e, i) => reportColLabel(e, i))]; + if (twoWay) header.push("Delta"); + L(`| ${header.join(" | ")} |`); + L(`| ${header.map((_, i) => (i === 0 ? ":---" : "---:")).join(" | ")} |`); + + for (const key of metricKeys) { + const isTime = isTimeMetric(key); + const best = analysis?.metricAnalysis[key]?.bestIdx ?? null; + const cells = [escPipe(formatMetricName(key))]; + + experiments.forEach((exp, i) => { + const val = exp.metrics[key]; + let cell = formatMetricValue(key, val); + if (val !== null && i === best && experiments.length > 1) cell = `**${cell}** ⭐`; + cells.push(cell); + }); + + if (twoWay) { + const v0 = experiments[0].metrics[key]; + const v1 = experiments[1].metrics[key]; + if (v0 === null || v1 === null) { + cells.push("—"); + } else { + const delta = v1 - v0; + if (Math.abs(delta) < 1e-9) { + cells.push("0"); + } else { + const isImprovement = isTime ? delta < 0 : delta > 0; + const sign = delta > 0 ? "+" : ""; + const formatted = isTime + ? `${sign}${delta.toFixed(2)}s` + : `${sign}${(delta * 100).toFixed(2)}%`; + cells.push(`${formatted} ${isImprovement ? "▲" : "▼"}`); + } + } + } + + L(`| ${cells.join(" | ")} |`); + } + L(); + L(`⭐ marks the best value for each metric. ${twoWay ? "Delta = [2] − [1] (▲ improvement, ▼ regression)." : ""}`); + L(); + } + + // ── Analysis ───────────────────────────────────────────────────────────── + if (analysis) { + L(`## Analysis`); + L(); + + const nonTime = metricKeys.filter((k) => !isTimeMetric(k)); + L(`### Metric Wins (excluding time)`); + L(); + experiments.forEach((exp, i) => { + const w = analysis.wins[i]; + const pct = nonTime.length > 0 ? ((w / nonTime.length) * 100).toFixed(0) : "0"; + L(`- **${reportColLabel(exp, i)}:** ${w} of ${nonTime.length} (${pct}%)`); + }); + L(); + + // Speed comparison (only meaningful for a 2-way comparison). + if (experiments.length === 2) { + const timeKey = metricKeys.find((k) => isTimeMetric(k)); + const t0 = timeKey ? experiments[0].metrics[timeKey] : null; + const t1 = timeKey ? experiments[1].metrics[timeKey] : null; + if (t0 !== null && t1 !== null) { + const faster = t0 < t1 ? 0 : 1; + const slower = 1 - faster; + const speedup = ((Math.max(t0, t1) - Math.min(t0, t1)) / Math.max(t0, t1)) * 100; + if (speedup >= 0.1) { + L(`### Speed`); + L(); + L( + `**${reportColLabel(experiments[faster], faster)}** is **${speedup.toFixed(1)}% faster** than ` + + `**${reportColLabel(experiments[slower], slower)}** ` + + `(${Math.min(t0, t1).toFixed(2)}s vs ${Math.max(t0, t1).toFixed(2)}s).`, + ); + L(); + } + } + } + + // Largest differences. + const diffs = nonTime + .map((k) => ({ key: k, spread: analysis.metricAnalysis[k]?.spread ?? null })) + .filter((d) => d.spread !== null && d.spread > 0) + .sort((a, b) => (b.spread ?? 0) - (a.spread ?? 0)) + .slice(0, 5); + if (diffs.length > 0) { + L(`### Largest Differences`); + L(); + for (const d of diffs) { + L(`- **${escPipe(formatMetricName(d.key))}:** ${(d.spread! * 100).toFixed(2)}% spread`); + } + L(); + } + } + + // ── Per-query analysis ───────────────────────────────────────────────────── + if (queries && !queries.warning && queries.rows.length > 0) { + const { counts, sharedByAll, totalQuestions, mode } = queries; + const qExps = queries.experiments; + + // Outcome label + sort weight (disagreements first, then all-wrong, then all-correct). + const outcomeLabel: Record = { + mixed: "Disagreement", + all_wrong: "All wrong", + all_correct: "All correct", + }; + + /** Compact one-cell status used in the overview table. */ + const cellStatus = (cell: ComparisonCell | null): string => { + if (!cell) return "—"; + if (mode === "search") return cell.correct ? `${cell.overlap ?? 0} hit` : "0 hit"; + if (cell.is_error) return "err"; + if (cell.correct === null) return "?"; + return cell.correct ? "✓" : "✗"; + }; + + L(`## Per-Query Analysis`); + L(); + L(`Queries aligned by question text (mode: ${mode}).`); + L(); + L(`- **Comparable** (present in ≥2 experiments): ${counts.comparable}`); + L(`- **Shared by all:** ${sharedByAll}`); + L(`- **Total distinct questions:** ${totalQuestions}`); + L(`- **All correct:** ${counts.allCorrect}`); + L(`- **All wrong:** ${counts.allWrong}`); + L(`- **Disagreements:** ${counts.mixed}`); + L(); + + // Comparable rows are already sorted disagreements-first by buildQueryComparison. + const comparable = queries.rows.filter((r) => r.presentCount >= 2); + const singleOnly = queries.rows.length - comparable.length; + + if (comparable.length > 0) { + // ── Overview table — every comparable query at a glance ──────────────── + L(`### Results Overview`); + L(); + const head = ["#", "Outcome", ...qExps.map((e, i) => reportColLabel(e, i)), "Question"]; + L(`| ${head.join(" | ")} |`); + L(`| ${head.map((_, i) => (i === 0 ? "---:" : ":---")).join(" | ")} |`); + comparable.forEach((row, idx) => { + const statusCells = row.cells.map((c) => cellStatus(c)); + const q = escPipe(row.question.replace(/\s+/g, " ").trim()); + const qShort = q.length > 100 ? `${q.slice(0, 100)}…` : q; + L(`| ${idx + 1} | ${outcomeLabel[row.outcome]} | ${statusCells.join(" | ")} | ${qShort} |`); + }); + L(); + if (mode === "search") { + L(`_Cells show relevant-document hits per experiment._`); + } else { + L(`_Cells: ✓ correct · ✗ wrong · ? unscored · err error._`); + } + L(); + + // ── Full detail for every comparable query ───────────────────────────── + L(`### Query Details`); + L(); + comparable.forEach((row, idx) => { + L(`#### ${idx + 1}. [${outcomeLabel[row.outcome]}] ${row.question}`); + L(); + if (mode === "ask" && row.ground_truth_answer) { + L(`**Ground truth:** ${row.ground_truth_answer}`); + L(); + } + if (mode === "search" && row.ground_truth_ids) { + L( + `**Ground truth IDs (${row.ground_truth_ids.length}):** ${row.ground_truth_ids.join(", ") || "—"}`, + ); + L(); + } + + row.cells.forEach((cell, i) => { + const name = reportColLabel(qExps[i], i); + if (!cell) { + L(`- ${name}: _not present in this experiment_`); + return; + } + const time = cell.time_taken != null ? ` · ${cell.time_taken.toFixed(2)}s` : ""; + + if (mode === "ask") { + const status = cell.is_error + ? "error" + : cell.correct === true + ? "✓ correct" + : cell.correct === null + ? "? unscored" + : "✗ wrong"; + L(`- **${name}** — ${status}${time}`); + L(` - Answer: ${cell.system_answer ? cell.system_answer : "_(empty)_"}`); + if (cell.judge_reasoning) L(` - Judge: ${cell.judge_reasoning}`); + } else { + const status = cell.correct ? "✓ hit" : "✗ miss"; + const gt = new Set(row.ground_truth_ids ?? []); + const retrieved = cell.retrieved_ids ?? []; + L( + `- **${name}** — ${status} · ${cell.overlap ?? 0} relevant of ${cell.num_retrieved ?? retrieved.length} retrieved${time}`, + ); + if (retrieved.length > 0) { + const shown = retrieved + .slice(0, 20) + .map((id) => (gt.has(id) ? `**${id}**` : id)); + const more = retrieved.length > 20 ? ` … +${retrieved.length - 20} more` : ""; + L(` - Retrieved: ${shown.join(", ")}${more}`); + } + if (cell.searches && cell.searches.length > 0) { + const plan = cell.searches + .map((s) => { + const q = s.query ? `"${s.query}"` : s.uuid_value ? `uuid:${s.uuid_value}` : "—"; + const filt = s.filters ? " +filter" : ""; + const sort = s.sort_property ? ` +sort(${s.sort_property.property_name})` : ""; + return `${s.collection}:${q}${filt}${sort}`; + }) + .join("; "); + L(` - Search plan (${cell.searches.length}): ${plan}`); + } + } + }); + L(); + }); + + if (singleOnly > 0) { + L( + `_${singleOnly} additional ${singleOnly === 1 ? "query was" : "queries were"} present in only one experiment and omitted from the comparison._`, + ); + L(); + } + } + } + + L(`---`); + L(`_Report generated by the Query Agent Benchmarking console._`); + L(); + + return lines.join("\n"); +} + +/** Trigger a client-side download of a markdown file. */ +function downloadMarkdown(content: string, filename: string): void { + const blob = new Blob([content], { type: "text/markdown;charset=utf-8" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); +} + export default function ComparePage() { return ( (null); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); + const [exporting, setExporting] = useState(false); useEffect(() => { if (!ids) { @@ -123,6 +452,30 @@ function ComparePageInner() { return { metricAnalysis, wins }; }, [data]); + const handleExport = async () => { + if (!data || exporting) return; + setExporting(true); + try { + // Fetch per-query data so the report is complete regardless of whether + // the user expanded the per-query section. It's optional — if it fails + // we still export the metrics + analysis. + let queries: QueryComparison | null = null; + try { + const r = await fetch(`/api/compare/queries?ids=${ids}`); + if (r.ok) queries = (await r.json()) as QueryComparison; + } catch { + // per-query analysis is best-effort + } + + const now = new Date(); + const md = buildMarkdownReport(data, analysis, queries, now.toLocaleString()); + const stamp = now.toISOString().slice(0, 10); + downloadMarkdown(md, `experiment-comparison-${stamp}.md`); + } finally { + setExporting(false); + } + }; + if (loading) { return (
@@ -156,9 +509,20 @@ function ComparePageInner() { Compare -

- Compare Experiments -

+
+

+ Compare Experiments +

+ +
{/* ── Experiment legend ─────────────────────────────────────────────── */}
)} + + {/* ── Per-query analysis ────────────────────────────────────────────── */} + +
+ ); +} + +/* ═══════════════════════════════════════════════════════════════════════════ + Per-query analysis: what each experiment predicted for individual queries. + Loaded on demand (trial files are large) and aligned by question text. + ═══════════════════════════════════════════════════════════════════════════ */ + +type QueryFilter = "all" | "mixed" | "all_correct" | "all_wrong"; + +const OUTCOME_META: Record< + ComparisonOutcome, + { label: string; fg: string; bg: string } +> = { + mixed: { label: "Disagreement", fg: "var(--color-coral)", bg: "rgba(244,64,78,0.12)" }, + all_correct: { label: "All correct", fg: "var(--color-green)", bg: "rgba(97,189,115,0.15)" }, + all_wrong: { label: "All wrong", fg: "#b8a900", bg: "rgba(249,241,93,0.15)" }, +}; + +function PerQuerySection({ ids }: { ids: string }) { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [loaded, setLoaded] = useState(false); + + const [filter, setFilter] = useState("all"); + const [search, setSearch] = useState(""); + + const load = () => { + setLoading(true); + setError(null); + fetch(`/api/compare/queries?ids=${ids}`) + .then((r) => { + if (!r.ok) throw new Error("Failed to load per-query comparison"); + return r.json(); + }) + .then((d: QueryComparison) => { + setData(d); + setLoaded(true); + setLoading(false); + }) + .catch((e) => { + setError(e.message); + setLoading(false); + }); + }; + + const filteredRows = useMemo(() => { + if (!data) return []; + const term = search.trim().toLowerCase(); + return data.rows.filter((row) => { + if (row.presentCount < 2) return false; // only comparable rows + if (filter !== "all" && row.outcome !== filter) return false; + if (term && !row.question.toLowerCase().includes(term)) return false; + return true; + }); + }, [data, filter, search]); + + return ( +
+
+

+ Per-Query Analysis +

+ {!loaded && !loading && ( + + )} +
+ + {!loaded && !loading && ( +

+ Compare what each experiment predicted for individual queries, aligned by question text. + Surfaces disagreements where one experiment succeeds and another fails. +

+ )} + + {loading && ( +
+
+

Loading per-query data...

+
+ )} + + {error && ( +

+ {error} +

+ )} + + {data && data.warning && ( +
+ {data.warning} +
+ )} + + {data && !data.warning && data.rows.length === 0 && ( +

+ No overlapping queries found across these experiments (they may use different datasets). +

+ )} + + {data && !data.warning && data.rows.length > 0 && ( + + )} +
+ ); +} + +function PerQueryBody({ + data, + filteredRows, + filter, + setFilter, + search, + setSearch, +}: { + data: QueryComparison; + filteredRows: QueryComparisonRow[]; + filter: QueryFilter; + setFilter: (f: QueryFilter) => void; + search: string; + setSearch: (s: string) => void; +}) { + const { experiments, counts, sharedByAll, totalQuestions, mode } = data; + + const summary: { key: QueryFilter; label: string; value: number; fg: string }[] = [ + { key: "mixed", label: "Disagreements", value: counts.mixed, fg: "var(--color-coral)" }, + { key: "all_correct", label: "All correct", value: counts.allCorrect, fg: "var(--color-green)" }, + { key: "all_wrong", label: "All wrong", value: counts.allWrong, fg: "#b8a900" }, + ]; + + return ( +
+ {/* Summary strip */} +
+
+
Comparable
+
+ {counts.comparable} +
+
+ {sharedByAll} in all · {totalQuestions} total +
+
+ {summary.map((s) => ( + + ))} +
+ + {/* Controls */} +
+ Filter + {(["all", "mixed", "all_correct", "all_wrong"] as const).map((f) => ( + + ))} + setSearch(e.target.value)} + placeholder="Search question..." + className="rounded-md px-2.5 py-1 text-xs ml-2" + style={{ + background: "var(--bg-surface)", + border: "1px solid var(--border-default)", + color: "var(--text-primary)", + width: "220px", + }} + /> + + {filteredRows.length} of {counts.comparable} + +
+ + {/* Comparison table */} +
+ + + + + + {experiments.map((exp, i) => { + const c = EXP_COLORS[i % EXP_COLORS.length]; + return ( + + ); + })} + + + + {filteredRows.map((row, ri) => ( + + ))} + +
Question + + {i + 1} + +
+
+
+ ); +} + +function ComparisonRowView({ + row, + mode, + expCount, +}: { + row: QueryComparisonRow; + mode: QueryComparison["mode"]; + expCount: number; +}) { + const [open, setOpen] = useState(false); + const meta = OUTCOME_META[row.outcome]; + const colSpan = 2 + expCount; + + return ( + + setOpen((v) => !v)} + style={{ + cursor: "pointer", + background: row.outcome === "mixed" ? "rgba(244,64,78,0.05)" : undefined, + }} + > + + + ▶ + + + +
+ + {row.outcome === "mixed" ? "≠" : row.outcome === "all_correct" ? "✓" : "✗"} + + {row.question} +
+ + {row.cells.map((cell, i) => ( + + + + ))} + + {open && ( + + +
+ +
+ + + )} + + ); +} + +/** Compact per-experiment cell shown in the main comparison row. */ +function CellBadge({ cell, mode }: { cell: ComparisonCell | null; mode: QueryComparison["mode"] }) { + if (!cell) { + return —; + } + const ok = cell.correct === true; + const unknown = cell.correct === null; + const fg = unknown ? "var(--text-muted)" : ok ? "var(--color-green)" : "var(--color-coral)"; + const bg = unknown ? "var(--border-subtle)" : ok ? "rgba(97,189,115,0.15)" : "rgba(244,64,78,0.12)"; + + let label: string; + if (mode === "search") { + const gt = cell.overlap ?? 0; + label = unknown ? "?" : `${gt} hit`; + } else { + label = cell.is_error ? "err" : unknown ? "?" : ok ? "✓" : "✗"; + } + + return ( + + {label} + + ); +} + +/** Expanded full-width detail comparing each experiment's prediction. */ +function RowDetail({ row, mode }: { row: QueryComparison["rows"][number]; mode: QueryComparison["mode"] }) { + return ( +
+ {/* Ground truth */} +
+
Question
+

{row.question}

+ {mode === "ask" && row.ground_truth_answer && ( + <> +
Ground Truth Answer
+

+ {row.ground_truth_answer} +

+ + )} + {mode === "search" && row.ground_truth_ids && ( +
+ Ground truth ({row.ground_truth_ids.length}): {row.ground_truth_ids.join(", ")} +
+ )} +
+ + {/* Per-experiment predictions */} +
+ {row.cells.map((cell, i) => { + const c = EXP_COLORS[i % EXP_COLORS.length]; + return ( +
+
+ + {i + 1} + + {cell ? ( + + ) : ( + not present + )} + {cell?.time_taken != null && ( + + {cell.time_taken.toFixed(2)}s + + )} +
+ {cell && } +
+ ); + })} +
+
+ ); +} + +function CellDetail({ + cell, + mode, + groundTruthIds, +}: { + cell: ComparisonCell; + mode: QueryComparison["mode"]; + groundTruthIds?: string[]; +}) { + if (mode === "ask") { + return ( +
+
Answer
+

+ {cell.system_answer || (empty)} +

+ {cell.judge_reasoning && ( +
+ + Judge reasoning + +

+ {cell.judge_reasoning} +

+
+ )} +
+ ); + } + + // search mode + const gt = new Set(groundTruthIds ?? []); + const retrieved = cell.retrieved_ids ?? []; + return ( +
+
+
+ Retrieved ({cell.num_retrieved ?? retrieved.length}) · {cell.overlap ?? 0} relevant +
+
+ {retrieved.length === 0 && ( + none + )} + {retrieved.slice(0, 20).map((id, i) => { + const hit = gt.has(id); + return ( + + {id} + + ); + })} + {retrieved.length > 20 && ( + +{retrieved.length - 20} more + )} +
+
+ {cell.searches != null && cell.searches.length > 0 && ( +
+
Search Plan
+ +
+ )}
); } diff --git a/console/app/results/experiments/[id]/trial/[trialNum]/page.tsx b/console/app/results/experiments/[id]/trial/[trialNum]/page.tsx index 8f52ae1..805c32c 100644 --- a/console/app/results/experiments/[id]/trial/[trialNum]/page.tsx +++ b/console/app/results/experiments/[id]/trial/[trialNum]/page.tsx @@ -1,7 +1,8 @@ "use client"; -import { useEffect, useState, useCallback, useMemo } from "react"; +import { Fragment, useEffect, useState, useCallback, useMemo } from "react"; import type { TrialResultFile, SearchQuery, AskQuery } from "@/lib/results"; +import { SearchPlan } from "@/app/components/SearchPlan"; function CopyButton({ text }: { text: string }) { const [copied, setCopied] = useState(false); @@ -498,15 +499,30 @@ function AskQueriesView({ queries }: { queries: AskQuery[] }) { } function SearchQueriesView({ queries }: { queries: SearchQuery[] }) { + const [expanded, setExpanded] = useState>(new Set()); + + const toggle = useCallback((id: string) => { + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + }, []); + + const anyPlans = queries.some((q) => (q.num_searches ?? q.searches?.length ?? 0) > 0); + return (
+ + {anyPlans && } @@ -516,29 +532,80 @@ function SearchQueriesView({ queries }: { queries: SearchQuery[] }) { q.ground_truth_ids.includes(id) ).length; const hasOverlap = overlap > 0; + const searches = q.searches ?? []; + const numSearches = q.num_searches ?? searches.length; + const canExpand = numSearches > 0; + const isOpen = expanded.has(q.query_id); + const colSpan = anyPlans ? 7 : 6; return ( - - - - - - - + + toggle(q.query_id) : undefined} + > + + + + + + {anyPlans && ( + + )} + + + {isOpen && ( + + + + )} + ); })} diff --git a/console/lib/results.ts b/console/lib/results.ts index ebc88cc..a1cb987 100644 --- a/console/lib/results.ts +++ b/console/lib/results.ts @@ -16,6 +16,49 @@ export interface TrialMetadata { mode: "search" | "ask"; } +/** + * A leaf property filter, mirroring the Query Agent's normalized filter shape. + * e.g. { filter_type: "integer", property_name: "price", operator: "<", value: 30000 } + * Date filters carry a structured `value` object instead of a scalar. + */ +export interface PropertyFilterLeaf { + filter_type?: string | null; + property_name?: string; + operator?: string; + value?: unknown; + is_null?: boolean; + // geo filters + latitude?: number; + longitude?: number; + max_distance_meters?: number; +} + +/** A boolean group combining nested filters with AND / OR. */ +export interface FilterGroup { + combine: "AND" | "OR"; + filters: FilterNode[]; +} + +export type FilterNode = PropertyFilterLeaf | FilterGroup; + +export interface QuerySort { + property_name: string; + order: "ascending" | "descending"; + tie_break?: QuerySort | null; +} + +/** + * One structured sub-search the agent issued for a single benchmark query — + * its search plan. Mirrors the backend `AgentSearch` model. + */ +export interface AgentSearch { + collection: string; + query?: string | null; + filters?: FilterNode | null; + sort_property?: QuerySort | null; + uuid_value?: string | null; +} + export interface SearchQuery { query_id: string; question: string; @@ -24,6 +67,9 @@ export interface SearchQuery { num_retrieved: number; num_ground_truth: number; time_taken: number; + /** The agent's search plan. Absent/empty for direct or BYOS retrievers. */ + num_searches?: number; + searches?: AgentSearch[]; } export interface AskQuery { @@ -242,6 +288,239 @@ export function loadExperiment(id: string): Experiment | null { return experiments.find((e) => e.id === id) || null; } +// ============================================================================ +// Per-query comparison across K experiments +// ============================================================================ + +export type ComparisonOutcome = "all_correct" | "all_wrong" | "mixed"; + +/** One experiment's prediction for a single aligned query. */ +export interface ComparisonCell { + present: boolean; + /** Unified success signal — search: retrieved a relevant doc; ask: judged correct. */ + correct: boolean | null; + time_taken?: number; + // search-mode fields + retrieved_ids?: string[]; + num_retrieved?: number; + overlap?: number; + searches?: AgentSearch[] | null; + // ask-mode fields + system_answer?: string; + score?: number; + is_error?: boolean; + judge_reasoning?: string; +} + +/** One query aligned across all compared experiments (joined by question text). */ +export interface QueryComparisonRow { + question: string; + ground_truth_ids?: string[]; + ground_truth_answer?: string; + question_type?: string; + tenant_id?: string; + /** Indexed by experiment order; `null` when the query is absent from that experiment. */ + cells: (ComparisonCell | null)[]; + presentCount: number; + correctCount: number; + /** Classification over present cells (only meaningful when presentCount >= 2). */ + outcome: ComparisonOutcome; +} + +export interface QueryComparisonExperiment { + id: string; + label: string; + agent_name: string; + dataset: string; + mode: "search" | "ask" | "unknown"; + trialNumber: number | null; +} + +export interface QueryComparison { + mode: "search" | "ask" | "mixed" | "unknown"; + experiments: QueryComparisonExperiment[]; + rows: QueryComparisonRow[]; + /** Total distinct questions across the union of experiments. */ + totalQuestions: number; + /** Questions present in every compared experiment. */ + sharedByAll: number; + /** Summary counts over comparable rows (present in >= 2 experiments). */ + counts: { comparable: number; allCorrect: number; allWrong: number; mixed: number }; + warning?: string; +} + +function firstTrialWithResults( + exp: Experiment +): { trialNumber: number; results: TrialResultFile } | null { + const trial = exp.trials.find((t) => t.results != null); + return trial && trial.results ? { trialNumber: trial.trialNumber, results: trial.results } : null; +} + +function buildSearchCell(q: SearchQuery): ComparisonCell { + const gt = new Set(q.ground_truth_ids ?? []); + const overlap = (q.retrieved_ids ?? []).filter((id) => gt.has(id)).length; + return { + present: true, + correct: overlap > 0, + time_taken: q.time_taken, + retrieved_ids: q.retrieved_ids, + num_retrieved: q.num_retrieved, + overlap, + searches: q.searches ?? null, + }; +} + +function buildAskCell(q: AskQuery): ComparisonCell { + const correct = q.is_error ? false : q.score === undefined ? null : q.score === 1; + return { + present: true, + correct, + time_taken: q.time_taken, + system_answer: q.system_answer, + score: q.score, + is_error: q.is_error, + judge_reasoning: q.judge_reasoning, + }; +} + +/** + * Build a per-query comparison across K experiments. + * + * Queries are aligned by **question text** (robust to per-run query-id + * reordering from subsetting/shuffling). Uses the first trial with results for + * each experiment. Requires all experiments to share a single mode (search or + * ask); otherwise returns a warning and no rows. + */ +export function buildQueryComparison(ids: string[]): QueryComparison | null { + const all = loadAllExperiments(); + const matched = ids + .map((id) => all.find((e) => e.id === id)) + .filter((e): e is Experiment => Boolean(e)); + + if (matched.length < 2) return null; + + const labels = loadLabels(); + const trials = matched.map(firstTrialWithResults); + + const experiments: QueryComparisonExperiment[] = matched.map((exp, i) => ({ + id: exp.id, + label: labels[exp.id] ?? "", + agent_name: exp.agent_name, + dataset: exp.dataset, + mode: exp.mode, + trialNumber: trials[i]?.trialNumber ?? null, + })); + + // Determine a single shared mode for the comparison. + const modes = new Set(matched.map((e) => e.mode)); + let mode: QueryComparison["mode"]; + let warning: string | undefined; + if (modes.size > 1) { + mode = "mixed"; + warning = "Selected experiments use different modes; per-query comparison requires a single mode."; + } else { + mode = [...modes][0] as QueryComparison["mode"]; + } + + const K = matched.length; + const emptyResult = (w?: string): QueryComparison => ({ + mode, + experiments, + rows: [], + totalQuestions: 0, + sharedByAll: 0, + counts: { comparable: 0, allCorrect: 0, allWrong: 0, mixed: 0 }, + warning: w ?? warning, + }); + + if (mode === "mixed" || mode === "unknown") return emptyResult(); + + // Align by question text, preserving first-seen order. + const rowMap = new Map(); + + const ensureRow = (question: string): QueryComparisonRow => { + let row = rowMap.get(question); + if (!row) { + row = { + question, + cells: new Array(K).fill(null), + presentCount: 0, + correctCount: 0, + outcome: "all_wrong", + }; + rowMap.set(question, row); + } + return row; + }; + + trials.forEach((trial, expIdx) => { + if (!trial) return; + const queries = trial.results.queries ?? []; + for (const q of queries) { + const question = q.question?.trim(); + if (!question) continue; + const row = ensureRow(question); + if (mode === "search") { + const sq = q as SearchQuery; + row.cells[expIdx] = buildSearchCell(sq); + if (!row.ground_truth_ids) row.ground_truth_ids = sq.ground_truth_ids; + } else { + const aq = q as AskQuery; + row.cells[expIdx] = buildAskCell(aq); + if (!row.ground_truth_answer) row.ground_truth_answer = aq.ground_truth_answer; + if (!row.question_type && aq.question_type) row.question_type = aq.question_type; + if (!row.tenant_id && aq.tenant_id) row.tenant_id = aq.tenant_id; + } + } + }); + + // Finalize per-row aggregates. + const rows = [...rowMap.values()]; + let sharedByAll = 0; + const counts = { comparable: 0, allCorrect: 0, allWrong: 0, mixed: 0 }; + + for (const row of rows) { + const present = row.cells.filter((c): c is ComparisonCell => c != null); + row.presentCount = present.length; + // Treat unknown (null) correctness as not-correct for counting. + row.correctCount = present.filter((c) => c.correct === true).length; + row.outcome = + row.correctCount === row.presentCount + ? "all_correct" + : row.correctCount === 0 + ? "all_wrong" + : "mixed"; + + if (row.presentCount === K) sharedByAll++; + if (row.presentCount >= 2) { + counts.comparable++; + if (row.outcome === "all_correct") counts.allCorrect++; + else if (row.outcome === "all_wrong") counts.allWrong++; + else counts.mixed++; + } + } + + // Surface disagreements first, then all-wrong, then all-correct; stable within. + const outcomeRank: Record = { mixed: 0, all_wrong: 1, all_correct: 2 }; + rows.sort((a, b) => { + // Comparable rows (>=2 present) before partial ones. + const aCmp = a.presentCount >= 2 ? 0 : 1; + const bCmp = b.presentCount >= 2 ? 0 : 1; + if (aCmp !== bCmp) return aCmp - bCmp; + return outcomeRank[a.outcome] - outcomeRank[b.outcome]; + }); + + return { + mode, + experiments, + rows, + totalQuestions: rows.length, + sharedByAll, + counts, + warning, + }; +} + /** Return all JSON filenames on disk that belong to the given experiment id. */ export function getExperimentFiles(id: string): string[] { const baseName = decodeURIComponent(id); diff --git a/query_agent_benchmarking/__init__.py b/query_agent_benchmarking/__init__.py index 69252d6..0f31cd9 100644 --- a/query_agent_benchmarking/__init__.py +++ b/query_agent_benchmarking/__init__.py @@ -33,6 +33,8 @@ # Search-specific InMemorySearchQuery, SearchResult, + AgentSearch, + SearchAgentResponse, # Ask-specific InMemoryAskQuery, AskResult, @@ -111,6 +113,8 @@ "QueryResult", "InMemorySearchQuery", "SearchResult", + "AgentSearch", + "SearchAgentResponse", "InMemoryAskQuery", "AskResult", "AskQueriesCollection", diff --git a/query_agent_benchmarking/internal/adapters/agents/external_service.py b/query_agent_benchmarking/internal/adapters/agents/external_service.py index 273c009..d19ffab 100644 --- a/query_agent_benchmarking/internal/adapters/agents/external_service.py +++ b/query_agent_benchmarking/internal/adapters/agents/external_service.py @@ -4,19 +4,38 @@ HTTP POST requests to a user-provided endpoint. """ -from typing import Optional +from typing import Any, Optional import httpx -from query_agent_benchmarking.internal.core.domain.models import ObjectID +from query_agent_benchmarking.internal.core.domain.models import ( + AgentSearch, + ObjectID, + SearchAgentResponse, +) from query_agent_benchmarking.internal.core.ports.ask_agent import AskResponse +def _parse_searches(raw: Any) -> Optional[list[AgentSearch]]: + """Parse an optional external ``searches`` payload into ``AgentSearch``. + + Returns ``None`` when the service does not report a search plan, so the + distinction between "not reported" and "empty plan" is preserved. + """ + if raw is None: + return None + return [AgentSearch.model_validate(s) for s in raw] + + class ExternalSearchService: """SearchAgent adapter that delegates to an external HTTP service. Expected request: ``{"query": "..."}`` Expected response: ``{"results": ["id1", "id2", ...]}`` + + The service may optionally include a ``"searches"`` key describing the + structured search plan it executed; when absent, the search plan is reported + as ``None`` (not yet implemented), which serializes and visualizes cleanly. """ def __init__(self, host: str): @@ -24,19 +43,26 @@ def __init__(self, host: str): raise ValueError("host is required for ExternalSearchService") self.host = host - def run(self, query: str, tenant: Optional[str] = None) -> list[ObjectID]: + def _build_response(self, data: dict[str, Any]) -> SearchAgentResponse: + retrieved_ids = [ObjectID(object_id=str(doc_id)) for doc_id in data.get("results", [])] + return SearchAgentResponse( + retrieved_ids=retrieved_ids, + searches=_parse_searches(data.get("searches")), + ) + + def run(self, query: str, tenant: Optional[str] = None) -> SearchAgentResponse: with httpx.Client(timeout=300.0) as client: response = client.post(self.host, json={"query": query}) response.raise_for_status() data = response.json() - return [ObjectID(object_id=str(doc_id)) for doc_id in data.get("results", [])] + return self._build_response(data) - async def run_async(self, query: str, tenant: Optional[str] = None) -> list[ObjectID]: + async def run_async(self, query: str, tenant: Optional[str] = None) -> SearchAgentResponse: async with httpx.AsyncClient(timeout=300.0) as client: response = await client.post(self.host, json={"query": query}) response.raise_for_status() data = response.json() - return [ObjectID(object_id=str(doc_id)) for doc_id in data.get("results", [])] + return self._build_response(data) async def initialize_async(self) -> None: pass diff --git a/query_agent_benchmarking/internal/adapters/agents/weaviate_query_agent.py b/query_agent_benchmarking/internal/adapters/agents/weaviate_query_agent.py index bf556d0..fc244b3 100644 --- a/query_agent_benchmarking/internal/adapters/agents/weaviate_query_agent.py +++ b/query_agent_benchmarking/internal/adapters/agents/weaviate_query_agent.py @@ -12,7 +12,11 @@ from weaviate.config import AdditionalConfig, Timeout from weaviate.agents.query import QueryAgent, AsyncQueryAgent -from query_agent_benchmarking.internal.core.domain.models import ObjectID +from query_agent_benchmarking.internal.core.domain.models import ( + AgentSearch, + ObjectID, + SearchAgentResponse, +) from query_agent_benchmarking.internal.core.ports.ask_agent import AskResponse from query_agent_benchmarking.internal.adapters.agents.collection_resolver import resolve_collection_info from query_agent_benchmarking.internal.adapters.clients.provider_headers import ( @@ -38,6 +42,41 @@ def _validate_filtering(filtering: Optional[str]) -> Filtering: return filtering +def _dump_optional(value: Any) -> Optional[Any]: + """Dump a (possibly None) pydantic model to a JSON-serializable value. + + The SDK's filter/sort structures are nested pydantic models; we store them + as plain JSON so the domain stays SDK-agnostic and they persist cleanly. + """ + if value is None: + return None + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + return value + + +def _extract_searches(response: Any) -> list[AgentSearch]: + """Convert a SearchModeResponse's ``searches`` into domain ``AgentSearch``. + + Defensive: the SDK field is optional and may be ``None``; older/other + response shapes simply yield an empty list. + """ + raw_searches = getattr(response, "searches", None) or [] + searches: list[AgentSearch] = [] + for s in raw_searches: + uuid_value = getattr(s, "uuid_value", None) + searches.append( + AgentSearch( + collection=getattr(s, "collection", ""), + query=getattr(s, "query", None), + filters=_dump_optional(getattr(s, "filters", None)), + sort_property=_dump_optional(getattr(s, "sort_property", None)), + uuid_value=str(uuid_value) if uuid_value is not None else None, + ) + ) + return searches + + class WeaviateQueryAgentSearch: """SearchAgent adapter wrapping Weaviate QueryAgent in search-only mode.""" @@ -100,21 +139,31 @@ async def initialize_async(self): agents_host=self.agents_host, ) - def run(self, query: str, tenant: Optional[str] = None) -> list[ObjectID]: - if self._agent is None: - self.initialize_sync() - response = self._agent.search(query, limit=20, filtering=self.filtering) - return [ + def _build_response(self, response: Any) -> SearchAgentResponse: + """Map a Weaviate ``SearchModeResponse`` to the domain response. + + Captures both the ranked document IDs and the structured ``searches`` + the QueryAgent issued (query text, filters, sort, uuid lookups) so the + agent's search plan can be persisted and visualized per query. + """ + retrieved_ids = [ ObjectID(object_id=obj.properties[self.id_property]) for obj in response.search_results.objects ] + return SearchAgentResponse( + retrieved_ids=retrieved_ids, + searches=_extract_searches(response), + ) - async def run_async(self, query: str, tenant: Optional[str] = None) -> list[ObjectID]: + def run(self, query: str, tenant: Optional[str] = None) -> SearchAgentResponse: + if self._agent is None: + self.initialize_sync() + response = self._agent.search(query, limit=20, filtering=self.filtering) + return self._build_response(response) + + async def run_async(self, query: str, tenant: Optional[str] = None) -> SearchAgentResponse: response = await self._agent.search(query, limit=20, filtering=self.filtering) - return [ - ObjectID(object_id=obj.properties[self.id_property]) - for obj in response.search_results.objects - ] + return self._build_response(response) async def close_async(self): if self._client: diff --git a/query_agent_benchmarking/internal/adapters/metrics/ir_metrics.py b/query_agent_benchmarking/internal/adapters/metrics/ir_metrics.py index 2108105..939ac1a 100644 --- a/query_agent_benchmarking/internal/adapters/metrics/ir_metrics.py +++ b/query_agent_benchmarking/internal/adapters/metrics/ir_metrics.py @@ -42,6 +42,48 @@ def calculate_recall_at_k( return recall +def calculate_precision_at_k( + target_ids: list[str], + retrieved_ids: list[str], + k: int, + verbose: bool = False +) -> float: + """Calculate precision@k for retrieved documents. + + Precision@k is the proportion of the top-k retrieved documents that are + relevant. For filtered retrieval (where fewer than k documents may be + returned) the denominator is the number of documents actually retrieved + in the top-k, so returning extra irrelevant documents is penalized while + returning fewer relevant-only documents is not. + + Args: + target_ids: List of target document IDs (ground truth). + retrieved_ids: List of retrieved document IDs. + k: The number of top results to consider. + + Returns: + float: Precision@k score (0.0 to 1.0) - proportion of the retrieved + top-k documents that are relevant. + """ + target_id_set = {str(id) for id in target_ids} + retrieved_ids = [str(id) for id in retrieved_ids] if retrieved_ids else [] + + retrieved_ids_at_k = retrieved_ids[:k] + + if not retrieved_ids_at_k: + return 0.0 + + found_count = sum(1 for retrieved_id in retrieved_ids_at_k if retrieved_id in target_id_set) + precision = found_count / len(retrieved_ids_at_k) + + if verbose: + print(f"\033[96mTarget IDs: {target_id_set}\033[0m") + print(f"\033[92mRetrieved IDs @{k}: {retrieved_ids_at_k}\033[0m") + print(f"\033[96mPrecision@{k}: {found_count}/{len(retrieved_ids_at_k)} = {precision:.2f}\033[0m") + + return precision + + def calculate_success_at_k( target_ids: list[str], retrieved_ids: list[str], diff --git a/query_agent_benchmarking/internal/adapters/metrics/ir_metrics_calculator.py b/query_agent_benchmarking/internal/adapters/metrics/ir_metrics_calculator.py index d901dc4..caa0bf7 100644 --- a/query_agent_benchmarking/internal/adapters/metrics/ir_metrics_calculator.py +++ b/query_agent_benchmarking/internal/adapters/metrics/ir_metrics_calculator.py @@ -13,6 +13,7 @@ from query_agent_benchmarking.internal.core.domain.metrics_config import MetricSpec, resolve_metrics_profile, parse_extra_metrics from query_agent_benchmarking.internal.adapters.metrics.ir_metrics import ( calculate_recall_at_k, + calculate_precision_at_k, calculate_success_at_k, calculate_nDCG_at_k, calculate_coverage, @@ -22,6 +23,7 @@ # Maps MetricSpec.name to the concrete metric function _METRIC_FUNCTIONS = { "recall": calculate_recall_at_k, + "precision": calculate_precision_at_k, "success": calculate_success_at_k, "nDCG": calculate_nDCG_at_k, "coverage": calculate_coverage, @@ -121,7 +123,7 @@ def _compute_single_metric(func, spec: MetricSpec, retrieved_ids, ground_truth): """Dispatch a single metric computation based on its name.""" params = spec.params - if spec.name in ("recall", "success", "nDCG"): + if spec.name in ("recall", "precision", "success", "nDCG"): return func( target_ids=ground_truth.dataset_ids, retrieved_ids=retrieved_ids, diff --git a/query_agent_benchmarking/internal/adapters/results/serialization.py b/query_agent_benchmarking/internal/adapters/results/serialization.py index 11b0fe9..2cf34b8 100644 --- a/query_agent_benchmarking/internal/adapters/results/serialization.py +++ b/query_agent_benchmarking/internal/adapters/results/serialization.py @@ -74,15 +74,7 @@ def save_trial_results( "mode": "search", }, "queries": [ - { - "query_id": f"q{idx}", - "question": result.query.question, - "ground_truth_ids": result.query_ground_truth_id, - "retrieved_ids": [obj.object_id for obj in result.retrieved_ids], - "num_retrieved": len(result.retrieved_ids), - "num_ground_truth": len(result.query_ground_truth_id), - "time_taken": result.time_taken, - } + _search_query_to_dict(idx, result) for idx, result in enumerate(results) ] } @@ -91,6 +83,31 @@ def save_trial_results( json.dump(trial_data, f, indent=2) +def _search_query_to_dict(idx: int, result: QueryResult) -> dict[str, Any]: + """Serialize a single search QueryResult to a JSON-ready dict. + + The agent's search plan (``searches``/``num_searches``) is only included + when the agent reported one. Agents that don't expose a plan (direct + hybrid/vector/BM25 or external services that haven't implemented it) leave + ``result.searches`` as ``None``, and these keys are omitted entirely. + """ + query_dict: dict[str, Any] = { + "query_id": f"q{idx}", + "question": result.query.question, + "ground_truth_ids": result.query_ground_truth_id, + "retrieved_ids": [obj.object_id for obj in result.retrieved_ids], + "num_retrieved": len(result.retrieved_ids), + "num_ground_truth": len(result.query_ground_truth_id), + "time_taken": result.time_taken, + } + if result.searches is not None: + # The agent's search plan: how it decomposed this query into structured + # sub-searches (query text, filters, sort, uuid). + query_dict["num_searches"] = len(result.searches) + query_dict["searches"] = [s.model_dump(mode="json") for s in result.searches] + return query_dict + + def save_ask_trial_results( results: list[AskResult], config: dict[str, Any], diff --git a/query_agent_benchmarking/internal/core/domain/metrics_config.py b/query_agent_benchmarking/internal/core/domain/metrics_config.py index 3b0d7e9..afb8ae3 100644 --- a/query_agent_benchmarking/internal/core/domain/metrics_config.py +++ b/query_agent_benchmarking/internal/core/domain/metrics_config.py @@ -83,10 +83,13 @@ class MetricsProfile: MetricSpec("recall", {"k": 20}), ), primary_metric="recall_at_5"), MetricsProfile("filtered-cars", ( + MetricSpec("precision", {"k": 1}), + MetricSpec("precision", {"k": 5}), + MetricSpec("precision", {"k": 20}), MetricSpec("recall", {"k": 1}), MetricSpec("recall", {"k": 5}), MetricSpec("recall", {"k": 20}), - ), primary_metric="recall_at_5"), + ), primary_metric="precision_at_5"), MetricsProfile("vidore_v3_hr", ( MetricSpec("recall", {"k": 1}), MetricSpec("recall", {"k": 5}), diff --git a/query_agent_benchmarking/internal/core/domain/models.py b/query_agent_benchmarking/internal/core/domain/models.py index 8b1478b..efa098f 100644 --- a/query_agent_benchmarking/internal/core/domain/models.py +++ b/query_agent_benchmarking/internal/core/domain/models.py @@ -52,17 +52,63 @@ class InMemoryAskQuery(BaseModel): # Result Models # ============================================================================ +class AgentSearch(BaseModel): + """A single sub-query a search agent issued under the hood. + + Infrastructure-agnostic mirror of the Weaviate SDK's + ``QueryResultWithCollectionNormalized``. It records *how* an agent + decomposed one natural-language query into a structured search against a + collection: an optional semantic query string, an optional (recursive) + filter tree, an optional sort, and an optional direct UUID lookup. + + ``filters`` and ``sort_property`` are stored as JSON-serializable values + (the SDK models dumped with ``mode="json"``) so the domain layer stays free + of SDK types and the structures persist/serialize cleanly. ``filters`` is + either a leaf filter (``{"filter_type", "property_name", "operator", + "value", ...}``) or a boolean group (``{"combine": "AND"|"OR", "filters": + [...]}``). + """ + collection: str + query: Optional[str] = None + filters: Optional[Any] = None + sort_property: Optional[Any] = None + uuid_value: Optional[str] = None + + class QueryResult(BaseModel): - """Result from a search query - kept for backwards compatibility.""" + """Result from a search query - kept for backwards compatibility. + + ``searches`` is optional: ``None`` means the agent does not report a search + plan (e.g. direct hybrid/vector/BM25 or an external service that hasn't + implemented it), which is distinct from an empty list (a plan with zero + sub-searches). + """ query: InMemoryQuery query_ground_truth_id: list[str] retrieved_ids: list[ObjectID] time_taken: float + searches: Optional[list[AgentSearch]] = None SearchResult = QueryResult +class SearchAgentResponse(BaseModel): + """Optional richer return type for ``SearchAgent.run`` / ``run_async``. + + Agents may return a plain ``list[ObjectID]`` (the legacy contract, still + fully supported for BYOS and direct-search adapters) or this object to + additionally expose the structured ``searches`` the agent performed. The + query-execution layer normalizes both shapes, so returning this is purely + additive. + + ``searches`` is optional: leave it ``None`` to signal that this agent does + not (yet) report a search plan, while still using the richer response type. + """ + retrieved_ids: list[ObjectID] + searches: Optional[list[AgentSearch]] = None + + class AskResult(BaseModel): """Result from an ask benchmark query.""" query: InMemoryAskQuery diff --git a/query_agent_benchmarking/internal/core/domain/query_execution.py b/query_agent_benchmarking/internal/core/domain/query_execution.py index 1883b61..3b55757 100644 --- a/query_agent_benchmarking/internal/core/domain/query_execution.py +++ b/query_agent_benchmarking/internal/core/domain/query_execution.py @@ -7,14 +7,17 @@ import asyncio import time -from typing import Any +from typing import Any, Optional from tqdm import tqdm from query_agent_benchmarking.internal.core.domain.models import ( + AgentSearch, InMemoryQuery, InMemoryAskQuery, + ObjectID, QueryResult, + SearchAgentResponse, AskResult, ) @@ -23,6 +26,20 @@ # Search Mode Query Execution # ============================================================================ +def _unpack_search_response( + response: object, +) -> tuple[list[ObjectID], Optional[list[AgentSearch]]]: + """Normalize a search agent's return value into (retrieved_ids, searches). + + Agents may return either a plain ``list[ObjectID]`` (legacy / BYOS / + direct-search adapters) or a ``SearchAgentResponse`` that also carries the + structured searches the agent performed. ``searches`` is ``None`` when the + agent does not report a search plan. + """ + if isinstance(response, SearchAgentResponse): + return response.retrieved_ids, response.searches + return response, None # type: ignore[return-value] + def run_search_queries( queries: list[InMemoryQuery], query_agent: Any, @@ -34,13 +51,15 @@ def run_search_queries( query_start_time = time.time() stringified_ids = [str(dataset_id) for dataset_id in query.dataset_ids] response = query_agent.run(query.question, tenant=query.tenant_id) + retrieved_ids, searches = _unpack_search_response(response) query_time_taken = time.time() - query_start_time results.append(QueryResult( query=query, query_ground_truth_id=stringified_ids, - retrieved_ids=response, + retrieved_ids=retrieved_ids, time_taken=query_time_taken, + searches=searches, )) if i % 10 == 0: @@ -79,13 +98,15 @@ async def process_query(query, index, retry_count=0, max_retries=3): print(f"Running search query {index}: {query.question}") response = await query_agent.run_async(query.question, tenant=query.tenant_id) + retrieved_ids, searches = _unpack_search_response(response) query_time_taken = time.time() - query_start_time return QueryResult( query=query, query_ground_truth_id=stringified_ids, - retrieved_ids=response, + retrieved_ids=retrieved_ids, time_taken=query_time_taken, + searches=searches, ) except Exception as e: query_time_taken = time.time() - query_start_time diff --git a/query_agent_benchmarking/internal/core/ports/search_agent.py b/query_agent_benchmarking/internal/core/ports/search_agent.py index be3282a..f6e7303 100644 --- a/query_agent_benchmarking/internal/core/ports/search_agent.py +++ b/query_agent_benchmarking/internal/core/ports/search_agent.py @@ -1,19 +1,29 @@ """Port interface for search agents.""" -from typing import Optional, Protocol, runtime_checkable +from typing import Optional, Protocol, Union, runtime_checkable -from query_agent_benchmarking.internal.core.domain.models import ObjectID +from query_agent_benchmarking.internal.core.domain.models import ( + ObjectID, + SearchAgentResponse, +) + +# A search agent may return either the legacy ranked list of document IDs, or a +# richer SearchAgentResponse that also exposes the structured searches the agent +# performed. The query-execution layer normalizes both shapes. +SearchAgentResult = Union[list[ObjectID], SearchAgentResponse] @runtime_checkable class SearchAgent(Protocol): """Protocol for search agent implementations. - Implementations must support both sync and async query execution, - returning ranked lists of document IDs. + Implementations must support both sync and async query execution. They may + return either a ranked ``list[ObjectID]`` (the legacy contract) or a + ``SearchAgentResponse`` carrying both the ranked IDs and the structured + ``searches`` the agent issued under the hood. """ - def run(self, query: str, tenant: Optional[str] = None) -> list[ObjectID]: + def run(self, query: str, tenant: Optional[str] = None) -> SearchAgentResult: """Execute a synchronous search query. Args: @@ -21,11 +31,12 @@ def run(self, query: str, tenant: Optional[str] = None) -> list[ObjectID]: tenant: Optional tenant ID for multi-tenant datasets. Returns: - Ranked list of retrieved document ObjectIDs. + Ranked list of retrieved document ObjectIDs, or a + ``SearchAgentResponse`` wrapping those IDs plus per-query searches. """ ... - async def run_async(self, query: str, tenant: Optional[str] = None) -> list[ObjectID]: + async def run_async(self, query: str, tenant: Optional[str] = None) -> SearchAgentResult: """Execute an asynchronous search query. Args: @@ -33,7 +44,8 @@ async def run_async(self, query: str, tenant: Optional[str] = None) -> list[Obje tenant: Optional tenant ID for multi-tenant datasets. Returns: - Ranked list of retrieved document ObjectIDs. + Ranked list of retrieved document ObjectIDs, or a + ``SearchAgentResponse`` wrapping those IDs plus per-query searches. """ ... diff --git a/tests/adapters/test_agents.py b/tests/adapters/test_agents.py index b200848..eceb69f 100644 --- a/tests/adapters/test_agents.py +++ b/tests/adapters/test_agents.py @@ -117,11 +117,14 @@ def test_run_sync(self): mock_client.post.return_value = mock_response mock_client_cls.return_value = mock_client - results = svc.run("test query") + response = svc.run("test query") + results = response.retrieved_ids assert len(results) == 3 assert all(isinstance(r, ObjectID) for r in results) assert results[0].object_id == "doc1" + # No search plan reported by the external service. + assert response.searches is None mock_client.post.assert_called_once_with( "http://localhost:8080/search", json={"query": "test query"} ) @@ -140,9 +143,10 @@ def test_run_empty_results(self): mock_client.post.return_value = mock_response mock_client_cls.return_value = mock_client - results = svc.run("test query") + response = svc.run("test query") - assert results == [] + assert response.retrieved_ids == [] + assert response.searches is None @pytest.mark.asyncio async def test_run_async(self): @@ -159,10 +163,12 @@ async def test_run_async(self): mock_client.post = AsyncMock(return_value=mock_response) mock_client_cls.return_value = mock_client - results = await svc.run_async("async query") + response = await svc.run_async("async query") + results = response.retrieved_ids assert len(results) == 2 assert results[0].object_id == "a" + assert response.searches is None def test_conforms_to_search_agent_protocol(self): svc = ExternalSearchService(host="http://example.com") diff --git a/tests/adapters/test_json_repository.py b/tests/adapters/test_json_repository.py index 91a6c1b..ab53e9c 100644 --- a/tests/adapters/test_json_repository.py +++ b/tests/adapters/test_json_repository.py @@ -8,6 +8,7 @@ JsonFileResultRepository, ) from query_agent_benchmarking.internal.core.domain.models import ( + AgentSearch, InMemoryQuery, QueryResult, ObjectID, @@ -54,6 +55,50 @@ def test_save_trial_results(self, repo_with_tmp_dir, sample_config): assert data["metadata"]["mode"] == "search" assert len(data["queries"]) == 1 assert data["queries"][0]["question"] == "Q1" + # Agents that don't report a search plan (searches=None) omit the keys + # entirely, so the serialized shape is unchanged from before the feature. + assert "num_searches" not in data["queries"][0] + assert "searches" not in data["queries"][0] + + def test_save_trial_results_with_search_plan(self, repo_with_tmp_dir, sample_config): + repo, tmp_path = repo_with_tmp_dir + query = InMemoryQuery(question="red cars under 30k", dataset_ids=["d1"]) + results = [ + QueryResult( + query=query, + query_ground_truth_id=["d1"], + retrieved_ids=[ObjectID(object_id="d1")], + time_taken=0.5, + searches=[ + AgentSearch( + collection="FilteredCars", + query="red cars", + filters={ + "combine": "AND", + "filters": [ + { + "filter_type": "integer", + "property_name": "price", + "operator": "<", + "value": 30000, + } + ], + }, + ) + ], + ) + ] + + repo.save_trial_results(results, sample_config, trial_number=1) + + files = list(tmp_path.glob("*.json")) + data = json.loads(files[0].read_text()) + q = data["queries"][0] + assert q["num_searches"] == 1 + assert q["searches"][0]["collection"] == "FilteredCars" + assert q["searches"][0]["query"] == "red cars" + assert q["searches"][0]["filters"]["combine"] == "AND" + assert q["searches"][0]["filters"]["filters"][0]["property_name"] == "price" def test_save_trial_metrics(self, repo_with_tmp_dir, sample_config): repo, tmp_path = repo_with_tmp_dir From ad69649e4003786bd4aa8003b885b45368c24988 Mon Sep 17 00:00:00 2001 From: Connor Shorten Date: Tue, 16 Jun 2026 15:31:28 -0400 Subject: [PATCH 4/5] export comparison --- console/app/results/compare/page.tsx | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/console/app/results/compare/page.tsx b/console/app/results/compare/page.tsx index 228d05d..70ac1d8 100644 --- a/console/app/results/compare/page.tsx +++ b/console/app/results/compare/page.tsx @@ -1034,10 +1034,11 @@ function PerQueryBody({ {experiments.map((exp, i) => { const c = EXP_COLORS[i % EXP_COLORS.length]; return ( - ); })} @@ -1045,7 +1046,7 @@ function PerQueryBody({ {filteredRows.map((row, ri) => ( - + ))}
ID Question Ground Truth RetrievedPlanTime
{q.query_id}{q.question}{q.num_ground_truth} IDs - {q.num_retrieved} IDs - - {overlap} match - - {q.time_taken.toFixed(2)}s
+ {canExpand && ( + + ▶ + + )} + {q.query_id}{q.question}{q.num_ground_truth} IDs + {q.num_retrieved} IDs + + {overlap} match + + + {numSearches > 0 ? ( + + {numSearches} {numSearches === 1 ? "search" : "searches"} + + ) : ( + — + )} + {q.time_taken.toFixed(2)}s
+
+
Agent Search Plan
+ +
+
- + + {i + 1} + {exp.label || exp.agent_name}
@@ -1057,15 +1058,15 @@ function PerQueryBody({ function ComparisonRowView({ row, mode, - expCount, + experiments, }: { row: QueryComparisonRow; mode: QueryComparison["mode"]; - expCount: number; + experiments: QueryComparison["experiments"]; }) { const [open, setOpen] = useState(false); const meta = OUTCOME_META[row.outcome]; - const colSpan = 2 + expCount; + const colSpan = 2 + experiments.length; return ( @@ -1113,7 +1114,7 @@ function ComparisonRowView({ className="px-4 py-4" style={{ background: "var(--bg-card)", borderTop: "1px solid var(--border-subtle)" }} > - +
@@ -1148,7 +1149,15 @@ function CellBadge({ cell, mode }: { cell: ComparisonCell | null; mode: QueryCom } /** Expanded full-width detail comparing each experiment's prediction. */ -function RowDetail({ row, mode }: { row: QueryComparison["rows"][number]; mode: QueryComparison["mode"] }) { +function RowDetail({ + row, + mode, + experiments, +}: { + row: QueryComparison["rows"][number]; + mode: QueryComparison["mode"]; + experiments: QueryComparison["experiments"]; +}) { return (
{/* Ground truth */} @@ -1184,6 +1193,9 @@ function RowDetail({ row, mode }: { row: QueryComparison["rows"][number]; mode: {i + 1} + + {experiments[i]?.label || experiments[i]?.agent_name} + {cell ? ( ) : ( From 15f4730d300690abbf5e75d72efa6c6fd04e8095 Mon Sep 17 00:00:00 2001 From: Connor Shorten Date: Tue, 16 Jun 2026 17:58:35 -0400 Subject: [PATCH 5/5] misc fixes --- console/app/results/experiments/[id]/page.tsx | 295 +++++++++++++++++- .../internal/adapters/metrics/ir_metrics.py | 21 +- .../adapters/metrics/ir_metrics_calculator.py | 8 +- tests/adapters/test_ir_metrics.py | 43 +++ tests/domain/test_metrics_config.py | 2 +- tests/integration/test_search_e2e.py | 59 ++++ 6 files changed, 404 insertions(+), 24 deletions(-) diff --git a/console/app/results/experiments/[id]/page.tsx b/console/app/results/experiments/[id]/page.tsx index d0f9e91..fdf5e09 100644 --- a/console/app/results/experiments/[id]/page.tsx +++ b/console/app/results/experiments/[id]/page.tsx @@ -1,6 +1,7 @@ "use client"; import { useEffect, useState, useCallback } from "react"; +import type { TrialResultFile, SearchQuery, AskQuery } from "@/lib/results"; interface TrialSummary { trialNumber: number; @@ -26,8 +27,8 @@ const POLL_INTERVAL_MS = 5_000; const KEY_SCORE_KEYS = [ "avg_alignment_score", "avg_exact_match_accuracy", - "avg_recall@5", - "avg_nDCG@10", + "avg_recall_at_5", + "avg_nDCG_at_10", ]; function parseMetricValue(value: string): number | null { @@ -35,6 +36,243 @@ function parseMetricValue(value: string): number | null { return isNaN(num) ? null : num; } +/* ═══════════════════════════════════════════════════════════════════════════ + Markdown report export — single experiment + ═══════════════════════════════════════════════════════════════════════════ */ + +/** Escape pipe characters so values can't break markdown table rows. */ +function escPipe(s: string): string { + return s.replace(/\|/g, "\\|"); +} + +/** Collapse whitespace and truncate for compact display in a table cell. */ +function truncateCell(s: string, max = 100): string { + const clean = escPipe(s.replace(/\s+/g, " ").trim()); + return clean.length > max ? `${clean.slice(0, max)}…` : clean; +} + +/** Count how many retrieved IDs are relevant (present in the ground truth). */ +function relevantHits(q: SearchQuery): number { + const gt = new Set(q.ground_truth_ids); + return q.retrieved_ids.filter((id) => gt.has(id)).length; +} + +/** A representative trial whose per-query results are embedded in the report. */ +interface ReportTrial { + trialNumber: number; + data: TrialResultFile; +} + +/** + * Append a per-query results section for one trial: a compact overview table + * followed by full details for every query. Mirrors the comparison report's + * per-query layout. No-op if the trial has no queries. + */ +function appendPerQueryResults( + L: (s?: string) => void, + trial: ReportTrial, + totalTrials: number, +): void { + const mode = trial.data.metadata.mode; + const queries = trial.data.queries; + if (queries.length === 0) return; + + L(`## Per-Query Results`); + L(); + L(`_All ${queries.length} queries from Trial ${trial.trialNumber} (mode: ${mode})._`); + if (totalTrials > 1) { + L(); + L( + `_This experiment ran ${totalTrials} trials; the queries below are from a single representative trial. ` + + `Per-trial aggregate scores are in the Trials table above._`, + ); + } + L(); + + if (mode === "ask") { + const asks = queries as AskQuery[]; + const askStatus = (q: AskQuery): string => + q.is_error ? "error" : q.score === 1 ? "✓ correct" : q.score === 0 ? "✗ wrong" : "? unscored"; + + L(`### Results Overview`); + L(); + L(`| # | Status | Time | Question |`); + L(`| ---: | :--- | ---: | :--- |`); + asks.forEach((q, i) => { + const status = q.is_error ? "error" : q.score === 1 ? "✓" : q.score === 0 ? "✗" : "?"; + L(`| ${i + 1} | ${status} | ${q.time_taken.toFixed(2)}s | ${truncateCell(q.question)} |`); + }); + L(); + L(`_Status: ✓ correct · ✗ wrong · ? unscored · error._`); + L(); + + L(`### Query Details`); + L(); + asks.forEach((q, i) => { + L(`#### ${i + 1}. [${askStatus(q)}] \`${escPipe(q.query_id)}\``); + L(); + const meta: string[] = []; + if (q.question_type) meta.push(`type: ${q.question_type}`); + if (q.tenant_id) meta.push(`tenant: ${q.tenant_id}`); + meta.push(`${q.time_taken.toFixed(2)}s`); + L(`_${meta.join(" · ")}_`); + L(); + L(`- **Question:** ${q.question}`); + L(`- **Ground truth:** ${q.ground_truth_answer || "_(empty)_"}`); + L(`- **System answer:** ${q.system_answer || "_(empty)_"}`); + if (q.judge_reasoning) L(`- **Judge:** ${q.judge_reasoning}`); + L(); + }); + } else { + const searches = queries as SearchQuery[]; + + L(`### Results Overview`); + L(); + L(`| # | Hits | Retrieved | Time | Question |`); + L(`| ---: | ---: | ---: | ---: | :--- |`); + searches.forEach((q, i) => { + const hits = relevantHits(q); + L( + `| ${i + 1} | ${hits}/${q.num_ground_truth} | ${q.num_retrieved} | ${q.time_taken.toFixed(2)}s | ${truncateCell(q.question)} |`, + ); + }); + L(); + L(`_Hits = relevant documents retrieved of total ground-truth relevant._`); + L(); + + L(`### Query Details`); + L(); + searches.forEach((q, i) => { + const hits = relevantHits(q); + const status = hits > 0 ? "✓ hit" : "✗ miss"; + L(`#### ${i + 1}. [${status}] \`${escPipe(q.query_id)}\``); + L(); + L(`- **Question:** ${q.question}`); + L(`- **Ground truth IDs (${q.num_ground_truth}):** ${q.ground_truth_ids.join(", ") || "—"}`); + L( + `- **Retrieved (${q.num_retrieved}) · ${hits} relevant · ${q.time_taken.toFixed(2)}s:**`, + ); + const gt = new Set(q.ground_truth_ids); + const shown = q.retrieved_ids.slice(0, 20).map((id) => (gt.has(id) ? `**${id}**` : id)); + const more = q.retrieved_ids.length > 20 ? ` … +${q.retrieved_ids.length - 20} more` : ""; + L(` - Retrieved: ${shown.join(", ") || "—"}${more}`); + const plan = q.searches ?? []; + if (plan.length > 0) { + const planStr = plan + .map((s) => { + const qq = s.query ? `"${s.query}"` : s.uuid_value ? `uuid:${s.uuid_value}` : "—"; + const filt = s.filters ? " +filter" : ""; + const sort = s.sort_property ? ` +sort(${s.sort_property.property_name})` : ""; + return `${s.collection}:${qq}${filt}${sort}`; + }) + .join("; "); + L(` - Search plan (${plan.length}): ${planStr}`); + } + L(); + }); + } +} + +/** + * Build a detailed markdown report for a single experiment. When a representative + * trial is provided, its per-query results are embedded. + */ +function buildExperimentReport( + exp: ExperimentDetail, + trial: ReportTrial | null, + generatedAt: string, +): string { + const lines: string[] = []; + const L = (s = "") => lines.push(s); + + L(`# Experiment Report: ${exp.dataset}`); + L(); + L(`_Generated ${generatedAt}_`); + L(); + + // ── Overview ─────────────────────────────────────────────────────────────── + L(`## Overview`); + L(); + L(`- **Dataset:** ${exp.dataset}`); + L(`- **Agent:** \`${exp.agent_name}\``); + L(`- **Mode:** ${exp.mode}`); + L(`- **Trials:** ${exp.num_trials}`); + L(`- **Timestamp:** ${exp.timestamp ? new Date(exp.timestamp).toLocaleString() : "—"}`); + L(`- **ID:** \`${decodeURIComponent(exp.id)}\``); + L(); + + // ── Aggregated metrics ─────────────────────────────────────────────────────── + L(`## Aggregated Metrics`); + L(); + if (exp.metricEntries.length === 0) { + L(`_No aggregated metrics available._`); + L(); + } else { + L(`| Metric | Value |`); + L(`| :--- | ---: |`); + for (const { key, value } of exp.metricEntries) { + L(`| ${escPipe(key.replace(/_/g, " "))} | ${escPipe(value)} |`); + } + L(); + } + + // ── Per-trial summary ───────────────────────────────────────────────────────── + L(`## Trials`); + L(); + if (exp.trials.length === 0) { + L(`_No trial data available._`); + L(); + } else { + L(`| Trial | Queries | Avg Query Time | Key Score |`); + L(`| :--- | ---: | ---: | ---: |`); + for (const trial of exp.trials) { + const totalQueries = trial.totalQueries ?? "—"; + const avgTime = trial.avgQueryTime != null ? `${trial.avgQueryTime.toFixed(2)}s` : "—"; + + let keyScore = "—"; + if (trial.metrics) { + for (const k of KEY_SCORE_KEYS) { + if (typeof trial.metrics[k] === "number") { + keyScore = `${((trial.metrics[k] as number) * 100).toFixed(1)}%`; + break; + } + } + } + L(`| Trial ${trial.trialNumber} | ${totalQueries} | ${avgTime} | ${keyScore} |`); + } + L(); + } + + // ── Per-query results (representative trial) ─────────────────────────────── + if (trial) { + appendPerQueryResults(L, trial, exp.num_trials); + } + + L(`---`); + L(`_Report generated by the Query Agent Benchmarking console._`); + L(); + + return lines.join("\n"); +} + +/** Trigger a client-side download of a markdown file. */ +function downloadMarkdown(content: string, filename: string): void { + const blob = new Blob([content], { type: "text/markdown;charset=utf-8" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); +} + +/** Slugify a string for use in a filename. */ +function slugify(s: string): string { + return s.replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").toLowerCase() || "experiment"; +} + export default function ExperimentDetailPage({ params: paramsPromise, }: { @@ -43,6 +281,38 @@ export default function ExperimentDetailPage({ const [id, setId] = useState(null); const [experiment, setExperiment] = useState(null); const [notFound, setNotFound] = useState(false); + const [exporting, setExporting] = useState(false); + + const handleExport = async () => { + if (!experiment || exporting) return; + setExporting(true); + try { + // Fetch the first trial with results so the report can embed per-query + // detail. Best-effort — if it fails we still export the metrics + summary. + let trial: ReportTrial | null = null; + const firstWithResults = experiment.trials.find((t) => t.hasResults); + if (firstWithResults) { + try { + const r = await fetch( + `/api/trial?id=${experiment.id}&trial=${firstWithResults.trialNumber}`, + ); + if (r.ok) { + const data = (await r.json()) as TrialResultFile; + trial = { trialNumber: firstWithResults.trialNumber, data }; + } + } catch { + // per-query detail is best-effort + } + } + + const now = new Date(); + const md = buildExperimentReport(experiment, trial, now.toLocaleString()); + const stamp = now.toISOString().slice(0, 10); + downloadMarkdown(md, `experiment-${slugify(experiment.dataset)}-${stamp}.md`); + } finally { + setExporting(false); + } + }; useEffect(() => { paramsPromise.then((p) => setId(p.id)); @@ -113,11 +383,22 @@ export default function ExperimentDetailPage({ {experiment.num_trials} trial{experiment.num_trials !== 1 ? "s" : ""}
- {experiment.timestamp && ( - - {new Date(experiment.timestamp).toLocaleString()} - - )} +
+ + {experiment.timestamp && ( + + {new Date(experiment.timestamp).toLocaleString()} + + )} +
diff --git a/query_agent_benchmarking/internal/adapters/metrics/ir_metrics.py b/query_agent_benchmarking/internal/adapters/metrics/ir_metrics.py index 939ac1a..541c072 100644 --- a/query_agent_benchmarking/internal/adapters/metrics/ir_metrics.py +++ b/query_agent_benchmarking/internal/adapters/metrics/ir_metrics.py @@ -51,35 +51,32 @@ def calculate_precision_at_k( """Calculate precision@k for retrieved documents. Precision@k is the proportion of the top-k retrieved documents that are - relevant. For filtered retrieval (where fewer than k documents may be - returned) the denominator is the number of documents actually retrieved - in the top-k, so returning extra irrelevant documents is penalized while - returning fewer relevant-only documents is not. + relevant, using the standard IR convention with a constant denominator of + ``k`` (never ``min(k, retrieved)`` and never the retrieved count). This + keeps the metric comparable across queries with different retrieved counts + and yields a well-defined ``0/k = 0`` for zero-retrieval queries. Args: target_ids: List of target document IDs (ground truth). retrieved_ids: List of retrieved document IDs. - k: The number of top results to consider. + k: The number of top results to consider (the denominator). Returns: - float: Precision@k score (0.0 to 1.0) - proportion of the retrieved - top-k documents that are relevant. + float: Precision@k score (0.0 to 1.0) - number of relevant documents + in the top-k retrieved divided by k. """ target_id_set = {str(id) for id in target_ids} retrieved_ids = [str(id) for id in retrieved_ids] if retrieved_ids else [] retrieved_ids_at_k = retrieved_ids[:k] - if not retrieved_ids_at_k: - return 0.0 - found_count = sum(1 for retrieved_id in retrieved_ids_at_k if retrieved_id in target_id_set) - precision = found_count / len(retrieved_ids_at_k) + precision = found_count / k if verbose: print(f"\033[96mTarget IDs: {target_id_set}\033[0m") print(f"\033[92mRetrieved IDs @{k}: {retrieved_ids_at_k}\033[0m") - print(f"\033[96mPrecision@{k}: {found_count}/{len(retrieved_ids_at_k)} = {precision:.2f}\033[0m") + print(f"\033[96mPrecision@{k}: {found_count}/{k} = {precision:.2f}\033[0m") return precision diff --git a/query_agent_benchmarking/internal/adapters/metrics/ir_metrics_calculator.py b/query_agent_benchmarking/internal/adapters/metrics/ir_metrics_calculator.py index caa0bf7..9a87a49 100644 --- a/query_agent_benchmarking/internal/adapters/metrics/ir_metrics_calculator.py +++ b/query_agent_benchmarking/internal/adapters/metrics/ir_metrics_calculator.py @@ -72,10 +72,10 @@ def compute( for i, (result, ground_truth) in enumerate( tqdm(zip(results, ground_truths), desc="Analyzing search results") ): - if result.retrieved_ids == []: - print(f"\n\033[91mSkipping analysis for query {i} due to error.\033[0m") - continue - + # A zero-retrieval query is a real, meaningful outcome (precision + # mode may legitimately return nothing). It must be scored, not + # skipped: every metric falls out to 0 naturally from the arithmetic + # below, and the query stays in the aggregate denominator. retrieved_ids = [res.object_id for res in result.retrieved_ids] for spec in self.metric_specs: diff --git a/tests/adapters/test_ir_metrics.py b/tests/adapters/test_ir_metrics.py index 6dabaef..3c4b396 100644 --- a/tests/adapters/test_ir_metrics.py +++ b/tests/adapters/test_ir_metrics.py @@ -4,6 +4,7 @@ from query_agent_benchmarking.internal.adapters.metrics.ir_metrics import ( calculate_recall_at_k, + calculate_precision_at_k, calculate_success_at_k, calculate_nDCG_at_k, calculate_coverage, @@ -65,6 +66,48 @@ def test_string_coercion(self): assert calculate_recall_at_k(target_ids, retrieved_ids, k=3) == pytest.approx(2 / 3) +# ============================================================================ +# Precision@K Tests +# ============================================================================ + +class TestPrecisionAtK: + def test_perfect_precision(self): + target_ids = ["doc1", "doc2", "doc3"] + retrieved_ids = ["doc1", "doc2", "doc3"] + assert calculate_precision_at_k(target_ids, retrieved_ids, k=3) == 1.0 + + def test_constant_k_denominator_under_retrieval(self): + """precision@k uses k as the denominator, not the retrieved count. + + 6 retrieved, all relevant, at k=20 must be 6/20 = 0.30 -- not 1.0. + """ + target_ids = [f"doc{i}" for i in range(6)] + retrieved_ids = [f"doc{i}" for i in range(6)] + assert calculate_precision_at_k(target_ids, retrieved_ids, k=20) == pytest.approx(0.30) + + def test_partial_precision(self): + target_ids = ["doc1", "doc2"] + retrieved_ids = ["doc1", "doc3", "doc2", "doc4", "doc5"] + # 2 relevant out of k=5 + assert calculate_precision_at_k(target_ids, retrieved_ids, k=5) == pytest.approx(0.4) + + def test_empty_retrieved_scores_zero(self): + """Zero-retrieval is 0/k = 0 for every k; no 0/0 case.""" + target_ids = ["doc1", "doc2"] + assert calculate_precision_at_k(target_ids, [], k=1) == 0.0 + assert calculate_precision_at_k(target_ids, [], k=20) == 0.0 + + def test_precision_at_1(self): + target_ids = ["doc1"] + assert calculate_precision_at_k(target_ids, ["doc1"], k=1) == 1.0 + assert calculate_precision_at_k(target_ids, ["doc2"], k=1) == 0.0 + + def test_string_coercion(self): + target_ids = [1, 2] + retrieved_ids = ["1", "3"] + assert calculate_precision_at_k(target_ids, retrieved_ids, k=2) == pytest.approx(0.5) + + # ============================================================================ # Success@K Tests # ============================================================================ diff --git a/tests/domain/test_metrics_config.py b/tests/domain/test_metrics_config.py index 0435f14..8409d73 100644 --- a/tests/domain/test_metrics_config.py +++ b/tests/domain/test_metrics_config.py @@ -109,7 +109,7 @@ def test_all_profiles_have_metrics(self): assert len(profile.metrics) > 0, f"Empty metrics for {profile.dataset_pattern}" def test_all_metric_names_are_known(self): - known_names = {"recall", "success", "nDCG", "coverage", "alpha_ndcg"} + known_names = {"recall", "precision", "success", "nDCG", "coverage", "alpha_ndcg"} for profile in DATASET_METRICS_REGISTRY: for spec in profile.metrics: assert spec.name in known_names, f"Unknown metric {spec.name} in {profile.dataset_pattern}" diff --git a/tests/integration/test_search_e2e.py b/tests/integration/test_search_e2e.py index 6532821..94fe269 100644 --- a/tests/integration/test_search_e2e.py +++ b/tests/integration/test_search_e2e.py @@ -174,3 +174,62 @@ def test_different_dataset_metrics(self, sample_queries): enron_metrics = enron_calc.compute(results, sample_queries) assert "avg_nDCG_at_10" not in enron_metrics assert "avg_recall_at_1" in enron_metrics + + +class TestZeroAndUnderRetrievalScoring: + """Regression tests: zero-retrieval queries are scored (not skipped) and + precision@k uses a constant-k denominator (filtered-cars precision mode). + """ + + @staticmethod + def _query(ground_truth_ids): + return InMemoryQuery(question="q", dataset_ids=ground_truth_ids) + + @staticmethod + def _result(query, retrieved_ids): + return QueryResult( + query=query, + query_ground_truth_id=query.dataset_ids, + retrieved_ids=[ObjectID(object_id=i) for i in retrieved_ids], + time_taken=0.1, + ) + + def test_zero_retrieval_is_scored_and_counted(self): + """A zero-retrieval query scores 0 at every metric and stays in the + aggregate denominator -- it pulls the averages below 1.0.""" + # Two queries: one perfect, one retrieving nothing. + queries = [self._query(["doc1"]), self._query(["doc2"])] + results = [ + self._result(queries[0], ["doc1"]), # perfect + self._result(queries[1], []), # zero retrieval + ] + + calculator = IRMetricsCalculator(dataset_name="filtered-cars") + metrics = calculator.compute(results, queries) + + # Per-query scores: 2 entries (denominator = query count = 2), + # the zero-retrieval query contributes 0. + for key in ("precision_at_1", "precision_at_5", "precision_at_20", + "recall_at_1", "recall_at_5", "recall_at_20"): + scores = metrics[f"{key}_scores"] + assert len(scores) == 2, key + assert scores[1] == 0.0, key + + # Aggregates are the simple mean over both queries -> 0.5, never 1.0. + assert metrics["avg_precision_at_1"] == pytest.approx(0.5) + assert metrics["avg_recall_at_1"] == pytest.approx(0.5) + + def test_precision_uses_constant_k_denominator(self): + """A query retrieving 6 relevant docs scores precision@20 = 6/20 = 0.30, + not 1.0; recall@5 cannot reach 1.0 when 6 ground-truth docs exist.""" + ground_truth = [f"doc{i}" for i in range(6)] + query = self._query(ground_truth) + result = self._result(query, ground_truth) # 6 retrieved, all relevant + + calculator = IRMetricsCalculator(dataset_name="filtered-cars") + metrics = calculator.compute([result], [query]) + + assert metrics["precision_at_20_scores"][0] == pytest.approx(0.30) + assert metrics["precision_at_5_scores"][0] == pytest.approx(1.0) # 5/5 + # Top 5 cannot hold all 6 relevant docs. + assert metrics["recall_at_5_scores"][0] < 1.0