diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports.rs index e9807287..94aca5ea 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports.rs @@ -4,7 +4,6 @@ mod job_report_domain_metrics; mod job_report_evolution; mod job_report_misc; mod job_report_scoring; -mod local_organizer_reports; pub(super) use self::{ consolidation_reports::{ @@ -21,5 +20,5 @@ pub(super) use self::{ job_report_evolution::{EvolutionJobReport, EvolutionSummary}, job_report_misc::{FollowUpReport, PrivateCorpusRedaction}, job_report_scoring::{FailureCounts, JobMetrics, JobScoring, ScoreboardRankedMetrics}, - local_organizer_reports::LocalOrganizerJobReport, }; +pub(super) use crate::LocalOrganizerJobReport; diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/local_organizer_reports.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/local_organizer_reports.rs deleted file mode 100644 index bee40f09..00000000 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/local_organizer_reports.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) use crate::LocalOrganizerJobReport; diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking.rs index e9d7dbf7..86c4949f 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking.rs @@ -1,14 +1,21 @@ -mod average_precision; -mod ndcg; -mod reciprocal_rank; - -use crate::BTreeMap; +use crate::{BTreeMap, BTreeSet}; pub(super) fn reciprocal_rank( candidates: &[String], relevance: &BTreeMap, ) -> Option { - reciprocal_rank::reciprocal_rank(candidates, relevance) + if super::positive_qrel_count(relevance) == 0 { + return None; + } + + Some( + candidates + .iter() + .position(|candidate| { + relevance.get(candidate.as_str()).is_some_and(|grade| *grade > 0.0) + }) + .map_or(0.0, |index| 1.0 / (index + 1) as f64), + ) } pub(super) fn ndcg_at_k( @@ -16,12 +23,56 @@ pub(super) fn ndcg_at_k( relevance: &BTreeMap, k: usize, ) -> Option { - ndcg::ndcg_at_k(candidates, relevance, k) + if super::positive_qrel_count(relevance) == 0 { + return None; + } + + let dcg = candidates + .iter() + .take(k) + .enumerate() + .map(|(index, candidate)| { + relevance.get(candidate.as_str()).copied().unwrap_or(0.0).max(0.0) + / ((index + 2) as f64).log2() + }) + .sum::(); + let mut ideal = relevance.values().copied().filter(|grade| *grade > 0.0).collect::>(); + + ideal.sort_by(|left, right| right.total_cmp(left)); + + let idcg = ideal + .iter() + .take(k) + .enumerate() + .map(|(index, grade)| grade / ((index + 2) as f64).log2()) + .sum::(); + + Some(if idcg > 0.0 { dcg / idcg } else { 0.0 }) } pub(super) fn average_precision( candidates: &[String], relevance: &BTreeMap, ) -> Option { - average_precision::average_precision(candidates, relevance) + let positive_count = super::positive_qrel_count(relevance); + + if positive_count == 0 { + return None; + } + + let mut hit_count = 0; + let mut precision_sum = 0.0; + let mut seen = BTreeSet::new(); + + for (index, candidate) in candidates.iter().enumerate() { + if !seen.insert(candidate.as_str()) { + continue; + } + if relevance.get(candidate.as_str()).is_some_and(|grade| *grade > 0.0) { + hit_count += 1; + precision_sum += hit_count as f64 / (index + 1) as f64; + } + } + + Some(precision_sum / positive_count as f64) } diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking/average_precision.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking/average_precision.rs deleted file mode 100644 index 13c196ca..00000000 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking/average_precision.rs +++ /dev/null @@ -1,28 +0,0 @@ -use crate::{BTreeMap, BTreeSet, quantitative::metrics::per_query::query_metrics}; - -pub(super) fn average_precision( - candidates: &[String], - relevance: &BTreeMap, -) -> Option { - let positive_count = query_metrics::positive_qrel_count(relevance); - - if positive_count == 0 { - return None; - } - - let mut hit_count = 0; - let mut precision_sum = 0.0; - let mut seen = BTreeSet::new(); - - for (index, candidate) in candidates.iter().enumerate() { - if !seen.insert(candidate.as_str()) { - continue; - } - if relevance.get(candidate.as_str()).is_some_and(|grade| *grade > 0.0) { - hit_count += 1; - precision_sum += hit_count as f64 / (index + 1) as f64; - } - } - - Some(precision_sum / positive_count as f64) -} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking/ndcg.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking/ndcg.rs deleted file mode 100644 index 540d2f66..00000000 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking/ndcg.rs +++ /dev/null @@ -1,33 +0,0 @@ -use crate::{BTreeMap, quantitative::metrics::per_query::query_metrics}; - -pub(super) fn ndcg_at_k( - candidates: &[String], - relevance: &BTreeMap, - k: usize, -) -> Option { - if query_metrics::positive_qrel_count(relevance) == 0 { - return None; - } - - let dcg = candidates - .iter() - .take(k) - .enumerate() - .map(|(index, candidate)| { - relevance.get(candidate.as_str()).copied().unwrap_or(0.0).max(0.0) - / ((index + 2) as f64).log2() - }) - .sum::(); - let mut ideal = relevance.values().copied().filter(|grade| *grade > 0.0).collect::>(); - - ideal.sort_by(|left, right| right.total_cmp(left)); - - let idcg = ideal - .iter() - .take(k) - .enumerate() - .map(|(index, grade)| grade / ((index + 2) as f64).log2()) - .sum::(); - - Some(if idcg > 0.0 { dcg / idcg } else { 0.0 }) -} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking/reciprocal_rank.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking/reciprocal_rank.rs deleted file mode 100644 index 99956367..00000000 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/metrics/per_query/query_metrics/ranking/reciprocal_rank.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::{BTreeMap, quantitative::metrics::per_query::query_metrics}; - -pub(super) fn reciprocal_rank( - candidates: &[String], - relevance: &BTreeMap, -) -> Option { - if query_metrics::positive_qrel_count(relevance) == 0 { - return None; - } - - Some( - candidates - .iter() - .position(|candidate| { - relevance.get(candidate.as_str()).is_some_and(|grade| *grade > 0.0) - }) - .map_or(0.0, |index| 1.0 / (index + 1) as f64), - ) -} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/product_manifest/export/rows.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/product_manifest/export/rows.rs index e29f4f74..2e1923db 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/product_manifest/export/rows.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/product_manifest/export/rows.rs @@ -1,4 +1,55 @@ -mod per_query; -mod product; +use crate::{QuantitativeBenchmarkRow, QuantitativePerQueryRow, RealWorldReport}; -pub(super) use self::{per_query::exported_per_query_rows, product::exported_product_row}; +pub(super) fn exported_product_row( + source_row: &QuantitativeBenchmarkRow, + product: &str, + adapter_id: &str, + adapter_name: &str, +) -> QuantitativeBenchmarkRow { + let mut row = source_row.clone(); + + row.product = product.to_string(); + row.adapter_id = adapter_id.to_string(); + row.adapter_name = adapter_name.to_string(); + row.claim_boundary = concat!( + "Exported from a generated real_world_job_report quantitative row; ", + "import remains subject to same-corpus, per-query, explicit-qrel, and leaderboard gates." + ) + .to_string(); + + row +} + +pub(super) fn exported_per_query_rows( + report: &RealWorldReport, + source_product: &str, + source_adapter_id: &str, + product: &str, + adapter_id: &str, +) -> Vec { + report + .quantitative_scoreboard + .per_query_rows + .iter() + .filter(|row| row.product == source_product && row.adapter_id == source_adapter_id) + .map(|row| exported_per_query_row(row, product, adapter_id)) + .collect() +} + +fn exported_per_query_row( + source_row: &QuantitativePerQueryRow, + product: &str, + adapter_id: &str, +) -> QuantitativePerQueryRow { + let mut row = source_row.clone(); + + row.product = product.to_string(); + row.adapter_id = adapter_id.to_string(); + row.claim_boundary = concat!( + "Exported from generated report per-query quantitative evidence; ", + "import does not relax paired-significance or leaderboard gates." + ) + .to_string(); + + row +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/product_manifest/export/rows/per_query.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/product_manifest/export/rows/per_query.rs deleted file mode 100644 index fcc61d9e..00000000 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/product_manifest/export/rows/per_query.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::{QuantitativePerQueryRow, RealWorldReport}; - -pub(in crate::quantitative::product_manifest::export) fn exported_per_query_rows( - report: &RealWorldReport, - source_product: &str, - source_adapter_id: &str, - product: &str, - adapter_id: &str, -) -> Vec { - report - .quantitative_scoreboard - .per_query_rows - .iter() - .filter(|row| row.product == source_product && row.adapter_id == source_adapter_id) - .map(|row| exported_per_query_row(row, product, adapter_id)) - .collect() -} - -fn exported_per_query_row( - source_row: &QuantitativePerQueryRow, - product: &str, - adapter_id: &str, -) -> QuantitativePerQueryRow { - let mut row = source_row.clone(); - - row.product = product.to_string(); - row.adapter_id = adapter_id.to_string(); - row.claim_boundary = concat!( - "Exported from generated report per-query quantitative evidence; ", - "import does not relax paired-significance or leaderboard gates." - ) - .to_string(); - - row -} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/product_manifest/export/rows/product.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/product_manifest/export/rows/product.rs deleted file mode 100644 index 2551c2ff..00000000 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/quantitative/product_manifest/export/rows/product.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::QuantitativeBenchmarkRow; - -pub(in crate::quantitative::product_manifest::export) fn exported_product_row( - source_row: &QuantitativeBenchmarkRow, - product: &str, - adapter_id: &str, - adapter_name: &str, -) -> QuantitativeBenchmarkRow { - let mut row = source_row.clone(); - - row.product = product.to_string(); - row.adapter_id = adapter_id.to_string(); - row.adapter_name = adapter_name.to_string(); - row.claim_boundary = concat!( - "Exported from a generated real_world_job_report quantitative row; ", - "import remains subject to same-corpus, per-query, explicit-qrel, and leaderboard gates." - ) - .to_string(); - - row -}