From a14e75a737e08a2871ccc778e3799378460636f1 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Tue, 30 Jun 2026 14:43:36 -0400 Subject: [PATCH] {"schema":"decodex/commit/1","summary":"Split real-world benchmark report tests into modules","authority":"manual"} --- .../competitor_retest.rs | 249 ++++ .../dreaming_reports.rs | 1074 +---------------- .../external_adapters.rs | 798 +----------- .../real_world_job_benchmark/graph_rag.rs | 363 ++++++ .../real_world_job_benchmark/letta_core.rs | 105 ++ .../real_world_job_benchmark/loss_summary.rs | 97 ++ .../manifest_summary.rs | 245 ++++ .../openviking_trajectory.rs | 193 +++ .../qmd_debug_retest.rs | 180 +++ .../real_world_job_benchmark/review_queue.rs | 103 ++ .../service_native.rs | 180 +++ .../temporal_reconciliation.rs | 95 ++ .../real_world_job_benchmark/validation.rs | 89 ++ 13 files changed, 1926 insertions(+), 1845 deletions(-) create mode 100644 apps/elf-eval/tests/real_world_job_benchmark/competitor_retest.rs create mode 100644 apps/elf-eval/tests/real_world_job_benchmark/graph_rag.rs create mode 100644 apps/elf-eval/tests/real_world_job_benchmark/letta_core.rs create mode 100644 apps/elf-eval/tests/real_world_job_benchmark/loss_summary.rs create mode 100644 apps/elf-eval/tests/real_world_job_benchmark/manifest_summary.rs create mode 100644 apps/elf-eval/tests/real_world_job_benchmark/openviking_trajectory.rs create mode 100644 apps/elf-eval/tests/real_world_job_benchmark/qmd_debug_retest.rs create mode 100644 apps/elf-eval/tests/real_world_job_benchmark/review_queue.rs create mode 100644 apps/elf-eval/tests/real_world_job_benchmark/service_native.rs create mode 100644 apps/elf-eval/tests/real_world_job_benchmark/temporal_reconciliation.rs create mode 100644 apps/elf-eval/tests/real_world_job_benchmark/validation.rs diff --git a/apps/elf-eval/tests/real_world_job_benchmark/competitor_retest.rs b/apps/elf-eval/tests/real_world_job_benchmark/competitor_retest.rs new file mode 100644 index 00000000..8239aefd --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/competitor_retest.rs @@ -0,0 +1,249 @@ +use std::fs; + +use color_eyre::Result; +use serde_json::Value; + +use crate::support; + +#[test] +fn dreaming_competitor_strength_retest_report_closes_xy955_without_overclaims() -> Result<()> { + let report = serde_json::from_str::(&fs::read_to_string( + support::dreaming_competitor_strength_retest_report_json_path()?, + )?)?; + let markdown = + fs::read_to_string(support::dreaming_competitor_strength_retest_report_markdown_path()?)?; + let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; + let readme = fs::read_to_string(support::readme_path()?)?; + + assert_eq!( + report.pointer("/schema").and_then(Value::as_str), + Some("elf.dreaming_competitor_strength_retest_report/v1") + ); + assert_eq!(report.pointer("/authority").and_then(Value::as_str), Some("XY-955")); + assert_eq!( + report.pointer("/summary/overall_judgment").and_then(Value::as_str), + Some("locally_and_partially_stronger_only") + ); + assert_eq!( + report.pointer("/summary/broader_superiority").and_then(Value::as_str), + Some("not_proven") + ); + assert_eq!(report.pointer("/summary/regressed_stage_count").and_then(Value::as_u64), Some(0)); + assert!(support::array_contains_str(&report, "/status_terms", "typed_non_pass")?); + assert!(support::array_contains_str( + &report, + "/summary/unsupported_claims_rejected", + "ELF does not broadly beat qmd from this retest." + )?); + + assert_xy955_commands(&report)?; + assert_xy955_stage_closeout(&report)?; + assert_xy955_scenario_retests(&report)?; + assert_xy955_optimization_queue(&report)?; + assert_xy955_follow_up_issue_briefs(&report)?; + + assert!(markdown.contains("ELF is locally and partially stronger")); + assert!( + markdown.contains("The full live-adapter command now has fresh ELF and qmd scored reports") + ); + assert!( + markdown.contains( + "Do not treat qmd full-suite wrong_result counts as a regression of qmd debug" + ) + ); + assert!(markdown.contains("## Follow-Up Issue Briefs")); + assert!(markdown.contains( + "| GraphRAG/LightRAG/RAGFlow/llm-wiki/gbrain/graphify citation/navigation/knowledge surfaces |" + )); + assert!( + benchmarking_index.contains("2026-06-17-dreaming-competitor-strength-retest-report.md") + ); + assert!(readme.contains("Dreaming Competitor-Strength Retest Report - June 17, 2026")); + assert!(readme.contains("17 competitor-strength closeout")); + + Ok(()) +} + +fn assert_xy955_commands(report: &Value) -> Result<()> { + let commands = support::array_at(report, "/commands")?; + let aggregate = support::find_by_field(commands, "/command", "cargo make real-world-memory")?; + let graph_rag = + support::find_by_field(commands, "/command", "cargo make real-world-memory-graph-rag")?; + let first_generation = + support::find_by_field(commands, "/command", "cargo make real-world-first-generation-oss")?; + let live = + support::find_by_field(commands, "/command", "cargo make real-world-memory-live-adapters")?; + + assert_eq!(aggregate.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(aggregate.pointer("/summary/pass").and_then(Value::as_u64), Some(53)); + assert_eq!(aggregate.pointer("/summary/blocked").and_then(Value::as_u64), Some(7)); + assert_eq!(graph_rag.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(graph_rag.pointer("/summary/wrong_result").and_then(Value::as_u64), Some(1)); + assert_eq!(graph_rag.pointer("/summary/incomplete").and_then(Value::as_u64), Some(1)); + assert_eq!(graph_rag.pointer("/summary/blocked").and_then(Value::as_u64), Some(3)); + assert_eq!(first_generation.pointer("/summary/pass").and_then(Value::as_u64), Some(4)); + assert_eq!(first_generation.pointer("/summary/blocked").and_then(Value::as_u64), Some(2)); + assert_eq!(live.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!( + live.pointer("/partial_summary/elf_live_real_world/pass").and_then(Value::as_u64), + Some(40) + ); + assert_eq!( + live.pointer("/partial_summary/elf_live_real_world/wrong_result").and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + live.pointer("/partial_summary/qmd_live_real_world/pass").and_then(Value::as_u64), + Some(17) + ); + assert_eq!( + live.pointer("/partial_summary/qmd_live_real_world/wrong_result").and_then(Value::as_u64), + Some(13) + ); + + Ok(()) +} + +fn assert_xy955_stage_closeout(report: &Value) -> Result<()> { + let stages = support::array_at(report, "/stage_closeout")?; + + assert_eq!(stages.len(), 8); + + let current = support::find_by_field(stages, "/stage_id", "current_vs_historical_correctness")?; + let proactive = support::find_by_field(stages, "/stage_id", "proactive_brief_readiness")?; + let scheduled = support::find_by_field(stages, "/stage_id", "scheduled_memory_task_readiness")?; + let final_retest = + support::find_by_field(stages, "/stage_id", "final_competitor_retest_status")?; + + assert_eq!(current.pointer("/judgment").and_then(Value::as_str), Some("improved")); + assert_eq!(current.pointer("/current_counts/pass").and_then(Value::as_u64), Some(6)); + assert_eq!(current.pointer("/current_counts/wrong_result").and_then(Value::as_u64), Some(0)); + assert_eq!(proactive.pointer("/judgment").and_then(Value::as_str), Some("improved")); + assert_eq!(proactive.pointer("/current_counts/blocked").and_then(Value::as_u64), Some(1)); + assert_eq!(scheduled.pointer("/current_counts/pass").and_then(Value::as_u64), Some(4)); + assert_eq!(scheduled.pointer("/current_counts/blocked").and_then(Value::as_u64), Some(1)); + assert_eq!(final_retest.pointer("/judgment").and_then(Value::as_str), Some("unchanged")); + assert_eq!(final_retest.pointer("/current_counts/pass").and_then(Value::as_u64), Some(40)); + assert_eq!( + final_retest.pointer("/current_counts/wrong_result").and_then(Value::as_u64), + Some(0) + ); + assert_eq!(final_retest.pointer("/current_counts/blocked").and_then(Value::as_u64), Some(7)); + assert_eq!( + final_retest.pointer("/current_counts/not_encoded").and_then(Value::as_u64), + Some(19) + ); + assert!(final_retest.pointer("/boundary").and_then(Value::as_str).is_some_and(|boundary| { + boundary.contains("qmd now has a fresh scored live report") + && boundary.contains("broader superiority is not proven") + })); + assert_eq!(final_retest.pointer("/qmd_current_counts/pass").and_then(Value::as_u64), Some(17)); + assert_eq!( + final_retest.pointer("/qmd_current_counts/wrong_result").and_then(Value::as_u64), + Some(13) + ); + + Ok(()) +} + +fn assert_xy955_scenario_retests(report: &Value) -> Result<()> { + let scenarios = support::array_at(report, "/scenario_retests")?; + let qmd = support::find_by_field(scenarios, "/scenario_id", "qmd_debug_ergonomics")?; + let mem0 = support::find_by_field( + scenarios, + "/scenario_id", + "mem0_openmemory_preference_history_export", + )?; + let letta = support::find_by_field(scenarios, "/scenario_id", "letta_core_archive")?; + let graph_rag = support::find_by_field( + scenarios, + "/scenario_id", + "graph_rag_citation_navigation_knowledge_surfaces", + )?; + let private_provider = + support::find_by_field(scenarios, "/scenario_id", "private_provider_production_gates")?; + + assert_eq!(qmd.pointer("/current_outcome").and_then(Value::as_str), Some("unchanged")); + assert_eq!(qmd.pointer("/current_status").and_then(Value::as_str), Some("pass")); + assert!(qmd.pointer("/evidence").and_then(Value::as_str).is_some_and(|evidence| { + evidence.contains("17 pass") + && evidence.contains("13 wrong_result") + && evidence.contains("does not retest or erase") + })); + assert_eq!(mem0.pointer("/current_outcome").and_then(Value::as_str), Some("unchanged")); + assert!(mem0.pointer("/evidence").and_then(Value::as_str).is_some_and(|evidence| { + evidence.contains("mem0/OpenMemory local OSS history") + && evidence.contains("OpenMemory UI/export remains setup-blocked") + })); + assert_eq!(letta.pointer("/current_status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + graph_rag.pointer("/current_status").and_then(Value::as_str), + Some("typed_non_pass") + ); + assert!(graph_rag.pointer("/evidence").and_then(Value::as_str).is_some_and(|evidence| { + evidence.contains("0 pass") + && evidence.contains("1 wrong_result") + && evidence.contains("3 blocked") + })); + assert_eq!(private_provider.pointer("/follow_up").and_then(Value::as_str), Some("XY-930")); + + Ok(()) +} + +fn assert_xy955_optimization_queue(report: &Value) -> Result<()> { + let queue = support::array_at(report, "/optimization_queue")?; + let qmd = support::find_by_field(queue, "/issue", "XY-923")?; + let private_provider = support::find_by_field(queue, "/issue", "XY-930")?; + let openviking = support::find_by_field(queue, "/issue", "XY-928")?; + let letta = support::find_by_field(queue, "/issue", "letta-core-archive-adapter-brief")?; + let service_native = + support::find_by_field(queue, "/issue", "service-native-dreaming-outputs-brief")?; + + assert_eq!(qmd.pointer("/status").and_then(Value::as_str), Some("existing")); + assert_eq!(private_provider.pointer("/status").and_then(Value::as_str), Some("existing")); + assert_eq!(openviking.pointer("/status").and_then(Value::as_str), Some("existing")); + assert_eq!(letta.pointer("/status").and_then(Value::as_str), Some("proposed")); + assert_eq!(service_native.pointer("/status").and_then(Value::as_str), Some("proposed")); + assert!(support::array_contains_str( + report, + "/claim_boundaries/not_allowed", + "Do not treat qmd full-suite wrong_result counts as a regression of qmd debug ergonomics." + )?); + + Ok(()) +} + +fn assert_xy955_follow_up_issue_briefs(report: &Value) -> Result<()> { + let existing = support::array_at(report, "/follow_up_issue_briefs/existing")?; + let proposed = support::array_at(report, "/follow_up_issue_briefs/proposed")?; + let qmd = support::find_by_field(existing, "/issue", "XY-923")?; + let private_provider = support::find_by_field(existing, "/issue", "XY-930")?; + let letta = support::find_by_field(proposed, "/issue", "letta-core-archive-adapter-brief")?; + let service_native = + support::find_by_field(proposed, "/issue", "service-native-dreaming-outputs-brief")?; + + assert!(qmd.pointer("/scope").and_then(Value::as_str).is_some_and(|scope| { + scope.contains("immediate top-k") && scope.contains("candidate-drop artifacts") + })); + assert!(qmd.pointer("/non_goal").and_then(Value::as_str).is_some_and(|non_goal| { + non_goal.contains("qmd full-suite wrong_result counts") + && non_goal.contains("debug ergonomics") + })); + assert!( + private_provider + .pointer("/non_goal") + .and_then(Value::as_str) + .is_some_and(|non_goal| non_goal.contains("Do not infer credentials")) + ); + assert!(letta.pointer("/validation").and_then(Value::as_str).is_some_and(|validation| { + validation.contains("Letta core block JSON") && validation.contains("typed outcome states") + })); + assert!( + service_native + .pointer("/non_goal") + .and_then(Value::as_str) + .is_some_and(|non_goal| non_goal.contains("Pulse clone")) + ); + + Ok(()) +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/dreaming_reports.rs b/apps/elf-eval/tests/real_world_job_benchmark/dreaming_reports.rs index a4bce786..248e79f1 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/dreaming_reports.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/dreaming_reports.rs @@ -1,11 +1,16 @@ -use std::{fs, path::Path}; +mod competitor_retest; +mod letta_core; +mod openviking_trajectory; +mod qmd_debug_retest; +mod review_queue; +mod service_native; +mod temporal_reconciliation; -use color_eyre::{self, Result, eyre}; -use serde_json::Value; +use std::{fs, path::Path}; -use crate::support; +use color_eyre::Result; -fn read_rust_module_sources(src_dir: &Path, module_name: &str) -> Result { +pub(crate) fn read_rust_module_sources(src_dir: &Path, module_name: &str) -> Result { let module_root = src_dir.join(format!("{module_name}.rs")); let module_dir = src_dir.join(module_name); let mut source = fs::read_to_string(module_root)?; @@ -37,1062 +42,3 @@ fn append_rust_sources(dir: &Path, source: &mut String) -> Result<()> { Ok(()) } - -#[test] -fn live_temporal_reconciliation_report_records_xy905_before_after() -> Result<()> { - let report = serde_json::from_str::(&fs::read_to_string( - support::live_temporal_reconciliation_report_json_path()?, - )?)?; - let markdown = - fs::read_to_string(support::live_temporal_reconciliation_report_markdown_path()?)?; - let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; - let readme = fs::read_to_string(support::readme_path()?)?; - - assert_eq!( - report.pointer("/schema").and_then(Value::as_str), - Some("elf.live_temporal_reconciliation_report/v1") - ); - assert_eq!(report.pointer("/authority").and_then(Value::as_str), Some("XY-905")); - assert_eq!( - report - .pointer("/baseline/elf_memory_evolution/job_status_counts/pass") - .and_then(Value::as_u64), - Some(1) - ); - assert_eq!( - report - .pointer("/baseline/elf_memory_evolution/job_status_counts/wrong_result") - .and_then(Value::as_u64), - Some(5) - ); - assert_eq!( - report - .pointer("/post_stage/elf_memory_evolution/job_status_counts/pass") - .and_then(Value::as_u64), - Some(6) - ); - assert_eq!( - report - .pointer("/post_stage/elf_memory_evolution/job_status_counts/wrong_result") - .and_then(Value::as_u64), - Some(0) - ); - assert_eq!( - report.pointer("/post_stage/elf_memory_evolution/suite_status").and_then(Value::as_str), - Some("pass") - ); - assert_eq!( - report.pointer("/post_stage/qmd_memory_evolution/suite_status").and_then(Value::as_str), - Some("wrong_result") - ); - assert_eq!( - report - .pointer("/comparison_judgment/current_vs_historical_correctness") - .and_then(Value::as_str), - Some("improved") - ); - assert_eq!( - report - .pointer("/comparison_judgment/deletion_ttl_tombstone_behavior") - .and_then(Value::as_str), - Some("unchanged") - ); - assert!(support::array_contains_str( - &report, - "/trace_contract/answer_fields", - "selected_historical_evidence" - )?); - assert!(support::array_contains_str( - &report, - "/trace_contract/materialization_fields", - "current_winner_evidence_ids" - )?); - assert!(support::array_contains_str( - &report, - "/trace_contract/trace_stages", - "temporal_reconciliation.conflict_candidates" - )?); - assert!(report.pointer("/trace_contract/negative_gate").and_then(Value::as_str).is_some_and( - |gate| gate.contains("selected conflict evidence id") && gate.contains("wrong_result") - )); - assert!(markdown.contains("ELF passing all six memory-evolution jobs")); - assert!(markdown.contains("selected-but-not-narrated conflicts as `wrong_result`")); - assert!(markdown.contains("Do not claim ELF beats Graphiti/Zep")); - assert!(benchmarking_index.contains("2026-06-16-live-temporal-reconciliation-report.md")); - assert!( - readme.contains("Live Temporal Reconciliation Report - June 16, 2026") - && readme.contains("now reports ELF live `memory_evolution` as 6/6 pass") - ); - - Ok(()) -} - -#[test] -fn dreaming_competitor_strength_retest_report_closes_xy955_without_overclaims() -> Result<()> { - let report = serde_json::from_str::(&fs::read_to_string( - support::dreaming_competitor_strength_retest_report_json_path()?, - )?)?; - let markdown = - fs::read_to_string(support::dreaming_competitor_strength_retest_report_markdown_path()?)?; - let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; - let readme = fs::read_to_string(support::readme_path()?)?; - - assert_eq!( - report.pointer("/schema").and_then(Value::as_str), - Some("elf.dreaming_competitor_strength_retest_report/v1") - ); - assert_eq!(report.pointer("/authority").and_then(Value::as_str), Some("XY-955")); - assert_eq!( - report.pointer("/summary/overall_judgment").and_then(Value::as_str), - Some("locally_and_partially_stronger_only") - ); - assert_eq!( - report.pointer("/summary/broader_superiority").and_then(Value::as_str), - Some("not_proven") - ); - assert_eq!(report.pointer("/summary/regressed_stage_count").and_then(Value::as_u64), Some(0)); - assert!(support::array_contains_str(&report, "/status_terms", "typed_non_pass")?); - assert!(support::array_contains_str( - &report, - "/summary/unsupported_claims_rejected", - "ELF does not broadly beat qmd from this retest." - )?); - - assert_xy955_commands(&report)?; - assert_xy955_stage_closeout(&report)?; - assert_xy955_scenario_retests(&report)?; - assert_xy955_optimization_queue(&report)?; - assert_xy955_follow_up_issue_briefs(&report)?; - - assert!(markdown.contains("ELF is locally and partially stronger")); - assert!( - markdown.contains("The full live-adapter command now has fresh ELF and qmd scored reports") - ); - assert!( - markdown.contains( - "Do not treat qmd full-suite wrong_result counts as a regression of qmd debug" - ) - ); - assert!(markdown.contains("## Follow-Up Issue Briefs")); - assert!(markdown.contains( - "| GraphRAG/LightRAG/RAGFlow/llm-wiki/gbrain/graphify citation/navigation/knowledge surfaces |" - )); - assert!( - benchmarking_index.contains("2026-06-17-dreaming-competitor-strength-retest-report.md") - ); - assert!(readme.contains("Dreaming Competitor-Strength Retest Report - June 17, 2026")); - assert!(readme.contains("17 competitor-strength closeout")); - - Ok(()) -} - -#[test] -fn qmd_debug_ergonomics_dreaming_retest_report_preserves_qmd_edge() -> Result<()> { - let report = serde_json::from_str::(&fs::read_to_string( - support::qmd_debug_ergonomics_dreaming_retest_report_json_path()?, - )?)?; - let markdown = - fs::read_to_string(support::qmd_debug_ergonomics_dreaming_retest_report_markdown_path()?)?; - let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; - let readme = fs::read_to_string(support::readme_path()?)?; - - assert_qmd_debug_retest_summary(&report)?; - assert_qmd_debug_retest_command_and_adapters(&report)?; - assert_qmd_debug_retest_scenarios(&report)?; - assert_qmd_debug_retest_boundaries(&report)?; - assert_qmd_debug_retest_markdown_and_indexes(&markdown, &benchmarking_index, &readme); - - Ok(()) -} - -fn assert_qmd_debug_retest_summary(report: &Value) -> Result<()> { - assert_eq!( - report.pointer("/schema").and_then(Value::as_str), - Some("elf.qmd_debug_ergonomics_dreaming_retest_report/v1") - ); - assert_eq!(report.pointer("/authority").and_then(Value::as_str), Some("XY-982")); - assert_eq!( - report.pointer("/summary/overall_judgment").and_then(Value::as_str), - Some("unchanged_with_live_operator_debug_confirmation") - ); - assert_eq!( - report.pointer("/summary/debug_ergonomics_edge").and_then(Value::as_str), - Some("qmd_default_top10_and_short_cli_replay_preserved") - ); - assert_eq!( - report.pointer("/summary/broader_superiority").and_then(Value::as_str), - Some("not_proven") - ); - assert_eq!(report.pointer("/summary/improved_scenario_count").and_then(Value::as_u64), Some(0)); - assert_eq!( - report.pointer("/summary/regressed_scenario_count").and_then(Value::as_u64), - Some(0) - ); - assert_eq!( - report.pointer("/summary/unchanged_scenario_count").and_then(Value::as_u64), - Some(6) - ); - assert!(support::array_contains_str( - report, - "/summary/unsupported_claims_rejected", - "qmd's live operator-debug wrong_result rows do not erase qmd's default top-k and short CLI replay edge." - )?); - - Ok(()) -} - -fn assert_qmd_debug_retest_command_and_adapters(report: &Value) -> Result<()> { - let command = support::find_by_field( - support::array_at(report, "/commands")?, - "/command", - "cargo make real-world-job-operator-ux-live-adapters", - )?; - - assert_eq!(command.pointer("/status").and_then(Value::as_str), Some("pass")); - assert_eq!( - command.pointer("/summary/schema").and_then(Value::as_str), - Some("elf.real_world_operator_debug_live_adapter_sweep/v1") - ); - - let adapters = support::array_at(report, "/adapter_summaries")?; - let elf = support::find_by_field(adapters, "/adapter_id", "elf_operator_debug_live")?; - let qmd = support::find_by_field(adapters, "/adapter_id", "qmd_operator_debug_live")?; - - assert_eq!(elf.pointer("/job_count").and_then(Value::as_u64), Some(6)); - assert_eq!(elf.pointer("/pass").and_then(Value::as_u64), Some(6)); - assert_eq!(elf.pointer("/wrong_result").and_then(Value::as_u64), Some(0)); - assert_eq!(elf.pointer("/trace_available_count").and_then(Value::as_u64), Some(6)); - assert_eq!(elf.pointer("/replay_command_available_count").and_then(Value::as_u64), Some(6)); - assert_eq!(qmd.pointer("/job_count").and_then(Value::as_u64), Some(6)); - assert_eq!(qmd.pointer("/pass").and_then(Value::as_u64), Some(0)); - assert_eq!(qmd.pointer("/wrong_result").and_then(Value::as_u64), Some(6)); - assert_eq!(qmd.pointer("/trace_available_count").and_then(Value::as_u64), Some(0)); - assert_eq!(qmd.pointer("/trace_incomplete_count").and_then(Value::as_u64), Some(6)); - assert_eq!(qmd.pointer("/replay_command_available_count").and_then(Value::as_u64), Some(6)); - - Ok(()) -} - -fn assert_qmd_debug_retest_scenarios(report: &Value) -> Result<()> { - let scenarios = support::array_at(report, "/scenario_retests")?; - let top10 = - support::find_by_field(scenarios, "/scenario_id", "qmd_default_top10_candidate_artifact")?; - let replay = support::find_by_field(scenarios, "/scenario_id", "qmd_short_cli_replay")?; - let trace = - support::find_by_field(scenarios, "/scenario_id", "elf_operator_debug_trace_hydration")?; - let candidate = support::find_by_field( - scenarios, - "/scenario_id", - "operator_debug_candidate_drop_visibility", - )?; - let expansion = - support::find_by_field(scenarios, "/scenario_id", "query_expansion_attribution")?; - let fusion = support::find_by_field(scenarios, "/scenario_id", "fusion_attribution")?; - let rerank = support::find_by_field(scenarios, "/scenario_id", "rerank_attribution")?; - - assert_eq!(scenarios.len(), 10); - assert_eq!(top10.pointer("/judgment").and_then(Value::as_str), Some("unchanged")); - assert_eq!(top10.pointer("/current_outcome").and_then(Value::as_str), Some("loss")); - assert_eq!(replay.pointer("/current_outcome").and_then(Value::as_str), Some("loss")); - assert_eq!( - trace.pointer("/current_counts/elf_trace_available").and_then(Value::as_u64), - Some(6) - ); - assert_eq!( - trace.pointer("/current_counts/qmd_trace_available").and_then(Value::as_u64), - Some(0) - ); - assert_eq!( - candidate - .pointer("/current_counts/qmd_intermediate_stage_visible_jobs") - .and_then(Value::as_u64), - Some(0) - ); - assert!(support::array_contains_str( - candidate, - "/typed_non_pass_states", - "retrieved_but_dropped" - )?); - assert_eq!(expansion.pointer("/judgment").and_then(Value::as_str), Some("not_tested")); - assert_eq!(fusion.pointer("/judgment").and_then(Value::as_str), Some("not_tested")); - assert_eq!(rerank.pointer("/judgment").and_then(Value::as_str), Some("non_goal")); - - Ok(()) -} - -fn assert_qmd_debug_retest_boundaries(report: &Value) -> Result<()> { - assert!(support::array_contains_str( - report, - "/claim_boundaries/allowed", - "qmd's default local-debug edge remains: top-10 candidate rows plus short CLI replay." - )?); - assert!(support::array_contains_str( - report, - "/claim_boundaries/not_allowed", - "Do not claim ELF broadly beats qmd from this retest." - )?); - assert!(support::array_contains_str( - report, - "/next_optimization_direction/required_fields", - "fusion_rank_deltas" - )?); - - Ok(()) -} - -fn assert_qmd_debug_retest_markdown_and_indexes( - markdown: &str, - benchmarking_index: &str, - readme: &str, -) { - assert!(markdown.contains("The qmd debug-ergonomics outcome is unchanged")); - assert!(markdown.contains("ELF 6 pass/0 wrong_result; qmd 0 pass/6 wrong_result")); - assert!( - markdown.contains("Do not treat qmd's 0 pass/6 wrong_result live operator-debug slice") - ); - assert!(markdown.contains("Immediate top-k rows with source id")); - assert!( - benchmarking_index.contains("2026-06-19-qmd-debug-ergonomics-dreaming-retest-report.md") - ); - assert!(readme.contains("qmd Debug-Ergonomics Dreaming Retest Report - June 19, 2026")); - assert!(readme.contains("Temporal and Trajectory Adapter Coverage Report - June 23, 2026")); - assert!(readme.contains("Latest real-world benchmark report: June 27, 2026")); - assert!(readme.contains("keeps the qmd edge unchanged")); -} - -#[test] -fn openviking_trajectory_materialization_report_preserves_blocked_gates() -> Result<()> { - let report = serde_json::from_str::(&fs::read_to_string( - support::openviking_trajectory_materialization_report_json_path()?, - )?)?; - let markdown = - fs::read_to_string(support::openviking_trajectory_materialization_report_markdown_path()?)?; - let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; - let readme = fs::read_to_string(support::readme_path()?)?; - - assert_openviking_trajectory_materialization_summary(&report)?; - assert_openviking_trajectory_materialization_command(&report)?; - assert_openviking_trajectory_materialization_scenarios(&report)?; - assert_openviking_trajectory_materialization_boundaries(&report)?; - assert_openviking_trajectory_materialization_markdown_and_indexes( - &markdown, - &benchmarking_index, - &readme, - ); - - Ok(()) -} - -#[test] -fn letta_core_archive_export_readback_report_preserves_blocked_gates() -> Result<()> { - let report = serde_json::from_str::(&fs::read_to_string( - support::letta_core_archive_export_readback_report_json_path()?, - )?)?; - let markdown = - fs::read_to_string(support::letta_core_archive_export_readback_report_markdown_path()?)?; - let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; - let readme = fs::read_to_string(support::readme_path()?)?; - - assert_eq!( - report.pointer("/schema").and_then(Value::as_str), - Some("elf.letta_core_archive_export_readback_summary/v1") - ); - assert_eq!( - report.pointer("/adapter_id").and_then(Value::as_str), - Some("letta_core_archive_export_readback") - ); - assert_eq!( - report.pointer("/materialization/status/failure_class").and_then(Value::as_str), - Some("letta_live_run_disabled") - ); - assert_eq!( - report.pointer("/materialization/status/overall").and_then(Value::as_str), - Some("blocked") - ); - assert_eq!( - report.pointer("/materialization/scored_benchmark/status").and_then(Value::as_str), - Some("blocked") - ); - assert_eq!( - report.pointer("/materialization/scored_benchmark/counts/blocked").and_then(Value::as_u64), - Some(6) - ); - assert_eq!( - report.pointer("/materialization/scored_benchmark/counts/pass").and_then(Value::as_u64), - Some(0) - ); - assert_eq!( - report - .pointer("/materialization/scored_benchmark/counts/wrong_result") - .and_then(Value::as_u64), - Some(0) - ); - assert_eq!( - report - .pointer("/materialization/scored_benchmark/evidence_coverage") - .and_then(Value::as_f64), - Some(1.0) - ); - assert_eq!( - report - .pointer("/materialization/benchmark_input/core_blocks") - .and_then(Value::as_array) - .map(Vec::len), - Some(9) - ); - assert_eq!( - report - .pointer("/materialization/benchmark_input/archival_passages") - .and_then(Value::as_array) - .map(Vec::len), - Some(6) - ); - assert_eq!( - report - .pointer("/materialization/evidence_mapping/expected_evidence_ids") - .and_then(Value::as_array) - .map(Vec::len), - Some(14) - ); - assert_eq!( - report - .pointer("/materialization/evidence_mapping/mapped_evidence_ids") - .and_then(Value::as_array) - .map(Vec::len), - Some(0) - ); - assert_eq!( - report - .pointer("/materialization/improvement_regression_readback/judgment") - .and_then(Value::as_str), - Some("unchanged") - ); - assert!(support::array_contains_str( - &report, - "/materialization/claim_boundaries/not_allowed", - "Do not claim ELF beats Letta on core-vs-archival memory from fixture-only ELF evidence." - )?); - assert!(markdown.contains("The Letta follow-up is now reproducible")); - assert!(markdown.contains("6 typed blocked")); - assert!(markdown.contains("competitive status is unchanged")); - assert!(benchmarking_index.contains("2026-06-19-letta-core-archive-export-readback-report.md")); - assert!(readme.contains("Letta core/archive materialization after XY-984")); - assert!(readme.contains("smoke-letta-core-archive-export-readback")); - - Ok(()) -} - -#[test] -fn service_native_dreaming_readback_report_materializes_public_jobs() -> Result<()> { - let report = serde_json::from_str::(&fs::read_to_string( - support::service_native_dreaming_readback_report_json_path()?, - )?)?; - let materialization = serde_json::from_str::(&fs::read_to_string( - support::service_native_dreaming_readback_materialization_json_path()?, - )?)?; - let markdown = - fs::read_to_string(support::service_native_dreaming_readback_report_markdown_path()?)?; - let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; - let readme = fs::read_to_string(support::readme_path()?)?; - - assert_service_native_dreaming_report_summary(&report)?; - assert_service_native_dreaming_report_jobs(&report)?; - assert_service_native_dreaming_materialization(&materialization)?; - assert_service_native_dreaming_docs(&markdown, &benchmarking_index, &readme); - - Ok(()) -} - -fn assert_service_native_dreaming_report_summary(report: &Value) -> Result<()> { - assert_eq!( - report.pointer("/adapter/adapter_id").and_then(Value::as_str), - Some("elf_service_native_dreaming") - ); - assert_eq!( - report.pointer("/adapter/behavior").and_then(Value::as_str), - Some("service_native_dreaming_readback") - ); - assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(11)); - assert_eq!(report.pointer("/summary/pass").and_then(Value::as_u64), Some(9)); - assert_eq!(report.pointer("/summary/wrong_result").and_then(Value::as_u64), Some(0)); - assert_eq!(report.pointer("/summary/blocked").and_then(Value::as_u64), Some(2)); - assert_eq!(report.pointer("/summary/wrong_result_count").and_then(Value::as_u64), Some(0)); - assert_eq!(report.pointer("/summary/evidence_coverage").and_then(Value::as_f64), Some(1.0)); - assert_eq!(report.pointer("/summary/source_ref_coverage").and_then(Value::as_f64), Some(1.0)); - assert_eq!(report.pointer("/summary/quote_coverage").and_then(Value::as_f64), Some(1.0)); - assert_eq!( - report.pointer("/summary/memory_summary/source_ref_coverage").and_then(Value::as_f64), - Some(1.0) - ); - assert_eq!( - report.pointer("/summary/proactive_brief/evidence_ref_coverage").and_then(Value::as_f64), - Some(1.0) - ); - assert_eq!( - report.pointer("/summary/scheduled_memory/trace_coverage").and_then(Value::as_f64), - Some(1.0) - ); - assert_eq!( - report.pointer("/summary/scheduled_memory/source_mutation_count").and_then(Value::as_u64), - Some(0) - ); - - let suites = support::array_at(report, "/suites")?; - let memory = support::find_by_field(suites, "/suite_id", "memory_summary")?; - let proactive = support::find_by_field(suites, "/suite_id", "proactive_brief")?; - let scheduled = support::find_by_field(suites, "/suite_id", "scheduled_memory")?; - - assert_eq!(memory.pointer("/status").and_then(Value::as_str), Some("pass")); - assert_eq!(proactive.pointer("/status").and_then(Value::as_str), Some("blocked")); - assert_eq!(scheduled.pointer("/status").and_then(Value::as_str), Some("blocked")); - - Ok(()) -} - -fn assert_service_native_dreaming_report_jobs(report: &Value) -> Result<()> { - let jobs = support::array_at(report, "/jobs")?; - let memory = support::find_by_field(jobs, "/job_id", "memory-summary-source-trace-001")?; - let daily = support::find_by_field(jobs, "/job_id", "proactive-daily-project-brief-001")?; - let private_brief = - support::find_by_field(jobs, "/job_id", "proactive-private-corpus-refresh-blocked-001")?; - let weekly = - support::find_by_field(jobs, "/job_id", "scheduled-weekly-project-status-summary-001")?; - let private_scheduled = support::find_by_field( - jobs, - "/job_id", - "scheduled-private-provider-scheduler-blocked-001", - )?; - - assert_eq!(memory.pointer("/status").and_then(Value::as_str), Some("pass")); - assert_eq!(daily.pointer("/status").and_then(Value::as_str), Some("pass")); - assert_eq!(weekly.pointer("/status").and_then(Value::as_str), Some("pass")); - assert_eq!(private_brief.pointer("/status").and_then(Value::as_str), Some("blocked")); - assert_eq!(private_scheduled.pointer("/status").and_then(Value::as_str), Some("blocked")); - assert!(!support::array_contains_str(memory, "/produced_evidence", "stale-summary-gap")?); - assert!(!support::array_contains_str(memory, "/produced_evidence", "summary-temporary-claim")?); - assert!(!support::array_contains_str(daily, "/produced_evidence", "daily-old-parity-trap")?); - assert!(!support::array_contains_str( - weekly, - "/produced_evidence", - "scheduled-weekly-hosted-parity-trap" - )?); - - Ok(()) -} - -fn assert_service_native_dreaming_materialization(materialization: &Value) -> Result<()> { - assert_eq!( - materialization.pointer("/schema").and_then(Value::as_str), - Some("elf.real_world_live_adapter_materialization/v1") - ); - assert_eq!( - materialization.pointer("/adapter_id").and_then(Value::as_str), - Some("elf_service_native_dreaming") - ); - assert_eq!(materialization.pointer("/status").and_then(Value::as_str), Some("blocked")); - - let jobs = support::array_at(materialization, "/jobs")?; - let memory = support::find_by_field(jobs, "/job_id", "memory-summary-source-trace-001")?; - let daily = support::find_by_field(jobs, "/job_id", "proactive-daily-project-brief-001")?; - let private_brief = - support::find_by_field(jobs, "/job_id", "proactive-private-corpus-refresh-blocked-001")?; - - for job in jobs { - match job.pointer("/status").and_then(Value::as_str) { - Some("pass") => { - assert_eq!( - job.pointer("/dreaming_readback/runtime_path").and_then(Value::as_str), - Some("ElfService::add_note -> ElfService::list -> derived readback artifact") - ); - assert!( - support::array_at(job, "/dreaming_readback/missing_source_refs")?.is_empty() - ); - assert_eq!( - job.pointer("/dreaming_readback/source_mutation_count").and_then(Value::as_u64), - Some(0) - ); - assert_eq!( - job.pointer("/dreaming_readback/no_source_mutation_checked") - .and_then(Value::as_bool), - Some(true) - ); - }, - Some("blocked") => { - assert!(job.pointer("/dreaming_readback").is_none_or(Value::is_null)); - }, - status => { - return Err(eyre::eyre!( - "unexpected service-native materialization status: {status:?}" - )); - }, - } - } - - assert!(support::array_contains_str( - memory, - "/dreaming_readback/selected_source_refs", - "stale-summary-gap" - )?); - assert!(!support::array_contains_str(memory, "/evidence_ids", "stale-summary-gap")?); - assert!(support::array_contains_str( - daily, - "/dreaming_readback/selected_source_refs", - "daily-old-parity-trap" - )?); - assert!(!support::array_contains_str(daily, "/evidence_ids", "daily-old-parity-trap")?); - assert!(private_brief.pointer("/dreaming_readback").is_none_or(Value::is_null)); - - Ok(()) -} - -fn assert_service_native_dreaming_docs(markdown: &str, benchmarking_index: &str, readme: &str) { - assert!(markdown.contains("9 pass")); - assert!(markdown.contains("0 wrong_result")); - assert!(markdown.contains("2 typed blocked")); - assert!(markdown.contains("ElfService::add_note -> ElfService::list")); - assert!(markdown.contains("Do not claim ELF broadly beats OpenAI Pulse")); - assert!(benchmarking_index.contains("2026-06-19-service-native-dreaming-readback-report.md")); - assert!(readme.contains("Service-native Dreaming readback after XY-986")); - assert!(readme.contains("real-world-memory-service-native-dreaming")); -} - -#[test] -fn dreaming_review_queue_report_wires_reviewable_policy_contract() -> Result<()> { - let report = serde_json::from_str::(&fs::read_to_string( - support::dreaming_review_queue_report_json_path()?, - )?)?; - let markdown = fs::read_to_string(support::dreaming_review_queue_report_markdown_path()?)?; - let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; - let readme = fs::read_to_string(support::readme_path()?)?; - let workspace = support::workspace_root()?; - let service = read_rust_module_sources( - &workspace.join("packages/elf-service/src"), - "dreaming_review_queue", - )?; - let service_lib = fs::read_to_string(workspace.join("packages/elf-service/src/lib.rs"))?; - let routes = read_rust_module_sources(&workspace.join("apps/elf-api/src"), "routes")?; - let mcp = read_rust_module_sources(&workspace.join("apps/elf-mcp/src/app"), "server")?; - let consolidation_spec = - fs::read_to_string(workspace.join("docs/spec/system_consolidation_proposals_v1.md"))?; - let service_spec = - fs::read_to_string(workspace.join("docs/spec/system_elf_memory_service_v2.md"))?; - - assert_eq!( - report.pointer("/schema").and_then(Value::as_str), - Some("elf.dreaming_review_queue_report/v1") - ); - assert_eq!(report.pointer("/authority").and_then(Value::as_str), Some("XY-1021")); - assert_eq!( - report.pointer("/summary/queue_schema").and_then(Value::as_str), - Some("elf.dreaming_review_queue/v1") - ); - assert_eq!( - report.pointer("/summary/source_mutation_allowed").and_then(Value::as_bool), - Some(false) - ); - assert_eq!( - report.pointer("/summary/high_impact_requires_review").and_then(Value::as_bool), - Some(true) - ); - assert_eq!(report.pointer("/summary/variant_count").and_then(Value::as_u64), Some(9)); - - for suite in ["memory_summary", "proactive_brief", "scheduled_memory", "consolidation"] { - assert!(support::array_contains_str(&report, "/summary/covered_existing_suites", suite)?); - } - for variant in - ["tag", "duplicate_merge", "page_rebuild", "memory_promotion", "graph_fact", "correction"] - { - assert!(support::array_contains_str(&report, "/summary/covered_future_variants", variant)?); - - support::find_by_field( - support::array_at(&report, "/queue_variants")?, - "/variant", - variant, - )?; - } - for field in [ - "source_refs", - "affected_refs", - "confidence", - "unsupported_claim_flags", - "diff", - "policy", - "review_audit", - ] { - assert!(support::array_contains_str(&report, "/required_item_fields", field)?); - } - - assert!(service.contains("ELF_DREAMING_REVIEW_QUEUE_SCHEMA_V1")); - assert!(service.contains("pub async fn dreaming_review_queue")); - assert!(service.contains("source_mutation_allowed: false")); - assert!(service.contains("low_risk_derived_organization")); - assert!(service.contains("available_review_actions")); - assert!(service_lib.contains("pub mod dreaming_review_queue")); - assert!(service_lib.contains("DreamingReviewQueueResponse")); - assert!(routes.contains("/v2/admin/dreaming/review-queue")); - assert!(routes.contains("DreamingReviewQueueRequest")); - assert!(routes.contains("async fn dreaming_review_queue")); - assert!(mcp.contains("elf_dreaming_review_queue")); - assert!(mcp.contains("dreaming_review_queue_schema")); - assert!(mcp.contains("/v2/admin/dreaming/review-queue")); - assert!(consolidation_spec.contains("elf.dreaming_review_queue/v1")); - assert!(consolidation_spec.contains("source_mutation_allowed")); - assert!(consolidation_spec.contains("duplicate_merge")); - assert!(service_spec.contains("GET /v2/admin/dreaming/review-queue")); - assert!(service_spec.contains("source refs, affected refs, confidence")); - assert!(markdown.contains("Dreaming Review Queue Report")); - assert!(markdown.contains("Auto-apply is limited to approved low-risk")); - assert!(benchmarking_index.contains("2026-06-20-dreaming-review-queue-report.md")); - assert!(readme.contains("Dreaming review queue after XY-1021")); - assert!(readme.contains("elf.dreaming_review_queue/v1")); - - Ok(()) -} - -fn assert_openviking_trajectory_materialization_summary(report: &Value) -> Result<()> { - assert_eq!( - report.pointer("/schema").and_then(Value::as_str), - Some("elf.openviking_trajectory_materialization_report/v1") - ); - assert_eq!(report.pointer("/authority").and_then(Value::as_str), Some("XY-983")); - assert_eq!( - report.pointer("/summary/overall_judgment").and_then(Value::as_str), - Some("materialized_blocked_context_trajectory_evidence") - ); - assert_eq!( - report.pointer("/summary/broader_superiority").and_then(Value::as_str), - Some("not_proven") - ); - assert_eq!(report.pointer("/summary/blockers_removed_count").and_then(Value::as_u64), Some(0)); - assert_eq!(report.pointer("/summary/blocked_scenario_count").and_then(Value::as_u64), Some(3)); - assert_eq!(report.pointer("/summary/pass_count").and_then(Value::as_u64), Some(0)); - assert_eq!(report.pointer("/summary/wrong_result_count").and_then(Value::as_u64), Some(0)); - assert_eq!( - report.pointer("/summary/regressed_scenario_count").and_then(Value::as_u64), - Some(0) - ); - assert_eq!(report.pointer("/summary/evidence_coverage").and_then(Value::as_f64), Some(1.0)); - assert!(support::array_contains_str( - report, - "/summary/unsupported_claims_rejected", - "ELF does not beat OpenViking staged retrieval trajectory from fixture-only blocked rows." - )?); - - Ok(()) -} - -fn assert_openviking_trajectory_materialization_command(report: &Value) -> Result<()> { - let command = support::find_by_field( - support::array_at(report, "/commands")?, - "/command", - "cargo make real-world-memory-context-trajectory", - )?; - let summary = - command.pointer("/summary").ok_or_else(|| eyre::eyre!("missing command summary"))?; - - assert_eq!(command.pointer("/status").and_then(Value::as_str), Some("pass")); - assert_eq!( - command.pointer("/artifact_json").and_then(Value::as_str), - Some("tmp/real-world-memory/context-trajectory/report.json") - ); - assert_eq!(summary.pointer("/job_count").and_then(Value::as_u64), Some(3)); - assert_eq!(summary.pointer("/pass").and_then(Value::as_u64), Some(0)); - assert_eq!(summary.pointer("/wrong_result").and_then(Value::as_u64), Some(0)); - assert_eq!(summary.pointer("/blocked").and_then(Value::as_u64), Some(3)); - assert_eq!(summary.pointer("/evidence_covered_count").and_then(Value::as_u64), Some(9)); - assert_eq!(summary.pointer("/source_ref_covered_count").and_then(Value::as_u64), Some(9)); - assert_eq!(summary.pointer("/quote_covered_count").and_then(Value::as_u64), Some(9)); - - Ok(()) -} - -fn assert_openviking_trajectory_materialization_scenarios(report: &Value) -> Result<()> { - let scenarios = support::array_at(report, "/scenario_materialization")?; - let staged = support::find_by_field( - scenarios, - "/scenario_id", - "openviking_staged_retrieval_trajectory", - )?; - let hierarchy = - support::find_by_field(scenarios, "/scenario_id", "openviking_hierarchy_selection")?; - let recursive = support::find_by_field( - scenarios, - "/scenario_id", - "openviking_recursive_context_expansion", - )?; - - assert_eq!(scenarios.len(), 3); - - for scenario in [staged, hierarchy, recursive] { - assert_eq!(scenario.pointer("/previous_status").and_then(Value::as_str), Some("blocked")); - assert_eq!(scenario.pointer("/current_status").and_then(Value::as_str), Some("blocked")); - assert_eq!(scenario.pointer("/judgment").and_then(Value::as_str), Some("unchanged")); - } - - assert!(support::array_contains_str( - staged, - "/produced_evidence", - "openviking-evidence-id-output-contract" - )?); - assert!(support::array_contains_str( - hierarchy, - "/produced_evidence", - "hierarchy-selection-output-contract" - )?); - assert!(support::array_contains_str( - recursive, - "/produced_evidence", - "recursive-expansion-output-contract" - )?); - assert_eq!( - staged.pointer("/claim_boundary").and_then(Value::as_str), - Some( - "No ELF win, tie, or loss is allowed until both systems publish comparable stage artifacts for the same context-trajectory scenario." - ) - ); - assert_eq!( - hierarchy.pointer("/blocker").and_then(Value::as_str), - Some("selected_hierarchy_nodes_and_evidence_ids_missing") - ); - assert_eq!( - recursive.pointer("/blocker").and_then(Value::as_str), - Some("expansion_paths_and_same_corpus_evidence_ids_missing") - ); - - Ok(()) -} - -fn assert_openviking_trajectory_materialization_boundaries(report: &Value) -> Result<()> { - assert_eq!( - report.pointer("/improvement_regression_readback/improved").and_then(Value::as_u64), - Some(0) - ); - assert_eq!( - report.pointer("/improvement_regression_readback/blocked").and_then(Value::as_u64), - Some(3) - ); - assert!(support::array_contains_str( - report, - "/claim_boundaries/allowed", - "The context-trajectory slice is now reproducible through cargo make real-world-memory-context-trajectory." - )?); - assert!(support::array_contains_str( - report, - "/claim_boundaries/not_allowed", - "Do not claim ELF beats OpenViking on staged retrieval trajectory." - )?); - assert!(support::array_contains_str( - report, - "/next_optimization_direction/required_fields", - "expansion_path" - )?); - assert_eq!( - report.pointer("/next_optimization_direction/non_goal").and_then(Value::as_str), - Some( - "No ELF product change or superiority claim is authorized by this materialization-only report." - ) - ); - - Ok(()) -} - -fn assert_openviking_trajectory_materialization_markdown_and_indexes( - markdown: &str, - benchmarking_index: &str, - readme: &str, -) { - assert!(markdown.contains("The OpenViking trajectory follow-up is now materialized")); - assert!(markdown.contains("3 encoded jobs, 0 pass, 3 blocked, 9/9 evidence coverage")); - assert!(markdown.contains("Do not claim ELF beats OpenViking on staged retrieval trajectory.")); - assert!(markdown.contains("OpenViking context-trajectory job can move from `blocked`")); - assert!( - benchmarking_index.contains("2026-06-19-openviking-trajectory-materialization-report.md") - ); - assert!(readme.contains("OpenViking Trajectory Materialization Report - June 19, 2026")); - assert!(readme.contains("cargo make real-world-memory-context-trajectory")); - assert!(readme.contains("3 typed blockers with 9/9 evidence coverage")); -} - -fn assert_xy955_commands(report: &Value) -> Result<()> { - let commands = support::array_at(report, "/commands")?; - let aggregate = support::find_by_field(commands, "/command", "cargo make real-world-memory")?; - let graph_rag = - support::find_by_field(commands, "/command", "cargo make real-world-memory-graph-rag")?; - let first_generation = - support::find_by_field(commands, "/command", "cargo make real-world-first-generation-oss")?; - let live = - support::find_by_field(commands, "/command", "cargo make real-world-memory-live-adapters")?; - - assert_eq!(aggregate.pointer("/status").and_then(Value::as_str), Some("pass")); - assert_eq!(aggregate.pointer("/summary/pass").and_then(Value::as_u64), Some(53)); - assert_eq!(aggregate.pointer("/summary/blocked").and_then(Value::as_u64), Some(7)); - assert_eq!(graph_rag.pointer("/status").and_then(Value::as_str), Some("pass")); - assert_eq!(graph_rag.pointer("/summary/wrong_result").and_then(Value::as_u64), Some(1)); - assert_eq!(graph_rag.pointer("/summary/incomplete").and_then(Value::as_u64), Some(1)); - assert_eq!(graph_rag.pointer("/summary/blocked").and_then(Value::as_u64), Some(3)); - assert_eq!(first_generation.pointer("/summary/pass").and_then(Value::as_u64), Some(4)); - assert_eq!(first_generation.pointer("/summary/blocked").and_then(Value::as_u64), Some(2)); - assert_eq!(live.pointer("/status").and_then(Value::as_str), Some("pass")); - assert_eq!( - live.pointer("/partial_summary/elf_live_real_world/pass").and_then(Value::as_u64), - Some(40) - ); - assert_eq!( - live.pointer("/partial_summary/elf_live_real_world/wrong_result").and_then(Value::as_u64), - Some(0) - ); - assert_eq!( - live.pointer("/partial_summary/qmd_live_real_world/pass").and_then(Value::as_u64), - Some(17) - ); - assert_eq!( - live.pointer("/partial_summary/qmd_live_real_world/wrong_result").and_then(Value::as_u64), - Some(13) - ); - - Ok(()) -} - -fn assert_xy955_stage_closeout(report: &Value) -> Result<()> { - let stages = support::array_at(report, "/stage_closeout")?; - - assert_eq!(stages.len(), 8); - - let current = support::find_by_field(stages, "/stage_id", "current_vs_historical_correctness")?; - let proactive = support::find_by_field(stages, "/stage_id", "proactive_brief_readiness")?; - let scheduled = support::find_by_field(stages, "/stage_id", "scheduled_memory_task_readiness")?; - let final_retest = - support::find_by_field(stages, "/stage_id", "final_competitor_retest_status")?; - - assert_eq!(current.pointer("/judgment").and_then(Value::as_str), Some("improved")); - assert_eq!(current.pointer("/current_counts/pass").and_then(Value::as_u64), Some(6)); - assert_eq!(current.pointer("/current_counts/wrong_result").and_then(Value::as_u64), Some(0)); - assert_eq!(proactive.pointer("/judgment").and_then(Value::as_str), Some("improved")); - assert_eq!(proactive.pointer("/current_counts/blocked").and_then(Value::as_u64), Some(1)); - assert_eq!(scheduled.pointer("/current_counts/pass").and_then(Value::as_u64), Some(4)); - assert_eq!(scheduled.pointer("/current_counts/blocked").and_then(Value::as_u64), Some(1)); - assert_eq!(final_retest.pointer("/judgment").and_then(Value::as_str), Some("unchanged")); - assert_eq!(final_retest.pointer("/current_counts/pass").and_then(Value::as_u64), Some(40)); - assert_eq!( - final_retest.pointer("/current_counts/wrong_result").and_then(Value::as_u64), - Some(0) - ); - assert_eq!(final_retest.pointer("/current_counts/blocked").and_then(Value::as_u64), Some(7)); - assert_eq!( - final_retest.pointer("/current_counts/not_encoded").and_then(Value::as_u64), - Some(19) - ); - assert!(final_retest.pointer("/boundary").and_then(Value::as_str).is_some_and(|boundary| { - boundary.contains("qmd now has a fresh scored live report") - && boundary.contains("broader superiority is not proven") - })); - assert_eq!(final_retest.pointer("/qmd_current_counts/pass").and_then(Value::as_u64), Some(17)); - assert_eq!( - final_retest.pointer("/qmd_current_counts/wrong_result").and_then(Value::as_u64), - Some(13) - ); - - Ok(()) -} - -fn assert_xy955_scenario_retests(report: &Value) -> Result<()> { - let scenarios = support::array_at(report, "/scenario_retests")?; - let qmd = support::find_by_field(scenarios, "/scenario_id", "qmd_debug_ergonomics")?; - let mem0 = support::find_by_field( - scenarios, - "/scenario_id", - "mem0_openmemory_preference_history_export", - )?; - let letta = support::find_by_field(scenarios, "/scenario_id", "letta_core_archive")?; - let graph_rag = support::find_by_field( - scenarios, - "/scenario_id", - "graph_rag_citation_navigation_knowledge_surfaces", - )?; - let private_provider = - support::find_by_field(scenarios, "/scenario_id", "private_provider_production_gates")?; - - assert_eq!(qmd.pointer("/current_outcome").and_then(Value::as_str), Some("unchanged")); - assert_eq!(qmd.pointer("/current_status").and_then(Value::as_str), Some("pass")); - assert!(qmd.pointer("/evidence").and_then(Value::as_str).is_some_and(|evidence| { - evidence.contains("17 pass") - && evidence.contains("13 wrong_result") - && evidence.contains("does not retest or erase") - })); - assert_eq!(mem0.pointer("/current_outcome").and_then(Value::as_str), Some("unchanged")); - assert!(mem0.pointer("/evidence").and_then(Value::as_str).is_some_and(|evidence| { - evidence.contains("mem0/OpenMemory local OSS history") - && evidence.contains("OpenMemory UI/export remains setup-blocked") - })); - assert_eq!(letta.pointer("/current_status").and_then(Value::as_str), Some("blocked")); - assert_eq!( - graph_rag.pointer("/current_status").and_then(Value::as_str), - Some("typed_non_pass") - ); - assert!(graph_rag.pointer("/evidence").and_then(Value::as_str).is_some_and(|evidence| { - evidence.contains("0 pass") - && evidence.contains("1 wrong_result") - && evidence.contains("3 blocked") - })); - assert_eq!(private_provider.pointer("/follow_up").and_then(Value::as_str), Some("XY-930")); - - Ok(()) -} - -fn assert_xy955_optimization_queue(report: &Value) -> Result<()> { - let queue = support::array_at(report, "/optimization_queue")?; - let qmd = support::find_by_field(queue, "/issue", "XY-923")?; - let private_provider = support::find_by_field(queue, "/issue", "XY-930")?; - let openviking = support::find_by_field(queue, "/issue", "XY-928")?; - let letta = support::find_by_field(queue, "/issue", "letta-core-archive-adapter-brief")?; - let service_native = - support::find_by_field(queue, "/issue", "service-native-dreaming-outputs-brief")?; - - assert_eq!(qmd.pointer("/status").and_then(Value::as_str), Some("existing")); - assert_eq!(private_provider.pointer("/status").and_then(Value::as_str), Some("existing")); - assert_eq!(openviking.pointer("/status").and_then(Value::as_str), Some("existing")); - assert_eq!(letta.pointer("/status").and_then(Value::as_str), Some("proposed")); - assert_eq!(service_native.pointer("/status").and_then(Value::as_str), Some("proposed")); - assert!(support::array_contains_str( - report, - "/claim_boundaries/not_allowed", - "Do not treat qmd full-suite wrong_result counts as a regression of qmd debug ergonomics." - )?); - - Ok(()) -} - -fn assert_xy955_follow_up_issue_briefs(report: &Value) -> Result<()> { - let existing = support::array_at(report, "/follow_up_issue_briefs/existing")?; - let proposed = support::array_at(report, "/follow_up_issue_briefs/proposed")?; - let qmd = support::find_by_field(existing, "/issue", "XY-923")?; - let private_provider = support::find_by_field(existing, "/issue", "XY-930")?; - let letta = support::find_by_field(proposed, "/issue", "letta-core-archive-adapter-brief")?; - let service_native = - support::find_by_field(proposed, "/issue", "service-native-dreaming-outputs-brief")?; - - assert!(qmd.pointer("/scope").and_then(Value::as_str).is_some_and(|scope| { - scope.contains("immediate top-k") && scope.contains("candidate-drop artifacts") - })); - assert!(qmd.pointer("/non_goal").and_then(Value::as_str).is_some_and(|non_goal| { - non_goal.contains("qmd full-suite wrong_result counts") - && non_goal.contains("debug ergonomics") - })); - assert!( - private_provider - .pointer("/non_goal") - .and_then(Value::as_str) - .is_some_and(|non_goal| non_goal.contains("Do not infer credentials")) - ); - assert!(letta.pointer("/validation").and_then(Value::as_str).is_some_and(|validation| { - validation.contains("Letta core block JSON") && validation.contains("typed outcome states") - })); - assert!( - service_native - .pointer("/non_goal") - .and_then(Value::as_str) - .is_some_and(|non_goal| non_goal.contains("Pulse clone")) - ); - - Ok(()) -} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/external_adapters.rs b/apps/elf-eval/tests/real_world_job_benchmark/external_adapters.rs index 33d0fbd1..9bba3e95 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/external_adapters.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/external_adapters.rs @@ -1,10 +1,9 @@ -use std::{ - env, fs, - path::Path, - process::{self, Command}, -}; +mod graph_rag; +mod loss_summary; +mod manifest_summary; +mod validation; -use color_eyre::{Result, eyre}; +use color_eyre::Result; use serde_json::Value; use crate::support; @@ -13,343 +12,13 @@ use crate::support; fn real_world_report_includes_external_adapter_coverage_manifest() -> Result<()> { let report = support::run_json_report_from(support::real_world_memory_fixture_dir())?; - assert_external_adapter_manifest_summary(&report); - assert_external_adapter_manifest_records(&report)?; - - Ok(()) -} - -#[test] -fn external_adapter_run_summarizes_nonzero_scenario_losses() -> Result<()> { - let manifest_path = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("real_world_external_adapters") - .join("memory_projects_manifest.json"); - let mut manifest = serde_json::from_str::(&fs::read_to_string(manifest_path)?)?; - let adapters = manifest - .pointer_mut("/adapters") - .and_then(Value::as_array_mut) - .ok_or_else(|| eyre::eyre!("missing manifest adapters"))?; - let adapter = adapters - .iter_mut() - .find(|adapter| { - adapter.pointer("/adapter_id").and_then(Value::as_str) - == Some("agentmemory_live_baseline") - }) - .ok_or_else(|| eyre::eyre!("missing agentmemory adapter"))?; - - support::set_json_pointer(adapter, "/scenarios/0/elf_position", serde_json::json!("loses"))?; - support::set_json_pointer( - adapter, - "/scenarios/0/comparison_outcome", - serde_json::json!("loss"), - )?; - - let temp_dir = - env::temp_dir().join(format!("elf-real-world-loss-manifest-test-{}", process::id())); + manifest_summary::assert_external_adapter_manifest_summary(&report); - fs::create_dir_all(&temp_dir)?; - - let manifest_path = temp_dir.join("memory_projects_manifest.json"); - - fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)?; - - let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) - .arg("run") - .arg("--fixtures") - .arg(support::fixture_dir()) - .arg("--external-adapter-manifest") - .arg(&manifest_path) - .output()?; - - assert!( - output.status.success(), - "real_world_job runner failed: {}", - String::from_utf8_lossy(&output.stderr), - ); - - let report = serde_json::from_slice::(&output.stdout)?; - - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_position_counts/loses") - .and_then(Value::as_u64), - Some(2) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_position_counts/untested") - .and_then(Value::as_u64), - Some(52) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_outcome_counts/loss") - .and_then(Value::as_u64), - Some(2) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_outcome_counts/not_tested") - .and_then(Value::as_u64), - Some(18) - ); - - let adapters = support::array_at(&report, "/external_adapters/adapters")?; - let agentmemory = support::find_by_field(adapters, "/adapter_id", "agentmemory_live_baseline")?; - - assert_eq!( - agentmemory.pointer("/scenarios/0/elf_position").and_then(Value::as_str), - Some("loses") - ); + assert_external_adapter_manifest_records(&report)?; Ok(()) } -fn assert_external_adapter_manifest_summary(report: &Value) { - assert_eq!( - report.pointer("/external_adapters/schema").and_then(Value::as_str), - Some("elf.real_world_external_adapter_report/v1") - ); - assert_eq!( - report.pointer("/external_adapters/manifest_id").and_then(Value::as_str), - Some( - "real-world-memory-project-adapters-2026-06-11-first-generation-continuity-source-store" - ) - ); - assert_eq!( - report.pointer("/external_adapters/docker_isolation/default").and_then(Value::as_bool), - Some(true) - ); - assert_eq!( - report - .pointer("/external_adapters/docker_isolation/host_global_installs_required") - .and_then(Value::as_bool), - Some(false) - ); - assert_eq!( - report.pointer("/external_adapters/summary/adapter_count").and_then(Value::as_u64), - Some(26) - ); - assert_eq!( - report.pointer("/external_adapters/summary/external_project_count").and_then(Value::as_u64), - Some(19) - ); - assert_eq!( - report.pointer("/external_adapters/summary/fixture_backed_count").and_then(Value::as_u64), - Some(1) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/live_baseline_only_count") - .and_then(Value::as_u64), - Some(6) - ); - assert_eq!( - report.pointer("/external_adapters/summary/live_real_world_count").and_then(Value::as_u64), - Some(5) - ); - assert_eq!( - report.pointer("/external_adapters/summary/research_gate_count").and_then(Value::as_u64), - Some(14) - ); - - assert_external_adapter_manifest_status_summary(report); - assert_external_adapter_manifest_scenario_summary(report); -} - -fn assert_external_adapter_manifest_status_summary(report: &Value) { - assert_eq!( - report - .pointer("/external_adapters/summary/overall_status_counts/pass") - .and_then(Value::as_u64), - Some(4) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/overall_status_counts/wrong_result") - .and_then(Value::as_u64), - Some(6) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/overall_status_counts/lifecycle_fail") - .and_then(Value::as_u64), - Some(1) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/overall_status_counts/incomplete") - .and_then(Value::as_u64), - Some(0) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/overall_status_counts/blocked") - .and_then(Value::as_u64), - Some(10) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/overall_status_counts/not_encoded") - .and_then(Value::as_u64), - Some(5) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/capability_status_counts/mocked") - .and_then(Value::as_u64), - Some(1) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/capability_status_counts/unsupported") - .and_then(Value::as_u64), - Some(6) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/suite_status_counts/blocked") - .and_then(Value::as_u64), - Some(29) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/suite_status_counts/pass") - .and_then(Value::as_u64), - Some(27) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/suite_status_counts/incomplete") - .and_then(Value::as_u64), - Some(0) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/suite_status_counts/not_encoded") - .and_then(Value::as_u64), - Some(37) - ); -} - -fn assert_external_adapter_manifest_scenario_summary(report: &Value) { - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_status_counts/real") - .and_then(Value::as_u64), - Some(0) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_status_counts/mocked") - .and_then(Value::as_u64), - Some(0) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_status_counts/unsupported") - .and_then(Value::as_u64), - Some(3) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_status_counts/blocked") - .and_then(Value::as_u64), - Some(24) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_status_counts/incomplete") - .and_then(Value::as_u64), - Some(5) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_status_counts/wrong_result") - .and_then(Value::as_u64), - Some(6) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_status_counts/lifecycle_fail") - .and_then(Value::as_u64), - Some(1) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_status_counts/pass") - .and_then(Value::as_u64), - Some(23) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_status_counts/not_encoded") - .and_then(Value::as_u64), - Some(13) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_position_counts/wins") - .and_then(Value::as_u64), - Some(10) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_position_counts/ties") - .and_then(Value::as_u64), - Some(11) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_position_counts/loses") - .and_then(Value::as_u64), - Some(1) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_position_counts/untested") - .and_then(Value::as_u64), - Some(53) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_outcome_counts/win") - .and_then(Value::as_u64), - Some(10) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_outcome_counts/tie") - .and_then(Value::as_u64), - Some(11) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_outcome_counts/loss") - .and_then(Value::as_u64), - Some(1) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_outcome_counts/not_tested") - .and_then(Value::as_u64), - Some(19) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_outcome_counts/blocked") - .and_then(Value::as_u64), - Some(29) - ); - assert_eq!( - report - .pointer("/external_adapters/summary/scenario_outcome_counts/non_goal") - .and_then(Value::as_u64), - Some(5) - ); -} - fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { let adapters = support::array_at(report, "/external_adapters/adapters")?; let elf = support::find_by_field(adapters, "/adapter_id", "elf_real_world_memory_fixture")?; @@ -411,8 +80,16 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { assert_graph_rag_research_gate_records(ragflow, lightrag, graphrag); assert_graphiti_zep_adapter(graphiti_zep); - assert_graphify_adapter(graphify)?; - assert_graph_rag_representative_scenarios(ragflow, lightrag, graphrag, graphiti_zep, graphify)?; + + graph_rag::assert_graphify_adapter(graphify)?; + graph_rag::assert_graph_rag_representative_scenarios( + ragflow, + lightrag, + graphrag, + graphiti_zep, + graphify, + )?; + assert_letta_core_archival_gate(letta)?; assert_qmd_deep_profile_gate(qmd_deep); @@ -969,447 +646,6 @@ fn assert_graphiti_zep_adapter(adapter: &Value) { ) ); } - -fn assert_graphify_adapter(adapter: &Value) -> Result<()> { - assert_eq!(adapter.pointer("/evidence_class").and_then(Value::as_str), Some("live_real_world")); - assert_eq!(adapter.pointer("/overall_status").and_then(Value::as_str), Some("wrong_result")); - assert_eq!(adapter.pointer("/setup/status").and_then(Value::as_str), Some("pass")); - assert_eq!(adapter.pointer("/run/status").and_then(Value::as_str), Some("pass")); - assert_eq!(adapter.pointer("/result/status").and_then(Value::as_str), Some("wrong_result")); - assert_eq!( - adapter.pointer("/setup/command").and_then(Value::as_str), - Some("cargo make smoke-graphify-docker-graph-report") - ); - assert_eq!( - adapter.pointer("/suites/0/suite_id").and_then(Value::as_str), - Some("knowledge_compilation") - ); - assert_eq!(adapter.pointer("/suites/0/status").and_then(Value::as_str), Some("wrong_result")); - assert_eq!(adapter.pointer("/suites/1/suite_id").and_then(Value::as_str), Some("retrieval")); - assert_eq!(adapter.pointer("/suites/1/status").and_then(Value::as_str), Some("blocked")); - assert_eq!( - adapter.pointer("/execution_metadata/research_depth").and_then(Value::as_str), - Some( - "D1 feasibility verdict plus XY-889 Docker graph/report smoke implementation and XY-900 scored smoke promotion; current Docker validation reaches graphify output and scores the tiny knowledge_compilation job as wrong_result" - ) - ); - - let capabilities = support::array_at(adapter, "/capabilities")?; - let quality = support::find_by_field(capabilities, "/capability", "quality_or_scale_claim")?; - - assert_eq!(quality.pointer("/status").and_then(Value::as_str), Some("not_encoded")); - assert!(support::array_at(adapter, "/notes")?.iter().any(|note| { - note.as_str().is_some_and(|text| text.contains("tiny smoke") && text.contains("non-pass")) - })); - - Ok(()) -} - -fn assert_graph_rag_representative_scenarios( - ragflow: &Value, - lightrag: &Value, - graphrag: &Value, - graphiti_zep: &Value, - graphify: &Value, -) -> Result<()> { - let ragflow_scenarios = support::array_at(ragflow, "/scenarios")?; - let lightrag_scenarios = support::array_at(lightrag, "/scenarios")?; - let graphrag_scenarios = support::array_at(graphrag, "/scenarios")?; - let graphiti_scenarios = support::array_at(graphiti_zep, "/scenarios")?; - let graphify_scenarios = support::array_at(graphify, "/scenarios")?; - let ragflow_chunk = support::find_by_field( - ragflow_scenarios, - "/scenario_id", - "reference_chunk_citation_mapping", - )?; - let lightrag_context = support::find_by_field( - lightrag_scenarios, - "/scenario_id", - "context_source_reference_mapping", - )?; - let graphrag_tables = support::find_by_field( - graphrag_scenarios, - "/scenario_id", - "output_table_citation_mapping", - )?; - let graphiti_temporal = support::find_by_field( - graphiti_scenarios, - "/scenario_id", - "temporal_validity_window_mapping", - )?; - let graphify_lint = - support::find_by_field(graphify_scenarios, "/scenario_id", "graph_report_navigation_lint")?; - - assert_eq!( - ragflow_chunk.pointer("/comparison_outcome").and_then(Value::as_str), - Some("blocked") - ); - assert_eq!(lightrag_context.pointer("/status").and_then(Value::as_str), Some("incomplete")); - assert_eq!( - lightrag_context.pointer("/comparison_outcome").and_then(Value::as_str), - Some("blocked") - ); - assert_eq!( - graphrag_tables.pointer("/artifact").and_then(Value::as_str), - Some( - "apps/elf-eval/fixtures/real_world_external_adapters/graph_rag/graphrag_output_tables_blocked.json" - ) - ); - assert_eq!( - graphiti_temporal.pointer("/comparison_outcome").and_then(Value::as_str), - Some("blocked") - ); - assert_eq!(graphify_lint.pointer("/status").and_then(Value::as_str), Some("wrong_result")); - assert_eq!( - graphify_lint.pointer("/comparison_outcome").and_then(Value::as_str), - Some("not_tested") - ); - assert!( - graphify_lint - .pointer("/evidence") - .and_then(Value::as_str) - .is_some_and(|evidence| evidence.contains("not an ELF victory claim")) - ); - - assert_adapter_matrix_rows( - ragflow_scenarios, - &[ - ("reference_chunk_citation_mapping", "blocked", "blocked"), - ("retrieval_quality_reference_recall", "blocked", "blocked"), - ("navigation_quality_document_chunks", "blocked", "blocked"), - ("answer_faithfulness_reference_chunks", "blocked", "blocked"), - ("stale_source_behavior", "not_encoded", "not_tested"), - ("knowledge_compilation_quality", "not_encoded", "not_tested"), - ], - )?; - assert_adapter_matrix_rows( - lightrag_scenarios, - &[ - ("context_source_reference_mapping", "incomplete", "blocked"), - ("retrieval_quality_context_recall", "incomplete", "blocked"), - ("citation_quality_context_references", "incomplete", "blocked"), - ("navigation_quality_graph_context", "incomplete", "blocked"), - ("answer_faithfulness_context_refs", "incomplete", "blocked"), - ("stale_source_behavior", "not_encoded", "not_tested"), - ("knowledge_compilation_quality", "not_encoded", "not_tested"), - ], - )?; - assert_adapter_matrix_rows( - graphrag_scenarios, - &[ - ("output_table_citation_mapping", "blocked", "blocked"), - ("retrieval_quality_local_search", "not_encoded", "not_tested"), - ("navigation_quality_community_graph", "blocked", "blocked"), - ("answer_faithfulness_output_tables", "blocked", "blocked"), - ("stale_source_behavior", "not_encoded", "not_tested"), - ("graph_summary_synthesis_quality", "not_encoded", "not_tested"), - ], - )?; - - Ok(()) -} - -fn assert_adapter_matrix_rows(scenarios: &[Value], expected: &[(&str, &str, &str)]) -> Result<()> { - for (scenario_id, status, outcome) in expected { - let row = support::find_by_field(scenarios, "/scenario_id", scenario_id)?; - - assert_eq!(row.pointer("/status").and_then(Value::as_str), Some(*status)); - assert_eq!(row.pointer("/comparison_outcome").and_then(Value::as_str), Some(*outcome)); - assert!( - row.pointer("/evidence") - .and_then(Value::as_str) - .is_some_and(|evidence| !evidence.trim().is_empty()) - ); - } - - Ok(()) -} - -#[test] -fn graphify_generated_manifest_keeps_retrieval_unscored() -> Result<()> { - let manifest = serde_json::json!({ - "schema": "elf.real_world_external_adapter_manifest/v1", - "manifest_id": "graphify-generated-manifest-test", - "docker_isolation": { - "default": true, - "compose_file": "docker-compose.baseline.yml", - "runner": "scripts/graphify-docker-graph-report-smoke.py", - "artifact_dir": "tmp/real-world-memory/graphify-smoke", - "host_global_installs_required": false, - "notes": ["Synthetic graphify generated-manifest regression test."] - }, - "adapters": [{ - "adapter_id": "graphify_docker_smoke", - "project": "graphify", - "adapter_kind": "docker_cli_graph_report_smoke", - "evidence_class": "live_real_world", - "docker_default": true, - "host_global_installs_required": false, - "overall_status": "wrong_result", - "setup": { - "status": "pass", - "evidence": "setup evidence", - "command": "cargo make smoke-graphify-docker-graph-report", - "artifact": "tmp/real-world-memory/graphify-smoke/graphify-smoke.json" - }, - "run": { - "status": "pass", - "evidence": "run evidence", - "command": "cargo make smoke-graphify-docker-graph-report", - "artifact": "tmp/real-world-memory/graphify-smoke/summary.json" - }, - "result": { - "status": "wrong_result", - "evidence": "result evidence", - "artifact": "tmp/real-world-memory/graphify-smoke/graphify-report.json" - }, - "capabilities": [{ - "capability": "quality_or_scale_claim", - "status": "not_encoded", - "evidence": "No broad graph quality claim." - }], - "suites": [ - { - "suite_id": "knowledge_compilation", - "status": "wrong_result", - "evidence": "Only the generated graph/report evidence-mapping job is represented." - }, - { - "suite_id": "retrieval", - "status": "blocked", - "evidence": "The smoke uses graphify query output only to support source mapping; broad retrieval quality is not scored." - } - ], - "evidence": [], - "execution_metadata": { - "setup_path": "cargo make smoke-graphify-docker-graph-report", - "runtime_boundary": "Docker-only generated graph/report smoke.", - "resource_expectation": "Tiny generated corpus only.", - "retry_guidance": [], - "sources": [{ - "label": "graphify", - "url": "https://github.com/safishamsi/graphify", - "evidence": "Synthetic generated-manifest regression source." - }], - "research_depth": "Generated smoke manifest path" - }, - "notes": ["tiny smoke non-pass"] - }] - }); - let temp_dir = - env::temp_dir().join(format!("elf-real-world-graphify-manifest-test-{}", process::id())); - - fs::create_dir_all(&temp_dir)?; - - let manifest_path = temp_dir.join("manifest.json"); - let report_path = temp_dir.join("report.json"); - - fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)?; - - let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) - .arg("run") - .arg("--fixtures") - .arg(support::fixture_dir()) - .arg("--out") - .arg(&report_path) - .arg("--external-adapter-manifest") - .arg(&manifest_path) - .output()?; - - assert!( - output.status.success(), - "real_world_job runner failed: {}", - String::from_utf8_lossy(&output.stderr), - ); - - let report: Value = serde_json::from_slice(&fs::read(&report_path)?)?; - let adapters = support::array_at(&report, "/external_adapters/adapters")?; - let graphify = support::find_by_field(adapters, "/adapter_id", "graphify_docker_smoke")?; - let suites = support::array_at(graphify, "/suites")?; - let retrieval = support::find_by_field(suites, "/suite_id", "retrieval")?; - - assert_eq!(retrieval.pointer("/status").and_then(Value::as_str), Some("blocked")); - assert!( - retrieval - .pointer("/evidence") - .and_then(Value::as_str) - .is_some_and(|text| { text.contains("broad retrieval quality is not scored") }) - ); - - Ok(()) -} - -#[test] -fn graph_rag_representative_fixtures_report_typed_non_pass_states() -> Result<()> { - let report = support::run_json_report_from(support::graph_rag_external_fixture_dir())?; - - assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(5)); - assert_eq!(report.pointer("/summary/pass").and_then(Value::as_u64), Some(0)); - assert_eq!(report.pointer("/summary/wrong_result").and_then(Value::as_u64), Some(1)); - assert_eq!(report.pointer("/summary/incomplete").and_then(Value::as_u64), Some(1)); - assert_eq!(report.pointer("/summary/blocked").and_then(Value::as_u64), Some(3)); - assert_eq!( - report.pointer("/summary/knowledge/citation_coverage").and_then(Value::as_f64), - Some(0.667) - ); - assert_eq!( - report.pointer("/summary/knowledge/stale_claim_detection").and_then(Value::as_f64), - Some(0.0) - ); - assert_eq!( - report.pointer("/summary/knowledge/unsupported_summary_count").and_then(Value::as_u64), - Some(1) - ); - assert_eq!( - report.pointer("/summary/temporal_validity_not_encoded_count").and_then(Value::as_u64), - Some(1) - ); - assert_eq!( - report.pointer("/summary/trace_explainability_count").and_then(Value::as_u64), - Some(1) - ); - - let jobs = support::array_at(&report, "/jobs")?; - let ragflow = - support::find_by_field(jobs, "/job_id", "graph-rag-ragflow-reference-chunks-001")?; - let lightrag = - support::find_by_field(jobs, "/job_id", "graph-rag-lightrag-context-sources-001")?; - let graphrag = support::find_by_field(jobs, "/job_id", "graph-rag-graphrag-output-tables-001")?; - let graphiti = - support::find_by_field(jobs, "/job_id", "graph-rag-graphiti-temporal-validity-001")?; - let graphify = support::find_by_field(jobs, "/job_id", "graph-rag-graphify-graph-report-001")?; - - assert_eq!(ragflow.pointer("/status").and_then(Value::as_str), Some("blocked")); - assert_eq!(lightrag.pointer("/status").and_then(Value::as_str), Some("incomplete")); - assert_eq!(graphrag.pointer("/status").and_then(Value::as_str), Some("blocked")); - assert_eq!(graphiti.pointer("/status").and_then(Value::as_str), Some("blocked")); - assert_eq!(graphify.pointer("/status").and_then(Value::as_str), Some("wrong_result")); - assert_eq!( - graphify.pointer("/knowledge/stale_claim_detection").and_then(Value::as_f64), - Some(0.0) - ); - assert_eq!( - graphify.pointer("/knowledge/unsupported_summary_count").and_then(Value::as_u64), - Some(1) - ); - assert_eq!( - graphiti.pointer("/evolution/temporal_validity_not_encoded").and_then(Value::as_bool), - Some(true) - ); - assert_eq!( - graphiti.pointer("/trace_explainability/failure_stage").and_then(Value::as_str), - Some("graphiti.provider_boundary") - ); - assert!(support::array_contains_str( - graphiti, - "/produced_evidence", - "graphiti-current-fact-contract" - )?); - assert!(support::array_contains_str( - graphiti, - "/produced_evidence", - "graphiti-historical-fact-contract" - )?); - assert!(support::array_contains_str( - graphiti, - "/produced_evidence", - "graphiti-provider-boundary" - )?); - assert!(support::array_contains_str( - graphify, - "/produced_evidence", - "graphify-source-location-output" - )?); - - Ok(()) -} - -#[test] -fn external_adapter_manifest_rejects_unmeasured_win_loss_scenario_outcomes() -> Result<()> { - let output = support::run_external_manifest_with_letta_attachment_mutation( - "invalid-scenario-outcome-test", - |scenario| { - support::set_json_pointer(scenario, "/status", serde_json::json!("not_encoded"))?; - - support::set_json_pointer(scenario, "/comparison_outcome", serde_json::json!("win")) - }, - )?; - - assert!(!output.status.success(), "invalid scenario outcome unexpectedly passed"); - assert!( - String::from_utf8_lossy(&output.stderr).contains("not_encoded status with win outcome") - ); - - Ok(()) -} - -#[test] -fn external_adapter_manifest_rejects_unmeasured_win_loss_scenario_positions() -> Result<()> { - let output = support::run_external_manifest_with_letta_attachment_mutation( - "invalid-scenario-position-test", - |scenario| { - support::set_json_pointer(scenario, "/status", serde_json::json!("not_encoded"))?; - support::set_json_pointer(scenario, "/elf_position", serde_json::json!("wins"))?; - - support::set_json_pointer( - scenario, - "/comparison_outcome", - serde_json::json!("not_tested"), - ) - }, - )?; - - assert!(!output.status.success(), "invalid scenario position unexpectedly passed"); - assert!( - String::from_utf8_lossy(&output.stderr).contains("not_encoded status with wins position") - ); - - Ok(()) -} - -#[test] -fn external_adapter_manifest_rejects_blocked_status_without_blocked_outcome() -> Result<()> { - let output = support::run_external_manifest_scenario_mutation( - "invalid-blocked-scenario-outcome-test", - "letta_research_gate", - "stale_core_detection", - |scenario| { - scenario - .as_object_mut() - .ok_or_else(|| eyre::eyre!("scenario is not an object"))? - .remove("comparison_outcome"); - - Ok(()) - }, - )?; - - assert!(!output.status.success(), "invalid blocked scenario unexpectedly passed"); - assert!( - String::from_utf8_lossy(&output.stderr) - .contains("blocked status without blocked comparison outcome") - ); - - Ok(()) -} - -#[test] -fn external_adapter_manifest_rejects_conflicting_scenario_position_and_outcome() -> Result<()> { - let output = support::run_external_manifest_with_letta_attachment_mutation( - "invalid-scenario-position-outcome-test", - |scenario| { - support::set_json_pointer(scenario, "/status", serde_json::json!("pass"))?; - support::set_json_pointer(scenario, "/elf_position", serde_json::json!("ties"))?; - - support::set_json_pointer(scenario, "/comparison_outcome", serde_json::json!("loss")) - }, - )?; - - assert!(!output.status.success(), "conflicting scenario unexpectedly passed"); - assert!(String::from_utf8_lossy(&output.stderr).contains("ties position with loss outcome")); - - Ok(()) -} - fn assert_live_sweep_record(adapter: &Value, production_ops_status: &str) -> Result<()> { let suites = support::array_at(adapter, "/suites")?; let capabilities = support::array_at(adapter, "/capabilities")?; diff --git a/apps/elf-eval/tests/real_world_job_benchmark/graph_rag.rs b/apps/elf-eval/tests/real_world_job_benchmark/graph_rag.rs new file mode 100644 index 00000000..d60b7975 --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/graph_rag.rs @@ -0,0 +1,363 @@ +use std::{ + env, fs, + process::{self, Command}, +}; + +use color_eyre::Result; +use serde_json::Value; + +use crate::support; + +pub(super) fn assert_graphify_adapter(adapter: &Value) -> Result<()> { + assert_eq!(adapter.pointer("/evidence_class").and_then(Value::as_str), Some("live_real_world")); + assert_eq!(adapter.pointer("/overall_status").and_then(Value::as_str), Some("wrong_result")); + assert_eq!(adapter.pointer("/setup/status").and_then(Value::as_str), Some("pass")); + assert_eq!(adapter.pointer("/run/status").and_then(Value::as_str), Some("pass")); + assert_eq!(adapter.pointer("/result/status").and_then(Value::as_str), Some("wrong_result")); + assert_eq!( + adapter.pointer("/setup/command").and_then(Value::as_str), + Some("cargo make smoke-graphify-docker-graph-report") + ); + assert_eq!( + adapter.pointer("/suites/0/suite_id").and_then(Value::as_str), + Some("knowledge_compilation") + ); + assert_eq!(adapter.pointer("/suites/0/status").and_then(Value::as_str), Some("wrong_result")); + assert_eq!(adapter.pointer("/suites/1/suite_id").and_then(Value::as_str), Some("retrieval")); + assert_eq!(adapter.pointer("/suites/1/status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + adapter.pointer("/execution_metadata/research_depth").and_then(Value::as_str), + Some( + "D1 feasibility verdict plus XY-889 Docker graph/report smoke implementation and XY-900 scored smoke promotion; current Docker validation reaches graphify output and scores the tiny knowledge_compilation job as wrong_result" + ) + ); + + let capabilities = support::array_at(adapter, "/capabilities")?; + let quality = support::find_by_field(capabilities, "/capability", "quality_or_scale_claim")?; + + assert_eq!(quality.pointer("/status").and_then(Value::as_str), Some("not_encoded")); + assert!(support::array_at(adapter, "/notes")?.iter().any(|note| { + note.as_str().is_some_and(|text| text.contains("tiny smoke") && text.contains("non-pass")) + })); + + Ok(()) +} + +pub(super) fn assert_graph_rag_representative_scenarios( + ragflow: &Value, + lightrag: &Value, + graphrag: &Value, + graphiti_zep: &Value, + graphify: &Value, +) -> Result<()> { + let ragflow_scenarios = support::array_at(ragflow, "/scenarios")?; + let lightrag_scenarios = support::array_at(lightrag, "/scenarios")?; + let graphrag_scenarios = support::array_at(graphrag, "/scenarios")?; + let graphiti_scenarios = support::array_at(graphiti_zep, "/scenarios")?; + let graphify_scenarios = support::array_at(graphify, "/scenarios")?; + let ragflow_chunk = support::find_by_field( + ragflow_scenarios, + "/scenario_id", + "reference_chunk_citation_mapping", + )?; + let lightrag_context = support::find_by_field( + lightrag_scenarios, + "/scenario_id", + "context_source_reference_mapping", + )?; + let graphrag_tables = support::find_by_field( + graphrag_scenarios, + "/scenario_id", + "output_table_citation_mapping", + )?; + let graphiti_temporal = support::find_by_field( + graphiti_scenarios, + "/scenario_id", + "temporal_validity_window_mapping", + )?; + let graphify_lint = + support::find_by_field(graphify_scenarios, "/scenario_id", "graph_report_navigation_lint")?; + + assert_eq!( + ragflow_chunk.pointer("/comparison_outcome").and_then(Value::as_str), + Some("blocked") + ); + assert_eq!(lightrag_context.pointer("/status").and_then(Value::as_str), Some("incomplete")); + assert_eq!( + lightrag_context.pointer("/comparison_outcome").and_then(Value::as_str), + Some("blocked") + ); + assert_eq!( + graphrag_tables.pointer("/artifact").and_then(Value::as_str), + Some( + "apps/elf-eval/fixtures/real_world_external_adapters/graph_rag/graphrag_output_tables_blocked.json" + ) + ); + assert_eq!( + graphiti_temporal.pointer("/comparison_outcome").and_then(Value::as_str), + Some("blocked") + ); + assert_eq!(graphify_lint.pointer("/status").and_then(Value::as_str), Some("wrong_result")); + assert_eq!( + graphify_lint.pointer("/comparison_outcome").and_then(Value::as_str), + Some("not_tested") + ); + assert!( + graphify_lint + .pointer("/evidence") + .and_then(Value::as_str) + .is_some_and(|evidence| evidence.contains("not an ELF victory claim")) + ); + + assert_adapter_matrix_rows( + ragflow_scenarios, + &[ + ("reference_chunk_citation_mapping", "blocked", "blocked"), + ("retrieval_quality_reference_recall", "blocked", "blocked"), + ("navigation_quality_document_chunks", "blocked", "blocked"), + ("answer_faithfulness_reference_chunks", "blocked", "blocked"), + ("stale_source_behavior", "not_encoded", "not_tested"), + ("knowledge_compilation_quality", "not_encoded", "not_tested"), + ], + )?; + assert_adapter_matrix_rows( + lightrag_scenarios, + &[ + ("context_source_reference_mapping", "incomplete", "blocked"), + ("retrieval_quality_context_recall", "incomplete", "blocked"), + ("citation_quality_context_references", "incomplete", "blocked"), + ("navigation_quality_graph_context", "incomplete", "blocked"), + ("answer_faithfulness_context_refs", "incomplete", "blocked"), + ("stale_source_behavior", "not_encoded", "not_tested"), + ("knowledge_compilation_quality", "not_encoded", "not_tested"), + ], + )?; + assert_adapter_matrix_rows( + graphrag_scenarios, + &[ + ("output_table_citation_mapping", "blocked", "blocked"), + ("retrieval_quality_local_search", "not_encoded", "not_tested"), + ("navigation_quality_community_graph", "blocked", "blocked"), + ("answer_faithfulness_output_tables", "blocked", "blocked"), + ("stale_source_behavior", "not_encoded", "not_tested"), + ("graph_summary_synthesis_quality", "not_encoded", "not_tested"), + ], + )?; + + Ok(()) +} + +fn assert_adapter_matrix_rows(scenarios: &[Value], expected: &[(&str, &str, &str)]) -> Result<()> { + for (scenario_id, status, outcome) in expected { + let row = support::find_by_field(scenarios, "/scenario_id", scenario_id)?; + + assert_eq!(row.pointer("/status").and_then(Value::as_str), Some(*status)); + assert_eq!(row.pointer("/comparison_outcome").and_then(Value::as_str), Some(*outcome)); + assert!( + row.pointer("/evidence") + .and_then(Value::as_str) + .is_some_and(|evidence| !evidence.trim().is_empty()) + ); + } + + Ok(()) +} + +#[test] +fn graphify_generated_manifest_keeps_retrieval_unscored() -> Result<()> { + let manifest = serde_json::json!({ + "schema": "elf.real_world_external_adapter_manifest/v1", + "manifest_id": "graphify-generated-manifest-test", + "docker_isolation": { + "default": true, + "compose_file": "docker-compose.baseline.yml", + "runner": "scripts/graphify-docker-graph-report-smoke.py", + "artifact_dir": "tmp/real-world-memory/graphify-smoke", + "host_global_installs_required": false, + "notes": ["Synthetic graphify generated-manifest regression test."] + }, + "adapters": [{ + "adapter_id": "graphify_docker_smoke", + "project": "graphify", + "adapter_kind": "docker_cli_graph_report_smoke", + "evidence_class": "live_real_world", + "docker_default": true, + "host_global_installs_required": false, + "overall_status": "wrong_result", + "setup": { + "status": "pass", + "evidence": "setup evidence", + "command": "cargo make smoke-graphify-docker-graph-report", + "artifact": "tmp/real-world-memory/graphify-smoke/graphify-smoke.json" + }, + "run": { + "status": "pass", + "evidence": "run evidence", + "command": "cargo make smoke-graphify-docker-graph-report", + "artifact": "tmp/real-world-memory/graphify-smoke/summary.json" + }, + "result": { + "status": "wrong_result", + "evidence": "result evidence", + "artifact": "tmp/real-world-memory/graphify-smoke/graphify-report.json" + }, + "capabilities": [{ + "capability": "quality_or_scale_claim", + "status": "not_encoded", + "evidence": "No broad graph quality claim." + }], + "suites": [ + { + "suite_id": "knowledge_compilation", + "status": "wrong_result", + "evidence": "Only the generated graph/report evidence-mapping job is represented." + }, + { + "suite_id": "retrieval", + "status": "blocked", + "evidence": "The smoke uses graphify query output only to support source mapping; broad retrieval quality is not scored." + } + ], + "evidence": [], + "execution_metadata": { + "setup_path": "cargo make smoke-graphify-docker-graph-report", + "runtime_boundary": "Docker-only generated graph/report smoke.", + "resource_expectation": "Tiny generated corpus only.", + "retry_guidance": [], + "sources": [{ + "label": "graphify", + "url": "https://github.com/safishamsi/graphify", + "evidence": "Synthetic generated-manifest regression source." + }], + "research_depth": "Generated smoke manifest path" + }, + "notes": ["tiny smoke non-pass"] + }] + }); + let temp_dir = + env::temp_dir().join(format!("elf-real-world-graphify-manifest-test-{}", process::id())); + + fs::create_dir_all(&temp_dir)?; + + let manifest_path = temp_dir.join("manifest.json"); + let report_path = temp_dir.join("report.json"); + + fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)?; + + let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) + .arg("run") + .arg("--fixtures") + .arg(support::fixture_dir()) + .arg("--out") + .arg(&report_path) + .arg("--external-adapter-manifest") + .arg(&manifest_path) + .output()?; + + assert!( + output.status.success(), + "real_world_job runner failed: {}", + String::from_utf8_lossy(&output.stderr), + ); + + let report: Value = serde_json::from_slice(&fs::read(&report_path)?)?; + let adapters = support::array_at(&report, "/external_adapters/adapters")?; + let graphify = support::find_by_field(adapters, "/adapter_id", "graphify_docker_smoke")?; + let suites = support::array_at(graphify, "/suites")?; + let retrieval = support::find_by_field(suites, "/suite_id", "retrieval")?; + + assert_eq!(retrieval.pointer("/status").and_then(Value::as_str), Some("blocked")); + assert!( + retrieval + .pointer("/evidence") + .and_then(Value::as_str) + .is_some_and(|text| { text.contains("broad retrieval quality is not scored") }) + ); + + Ok(()) +} + +#[test] +fn graph_rag_representative_fixtures_report_typed_non_pass_states() -> Result<()> { + let report = support::run_json_report_from(support::graph_rag_external_fixture_dir())?; + + assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(5)); + assert_eq!(report.pointer("/summary/pass").and_then(Value::as_u64), Some(0)); + assert_eq!(report.pointer("/summary/wrong_result").and_then(Value::as_u64), Some(1)); + assert_eq!(report.pointer("/summary/incomplete").and_then(Value::as_u64), Some(1)); + assert_eq!(report.pointer("/summary/blocked").and_then(Value::as_u64), Some(3)); + assert_eq!( + report.pointer("/summary/knowledge/citation_coverage").and_then(Value::as_f64), + Some(0.667) + ); + assert_eq!( + report.pointer("/summary/knowledge/stale_claim_detection").and_then(Value::as_f64), + Some(0.0) + ); + assert_eq!( + report.pointer("/summary/knowledge/unsupported_summary_count").and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report.pointer("/summary/temporal_validity_not_encoded_count").and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report.pointer("/summary/trace_explainability_count").and_then(Value::as_u64), + Some(1) + ); + + let jobs = support::array_at(&report, "/jobs")?; + let ragflow = + support::find_by_field(jobs, "/job_id", "graph-rag-ragflow-reference-chunks-001")?; + let lightrag = + support::find_by_field(jobs, "/job_id", "graph-rag-lightrag-context-sources-001")?; + let graphrag = support::find_by_field(jobs, "/job_id", "graph-rag-graphrag-output-tables-001")?; + let graphiti = + support::find_by_field(jobs, "/job_id", "graph-rag-graphiti-temporal-validity-001")?; + let graphify = support::find_by_field(jobs, "/job_id", "graph-rag-graphify-graph-report-001")?; + + assert_eq!(ragflow.pointer("/status").and_then(Value::as_str), Some("blocked")); + assert_eq!(lightrag.pointer("/status").and_then(Value::as_str), Some("incomplete")); + assert_eq!(graphrag.pointer("/status").and_then(Value::as_str), Some("blocked")); + assert_eq!(graphiti.pointer("/status").and_then(Value::as_str), Some("blocked")); + assert_eq!(graphify.pointer("/status").and_then(Value::as_str), Some("wrong_result")); + assert_eq!( + graphify.pointer("/knowledge/stale_claim_detection").and_then(Value::as_f64), + Some(0.0) + ); + assert_eq!( + graphify.pointer("/knowledge/unsupported_summary_count").and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + graphiti.pointer("/evolution/temporal_validity_not_encoded").and_then(Value::as_bool), + Some(true) + ); + assert_eq!( + graphiti.pointer("/trace_explainability/failure_stage").and_then(Value::as_str), + Some("graphiti.provider_boundary") + ); + assert!(support::array_contains_str( + graphiti, + "/produced_evidence", + "graphiti-current-fact-contract" + )?); + assert!(support::array_contains_str( + graphiti, + "/produced_evidence", + "graphiti-historical-fact-contract" + )?); + assert!(support::array_contains_str( + graphiti, + "/produced_evidence", + "graphiti-provider-boundary" + )?); + assert!(support::array_contains_str( + graphify, + "/produced_evidence", + "graphify-source-location-output" + )?); + + Ok(()) +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/letta_core.rs b/apps/elf-eval/tests/real_world_job_benchmark/letta_core.rs new file mode 100644 index 00000000..58a2e866 --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/letta_core.rs @@ -0,0 +1,105 @@ +use std::fs; + +use color_eyre::Result; +use serde_json::Value; + +use crate::support; + +#[test] +fn letta_core_archive_export_readback_report_preserves_blocked_gates() -> Result<()> { + let report = serde_json::from_str::(&fs::read_to_string( + support::letta_core_archive_export_readback_report_json_path()?, + )?)?; + let markdown = + fs::read_to_string(support::letta_core_archive_export_readback_report_markdown_path()?)?; + let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; + let readme = fs::read_to_string(support::readme_path()?)?; + + assert_eq!( + report.pointer("/schema").and_then(Value::as_str), + Some("elf.letta_core_archive_export_readback_summary/v1") + ); + assert_eq!( + report.pointer("/adapter_id").and_then(Value::as_str), + Some("letta_core_archive_export_readback") + ); + assert_eq!( + report.pointer("/materialization/status/failure_class").and_then(Value::as_str), + Some("letta_live_run_disabled") + ); + assert_eq!( + report.pointer("/materialization/status/overall").and_then(Value::as_str), + Some("blocked") + ); + assert_eq!( + report.pointer("/materialization/scored_benchmark/status").and_then(Value::as_str), + Some("blocked") + ); + assert_eq!( + report.pointer("/materialization/scored_benchmark/counts/blocked").and_then(Value::as_u64), + Some(6) + ); + assert_eq!( + report.pointer("/materialization/scored_benchmark/counts/pass").and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/materialization/scored_benchmark/counts/wrong_result") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/materialization/scored_benchmark/evidence_coverage") + .and_then(Value::as_f64), + Some(1.0) + ); + assert_eq!( + report + .pointer("/materialization/benchmark_input/core_blocks") + .and_then(Value::as_array) + .map(Vec::len), + Some(9) + ); + assert_eq!( + report + .pointer("/materialization/benchmark_input/archival_passages") + .and_then(Value::as_array) + .map(Vec::len), + Some(6) + ); + assert_eq!( + report + .pointer("/materialization/evidence_mapping/expected_evidence_ids") + .and_then(Value::as_array) + .map(Vec::len), + Some(14) + ); + assert_eq!( + report + .pointer("/materialization/evidence_mapping/mapped_evidence_ids") + .and_then(Value::as_array) + .map(Vec::len), + Some(0) + ); + assert_eq!( + report + .pointer("/materialization/improvement_regression_readback/judgment") + .and_then(Value::as_str), + Some("unchanged") + ); + assert!(support::array_contains_str( + &report, + "/materialization/claim_boundaries/not_allowed", + "Do not claim ELF beats Letta on core-vs-archival memory from fixture-only ELF evidence." + )?); + assert!(markdown.contains("The Letta follow-up is now reproducible")); + assert!(markdown.contains("6 typed blocked")); + assert!(markdown.contains("competitive status is unchanged")); + assert!(benchmarking_index.contains("2026-06-19-letta-core-archive-export-readback-report.md")); + assert!(readme.contains("Letta core/archive materialization after XY-984")); + assert!(readme.contains("smoke-letta-core-archive-export-readback")); + + Ok(()) +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/loss_summary.rs b/apps/elf-eval/tests/real_world_job_benchmark/loss_summary.rs new file mode 100644 index 00000000..199831c8 --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/loss_summary.rs @@ -0,0 +1,97 @@ +use std::{ + env, fs, + path::Path, + process::{self, Command}, +}; + +use color_eyre::{Result, eyre}; +use serde_json::Value; + +use crate::support; + +#[test] +fn external_adapter_run_summarizes_nonzero_scenario_losses() -> Result<()> { + let manifest_path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("fixtures") + .join("real_world_external_adapters") + .join("memory_projects_manifest.json"); + let mut manifest = serde_json::from_str::(&fs::read_to_string(manifest_path)?)?; + let adapters = manifest + .pointer_mut("/adapters") + .and_then(Value::as_array_mut) + .ok_or_else(|| eyre::eyre!("missing manifest adapters"))?; + let adapter = adapters + .iter_mut() + .find(|adapter| { + adapter.pointer("/adapter_id").and_then(Value::as_str) + == Some("agentmemory_live_baseline") + }) + .ok_or_else(|| eyre::eyre!("missing agentmemory adapter"))?; + + support::set_json_pointer(adapter, "/scenarios/0/elf_position", serde_json::json!("loses"))?; + support::set_json_pointer( + adapter, + "/scenarios/0/comparison_outcome", + serde_json::json!("loss"), + )?; + + let temp_dir = + env::temp_dir().join(format!("elf-real-world-loss-manifest-test-{}", process::id())); + + fs::create_dir_all(&temp_dir)?; + + let manifest_path = temp_dir.join("memory_projects_manifest.json"); + + fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)?; + + let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) + .arg("run") + .arg("--fixtures") + .arg(support::fixture_dir()) + .arg("--external-adapter-manifest") + .arg(&manifest_path) + .output()?; + + assert!( + output.status.success(), + "real_world_job runner failed: {}", + String::from_utf8_lossy(&output.stderr), + ); + + let report = serde_json::from_slice::(&output.stdout)?; + + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/loses") + .and_then(Value::as_u64), + Some(2) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/untested") + .and_then(Value::as_u64), + Some(52) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_outcome_counts/loss") + .and_then(Value::as_u64), + Some(2) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_outcome_counts/not_tested") + .and_then(Value::as_u64), + Some(18) + ); + + let adapters = support::array_at(&report, "/external_adapters/adapters")?; + let agentmemory = support::find_by_field(adapters, "/adapter_id", "agentmemory_live_baseline")?; + + assert_eq!( + agentmemory.pointer("/scenarios/0/elf_position").and_then(Value::as_str), + Some("loses") + ); + + Ok(()) +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/manifest_summary.rs b/apps/elf-eval/tests/real_world_job_benchmark/manifest_summary.rs new file mode 100644 index 00000000..6ec85cb5 --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/manifest_summary.rs @@ -0,0 +1,245 @@ +use serde_json::Value; + +pub(super) fn assert_external_adapter_manifest_summary(report: &Value) { + assert_eq!( + report.pointer("/external_adapters/schema").and_then(Value::as_str), + Some("elf.real_world_external_adapter_report/v1") + ); + assert_eq!( + report.pointer("/external_adapters/manifest_id").and_then(Value::as_str), + Some( + "real-world-memory-project-adapters-2026-06-11-first-generation-continuity-source-store" + ) + ); + assert_eq!( + report.pointer("/external_adapters/docker_isolation/default").and_then(Value::as_bool), + Some(true) + ); + assert_eq!( + report + .pointer("/external_adapters/docker_isolation/host_global_installs_required") + .and_then(Value::as_bool), + Some(false) + ); + assert_eq!( + report.pointer("/external_adapters/summary/adapter_count").and_then(Value::as_u64), + Some(26) + ); + assert_eq!( + report.pointer("/external_adapters/summary/external_project_count").and_then(Value::as_u64), + Some(19) + ); + assert_eq!( + report.pointer("/external_adapters/summary/fixture_backed_count").and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/live_baseline_only_count") + .and_then(Value::as_u64), + Some(6) + ); + assert_eq!( + report.pointer("/external_adapters/summary/live_real_world_count").and_then(Value::as_u64), + Some(5) + ); + assert_eq!( + report.pointer("/external_adapters/summary/research_gate_count").and_then(Value::as_u64), + Some(14) + ); + + assert_external_adapter_manifest_status_summary(report); + assert_external_adapter_manifest_scenario_summary(report); +} + +fn assert_external_adapter_manifest_status_summary(report: &Value) { + assert_eq!( + report + .pointer("/external_adapters/summary/overall_status_counts/pass") + .and_then(Value::as_u64), + Some(4) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/overall_status_counts/wrong_result") + .and_then(Value::as_u64), + Some(6) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/overall_status_counts/lifecycle_fail") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/overall_status_counts/incomplete") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/overall_status_counts/blocked") + .and_then(Value::as_u64), + Some(10) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/overall_status_counts/not_encoded") + .and_then(Value::as_u64), + Some(5) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/capability_status_counts/mocked") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/capability_status_counts/unsupported") + .and_then(Value::as_u64), + Some(6) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/suite_status_counts/blocked") + .and_then(Value::as_u64), + Some(29) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/suite_status_counts/pass") + .and_then(Value::as_u64), + Some(27) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/suite_status_counts/incomplete") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/suite_status_counts/not_encoded") + .and_then(Value::as_u64), + Some(37) + ); +} + +fn assert_external_adapter_manifest_scenario_summary(report: &Value) { + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/real") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/mocked") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/unsupported") + .and_then(Value::as_u64), + Some(3) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/blocked") + .and_then(Value::as_u64), + Some(24) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/incomplete") + .and_then(Value::as_u64), + Some(5) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/wrong_result") + .and_then(Value::as_u64), + Some(6) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/lifecycle_fail") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/pass") + .and_then(Value::as_u64), + Some(23) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/not_encoded") + .and_then(Value::as_u64), + Some(13) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/wins") + .and_then(Value::as_u64), + Some(10) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/ties") + .and_then(Value::as_u64), + Some(11) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/loses") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/untested") + .and_then(Value::as_u64), + Some(53) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_outcome_counts/win") + .and_then(Value::as_u64), + Some(10) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_outcome_counts/tie") + .and_then(Value::as_u64), + Some(11) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_outcome_counts/loss") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_outcome_counts/not_tested") + .and_then(Value::as_u64), + Some(19) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_outcome_counts/blocked") + .and_then(Value::as_u64), + Some(29) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_outcome_counts/non_goal") + .and_then(Value::as_u64), + Some(5) + ); +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/openviking_trajectory.rs b/apps/elf-eval/tests/real_world_job_benchmark/openviking_trajectory.rs new file mode 100644 index 00000000..170a6a5e --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/openviking_trajectory.rs @@ -0,0 +1,193 @@ +use std::fs; + +use color_eyre::{Result, eyre}; +use serde_json::Value; + +use crate::support; + +#[test] +fn openviking_trajectory_materialization_report_preserves_blocked_gates() -> Result<()> { + let report = serde_json::from_str::(&fs::read_to_string( + support::openviking_trajectory_materialization_report_json_path()?, + )?)?; + let markdown = + fs::read_to_string(support::openviking_trajectory_materialization_report_markdown_path()?)?; + let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; + let readme = fs::read_to_string(support::readme_path()?)?; + + assert_openviking_trajectory_materialization_summary(&report)?; + assert_openviking_trajectory_materialization_command(&report)?; + assert_openviking_trajectory_materialization_scenarios(&report)?; + assert_openviking_trajectory_materialization_boundaries(&report)?; + assert_openviking_trajectory_materialization_markdown_and_indexes( + &markdown, + &benchmarking_index, + &readme, + ); + + Ok(()) +} + +fn assert_openviking_trajectory_materialization_summary(report: &Value) -> Result<()> { + assert_eq!( + report.pointer("/schema").and_then(Value::as_str), + Some("elf.openviking_trajectory_materialization_report/v1") + ); + assert_eq!(report.pointer("/authority").and_then(Value::as_str), Some("XY-983")); + assert_eq!( + report.pointer("/summary/overall_judgment").and_then(Value::as_str), + Some("materialized_blocked_context_trajectory_evidence") + ); + assert_eq!( + report.pointer("/summary/broader_superiority").and_then(Value::as_str), + Some("not_proven") + ); + assert_eq!(report.pointer("/summary/blockers_removed_count").and_then(Value::as_u64), Some(0)); + assert_eq!(report.pointer("/summary/blocked_scenario_count").and_then(Value::as_u64), Some(3)); + assert_eq!(report.pointer("/summary/pass_count").and_then(Value::as_u64), Some(0)); + assert_eq!(report.pointer("/summary/wrong_result_count").and_then(Value::as_u64), Some(0)); + assert_eq!( + report.pointer("/summary/regressed_scenario_count").and_then(Value::as_u64), + Some(0) + ); + assert_eq!(report.pointer("/summary/evidence_coverage").and_then(Value::as_f64), Some(1.0)); + assert!(support::array_contains_str( + report, + "/summary/unsupported_claims_rejected", + "ELF does not beat OpenViking staged retrieval trajectory from fixture-only blocked rows." + )?); + + Ok(()) +} + +fn assert_openviking_trajectory_materialization_command(report: &Value) -> Result<()> { + let command = support::find_by_field( + support::array_at(report, "/commands")?, + "/command", + "cargo make real-world-memory-context-trajectory", + )?; + let summary = + command.pointer("/summary").ok_or_else(|| eyre::eyre!("missing command summary"))?; + + assert_eq!(command.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!( + command.pointer("/artifact_json").and_then(Value::as_str), + Some("tmp/real-world-memory/context-trajectory/report.json") + ); + assert_eq!(summary.pointer("/job_count").and_then(Value::as_u64), Some(3)); + assert_eq!(summary.pointer("/pass").and_then(Value::as_u64), Some(0)); + assert_eq!(summary.pointer("/wrong_result").and_then(Value::as_u64), Some(0)); + assert_eq!(summary.pointer("/blocked").and_then(Value::as_u64), Some(3)); + assert_eq!(summary.pointer("/evidence_covered_count").and_then(Value::as_u64), Some(9)); + assert_eq!(summary.pointer("/source_ref_covered_count").and_then(Value::as_u64), Some(9)); + assert_eq!(summary.pointer("/quote_covered_count").and_then(Value::as_u64), Some(9)); + + Ok(()) +} + +fn assert_openviking_trajectory_materialization_scenarios(report: &Value) -> Result<()> { + let scenarios = support::array_at(report, "/scenario_materialization")?; + let staged = support::find_by_field( + scenarios, + "/scenario_id", + "openviking_staged_retrieval_trajectory", + )?; + let hierarchy = + support::find_by_field(scenarios, "/scenario_id", "openviking_hierarchy_selection")?; + let recursive = support::find_by_field( + scenarios, + "/scenario_id", + "openviking_recursive_context_expansion", + )?; + + assert_eq!(scenarios.len(), 3); + + for scenario in [staged, hierarchy, recursive] { + assert_eq!(scenario.pointer("/previous_status").and_then(Value::as_str), Some("blocked")); + assert_eq!(scenario.pointer("/current_status").and_then(Value::as_str), Some("blocked")); + assert_eq!(scenario.pointer("/judgment").and_then(Value::as_str), Some("unchanged")); + } + + assert!(support::array_contains_str( + staged, + "/produced_evidence", + "openviking-evidence-id-output-contract" + )?); + assert!(support::array_contains_str( + hierarchy, + "/produced_evidence", + "hierarchy-selection-output-contract" + )?); + assert!(support::array_contains_str( + recursive, + "/produced_evidence", + "recursive-expansion-output-contract" + )?); + assert_eq!( + staged.pointer("/claim_boundary").and_then(Value::as_str), + Some( + "No ELF win, tie, or loss is allowed until both systems publish comparable stage artifacts for the same context-trajectory scenario." + ) + ); + assert_eq!( + hierarchy.pointer("/blocker").and_then(Value::as_str), + Some("selected_hierarchy_nodes_and_evidence_ids_missing") + ); + assert_eq!( + recursive.pointer("/blocker").and_then(Value::as_str), + Some("expansion_paths_and_same_corpus_evidence_ids_missing") + ); + + Ok(()) +} + +fn assert_openviking_trajectory_materialization_boundaries(report: &Value) -> Result<()> { + assert_eq!( + report.pointer("/improvement_regression_readback/improved").and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report.pointer("/improvement_regression_readback/blocked").and_then(Value::as_u64), + Some(3) + ); + assert!(support::array_contains_str( + report, + "/claim_boundaries/allowed", + "The context-trajectory slice is now reproducible through cargo make real-world-memory-context-trajectory." + )?); + assert!(support::array_contains_str( + report, + "/claim_boundaries/not_allowed", + "Do not claim ELF beats OpenViking on staged retrieval trajectory." + )?); + assert!(support::array_contains_str( + report, + "/next_optimization_direction/required_fields", + "expansion_path" + )?); + assert_eq!( + report.pointer("/next_optimization_direction/non_goal").and_then(Value::as_str), + Some( + "No ELF product change or superiority claim is authorized by this materialization-only report." + ) + ); + + Ok(()) +} + +fn assert_openviking_trajectory_materialization_markdown_and_indexes( + markdown: &str, + benchmarking_index: &str, + readme: &str, +) { + assert!(markdown.contains("The OpenViking trajectory follow-up is now materialized")); + assert!(markdown.contains("3 encoded jobs, 0 pass, 3 blocked, 9/9 evidence coverage")); + assert!(markdown.contains("Do not claim ELF beats OpenViking on staged retrieval trajectory.")); + assert!(markdown.contains("OpenViking context-trajectory job can move from `blocked`")); + assert!( + benchmarking_index.contains("2026-06-19-openviking-trajectory-materialization-report.md") + ); + assert!(readme.contains("OpenViking Trajectory Materialization Report - June 19, 2026")); + assert!(readme.contains("cargo make real-world-memory-context-trajectory")); + assert!(readme.contains("3 typed blockers with 9/9 evidence coverage")); +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/qmd_debug_retest.rs b/apps/elf-eval/tests/real_world_job_benchmark/qmd_debug_retest.rs new file mode 100644 index 00000000..8c19bafe --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/qmd_debug_retest.rs @@ -0,0 +1,180 @@ +use std::fs; + +use color_eyre::Result; +use serde_json::Value; + +use crate::support; + +#[test] +fn qmd_debug_ergonomics_dreaming_retest_report_preserves_qmd_edge() -> Result<()> { + let report = serde_json::from_str::(&fs::read_to_string( + support::qmd_debug_ergonomics_dreaming_retest_report_json_path()?, + )?)?; + let markdown = + fs::read_to_string(support::qmd_debug_ergonomics_dreaming_retest_report_markdown_path()?)?; + let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; + let readme = fs::read_to_string(support::readme_path()?)?; + + assert_qmd_debug_retest_summary(&report)?; + assert_qmd_debug_retest_command_and_adapters(&report)?; + assert_qmd_debug_retest_scenarios(&report)?; + assert_qmd_debug_retest_boundaries(&report)?; + assert_qmd_debug_retest_markdown_and_indexes(&markdown, &benchmarking_index, &readme); + + Ok(()) +} + +fn assert_qmd_debug_retest_summary(report: &Value) -> Result<()> { + assert_eq!( + report.pointer("/schema").and_then(Value::as_str), + Some("elf.qmd_debug_ergonomics_dreaming_retest_report/v1") + ); + assert_eq!(report.pointer("/authority").and_then(Value::as_str), Some("XY-982")); + assert_eq!( + report.pointer("/summary/overall_judgment").and_then(Value::as_str), + Some("unchanged_with_live_operator_debug_confirmation") + ); + assert_eq!( + report.pointer("/summary/debug_ergonomics_edge").and_then(Value::as_str), + Some("qmd_default_top10_and_short_cli_replay_preserved") + ); + assert_eq!( + report.pointer("/summary/broader_superiority").and_then(Value::as_str), + Some("not_proven") + ); + assert_eq!(report.pointer("/summary/improved_scenario_count").and_then(Value::as_u64), Some(0)); + assert_eq!( + report.pointer("/summary/regressed_scenario_count").and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report.pointer("/summary/unchanged_scenario_count").and_then(Value::as_u64), + Some(6) + ); + assert!(support::array_contains_str( + report, + "/summary/unsupported_claims_rejected", + "qmd's live operator-debug wrong_result rows do not erase qmd's default top-k and short CLI replay edge." + )?); + + Ok(()) +} + +fn assert_qmd_debug_retest_command_and_adapters(report: &Value) -> Result<()> { + let command = support::find_by_field( + support::array_at(report, "/commands")?, + "/command", + "cargo make real-world-job-operator-ux-live-adapters", + )?; + + assert_eq!(command.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!( + command.pointer("/summary/schema").and_then(Value::as_str), + Some("elf.real_world_operator_debug_live_adapter_sweep/v1") + ); + + let adapters = support::array_at(report, "/adapter_summaries")?; + let elf = support::find_by_field(adapters, "/adapter_id", "elf_operator_debug_live")?; + let qmd = support::find_by_field(adapters, "/adapter_id", "qmd_operator_debug_live")?; + + assert_eq!(elf.pointer("/job_count").and_then(Value::as_u64), Some(6)); + assert_eq!(elf.pointer("/pass").and_then(Value::as_u64), Some(6)); + assert_eq!(elf.pointer("/wrong_result").and_then(Value::as_u64), Some(0)); + assert_eq!(elf.pointer("/trace_available_count").and_then(Value::as_u64), Some(6)); + assert_eq!(elf.pointer("/replay_command_available_count").and_then(Value::as_u64), Some(6)); + assert_eq!(qmd.pointer("/job_count").and_then(Value::as_u64), Some(6)); + assert_eq!(qmd.pointer("/pass").and_then(Value::as_u64), Some(0)); + assert_eq!(qmd.pointer("/wrong_result").and_then(Value::as_u64), Some(6)); + assert_eq!(qmd.pointer("/trace_available_count").and_then(Value::as_u64), Some(0)); + assert_eq!(qmd.pointer("/trace_incomplete_count").and_then(Value::as_u64), Some(6)); + assert_eq!(qmd.pointer("/replay_command_available_count").and_then(Value::as_u64), Some(6)); + + Ok(()) +} + +fn assert_qmd_debug_retest_scenarios(report: &Value) -> Result<()> { + let scenarios = support::array_at(report, "/scenario_retests")?; + let top10 = + support::find_by_field(scenarios, "/scenario_id", "qmd_default_top10_candidate_artifact")?; + let replay = support::find_by_field(scenarios, "/scenario_id", "qmd_short_cli_replay")?; + let trace = + support::find_by_field(scenarios, "/scenario_id", "elf_operator_debug_trace_hydration")?; + let candidate = support::find_by_field( + scenarios, + "/scenario_id", + "operator_debug_candidate_drop_visibility", + )?; + let expansion = + support::find_by_field(scenarios, "/scenario_id", "query_expansion_attribution")?; + let fusion = support::find_by_field(scenarios, "/scenario_id", "fusion_attribution")?; + let rerank = support::find_by_field(scenarios, "/scenario_id", "rerank_attribution")?; + + assert_eq!(scenarios.len(), 10); + assert_eq!(top10.pointer("/judgment").and_then(Value::as_str), Some("unchanged")); + assert_eq!(top10.pointer("/current_outcome").and_then(Value::as_str), Some("loss")); + assert_eq!(replay.pointer("/current_outcome").and_then(Value::as_str), Some("loss")); + assert_eq!( + trace.pointer("/current_counts/elf_trace_available").and_then(Value::as_u64), + Some(6) + ); + assert_eq!( + trace.pointer("/current_counts/qmd_trace_available").and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + candidate + .pointer("/current_counts/qmd_intermediate_stage_visible_jobs") + .and_then(Value::as_u64), + Some(0) + ); + assert!(support::array_contains_str( + candidate, + "/typed_non_pass_states", + "retrieved_but_dropped" + )?); + assert_eq!(expansion.pointer("/judgment").and_then(Value::as_str), Some("not_tested")); + assert_eq!(fusion.pointer("/judgment").and_then(Value::as_str), Some("not_tested")); + assert_eq!(rerank.pointer("/judgment").and_then(Value::as_str), Some("non_goal")); + + Ok(()) +} + +fn assert_qmd_debug_retest_boundaries(report: &Value) -> Result<()> { + assert!(support::array_contains_str( + report, + "/claim_boundaries/allowed", + "qmd's default local-debug edge remains: top-10 candidate rows plus short CLI replay." + )?); + assert!(support::array_contains_str( + report, + "/claim_boundaries/not_allowed", + "Do not claim ELF broadly beats qmd from this retest." + )?); + assert!(support::array_contains_str( + report, + "/next_optimization_direction/required_fields", + "fusion_rank_deltas" + )?); + + Ok(()) +} + +fn assert_qmd_debug_retest_markdown_and_indexes( + markdown: &str, + benchmarking_index: &str, + readme: &str, +) { + assert!(markdown.contains("The qmd debug-ergonomics outcome is unchanged")); + assert!(markdown.contains("ELF 6 pass/0 wrong_result; qmd 0 pass/6 wrong_result")); + assert!( + markdown.contains("Do not treat qmd's 0 pass/6 wrong_result live operator-debug slice") + ); + assert!(markdown.contains("Immediate top-k rows with source id")); + assert!( + benchmarking_index.contains("2026-06-19-qmd-debug-ergonomics-dreaming-retest-report.md") + ); + assert!(readme.contains("qmd Debug-Ergonomics Dreaming Retest Report - June 19, 2026")); + assert!(readme.contains("Temporal and Trajectory Adapter Coverage Report - June 23, 2026")); + assert!(readme.contains("Latest real-world benchmark report: June 27, 2026")); + assert!(readme.contains("keeps the qmd edge unchanged")); +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/review_queue.rs b/apps/elf-eval/tests/real_world_job_benchmark/review_queue.rs new file mode 100644 index 00000000..bb1d37fd --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/review_queue.rs @@ -0,0 +1,103 @@ +use std::fs; + +use color_eyre::Result; +use serde_json::Value; + +use crate::{dreaming_reports, support}; + +#[test] +fn dreaming_review_queue_report_wires_reviewable_policy_contract() -> Result<()> { + let report = serde_json::from_str::(&fs::read_to_string( + support::dreaming_review_queue_report_json_path()?, + )?)?; + let markdown = fs::read_to_string(support::dreaming_review_queue_report_markdown_path()?)?; + let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; + let readme = fs::read_to_string(support::readme_path()?)?; + let workspace = support::workspace_root()?; + let service = dreaming_reports::read_rust_module_sources( + &workspace.join("packages/elf-service/src"), + "dreaming_review_queue", + )?; + let service_lib = fs::read_to_string(workspace.join("packages/elf-service/src/lib.rs"))?; + let routes = + dreaming_reports::read_rust_module_sources(&workspace.join("apps/elf-api/src"), "routes")?; + let mcp = dreaming_reports::read_rust_module_sources( + &workspace.join("apps/elf-mcp/src/app"), + "server", + )?; + let consolidation_spec = + fs::read_to_string(workspace.join("docs/spec/system_consolidation_proposals_v1.md"))?; + let service_spec = + fs::read_to_string(workspace.join("docs/spec/system_elf_memory_service_v2.md"))?; + + assert_eq!( + report.pointer("/schema").and_then(Value::as_str), + Some("elf.dreaming_review_queue_report/v1") + ); + assert_eq!(report.pointer("/authority").and_then(Value::as_str), Some("XY-1021")); + assert_eq!( + report.pointer("/summary/queue_schema").and_then(Value::as_str), + Some("elf.dreaming_review_queue/v1") + ); + assert_eq!( + report.pointer("/summary/source_mutation_allowed").and_then(Value::as_bool), + Some(false) + ); + assert_eq!( + report.pointer("/summary/high_impact_requires_review").and_then(Value::as_bool), + Some(true) + ); + assert_eq!(report.pointer("/summary/variant_count").and_then(Value::as_u64), Some(9)); + + for suite in ["memory_summary", "proactive_brief", "scheduled_memory", "consolidation"] { + assert!(support::array_contains_str(&report, "/summary/covered_existing_suites", suite)?); + } + for variant in + ["tag", "duplicate_merge", "page_rebuild", "memory_promotion", "graph_fact", "correction"] + { + assert!(support::array_contains_str(&report, "/summary/covered_future_variants", variant)?); + + support::find_by_field( + support::array_at(&report, "/queue_variants")?, + "/variant", + variant, + )?; + } + for field in [ + "source_refs", + "affected_refs", + "confidence", + "unsupported_claim_flags", + "diff", + "policy", + "review_audit", + ] { + assert!(support::array_contains_str(&report, "/required_item_fields", field)?); + } + + assert!(service.contains("ELF_DREAMING_REVIEW_QUEUE_SCHEMA_V1")); + assert!(service.contains("pub async fn dreaming_review_queue")); + assert!(service.contains("source_mutation_allowed: false")); + assert!(service.contains("low_risk_derived_organization")); + assert!(service.contains("available_review_actions")); + assert!(service_lib.contains("pub mod dreaming_review_queue")); + assert!(service_lib.contains("DreamingReviewQueueResponse")); + assert!(routes.contains("/v2/admin/dreaming/review-queue")); + assert!(routes.contains("DreamingReviewQueueRequest")); + assert!(routes.contains("async fn dreaming_review_queue")); + assert!(mcp.contains("elf_dreaming_review_queue")); + assert!(mcp.contains("dreaming_review_queue_schema")); + assert!(mcp.contains("/v2/admin/dreaming/review-queue")); + assert!(consolidation_spec.contains("elf.dreaming_review_queue/v1")); + assert!(consolidation_spec.contains("source_mutation_allowed")); + assert!(consolidation_spec.contains("duplicate_merge")); + assert!(service_spec.contains("GET /v2/admin/dreaming/review-queue")); + assert!(service_spec.contains("source refs, affected refs, confidence")); + assert!(markdown.contains("Dreaming Review Queue Report")); + assert!(markdown.contains("Auto-apply is limited to approved low-risk")); + assert!(benchmarking_index.contains("2026-06-20-dreaming-review-queue-report.md")); + assert!(readme.contains("Dreaming review queue after XY-1021")); + assert!(readme.contains("elf.dreaming_review_queue/v1")); + + Ok(()) +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/service_native.rs b/apps/elf-eval/tests/real_world_job_benchmark/service_native.rs new file mode 100644 index 00000000..9dd5a077 --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/service_native.rs @@ -0,0 +1,180 @@ +use std::fs; + +use color_eyre::{Result, eyre}; +use serde_json::Value; + +use crate::support; + +#[test] +fn service_native_dreaming_readback_report_materializes_public_jobs() -> Result<()> { + let report = serde_json::from_str::(&fs::read_to_string( + support::service_native_dreaming_readback_report_json_path()?, + )?)?; + let materialization = serde_json::from_str::(&fs::read_to_string( + support::service_native_dreaming_readback_materialization_json_path()?, + )?)?; + let markdown = + fs::read_to_string(support::service_native_dreaming_readback_report_markdown_path()?)?; + let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; + let readme = fs::read_to_string(support::readme_path()?)?; + + assert_service_native_dreaming_report_summary(&report)?; + assert_service_native_dreaming_report_jobs(&report)?; + assert_service_native_dreaming_materialization(&materialization)?; + assert_service_native_dreaming_docs(&markdown, &benchmarking_index, &readme); + + Ok(()) +} + +fn assert_service_native_dreaming_report_summary(report: &Value) -> Result<()> { + assert_eq!( + report.pointer("/adapter/adapter_id").and_then(Value::as_str), + Some("elf_service_native_dreaming") + ); + assert_eq!( + report.pointer("/adapter/behavior").and_then(Value::as_str), + Some("service_native_dreaming_readback") + ); + assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(11)); + assert_eq!(report.pointer("/summary/pass").and_then(Value::as_u64), Some(9)); + assert_eq!(report.pointer("/summary/wrong_result").and_then(Value::as_u64), Some(0)); + assert_eq!(report.pointer("/summary/blocked").and_then(Value::as_u64), Some(2)); + assert_eq!(report.pointer("/summary/wrong_result_count").and_then(Value::as_u64), Some(0)); + assert_eq!(report.pointer("/summary/evidence_coverage").and_then(Value::as_f64), Some(1.0)); + assert_eq!(report.pointer("/summary/source_ref_coverage").and_then(Value::as_f64), Some(1.0)); + assert_eq!(report.pointer("/summary/quote_coverage").and_then(Value::as_f64), Some(1.0)); + assert_eq!( + report.pointer("/summary/memory_summary/source_ref_coverage").and_then(Value::as_f64), + Some(1.0) + ); + assert_eq!( + report.pointer("/summary/proactive_brief/evidence_ref_coverage").and_then(Value::as_f64), + Some(1.0) + ); + assert_eq!( + report.pointer("/summary/scheduled_memory/trace_coverage").and_then(Value::as_f64), + Some(1.0) + ); + assert_eq!( + report.pointer("/summary/scheduled_memory/source_mutation_count").and_then(Value::as_u64), + Some(0) + ); + + let suites = support::array_at(report, "/suites")?; + let memory = support::find_by_field(suites, "/suite_id", "memory_summary")?; + let proactive = support::find_by_field(suites, "/suite_id", "proactive_brief")?; + let scheduled = support::find_by_field(suites, "/suite_id", "scheduled_memory")?; + + assert_eq!(memory.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(proactive.pointer("/status").and_then(Value::as_str), Some("blocked")); + assert_eq!(scheduled.pointer("/status").and_then(Value::as_str), Some("blocked")); + + Ok(()) +} + +fn assert_service_native_dreaming_report_jobs(report: &Value) -> Result<()> { + let jobs = support::array_at(report, "/jobs")?; + let memory = support::find_by_field(jobs, "/job_id", "memory-summary-source-trace-001")?; + let daily = support::find_by_field(jobs, "/job_id", "proactive-daily-project-brief-001")?; + let private_brief = + support::find_by_field(jobs, "/job_id", "proactive-private-corpus-refresh-blocked-001")?; + let weekly = + support::find_by_field(jobs, "/job_id", "scheduled-weekly-project-status-summary-001")?; + let private_scheduled = support::find_by_field( + jobs, + "/job_id", + "scheduled-private-provider-scheduler-blocked-001", + )?; + + assert_eq!(memory.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(daily.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(weekly.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(private_brief.pointer("/status").and_then(Value::as_str), Some("blocked")); + assert_eq!(private_scheduled.pointer("/status").and_then(Value::as_str), Some("blocked")); + assert!(!support::array_contains_str(memory, "/produced_evidence", "stale-summary-gap")?); + assert!(!support::array_contains_str(memory, "/produced_evidence", "summary-temporary-claim")?); + assert!(!support::array_contains_str(daily, "/produced_evidence", "daily-old-parity-trap")?); + assert!(!support::array_contains_str( + weekly, + "/produced_evidence", + "scheduled-weekly-hosted-parity-trap" + )?); + + Ok(()) +} + +fn assert_service_native_dreaming_materialization(materialization: &Value) -> Result<()> { + assert_eq!( + materialization.pointer("/schema").and_then(Value::as_str), + Some("elf.real_world_live_adapter_materialization/v1") + ); + assert_eq!( + materialization.pointer("/adapter_id").and_then(Value::as_str), + Some("elf_service_native_dreaming") + ); + assert_eq!(materialization.pointer("/status").and_then(Value::as_str), Some("blocked")); + + let jobs = support::array_at(materialization, "/jobs")?; + let memory = support::find_by_field(jobs, "/job_id", "memory-summary-source-trace-001")?; + let daily = support::find_by_field(jobs, "/job_id", "proactive-daily-project-brief-001")?; + let private_brief = + support::find_by_field(jobs, "/job_id", "proactive-private-corpus-refresh-blocked-001")?; + + for job in jobs { + match job.pointer("/status").and_then(Value::as_str) { + Some("pass") => { + assert_eq!( + job.pointer("/dreaming_readback/runtime_path").and_then(Value::as_str), + Some("ElfService::add_note -> ElfService::list -> derived readback artifact") + ); + assert!( + support::array_at(job, "/dreaming_readback/missing_source_refs")?.is_empty() + ); + assert_eq!( + job.pointer("/dreaming_readback/source_mutation_count").and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + job.pointer("/dreaming_readback/no_source_mutation_checked") + .and_then(Value::as_bool), + Some(true) + ); + }, + Some("blocked") => { + assert!(job.pointer("/dreaming_readback").is_none_or(Value::is_null)); + }, + status => { + return Err(eyre::eyre!( + "unexpected service-native materialization status: {status:?}" + )); + }, + } + } + + assert!(support::array_contains_str( + memory, + "/dreaming_readback/selected_source_refs", + "stale-summary-gap" + )?); + assert!(!support::array_contains_str(memory, "/evidence_ids", "stale-summary-gap")?); + assert!(support::array_contains_str( + daily, + "/dreaming_readback/selected_source_refs", + "daily-old-parity-trap" + )?); + assert!(!support::array_contains_str(daily, "/evidence_ids", "daily-old-parity-trap")?); + assert!(private_brief.pointer("/dreaming_readback").is_none_or(Value::is_null)); + + Ok(()) +} + +fn assert_service_native_dreaming_docs(markdown: &str, benchmarking_index: &str, readme: &str) { + assert!(markdown.contains("9 pass")); + assert!(markdown.contains("0 wrong_result")); + assert!(markdown.contains("2 typed blocked")); + assert!(markdown.contains("ElfService::add_note -> ElfService::list")); + assert!(markdown.contains("Do not claim ELF broadly beats OpenAI Pulse")); + assert!(benchmarking_index.contains("2026-06-19-service-native-dreaming-readback-report.md")); + assert!(readme.contains("Service-native Dreaming readback after XY-986")); + assert!(readme.contains("real-world-memory-service-native-dreaming")); +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/temporal_reconciliation.rs b/apps/elf-eval/tests/real_world_job_benchmark/temporal_reconciliation.rs new file mode 100644 index 00000000..40bbacc8 --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/temporal_reconciliation.rs @@ -0,0 +1,95 @@ +use std::fs; + +use color_eyre::Result; +use serde_json::Value; + +use crate::support; + +#[test] +fn live_temporal_reconciliation_report_records_xy905_before_after() -> Result<()> { + let report = serde_json::from_str::(&fs::read_to_string( + support::live_temporal_reconciliation_report_json_path()?, + )?)?; + let markdown = + fs::read_to_string(support::live_temporal_reconciliation_report_markdown_path()?)?; + let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; + let readme = fs::read_to_string(support::readme_path()?)?; + + assert_eq!( + report.pointer("/schema").and_then(Value::as_str), + Some("elf.live_temporal_reconciliation_report/v1") + ); + assert_eq!(report.pointer("/authority").and_then(Value::as_str), Some("XY-905")); + assert_eq!( + report + .pointer("/baseline/elf_memory_evolution/job_status_counts/pass") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/baseline/elf_memory_evolution/job_status_counts/wrong_result") + .and_then(Value::as_u64), + Some(5) + ); + assert_eq!( + report + .pointer("/post_stage/elf_memory_evolution/job_status_counts/pass") + .and_then(Value::as_u64), + Some(6) + ); + assert_eq!( + report + .pointer("/post_stage/elf_memory_evolution/job_status_counts/wrong_result") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report.pointer("/post_stage/elf_memory_evolution/suite_status").and_then(Value::as_str), + Some("pass") + ); + assert_eq!( + report.pointer("/post_stage/qmd_memory_evolution/suite_status").and_then(Value::as_str), + Some("wrong_result") + ); + assert_eq!( + report + .pointer("/comparison_judgment/current_vs_historical_correctness") + .and_then(Value::as_str), + Some("improved") + ); + assert_eq!( + report + .pointer("/comparison_judgment/deletion_ttl_tombstone_behavior") + .and_then(Value::as_str), + Some("unchanged") + ); + assert!(support::array_contains_str( + &report, + "/trace_contract/answer_fields", + "selected_historical_evidence" + )?); + assert!(support::array_contains_str( + &report, + "/trace_contract/materialization_fields", + "current_winner_evidence_ids" + )?); + assert!(support::array_contains_str( + &report, + "/trace_contract/trace_stages", + "temporal_reconciliation.conflict_candidates" + )?); + assert!(report.pointer("/trace_contract/negative_gate").and_then(Value::as_str).is_some_and( + |gate| gate.contains("selected conflict evidence id") && gate.contains("wrong_result") + )); + assert!(markdown.contains("ELF passing all six memory-evolution jobs")); + assert!(markdown.contains("selected-but-not-narrated conflicts as `wrong_result`")); + assert!(markdown.contains("Do not claim ELF beats Graphiti/Zep")); + assert!(benchmarking_index.contains("2026-06-16-live-temporal-reconciliation-report.md")); + assert!( + readme.contains("Live Temporal Reconciliation Report - June 16, 2026") + && readme.contains("now reports ELF live `memory_evolution` as 6/6 pass") + ); + + Ok(()) +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/validation.rs b/apps/elf-eval/tests/real_world_job_benchmark/validation.rs new file mode 100644 index 00000000..27df61cb --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/validation.rs @@ -0,0 +1,89 @@ +use color_eyre::{Result, eyre}; + +use crate::support; + +#[test] +fn external_adapter_manifest_rejects_unmeasured_win_loss_scenario_outcomes() -> Result<()> { + let output = support::run_external_manifest_with_letta_attachment_mutation( + "invalid-scenario-outcome-test", + |scenario| { + support::set_json_pointer(scenario, "/status", serde_json::json!("not_encoded"))?; + + support::set_json_pointer(scenario, "/comparison_outcome", serde_json::json!("win")) + }, + )?; + + assert!(!output.status.success(), "invalid scenario outcome unexpectedly passed"); + assert!( + String::from_utf8_lossy(&output.stderr).contains("not_encoded status with win outcome") + ); + + Ok(()) +} + +#[test] +fn external_adapter_manifest_rejects_unmeasured_win_loss_scenario_positions() -> Result<()> { + let output = support::run_external_manifest_with_letta_attachment_mutation( + "invalid-scenario-position-test", + |scenario| { + support::set_json_pointer(scenario, "/status", serde_json::json!("not_encoded"))?; + support::set_json_pointer(scenario, "/elf_position", serde_json::json!("wins"))?; + + support::set_json_pointer( + scenario, + "/comparison_outcome", + serde_json::json!("not_tested"), + ) + }, + )?; + + assert!(!output.status.success(), "invalid scenario position unexpectedly passed"); + assert!( + String::from_utf8_lossy(&output.stderr).contains("not_encoded status with wins position") + ); + + Ok(()) +} + +#[test] +fn external_adapter_manifest_rejects_blocked_status_without_blocked_outcome() -> Result<()> { + let output = support::run_external_manifest_scenario_mutation( + "invalid-blocked-scenario-outcome-test", + "letta_research_gate", + "stale_core_detection", + |scenario| { + scenario + .as_object_mut() + .ok_or_else(|| eyre::eyre!("scenario is not an object"))? + .remove("comparison_outcome"); + + Ok(()) + }, + )?; + + assert!(!output.status.success(), "invalid blocked scenario unexpectedly passed"); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("blocked status without blocked comparison outcome") + ); + + Ok(()) +} + +#[test] +fn external_adapter_manifest_rejects_conflicting_scenario_position_and_outcome() -> Result<()> { + let output = support::run_external_manifest_with_letta_attachment_mutation( + "invalid-scenario-position-outcome-test", + |scenario| { + support::set_json_pointer(scenario, "/status", serde_json::json!("pass"))?; + support::set_json_pointer(scenario, "/elf_position", serde_json::json!("ties"))?; + + support::set_json_pointer(scenario, "/comparison_outcome", serde_json::json!("loss")) + }, + )?; + + assert!(!output.status.success(), "conflicting scenario unexpectedly passed"); + assert!(String::from_utf8_lossy(&output.stderr).contains("ties position with loss outcome")); + + Ok(()) +}