Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,22 +1,20 @@
mod external_adapter_detail_reports;
mod external_adapter_manifest_reports;
mod external_adapter_misc_reports;
mod external_adapter_summary_reports;
mod details;
mod manifest;
mod misc;
mod summary;

pub(super) use self::{
external_adapter_detail_reports::{
details::{
AdapterScenarioJudgment, AdapterSource, AdapterSuiteCoverage, ExternalAdapterReport,
},
external_adapter_manifest_reports::{
ExternalAdapterManifest, ExternalAdapterSection, ExternalDockerIsolation,
},
external_adapter_misc_reports::{AdapterReport, CaptureIntegrationReport},
external_adapter_summary_reports::{
manifest::{ExternalAdapterManifest, ExternalAdapterSection, ExternalDockerIsolation},
misc::{AdapterReport, CaptureIntegrationReport},
summary::{
AdapterStatusCounts, ExternalAdapterSummary, ScenarioOutcomeCounts, ScenarioPositionCounts,
},
};
#[allow(unused_imports)]
pub(super) use external_adapter_detail_reports::{
pub(super) use details::{
AdapterCapabilityCoverage, AdapterEvidencePointer, AdapterExecutionEvidence,
AdapterExecutionMetadata,
};
10 changes: 4 additions & 6 deletions apps/elf-eval/src/bin/real_world_job_benchmark/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,14 @@ mod scoreboard;
mod scoreboard_reports;
mod scoring;
mod source_backed_quality;
mod source_backed_quality_reports;
mod summary;
mod summary_reports;
mod validation;

pub(crate) use source_backed_quality::{
SourceBackedQualityReport, source_backed_quality_report, validate_source_backed_quality_gate,
};

use std::{
collections::{BTreeMap, BTreeSet},
fs,
Expand Down Expand Up @@ -110,11 +113,6 @@ use scoreboard_reports::{
ScoreboardRow,
};
use scoring::{job_report, score_job};
use source_backed_quality::{source_backed_quality_report, validate_source_backed_quality_gate};
use source_backed_quality_reports::{
SourceBackedContextPackDecisionCounts, SourceBackedQualityMetrics, SourceBackedQualityReport,
SourceBackedScenarioCoverage,
};
use summary::{evolution_summary, follow_up_reports, report_summary, suite_reports};
use summary_reports::{
ConsolidationSummaryReport, KnowledgeSummary, MemorySummaryReport, ProactiveBriefSummaryReport,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
use crate::{
BTreeSet, JobReport, RealWorldJob, ReportSummary, ScoreboardReport,
mod reports;

pub(super) use reports::{
SourceBackedContextPackDecisionCounts, SourceBackedQualityMetrics, SourceBackedQualityReport,
SourceBackedScenarioCoverage, TypedStatus, formatting,
SourceBackedScenarioCoverage,
};

use crate::{
BTreeSet, JobReport, RealWorldJob, ReportSummary, ScoreboardReport, TypedStatus, formatting,
};

const SOURCE_BACKED_QUALITY_SCHEMA: &str = "elf.source_backed_memory_quality_benchmark/v1";
Expand Down
25 changes: 8 additions & 17 deletions apps/elf-eval/src/bin/real_world_live_adapter/dreaming_readback.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
mod dreaming_readback_artifacts;
mod artifacts;

use crate::{
AGENT_ID, DreamingReadbackMaterializationEvidence, DreamingReadbackOutput, ElfService,
Expand Down Expand Up @@ -85,10 +85,10 @@ pub(super) async fn materialize_elf_dreaming_readback(

let generated_at = OffsetDateTime::now_utc().format(&Rfc3339)?;
let service_evidence_ids = service_readback_evidence_ids(service, project_id).await?;
let mut artifacts = dreaming_readback_artifacts::dreaming_readback_template_artifacts(loaded)?;
let mut artifacts = artifacts::dreaming_readback_template_artifacts(loaded)?;

for artifact in &mut artifacts {
dreaming_readback_artifacts::stamp_dreaming_readback_artifact(
artifacts::stamp_dreaming_readback_artifact(
artifact,
loaded,
project_id,
Expand All @@ -100,10 +100,7 @@ pub(super) async fn materialize_elf_dreaming_readback(
let mut artifact_source_refs = Vec::new();

for artifact in &artifacts {
dreaming_readback_artifacts::collect_dreaming_artifact_source_refs(
artifact,
&mut artifact_source_refs,
);
artifacts::collect_dreaming_artifact_source_refs(artifact, &mut artifact_source_refs);
}

artifact_source_refs.sort();
Expand All @@ -119,10 +116,8 @@ pub(super) async fn materialize_elf_dreaming_readback(
.filter(|source_ref| service_evidence_ids.contains(*source_ref))
.cloned()
.collect::<Vec<_>>();
let scoring_evidence_ids = dreaming_readback_artifacts::dreaming_readback_scoring_evidence_ids(
loaded,
&service_evidence_ids,
);
let scoring_evidence_ids =
artifacts::dreaming_readback_scoring_evidence_ids(loaded, &service_evidence_ids);
let artifact_kind = match loaded.job.suite.as_str() {
"memory_summary" => "elf.memory_summary/v1",
"proactive_brief" => "elf.proactive_project_brief/v1",
Expand All @@ -141,12 +136,8 @@ pub(super) async fn materialize_elf_dreaming_readback(
source_mutation_count: 0,
no_source_mutation_checked: true,
};
let trace_stages =
dreaming_readback_artifacts::dreaming_readback_trace_stages(loaded, &materialization);
let content = dreaming_readback_artifacts::dreaming_readback_content(
loaded.job.suite.as_str(),
&artifacts,
);
let trace_stages = artifacts::dreaming_readback_trace_stages(loaded, &materialization);
let content = artifacts::dreaming_readback_content(loaded.job.suite.as_str(), &artifacts);
let (memory_summaries, proactive_briefs, scheduled_tasks) = match loaded.job.suite.as_str() {
"memory_summary" => (artifacts, Vec::new(), Vec::new()),
"proactive_brief" => (Vec::new(), artifacts, Vec::new()),
Expand Down
10 changes: 5 additions & 5 deletions apps/elf-eval/tests/real_world_job_benchmark/benchmark_core.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
mod benchmark_core_adversarial_scoreboard;
mod benchmark_core_blocker_rows;
mod benchmark_core_capture_sources;
mod benchmark_core_smoke_report;
#[path = "benchmark_core/adversarial_scoreboard.rs"] mod adversarial_scoreboard;
#[path = "benchmark_core/blocker_rows.rs"] mod blocker_rows;
#[path = "benchmark_core/capture_sources.rs"] mod capture_sources;
#[path = "benchmark_core/smoke_report.rs"] mod smoke_report;

pub(super) use benchmark_core_blocker_rows::assert_tracked_external_blocker_row;
pub(super) use blocker_rows::assert_tracked_external_blocker_row;
14 changes: 7 additions & 7 deletions apps/elf-eval/tests/real_world_job_benchmark/closeout_reports.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
mod closeout_reports_agent_knowledge;
mod closeout_reports_competitor_strength;
mod closeout_reports_final_source_backed;
mod closeout_reports_graph_rag;
mod closeout_reports_helpers;
mod closeout_reports_openmemory;
mod closeout_reports_workspace;
#[path = "closeout_reports/agent_knowledge.rs"] mod agent_knowledge;
#[path = "closeout_reports/competitor_strength.rs"] mod competitor_strength;
#[path = "closeout_reports/final_source_backed.rs"] mod final_source_backed;
#[path = "closeout_reports/graph_rag.rs"] mod graph_rag;
#[path = "closeout_reports/helpers.rs"] mod helpers;
#[path = "closeout_reports/openmemory.rs"] mod openmemory;
#[path = "closeout_reports/workspace.rs"] mod workspace;
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use std::fs;
use color_eyre::Result;
use serde_json::Value;

use crate::{closeout_reports::closeout_reports_helpers, support};
use crate::{closeout_reports::helpers, support};

#[test]
fn p3_competitor_strength_absorption_report_preserves_claim_boundaries() -> Result<()> {
Expand Down Expand Up @@ -93,7 +93,7 @@ fn p3_competitor_strength_absorption_report_preserves_claim_boundaries() -> Resu
assert_eq!(item.pointer("/queued_label_applied").and_then(Value::as_bool), Some(false));
}

closeout_reports_helpers::assert_product_queue_items_reference_queue(products, queue)?;
helpers::assert_product_queue_items_reference_queue(products, queue)?;

assert!(support::array_contains_str(
&report,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use std::fs;
use color_eyre::Result;
use serde_json::Value;

use crate::{closeout_reports::closeout_reports_helpers, support};
use crate::{closeout_reports::helpers, support};

#[test]
fn graph_rag_citation_navigation_promotion_preserves_typed_non_passes() -> Result<()> {
Expand Down Expand Up @@ -109,14 +109,10 @@ fn graph_rag_adapter_matrix_report_preserves_no_parity_claims() -> Result<()> {
);

let rows = support::array_at(&report, "/adapter_matrix")?;
let ragflow_citation =
closeout_reports_helpers::find_matrix_row(rows, "RAGFlow", "citation_quality")?;
let lightrag_retrieval =
closeout_reports_helpers::find_matrix_row(rows, "LightRAG", "retrieval_quality")?;
let graphrag_navigation =
closeout_reports_helpers::find_matrix_row(rows, "GraphRAG", "navigation_quality")?;
let graphrag_retrieval =
closeout_reports_helpers::find_matrix_row(rows, "GraphRAG", "retrieval_quality")?;
let ragflow_citation = helpers::find_matrix_row(rows, "RAGFlow", "citation_quality")?;
let lightrag_retrieval = helpers::find_matrix_row(rows, "LightRAG", "retrieval_quality")?;
let graphrag_navigation = helpers::find_matrix_row(rows, "GraphRAG", "navigation_quality")?;
let graphrag_retrieval = helpers::find_matrix_row(rows, "GraphRAG", "retrieval_quality")?;

assert_eq!(ragflow_citation.pointer("/status").and_then(Value::as_str), Some("blocked"));
assert_eq!(lightrag_retrieval.pointer("/status").and_then(Value::as_str), Some("incomplete"));
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
mod competitor_strength_live;
mod competitor_strength_profile;
#[path = "competitor_strength/profile.rs"] mod profile;
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
mod boundaries;
mod openviking;
mod qmd;
mod summary_terms;
#[path = "profile/boundaries.rs"] mod boundaries;
#[path = "profile/openviking.rs"] mod openviking;
#[path = "profile/qmd.rs"] mod qmd;
#[path = "profile/summary_terms.rs"] mod summary_terms;

use std::fs;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
pub(crate) mod consolidation_knowledge_tests_helpers;
#[path = "consolidation_knowledge/helpers.rs"] pub(crate) mod helpers;

mod consolidation_knowledge_tests_capture;
mod consolidation_knowledge_tests_fixtures;
mod consolidation_knowledge_tests_live;
#[path = "consolidation_knowledge/capture.rs"] mod capture;
#[path = "consolidation_knowledge/fixtures.rs"] mod fixtures;
#[path = "consolidation_knowledge/live.rs"] mod live;
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#[path = "fixtures/consolidation.rs"] mod consolidation;
#[path = "fixtures/knowledge.rs"] mod knowledge;
#[path = "fixtures/operator_debug.rs"] mod operator_debug;
#[path = "fixtures/project_decisions.rs"] mod project_decisions;
#[path = "fixtures/runner.rs"] mod runner;
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use std::fs;
use color_eyre::Result;
use serde_json::Value;

use crate::{consolidation_knowledge::consolidation_knowledge_tests_helpers, support};
use crate::{consolidation_knowledge::helpers, support};

#[test]
fn live_consolidation_report_preserves_reviewable_output_boundaries() -> Result<()> {
Expand All @@ -25,8 +25,7 @@ fn live_consolidation_report_preserves_reviewable_output_boundaries() -> Result<
let makefile = support::make_task_catalog()?;
let live_script =
fs::read_to_string(workspace.join("scripts/real-world-consolidation-live-adapter.sh"))?;
let live_adapter =
consolidation_knowledge_tests_helpers::real_world_live_adapter_sources(&workspace)?;
let live_adapter = helpers::real_world_live_adapter_sources(&workspace)?;

assert_eq!(
report.pointer("/schema").and_then(Value::as_str),
Expand Down Expand Up @@ -140,8 +139,7 @@ fn live_knowledge_page_rebuild_lint_has_dedicated_docker_task() -> Result<()> {
let docker_script = fs::read_to_string(workspace.join("scripts/real-world-docker.sh"))?;
let live_script =
fs::read_to_string(workspace.join("scripts/real-world-knowledge-live-adapter.sh"))?;
let live_adapter =
consolidation_knowledge_tests_helpers::real_world_live_adapter_sources(&workspace)?;
let live_adapter = helpers::real_world_live_adapter_sources(&workspace)?;
let knowledge_spec = fs::read_to_string(
workspace.join("docs").join("spec").join("system_knowledge_pages_v1.md"),
)?;
Expand Down Expand Up @@ -191,8 +189,7 @@ fn live_knowledge_page_rebuild_lint_has_dedicated_docker_task() -> Result<()> {
assert!(live_adapter.contains("KnowledgePageLintRequest"));
assert!(live_adapter.contains("KnowledgePageSearchRequest"));
assert!(
consolidation_knowledge_tests_helpers::real_world_job_benchmark_sources(&workspace)?
.contains("version_diff_coverage")
helpers::real_world_job_benchmark_sources(&workspace)?.contains("version_diff_coverage")
);
assert!(knowledge_spec.contains("elf.knowledge_page.version_diff/v1"));
assert!(
Expand Down

This file was deleted.

36 changes: 17 additions & 19 deletions apps/elf-eval/tests/real_world_job_benchmark/external_adapters.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
mod external_adapters_first_generation;
mod external_adapters_fixture;
mod external_adapters_graph_gates;
mod external_adapters_letta;
mod external_adapters_live_sweep;
mod external_adapters_operator_debug;
#[path = "external_adapters/first_generation.rs"] mod first_generation;
#[path = "external_adapters/fixture.rs"] mod fixture;
#[path = "external_adapters/graph_gates.rs"] mod graph_gates;
mod graph_rag;
#[path = "external_adapters/letta.rs"] mod letta;
#[path = "external_adapters/live_sweep.rs"] mod live_sweep;
mod loss_summary;
mod manifest_summary;
#[path = "external_adapters/operator_debug.rs"] mod operator_debug;
mod validation;

use color_eyre::Result;
Expand Down Expand Up @@ -51,39 +51,39 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> {
support::find_by_field(adapters, "/adapter_id", "openviking_deep_profile_gate")?;
let letta = support::find_by_field(adapters, "/adapter_id", "letta_research_gate")?;

external_adapters_fixture::assert_elf_fixture_adapter_record(elf)?;
fixture::assert_elf_fixture_adapter_record(elf)?;

assert_eq!(
elf_live.pointer("/evidence_class").and_then(Value::as_str),
Some("live_real_world")
);
assert_eq!(elf_live.pointer("/overall_status").and_then(Value::as_str), Some("wrong_result"));

external_adapters_live_sweep::assert_live_sweep_record(elf_live, "blocked")?;
external_adapters_operator_debug::assert_operator_debug_live_adapter_records(
live_sweep::assert_live_sweep_record(elf_live, "blocked")?;
operator_debug::assert_operator_debug_live_adapter_records(
elf_operator_debug,
qmd_operator_debug,
)?;

assert_eq!(qmd.pointer("/overall_status").and_then(Value::as_str), Some("pass"));
assert_eq!(qmd.pointer("/suites/0/status").and_then(Value::as_str), Some("not_encoded"));

external_adapters_fixture::assert_qmd_live_baseline_record(qmd);
fixture::assert_qmd_live_baseline_record(qmd);

assert_eq!(
qmd_live.pointer("/evidence_class").and_then(Value::as_str),
Some("live_real_world")
);
assert_eq!(qmd_live.pointer("/overall_status").and_then(Value::as_str), Some("wrong_result"));

external_adapters_live_sweep::assert_live_sweep_record(qmd_live, "blocked")?;
live_sweep::assert_live_sweep_record(qmd_live, "blocked")?;

assert_eq!(
agentmemory.pointer("/capabilities/1/status").and_then(Value::as_str),
Some("mocked")
);

external_adapters_first_generation::assert_first_generation_adapter_records(
first_generation::assert_first_generation_adapter_records(
agentmemory,
mem0,
memsearch,
Expand All @@ -92,10 +92,8 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> {

assert_eq!(openviking.pointer("/overall_status").and_then(Value::as_str), Some("wrong_result"));

external_adapters_graph_gates::assert_graph_rag_research_gate_records(
ragflow, lightrag, graphrag,
);
external_adapters_graph_gates::assert_graphiti_zep_adapter(graphiti_zep);
graph_gates::assert_graph_rag_research_gate_records(ragflow, lightrag, graphrag);
graph_gates::assert_graphiti_zep_adapter(graphiti_zep);
graph_rag::assert_graphify_adapter(graphify)?;
graph_rag::assert_graph_rag_representative_scenarios(
ragflow,
Expand All @@ -104,8 +102,8 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> {
graphiti_zep,
graphify,
)?;
external_adapters_letta::assert_letta_core_archival_gate(letta)?;
external_adapters_fixture::assert_qmd_deep_profile_gate(qmd_deep);
letta::assert_letta_core_archival_gate(letta)?;
fixture::assert_qmd_deep_profile_gate(qmd_deep);

assert_eq!(
qmd_deep.pointer("/capabilities/2/status").and_then(Value::as_str),
Expand All @@ -120,7 +118,7 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> {
Some("docker_local_embed_context_trajectory_gate")
);

external_adapters_fixture::assert_openviking_deep_profile_gate(openviking_deep);
fixture::assert_openviking_deep_profile_gate(openviking_deep);

assert_eq!(
openviking_deep.pointer("/result/artifact").and_then(Value::as_str),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
mod markdown_rendering_external_adapters;
mod markdown_rendering_generated;
#[path = "markdown_rendering/external_adapters.rs"] mod external_adapters;
#[path = "markdown_rendering/generated.rs"] mod generated;
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#[path = "production_ops/failure_cases.rs"] mod failure_cases;
mod production_ops_evidence;
mod production_ops_failure_cases;
mod production_ops_jobs;
mod production_ops_summary;

Expand Down
Loading