diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index b1d3014e..3ce54053 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -146,13 +146,13 @@ }, "run": { "status": "wrong_result", - "evidence": "ELF materializes 38 real_world_job adapter_response objects through ElfService, worker indexing, and search_raw before scoring; the full sweep includes typed wrong_result, incomplete, blocked, and not_encoded records.", + "evidence": "ELF materializes 38 real_world_job adapter_response objects through ElfService, worker indexing, and search_raw before scoring; the full sweep includes typed wrong_result, blocked, and not_encoded job records.", "command": "cargo make real-world-memory-live-adapters", "artifact": "tmp/real-world-memory/live-adapters/elf-report.json" }, "result": { "status": "wrong_result", - "evidence": "The full live sweep scores 38 jobs across all 11 encoded suites: 18 pass, 5 wrong_result, 1 incomplete, 2 blocked, and 12 not_encoded. This is not a full-suite live pass.", + "evidence": "The fresh full live sweep scores 38 jobs across all 11 encoded suites: 18 pass, 5 wrong_result, 0 incomplete, 2 blocked, and 13 not_encoded. This is not a full-suite live pass.", "command": "cargo make real-world-memory-live-adapters", "artifact": "tmp/real-world-memory/live-adapters/elf-report.md" }, @@ -180,7 +180,7 @@ { "capability": "full_suite_live_pass", "status": "wrong_result", - "evidence": "No full-suite live pass is claimed; generated reports preserve wrong_result, incomplete, blocked, and not_encoded outcomes." + "evidence": "No full-suite live pass is claimed; generated reports preserve wrong_result, blocked, and not_encoded job outcomes." }, { "capability": "typed_failure_reporting", @@ -236,8 +236,8 @@ }, { "suite_id": "production_ops", - "status": "incomplete", - "evidence": "The live adapter sweep does not run backup/restore, private corpus, provider credential, or backfill operations; existing production-ops credential and private-manifest boundaries remain blocked and the cold-start dependency fixture remains incomplete." + "status": "blocked", + "evidence": "The live adapter sweep does not run backup/restore, private corpus, provider credential, or backfill operations; existing production-ops credential and private-manifest boundaries remain blocked." }, { "suite_id": "personalization", @@ -264,7 +264,7 @@ ], "notes": [ "This Docker-isolated live real_world_job record now covers the full encoded fixture corpus, not only the original three-suite representative slice.", - "The record is a full-suite sweep, not a full-suite pass; wrong_result, incomplete, blocked, and not_encoded states remain visible.", + "The record is a full-suite sweep, not a full-suite pass; wrong_result, blocked, and not_encoded states remain visible.", "This record does not prove private-corpus production quality or provider-backed production operations." ] }, @@ -359,13 +359,13 @@ }, "run": { "status": "wrong_result", - "evidence": "qmd materializes 38 real_world_job adapter_response objects through collection add, update, embed, and query --json before scoring; the full sweep includes typed wrong_result, incomplete, blocked, and not_encoded records.", + "evidence": "qmd materializes 38 real_world_job adapter_response objects through collection add, update, embed, and query --json before scoring; the full sweep includes typed wrong_result, blocked, and not_encoded job records.", "command": "cargo make real-world-memory-live-adapters", "artifact": "tmp/real-world-memory/live-adapters/qmd-report.json" }, "result": { "status": "wrong_result", - "evidence": "The full qmd live sweep scores 38 jobs across all 11 encoded suites: 18 pass, 5 wrong_result, 1 incomplete, 2 blocked, and 12 not_encoded. This is not a full-suite live pass.", + "evidence": "The fresh full qmd live sweep scores 38 jobs across all 11 encoded suites: 17 pass, 6 wrong_result, 0 incomplete, 2 blocked, and 13 not_encoded. This is not a full-suite live pass.", "command": "cargo make real-world-memory-live-adapters", "artifact": "tmp/real-world-memory/live-adapters/qmd-report.md" }, @@ -393,7 +393,7 @@ { "capability": "full_suite_live_pass", "status": "wrong_result", - "evidence": "No full-suite live pass is claimed; generated reports preserve wrong_result, incomplete, blocked, and not_encoded outcomes." + "evidence": "No full-suite live pass is claimed; generated reports preserve wrong_result, blocked, and not_encoded job outcomes." }, { "capability": "typed_failure_reporting", @@ -449,8 +449,8 @@ }, { "suite_id": "production_ops", - "status": "incomplete", - "evidence": "The qmd live adapter sweep does not run backup/restore, private corpus, provider credential, or backfill operations; existing production-ops credential and private-manifest boundaries remain blocked and the cold-start dependency fixture remains incomplete." + "status": "blocked", + "evidence": "The qmd live adapter sweep does not run backup/restore, private corpus, provider credential, or backfill operations; existing production-ops credential and private-manifest boundaries remain blocked." }, { "suite_id": "personalization", @@ -477,7 +477,7 @@ ], "notes": [ "This qmd record is real-world job evidence and must not be conflated with the same-corpus qmd_live_baseline record.", - "The record is a full-suite sweep, not a full-suite pass; wrong_result, incomplete, blocked, and not_encoded states remain visible.", + "The record is a full-suite sweep, not a full-suite pass; wrong_result, blocked, and not_encoded states remain visible.", "This record does not prove broad RAG/graph adapter parity or private-corpus production quality." ] }, @@ -503,7 +503,7 @@ "result": { "status": "lifecycle_fail", "evidence": "agentmemory remains a reference for capture and continuity UX, but current Docker evidence is not a durable lifecycle pass.", - "artifact": "docs/guide/benchmarking/live_baseline_benchmark.md" + "artifact": "tmp/live-baseline/live-baseline-report.json" }, "capabilities": [ { @@ -544,6 +544,34 @@ "evidence": "Durable update/supersede/delete history is not proven by the in-memory adapter." } ], + "scenarios": [ + { + "scenario_id": "basic_same_corpus_retrieval", + "suite_id": "retrieval", + "status": "pass", + "elf_position": "untested", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports agentmemory retrieval_pass with 3/3 same-corpus retrieval checks through mem::remember and mem::search. This is live-baseline-only evidence through an in-memory mock, not a real_world_job suite pass.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "durable_update_reload_lifecycle", + "suite_id": "memory_evolution", + "status": "lifecycle_fail", + "elf_position": "wins", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports ELF passing 8/8 local lifecycle checks, while agentmemory update_replaces_note_text is lifecycle_fail and cold_start_recovery_search is blocked because the harness uses an in-memory SDK/KV mock. This is an ELF baseline win only at the local lifecycle-smoke evidence class.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "work_resume_capture_continuity", + "suite_id": "work_resume", + "status": "blocked", + "elf_position": "untested", + "evidence": "agentmemory's relevant strength is durable coding-agent continuity and capture, but the Docker harness has not proven a persistent session/capture path. Keep work_resume and capture claims blocked until a durable local adapter path exists.", + "artifact": "apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json" + } + ], "evidence": [ { "kind": "guide", @@ -571,7 +599,7 @@ "evidence_class": "live_baseline_only", "docker_default": true, "host_global_installs_required": false, - "overall_status": "wrong_result", + "overall_status": "pass", "setup": { "status": "pass", "evidence": "The live-baseline Docker runner can install mem0 and configure local FastEmbed/Qdrant paths.", @@ -579,14 +607,14 @@ "artifact": "tmp/live-baseline/mem0.log" }, "run": { - "status": "wrong_result", - "evidence": "The Docker runner exercises local OSS mem0 with FastEmbed, Qdrant path storage, Memory.update, Memory.delete, and cold-start reload; same-corpus retrieval remains typed wrong_result or incomplete when evidence is missed.", + "status": "pass", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 exercises local OSS mem0 with FastEmbed, Qdrant path storage, Memory.update, Memory.delete, and cold-start reload; mem0 passed 4/4 encoded checks.", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "result": { - "status": "wrong_result", - "evidence": "No real_world_job mem0/OpenMemory adapter is encoded; local same-corpus evidence must not be upgraded to suite coverage.", - "artifact": "docs/guide/research/comparison_external_projects.md" + "status": "pass", + "evidence": "The local OSS mem0 baseline now passes basic same-corpus/update/delete/reload smoke. No real_world_job mem0/OpenMemory adapter, OpenMemory UI, hosted Platform, entity-history, or graph-memory behavior is encoded.", + "artifact": "tmp/live-baseline/live-baseline-report.json" }, "capabilities": [ { @@ -596,13 +624,13 @@ }, { "capability": "same_corpus_retrieval", - "status": "wrong_result", - "evidence": "The checked-in smoke evidence did not prove a correct same-corpus result for mem0." + "status": "pass", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports mem0 retrieval_pass with 3/3 same-corpus retrieval checks." }, { "capability": "local_lifecycle_update_delete_reload", - "status": "real", - "evidence": "The Docker runner exercises public Memory.update, Memory.delete, and a new Memory.from_config over the same local Qdrant/history paths; any miss is reported as lifecycle_fail instead of pass." + "status": "pass", + "evidence": "The Docker runner exercises public Memory.update, Memory.delete, and a new Memory.from_config over the same local Qdrant/history paths; the fresh scoped run reports 4/4 encoded checks passing." }, { "capability": "openmemory_ui_readback", @@ -623,8 +651,8 @@ "suites": [ { "suite_id": "memory_evolution", - "status": "wrong_result", - "evidence": "Local lifecycle checks are encoded in the Docker baseline, but real_world_job memory-evolution prompts are not executed and missed local evidence must remain typed non-pass." + "status": "not_encoded", + "evidence": "Basic local lifecycle checks now pass in Docker, but real_world_job memory-evolution prompts, preference history, deletion audit readback, and entity history are not encoded for mem0/OpenMemory." }, { "suite_id": "personalization", @@ -637,6 +665,33 @@ "evidence": "OpenMemory inspection is not encoded in this runner." } ], + "scenarios": [ + { + "scenario_id": "basic_local_lifecycle", + "suite_id": "memory_evolution", + "status": "pass", + "elf_position": "ties", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports ELF passing 8/8 local lifecycle checks and mem0 passing 4/4 same-corpus retrieval, update, delete, and cold-start reload checks. This is a basic local lifecycle tie at the encoded smoke surface, not a claim about OpenMemory UI, hosted behavior, entity history, or graph memory.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "preference_entity_history", + "suite_id": "personalization", + "status": "not_encoded", + "elf_position": "untested", + "evidence": "mem0/OpenMemory's strongest next comparison is preference and entity-scoped history. The current local OSS Docker baseline does not inspect memory history events, correction chains, or entity-scoped readback under real_world_job scoring.", + "artifact": "docs/guide/benchmarking/2026-06-11-temporal-history-competitor-gap-report.md" + }, + { + "scenario_id": "openmemory_ui_export_readback", + "suite_id": "operator_debugging_ux", + "status": "not_encoded", + "elf_position": "untested", + "evidence": "OpenMemory UI/export readback is not exercised by the local OSS Docker baseline and hosted Platform behavior remains out of scope for local OSS evidence.", + "artifact": "docs/guide/benchmarking/2026-06-11-temporal-history-competitor-gap-report.md" + } + ], "evidence": [ { "kind": "runner", @@ -655,7 +710,7 @@ "evidence_class": "live_baseline_only", "docker_default": true, "host_global_installs_required": false, - "overall_status": "wrong_result", + "overall_status": "pass", "setup": { "status": "pass", "evidence": "The live-baseline Docker runner can install memsearch and run its CLI path.", @@ -663,14 +718,14 @@ "artifact": "tmp/live-baseline/memsearch.log" }, "run": { - "status": "wrong_result", - "evidence": "The Docker runner indexes a per-adapter corpus copy, rewrites and deletes files, reruns memsearch index, and records wrong_result or lifecycle_fail when expected evidence is missed.", + "status": "pass", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 indexes a per-adapter corpus copy, rewrites and deletes files, reruns memsearch index, and reports memsearch 4/4 encoded checks passing.", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "result": { - "status": "wrong_result", - "evidence": "No real_world_job memsearch adapter is encoded; Markdown-first behavior remains a design reference.", - "artifact": "docs/guide/research/comparison_external_projects.md" + "status": "pass", + "evidence": "memsearch now passes the local same-corpus/reindex/update/delete/reload smoke. No real_world_job memsearch prompt adapter is encoded, so Markdown-first behavior remains baseline scenario evidence rather than suite pass evidence.", + "artifact": "tmp/live-baseline/live-baseline-report.json" }, "capabilities": [ { @@ -680,13 +735,13 @@ }, { "capability": "same_corpus_retrieval", - "status": "wrong_result", - "evidence": "The checked-in smoke evidence did not prove correct same-corpus retrieval." + "status": "pass", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports memsearch retrieval_pass with 3/3 same-corpus retrieval checks." }, { "capability": "reindex_update_delete_reload", - "status": "real", - "evidence": "The runner rewrites auth-memory.md, deletes a second corpus file, reruns memsearch index, and starts fresh memsearch search processes for update/delete/cold-start checks." + "status": "pass", + "evidence": "The runner rewrites auth-memory.md, deletes a second corpus file, reruns memsearch index, and starts fresh memsearch search processes; the fresh scoped run reports update, delete, and cold-start reload passing." }, { "capability": "real_world_job_adapter", @@ -697,18 +752,45 @@ "suites": [ { "suite_id": "trust_source_of_truth", - "status": "incomplete", - "evidence": "The Markdown-first source model is relevant, but no real_world_job source-of-truth run is encoded." + "status": "not_encoded", + "evidence": "The Markdown-first source model passed the local reindex/reload smoke, but no real_world_job source-of-truth prompt run is encoded." }, { "suite_id": "retrieval", - "status": "wrong_result", - "evidence": "The Docker same-corpus check reaches memsearch search, but current evidence is not a clean retrieval pass and no job-level run is encoded." + "status": "not_encoded", + "evidence": "The Docker same-corpus check now passes, but no job-level real_world retrieval run is encoded for memsearch." }, { "suite_id": "memory_evolution", - "status": "wrong_result", - "evidence": "Update/delete reindex semantics are exercised in Docker; misses remain typed wrong_result or lifecycle_fail and do not become suite passes." + "status": "not_encoded", + "evidence": "Update/delete reindex semantics pass in Docker, but memory_evolution real_world_job prompts are not encoded for memsearch." + } + ], + "scenarios": [ + { + "scenario_id": "canonical_markdown_reindex_reload", + "suite_id": "trust_source_of_truth", + "status": "pass", + "elf_position": "untested", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports memsearch passed same-corpus retrieval, update reindex, delete suppression, and cold-start reload over a canonical Markdown corpus. ELF has no directly comparable canonical Markdown source-store scenario in this baseline, so the ELF position remains untested.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "ttl_expiry_lifecycle", + "suite_id": "memory_evolution", + "status": "unsupported", + "elf_position": "untested", + "evidence": "The encoded memsearch CLI path supports reindex/delete but no TTL or expiry behavior. Unsupported TTL behavior is preserved as unsupported competitor evidence and does not create an ELF win/loss claim without a directly comparable scenario artifact.", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "real_world_prompt_adapter", + "suite_id": "retrieval", + "status": "not_encoded", + "elf_position": "untested", + "evidence": "No memsearch adapter currently executes real_world_job prompts and answer scoring; baseline retrieval/reindex evidence must stay separate from suite pass claims.", + "artifact": "apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json" } ], "evidence": [ @@ -844,7 +926,7 @@ "result": { "status": "wrong_result", "evidence": "No real_world_job claude-mem adapter is encoded; progressive disclosure remains a design reference.", - "artifact": "docs/guide/research/comparison_external_projects.md" + "artifact": "tmp/live-baseline/live-baseline-report.json" }, "capabilities": [ { @@ -890,6 +972,43 @@ "evidence": "claude-mem hooks are not executed by this runner." } ], + "scenarios": [ + { + "scenario_id": "same_corpus_retrieval", + "suite_id": "retrieval", + "status": "wrong_result", + "elf_position": "wins", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports ELF retrieval_pass and claude-mem same_corpus_retrieval as wrong_result with 0/3 expected query checks passing, while its durable repository setup completed. This is an ELF baseline win for the narrow retrieval smoke scenario.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "repository_lifecycle_reload", + "suite_id": "memory_evolution", + "status": "pass", + "elf_position": "ties", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports ELF passing local lifecycle checks and claude-mem update, delete, and cold-start reload checks passing over a durable Docker-local SQLite repository. This is a local lifecycle-smoke tie, not a hook-driven work-resume or full progressive-disclosure job pass.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "progressive_disclosure_detail_hydration", + "suite_id": "operator_debugging_ux", + "status": "pass", + "elf_position": "untested", + "evidence": "claude-mem passed the repository-level search-to-detail/source hydration check, which is a useful progressive-disclosure signal. ELF does not have a directly comparable claude-mem-style progressive-disclosure scenario in this baseline, so the ELF position remains untested rather than a loss claim.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "hook_capture_viewer_workflow", + "suite_id": "capture_integration", + "status": "not_encoded", + "elf_position": "untested", + "evidence": "The Docker baseline uses repository classes only. claude-mem hooks, viewer, timeline, and observation workflows are not executed by the runner.", + "artifact": "apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json" + } + ], "evidence": [ { "kind": "runner", diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark.rs b/apps/elf-eval/src/bin/real_world_job_benchmark.rs index d0482174..7635c0bd 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark.rs @@ -638,6 +638,15 @@ enum AdapterCoverageStatus { NotEncoded, } +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +enum ElfScenarioPosition { + Wins, + Ties, + Loses, + Untested, +} + #[derive(Debug, Deserialize)] struct ExternalAdapterManifest { schema: String, @@ -685,6 +694,8 @@ struct ExternalAdapterReport { #[serde(default)] suites: Vec, #[serde(default)] + scenarios: Vec, + #[serde(default)] evidence: Vec, #[serde(skip_serializing_if = "Option::is_none")] execution_metadata: Option, @@ -718,6 +729,20 @@ struct AdapterSuiteCoverage { evidence: String, } +#[derive(Clone, Debug, Deserialize, Serialize)] +struct AdapterScenarioJudgment { + scenario_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + suite_id: Option, + status: AdapterCoverageStatus, + elf_position: ElfScenarioPosition, + evidence: String, + #[serde(skip_serializing_if = "Option::is_none")] + command: Option, + #[serde(skip_serializing_if = "Option::is_none")] + artifact: Option, +} + #[derive(Clone, Debug, Deserialize, Serialize)] struct AdapterEvidencePointer { kind: String, @@ -760,6 +785,10 @@ struct ExternalAdapterSummary { overall_status_counts: AdapterStatusCounts, capability_status_counts: AdapterStatusCounts, suite_status_counts: AdapterStatusCounts, + #[serde(default)] + scenario_status_counts: AdapterStatusCounts, + #[serde(default)] + scenario_position_counts: ScenarioPositionCounts, } #[derive(Clone, Debug, Default, Deserialize, Serialize)] @@ -775,6 +804,14 @@ struct AdapterStatusCounts { not_encoded: usize, } +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +struct ScenarioPositionCounts { + wins: usize, + ties: usize, + loses: usize, + untested: usize, +} + #[derive(Clone, Debug, Default, Deserialize, Serialize)] struct CaptureIntegrationReport { #[serde(default)] @@ -3763,6 +3800,7 @@ fn validate_external_adapter(path: &Path, adapter: &ExternalAdapterReport) -> Re validate_adapter_execution(path, adapter)?; validate_adapter_capabilities(path, adapter)?; validate_adapter_suites(path, adapter)?; + validate_adapter_scenarios(path, adapter)?; validate_adapter_evidence(path, adapter)?; validate_adapter_execution_metadata(path, adapter)?; @@ -3833,6 +3871,36 @@ fn validate_adapter_suites(path: &Path, adapter: &ExternalAdapterReport) -> Resu Ok(()) } +fn validate_adapter_scenarios(path: &Path, adapter: &ExternalAdapterReport) -> Result<()> { + for scenario in &adapter.scenarios { + if scenario.scenario_id.trim().is_empty() + || scenario.evidence.trim().is_empty() + || scenario.command.as_deref().is_some_and(str::is_empty) + || scenario.artifact.as_deref().is_some_and(str::is_empty) + { + return Err(eyre::eyre!( + "{} adapter {} has incomplete scenario judgment.", + path.display(), + adapter.adapter_id + )); + } + + if let Some(suite_id) = &scenario.suite_id + && !SUITES.contains(&suite_id.as_str()) + { + return Err(eyre::eyre!( + "{} adapter {} scenario {} references unknown suite {}.", + path.display(), + adapter.adapter_id, + scenario.scenario_id, + suite_id + )); + } + } + + Ok(()) +} + fn validate_adapter_evidence(path: &Path, adapter: &ExternalAdapterReport) -> Result<()> { for evidence in &adapter.evidence { if evidence.kind.trim().is_empty() || evidence.reference.trim().is_empty() { @@ -3919,6 +3987,13 @@ fn accumulate_adapter_summary( for suite in &adapter.suites { increment_adapter_status_count(&mut summary.suite_status_counts, suite.status); } + for scenario in &adapter.scenarios { + increment_adapter_status_count(&mut summary.scenario_status_counts, scenario.status); + increment_scenario_position_count( + &mut summary.scenario_position_counts, + scenario.elf_position, + ); + } } fn increment_adapter_status_count(counts: &mut AdapterStatusCounts, status: AdapterCoverageStatus) { @@ -3935,6 +4010,18 @@ fn increment_adapter_status_count(counts: &mut AdapterStatusCounts, status: Adap } } +fn increment_scenario_position_count( + counts: &mut ScenarioPositionCounts, + position: ElfScenarioPosition, +) { + match position { + ElfScenarioPosition::Wins => counts.wins += 1, + ElfScenarioPosition::Ties => counts.ties += 1, + ElfScenarioPosition::Loses => counts.loses += 1, + ElfScenarioPosition::Untested => counts.untested += 1, + } +} + fn capture_integration_report(jobs: &[RealWorldJob]) -> CaptureIntegrationReport { let mut report = CaptureIntegrationReport::default(); @@ -4092,9 +4179,22 @@ fn render_markdown_external_adapters(out: &mut String, report: &RealWorldReport) adapter_status_counts_display(&summary.capability_status_counts) )); out.push_str(&format!( - "- Real-world suite statuses: `{}`\n\n", + "- Real-world suite statuses: `{}`\n", adapter_status_counts_display(&summary.suite_status_counts) )); + + if has_adapter_scenarios(report.external_adapters.adapters.as_slice()) { + out.push_str(&format!( + "- Scenario coverage statuses: `{}`\n", + adapter_status_counts_display(&summary.scenario_status_counts) + )); + out.push_str(&format!( + "- ELF scenario positions: `{}`\n", + scenario_position_counts_display(&summary.scenario_position_counts) + )); + } + + out.push('\n'); out.push_str("| Project | Adapter | Evidence Class | Overall | Setup | Run | Result | Docker | Suites | Evidence |\n"); out.push_str("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n"); @@ -4130,11 +4230,44 @@ fn render_markdown_external_adapters(out: &mut String, report: &RealWorldReport) } } + render_markdown_adapter_scenarios(out, report.external_adapters.adapters.as_slice()); render_markdown_adapter_execution_metadata(out, report.external_adapters.adapters.as_slice()); out.push('\n'); } +fn render_markdown_adapter_scenarios(out: &mut String, adapters: &[ExternalAdapterReport]) { + if !has_adapter_scenarios(adapters) { + return; + } + + out.push_str("\n### Adapter Scenario Judgments\n\n"); + out.push_str("| Adapter | Scenario | Suite | Status | ELF Position | Evidence |\n"); + out.push_str("| --- | --- | --- | --- | --- | --- |\n"); + + for adapter in adapters { + for scenario in &adapter.scenarios { + out.push_str(&format!( + "| `{}` | `{}` | {} | `{}` | `{}` | {} |\n", + md_inline(adapter.adapter_id.as_str()), + md_inline(scenario.scenario_id.as_str()), + scenario + .suite_id + .as_deref() + .map(|suite| format!("`{}`", md_inline(suite))) + .unwrap_or_else(|| "`none`".to_string()), + adapter_status_str(scenario.status), + scenario_position_str(scenario.elf_position), + adapter_scenario_evidence_cell(scenario) + )); + } + } +} + +fn has_adapter_scenarios(adapters: &[ExternalAdapterReport]) -> bool { + adapters.iter().any(|adapter| !adapter.scenarios.is_empty()) +} + fn render_markdown_adapter_execution_metadata( out: &mut String, adapters: &[ExternalAdapterReport], @@ -4773,6 +4906,15 @@ fn adapter_status_str(status: AdapterCoverageStatus) -> &'static str { } } +fn scenario_position_str(position: ElfScenarioPosition) -> &'static str { + match position { + ElfScenarioPosition::Wins => "wins", + ElfScenarioPosition::Ties => "ties", + ElfScenarioPosition::Loses => "loses", + ElfScenarioPosition::Untested => "untested", + } +} + fn adapter_status_counts_display(counts: &AdapterStatusCounts) -> String { [ ("real", counts.real), @@ -4792,6 +4934,20 @@ fn adapter_status_counts_display(counts: &AdapterStatusCounts) -> String { .join(", ") } +fn scenario_position_counts_display(counts: &ScenarioPositionCounts) -> String { + [ + ("wins", counts.wins), + ("ties", counts.ties), + ("loses", counts.loses), + ("untested", counts.untested), + ] + .into_iter() + .filter(|(_, count)| *count > 0) + .map(|(position, count)| format!("{position}={count}")) + .collect::>() + .join(", ") +} + fn adapter_suite_cell(suites: &[AdapterSuiteCoverage]) -> String { if suites.is_empty() { return "`none`".to_string(); @@ -4827,6 +4983,22 @@ fn adapter_evidence_cell(adapter: &ExternalAdapterReport) -> String { format!("setup: `{}`
result: `{}`", md_inline(setup), md_inline(result)) } +fn adapter_scenario_evidence_cell(scenario: &AdapterScenarioJudgment) -> String { + let evidence = md_cell(scenario.evidence.as_str()); + let command = scenario + .command + .as_deref() + .map(|command| format!("
command: `{}`", md_inline(command))) + .unwrap_or_default(); + let artifact = scenario + .artifact + .as_deref() + .map(|artifact| format!("
artifact: `{}`", md_inline(artifact))) + .unwrap_or_default(); + + format!("{evidence}{command}{artifact}") +} + fn adapter_sources_cell(sources: &[AdapterSource]) -> String { if sources.is_empty() { return "`none`".to_string(); diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 99c3a7ad..46fc4660 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -184,6 +184,76 @@ fn real_world_report_includes_external_adapter_coverage_manifest() -> Result<()> Ok(()) } +#[test] +fn external_adapter_run_summarizes_nonzero_scenario_losses() -> Result<()> { + let manifest_path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("fixtures") + .join("real_world_external_adapters") + .join("memory_projects_manifest.json"); + let mut manifest = serde_json::from_str::(&fs::read_to_string(manifest_path)?)?; + let adapters = manifest + .pointer_mut("/adapters") + .and_then(Value::as_array_mut) + .ok_or_else(|| eyre::eyre!("missing manifest adapters"))?; + let adapter = adapters + .iter_mut() + .find(|adapter| { + adapter.pointer("/adapter_id").and_then(Value::as_str) + == Some("agentmemory_live_baseline") + }) + .ok_or_else(|| eyre::eyre!("missing agentmemory adapter"))?; + + set_json_pointer(adapter, "/scenarios/0/elf_position", serde_json::json!("loses"))?; + + let temp_dir = + env::temp_dir().join(format!("elf-real-world-loss-manifest-test-{}", process::id())); + + fs::create_dir_all(&temp_dir)?; + + let manifest_path = temp_dir.join("memory_projects_manifest.json"); + + fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)?; + + let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) + .arg("run") + .arg("--fixtures") + .arg(fixture_dir()) + .arg("--external-adapter-manifest") + .arg(&manifest_path) + .output()?; + + assert!( + output.status.success(), + "real_world_job runner failed: {}", + String::from_utf8_lossy(&output.stderr), + ); + + let report = serde_json::from_slice::(&output.stdout)?; + + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/loses") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/untested") + .and_then(Value::as_u64), + Some(8) + ); + + let adapters = array_at(&report, "/external_adapters/adapters")?; + let agentmemory = find_by_field(adapters, "/adapter_id", "agentmemory_live_baseline")?; + + assert_eq!( + agentmemory.pointer("/scenarios/0/elf_position").and_then(Value::as_str), + Some("loses") + ); + + Ok(()) +} + fn assert_external_adapter_manifest_summary(report: &Value) { assert_eq!( report.pointer("/external_adapters/schema").and_then(Value::as_str), @@ -233,13 +303,13 @@ fn assert_external_adapter_manifest_summary(report: &Value) { report .pointer("/external_adapters/summary/overall_status_counts/pass") .and_then(Value::as_u64), - Some(1) + Some(3) ); assert_eq!( report .pointer("/external_adapters/summary/overall_status_counts/wrong_result") .and_then(Value::as_u64), - Some(7) + Some(5) ); assert_eq!( report @@ -281,7 +351,96 @@ fn assert_external_adapter_manifest_summary(report: &Value) { report .pointer("/external_adapters/summary/suite_status_counts/blocked") .and_then(Value::as_u64), - Some(10) + Some(12) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/suite_status_counts/incomplete") + .and_then(Value::as_u64), + Some(0) + ); + + assert_external_adapter_manifest_scenario_summary(report); +} + +fn assert_external_adapter_manifest_scenario_summary(report: &Value) { + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/real") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/mocked") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/unsupported") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/blocked") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/incomplete") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/wrong_result") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/lifecycle_fail") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/pass") + .and_then(Value::as_u64), + Some(5) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/not_encoded") + .and_then(Value::as_u64), + Some(4) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/wins") + .and_then(Value::as_u64), + Some(2) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/ties") + .and_then(Value::as_u64), + Some(2) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/loses") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/untested") + .and_then(Value::as_u64), + Some(9) ); } @@ -328,7 +487,7 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { Some("mocked") ); - assert_first_generation_adapter_records(mem0, memsearch, claude_mem); + assert_first_generation_adapter_records(agentmemory, mem0, memsearch, claude_mem); assert_eq!(openviking.pointer("/overall_status").and_then(Value::as_str), Some("wrong_result")); assert_eq!(ragflow.pointer("/evidence_class").and_then(Value::as_str), Some("research_gate")); @@ -383,18 +542,55 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { Ok(()) } -fn assert_first_generation_adapter_records(mem0: &Value, memsearch: &Value, claude_mem: &Value) { +fn assert_first_generation_adapter_records( + agentmemory: &Value, + mem0: &Value, + memsearch: &Value, + claude_mem: &Value, +) { + assert_eq!( + agentmemory.pointer("/scenarios/1/status").and_then(Value::as_str), + Some("lifecycle_fail") + ); + assert_eq!( + agentmemory.pointer("/scenarios/1/elf_position").and_then(Value::as_str), + Some("wins") + ); + assert_eq!(agentmemory.pointer("/scenarios/2/status").and_then(Value::as_str), Some("blocked")); assert_eq!( mem0.pointer("/capabilities/2/capability").and_then(Value::as_str), Some("local_lifecycle_update_delete_reload") ); - assert_eq!(mem0.pointer("/capabilities/2/status").and_then(Value::as_str), Some("real")); + assert_eq!(mem0.pointer("/capabilities/2/status").and_then(Value::as_str), Some("pass")); assert_eq!(mem0.pointer("/capabilities/4/status").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(mem0.pointer("/scenarios/0/status").and_then(Value::as_str), Some("pass")); + assert_eq!(mem0.pointer("/scenarios/0/elf_position").and_then(Value::as_str), Some("ties")); + assert_eq!( + mem0.pointer("/scenarios/2/scenario_id").and_then(Value::as_str), + Some("openmemory_ui_export_readback") + ); + assert_eq!(mem0.pointer("/scenarios/2/status").and_then(Value::as_str), Some("not_encoded")); assert_eq!( memsearch.pointer("/capabilities/2/capability").and_then(Value::as_str), Some("reindex_update_delete_reload") ); - assert_eq!(memsearch.pointer("/capabilities/2/status").and_then(Value::as_str), Some("real")); + assert_eq!(memsearch.pointer("/capabilities/2/status").and_then(Value::as_str), Some("pass")); + assert_eq!( + memsearch.pointer("/scenarios/0/scenario_id").and_then(Value::as_str), + Some("canonical_markdown_reindex_reload") + ); + assert_eq!( + memsearch.pointer("/scenarios/0/elf_position").and_then(Value::as_str), + Some("untested") + ); + assert_eq!( + memsearch.pointer("/scenarios/1/status").and_then(Value::as_str), + Some("unsupported") + ); + assert_eq!( + memsearch.pointer("/scenarios/1/elf_position").and_then(Value::as_str), + Some("untested") + ); assert_eq!(claude_mem.pointer("/capabilities/1/status").and_then(Value::as_str), Some("real")); assert_eq!( claude_mem.pointer("/capabilities/3/capability").and_then(Value::as_str), @@ -404,6 +600,11 @@ fn assert_first_generation_adapter_records(mem0: &Value, memsearch: &Value, clau claude_mem.pointer("/capabilities/4/status").and_then(Value::as_str), Some("not_encoded") ); + assert_eq!( + claude_mem.pointer("/scenarios/0/status").and_then(Value::as_str), + Some("wrong_result") + ); + assert_eq!(claude_mem.pointer("/scenarios/1/status").and_then(Value::as_str), Some("pass")); } fn assert_graphiti_zep_adapter(adapter: &Value) { @@ -634,7 +835,7 @@ fn assert_live_sweep_record(adapter: &Value) -> Result<()> { assert_eq!(full_pass.pointer("/status").and_then(Value::as_str), Some("wrong_result")); assert_eq!(work_resume.pointer("/status").and_then(Value::as_str), Some("pass")); assert_eq!(memory_evolution.pointer("/status").and_then(Value::as_str), Some("wrong_result")); - assert_eq!(production_ops.pointer("/status").and_then(Value::as_str), Some("incomplete")); + assert_eq!(production_ops.pointer("/status").and_then(Value::as_str), Some("blocked")); assert_eq!(consolidation.pointer("/status").and_then(Value::as_str), Some("not_encoded")); Ok(()) @@ -965,6 +1166,143 @@ fn generated_json_report_renders_markdown() -> Result<()> { assert!(markdown.contains("agentmemory-style hook capture")); assert!(markdown.contains("xy844-current-worktree")); assert!(markdown.contains("Existing live-baseline reports remain valid")); + assert!(markdown.contains("### Adapter Scenario Judgments")); + assert!(markdown.contains("ELF scenario positions: `wins=2, ties=2, untested=9`")); + assert!(markdown.contains("| `claude_mem_live_baseline` | `same_corpus_retrieval`")); + assert!(markdown.contains("| `memsearch_live_baseline` | `ttl_expiry_lifecycle`")); + + Ok(()) +} + +#[test] +fn external_adapter_markdown_renders_nonzero_scenario_losses() -> Result<()> { + let mut report = run_json_report()?; + let adapters = report + .pointer_mut("/external_adapters/adapters") + .and_then(Value::as_array_mut) + .ok_or_else(|| eyre::eyre!("missing external adapter records"))?; + let adapter = adapters + .iter_mut() + .find(|adapter| { + adapter.pointer("/adapter_id").and_then(Value::as_str) + == Some("agentmemory_live_baseline") + }) + .ok_or_else(|| eyre::eyre!("missing agentmemory adapter"))?; + + set_json_pointer(adapter, "/scenarios/0/elf_position", serde_json::json!("loses"))?; + set_json_pointer( + &mut report, + "/external_adapters/summary/scenario_position_counts", + serde_json::json!({ + "wins": 2, + "ties": 2, + "loses": 1, + "untested": 8 + }), + )?; + + let temp_dir = + env::temp_dir().join(format!("elf-real-world-loss-scenario-test-{}", process::id())); + + fs::create_dir_all(&temp_dir)?; + + let report_path = temp_dir.join("report.json"); + let markdown_path = temp_dir.join("report.md"); + + fs::write(&report_path, serde_json::to_vec_pretty(&report)?)?; + + let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) + .arg("publish") + .arg("--report") + .arg(&report_path) + .arg("--out") + .arg(&markdown_path) + .output()?; + + assert!( + output.status.success(), + "real_world_job publisher failed: {}", + String::from_utf8_lossy(&output.stderr), + ); + + let markdown = fs::read_to_string(markdown_path)?; + + assert!(markdown.contains("ELF scenario positions: `wins=2, ties=2, loses=1, untested=8`")); + assert!(markdown.contains( + "| `agentmemory_live_baseline` | `basic_same_corpus_retrieval` | `retrieval` | `pass` | `loses` |" + )); + + Ok(()) +} + +#[test] +fn external_adapter_markdown_omits_scenario_summary_when_manifest_has_no_scenarios() -> Result<()> { + let mut report = run_json_report()?; + let adapters = report + .pointer_mut("/external_adapters/adapters") + .and_then(Value::as_array_mut) + .ok_or_else(|| eyre::eyre!("missing external adapter records"))?; + + for adapter in adapters { + set_json_pointer(adapter, "/scenarios", serde_json::json!([]))?; + } + + set_json_pointer( + &mut report, + "/external_adapters/summary/scenario_status_counts", + serde_json::json!({ + "real": 0, + "mocked": 0, + "unsupported": 0, + "blocked": 0, + "incomplete": 0, + "wrong_result": 0, + "lifecycle_fail": 0, + "pass": 0, + "not_encoded": 0 + }), + )?; + set_json_pointer( + &mut report, + "/external_adapters/summary/scenario_position_counts", + serde_json::json!({ + "wins": 0, + "ties": 0, + "loses": 0, + "untested": 0 + }), + )?; + + let temp_dir = + env::temp_dir().join(format!("elf-real-world-no-scenario-test-{}", process::id())); + + fs::create_dir_all(&temp_dir)?; + + let report_path = temp_dir.join("report.json"); + let markdown_path = temp_dir.join("report.md"); + + fs::write(&report_path, serde_json::to_vec_pretty(&report)?)?; + + let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) + .arg("publish") + .arg("--report") + .arg(&report_path) + .arg("--out") + .arg(&markdown_path) + .output()?; + + assert!( + output.status.success(), + "real_world_job publisher failed: {}", + String::from_utf8_lossy(&output.stderr), + ); + + let markdown = fs::read_to_string(markdown_path)?; + + assert!(markdown.contains("External Adapter Coverage")); + assert!(!markdown.contains("Scenario coverage statuses:")); + assert!(!markdown.contains("ELF scenario positions:")); + assert!(!markdown.contains("### Adapter Scenario Judgments")); Ok(()) } diff --git a/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md b/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md index 97dcfb32..c78e50f3 100644 --- a/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md +++ b/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md @@ -26,8 +26,9 @@ is encoded and run at a comparable evidence class. Current boundary: - ELF and qmd have full-suite `live_real_world` sweeps, but neither has a full-suite - live pass. Each sweep produced 38 jobs with 18 pass, 5 wrong_result, 1 incomplete, - 2 blocked, and 12 not_encoded. + live pass. The fresh ELF sweep produced 38 jobs with 18 pass, 5 wrong_result, + 0 incomplete, 2 blocked, and 13 not_encoded; the fresh qmd sweep produced 17 pass, + 6 wrong_result, 0 incomplete, 2 blocked, and 13 not_encoded. - ELF fixture evidence is strong: `cargo make real-world-memory` reports 38 jobs across 11 suites with 36 pass and 2 blocked production-ops operator boundaries. That proves the fixture contract, not live-service parity. @@ -43,8 +44,8 @@ Current boundary: The current manifest has 21 adapter records across 16 external projects plus ELF. Evidence-class counts: 1 `fixture_backed`, 6 `live_baseline_only`, 3 -`live_real_world`, and 11 `research_gate`. Overall adapter-status counts: 1 `pass`, -7 `wrong_result`, 1 `lifecycle_fail`, 5 `blocked`, and 7 `not_encoded`. +`live_real_world`, and 11 `research_gate`. Overall adapter-status counts: 3 `pass`, +5 `wrong_result`, 1 `lifecycle_fail`, 5 `blocked`, and 7 `not_encoded`. ## State Taxonomy @@ -71,11 +72,11 @@ lifecycle-fail -> `lifecycle_fail`, and not-encoded -> `not_encoded`. | Project | Strongest user-facing scenario | Current evidence | Measured status and proof | Unsupported or blocked status | Required benchmark before ELF claim | Borrow if stronger | | --- | --- | --- | --- | --- | --- | --- | -| ELF | Evidence-linked source-of-truth memory service with real-world fixtures and live retrieval sweeps. | `live_real_world`; supporting `fixture_backed`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/elf-report.md`. Fixture contract: `cargo make real-world-memory`, `tmp/real-world-memory/real-world-memory-report.json`. | `blocked`: private manifest and provider credentials; broader live suites remain `wrong_result`, `incomplete`, or `not_encoded`. | Full-suite live pass plus separate private-corpus and credentialed production-ops proof. | Keep borrowing qmd debug knobs, OpenViking staged trajectory, mem0 history, Letta core memory, and graph/RAG navigation. | +| ELF | Evidence-linked source-of-truth memory service with real-world fixtures and live retrieval sweeps. | `live_real_world`; supporting `fixture_backed`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/elf-report.md`. Fixture contract: `cargo make real-world-memory`, `tmp/real-world-memory/real-world-memory-report.json`. | `blocked`: private manifest and provider credentials; broader live suites remain `wrong_result`, `blocked`, or `not_encoded`. | Full-suite live pass plus separate private-corpus and credentialed production-ops proof. | Keep borrowing qmd debug knobs, OpenViking staged trajectory, mem0 history, Letta core memory, and graph/RAG navigation. | | qmd | Local retrieval-debug workflow with transparent CLI indexing, querying, expansion, fusion, and rerank ergonomics. | `live_real_world`; supporting `live_baseline_only` and `research_gate`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/qmd-report.md`; targeted retrieval suites pass. | `not_encoded`: deep profile and non-retrieval live behavior are not encoded; memory_evolution is `wrong_result`. | qmd deep retrieval/debug profile plus full-suite live replay with trace-level diagnostics. | Weighted fusion, rerank explanation, local debug knobs, and command-line replay. | | agentmemory | Coding-agent continuity, MCP/REST packaging, viewer workflow, and durable cross-agent memory lifecycle. | `live_baseline_only`. | `lifecycle_fail`: `ELF_BASELINE_PROJECTS=agentmemory cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `blocked`: durable cold-start and real-world adapter coverage are missing. | Durable local adapter with update, delete, cold-start reload, work_resume, capture/write-policy, and lifecycle-staleness jobs. | Cross-agent hooks, packaging, continuity scenarios, and viewer affordances. | -| mem0/OpenMemory | Memory lifecycle, personalization, hosted/OpenMemory UI ergonomics, and optional graph memory. | `live_baseline_only`. | `wrong_result`: `ELF_BASELINE_PROJECTS=mem0 cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `not_encoded`: OpenMemory UI, hosted claims, and real-world personalization coverage are not encoded. | Fix local same-corpus result, then encode memory_evolution, personalization, UI readback, and optional graph-context jobs. | Entity-scoped history, lifecycle surfaces, async update ergonomics, and OpenMemory inspection UX. | -| memsearch | Markdown-first canonical store with rebuildable local index and practical hybrid retrieval. | `live_baseline_only`. | `wrong_result`: `ELF_BASELINE_PROJECTS=memsearch cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `incomplete`: source-of-truth and real-world reindex behavior are not cleanly scored. | Fix Docker same-corpus retrieval and reindex/update/delete reload evidence, then score source-of-truth and retrieval-debug jobs. | Canonical markdown store, local reindex clarity, and user-inspectable source files. | +| mem0/OpenMemory | Memory lifecycle, personalization, hosted/OpenMemory UI ergonomics, and optional graph memory. | `live_baseline_only`. | `pass`: fresh scoped run `ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`, with mem0 `4/4` local checks passing. | `not_encoded`: OpenMemory UI, hosted claims, entity/preference history, graph memory, and real-world personalization coverage are not encoded. | Encode memory_evolution preference/entity history, deletion audit readback, personalization, UI/export readback, and optional graph-context jobs. | Entity-scoped history, lifecycle surfaces, async update ergonomics, and OpenMemory inspection UX. | +| memsearch | Markdown-first canonical store with rebuildable local index and practical hybrid retrieval. | `live_baseline_only`. | `pass`: fresh scoped run `ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`, with memsearch `4/4` local checks passing. | `not_encoded`: real-world source-of-truth, retrieval, and memory-evolution prompt adapters are not encoded; TTL/expiry is unsupported by the current CLI path. | Score source-of-truth and retrieval-debug real-world jobs over the canonical Markdown store; keep TTL/expiry as unsupported unless a comparable path exists. | Canonical markdown store, local reindex clarity, and user-inspectable source files. | | OpenViking | Filesystem-like context trajectory, hierarchical retrieval, and staged context loading. | `live_baseline_only`; supporting `research_gate`. | `wrong_result`: `ELF_BASELINE_PROJECTS=OpenViking cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `not_encoded`: hierarchical context trajectory is not encoded; same-corpus output still misses expected evidence. | Make evidence-bearing same-corpus output pass, then score staged trajectory and hierarchy expansion. | `viking://`-style context model, trajectory readback, and staged retrieval planning. | | claude-mem | Progressive disclosure, automatic capture loop, repository-local lifecycle, and local viewer workflow. | `live_baseline_only`. | `wrong_result`: `ELF_BASELINE_PROJECTS=claude-mem cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `not_encoded`: progressive-disclosure real-world jobs are not encoded. | Durable repository-backed work_resume, operator_debugging_ux, capture/write-policy, and progressive-disclosure jobs. | Progressive disclosure, automatic capture review loops, and local viewer/operator comfort. | | RAGFlow | Full RAG application workflow with document, chunk, and reference evidence handles. | `research_gate`. | `blocked`: `ELF_RAGFLOW_SMOKE_START=1 ELF_RAGFLOW_SMOKE_ACCEPT_RESOURCE_ENVELOPE=1 cargo make ragflow-docker-smoke`, `tmp/real-world-memory/ragflow-smoke/ragflow-smoke.json`. | `blocked`: Docker resource envelope and adapter output mapping still need proof. | XY-885 tiny Docker evidence-smoke adapter mapping `reference.chunks` to scored evidence. | Document/chunk references, resource-envelope reporting, and RAG app evidence handles. | @@ -96,13 +97,13 @@ lifecycle-fail -> `lifecycle_fail`, and not-encoded -> `not_encoded`. | Retrieval/debug | Fixture retrieval passes; live retrieval passes. | qmd. | qmd live retrieval passes and live baseline passes, but full-suite live status is `wrong_result`. | Run qmd deep profile and ELF/qmd trace-level replay with expansion, fusion, rerank, and candidate-drop diagnostics. | | Work resume | Fixture and live work_resume pass. | agentmemory, claude-mem, OpenViking. | agentmemory `lifecycle_fail`, claude-mem `wrong_result`, OpenViking work_resume `not_encoded`. | Encode durable work_resume adapters or keep each blocked with lifecycle/setup evidence. | | Project decisions | Fixture and live project_decisions pass. | qmd, Letta. | qmd live project_decisions pass; Letta is `research_gate` `not_encoded`. | Add Letta core/archival decision jobs only after a contained export path exists. | -| Source-of-truth | Fixture and live trust_source_of_truth pass. | memsearch. | memsearch canonical-store evidence exists, but source-of-truth is `incomplete` and retrieval is `wrong_result`. | Fix memsearch reindex/retrieval evidence and score source-of-truth rebuild/reload jobs. | -| Temporal/current-vs-historical memory | Fixture memory_evolution passes; live memory_evolution is `wrong_result`. | Graphiti/Zep, mem0/OpenMemory. | Graphiti/Zep is `research_gate` `blocked`; mem0/OpenMemory is `wrong_result`. | Fix ELF/qmd live memory_evolution evidence links and run XY-888. | +| Source-of-truth | Fixture and live trust_source_of_truth pass. | memsearch. | memsearch canonical-store, reindex, delete, and reload smoke now passes, but source-of-truth real_world_job prompts are `not_encoded`. | Score memsearch source-of-truth rebuild/reload jobs before any suite-level win/loss claim. | +| Temporal/current-vs-historical memory | Fixture memory_evolution passes; live memory_evolution is `wrong_result`. | Graphiti/Zep, mem0/OpenMemory. | Graphiti/Zep is `research_gate` `blocked`; mem0/OpenMemory basic local lifecycle now passes, but preference/entity history, deletion audit, UI/export, and graph-memory scenarios are `not_encoded`. | Fix ELF/qmd live memory_evolution evidence links, encode mem0/OpenMemory history/UI jobs, and run XY-888. | | Consolidation | Fixture consolidation passes; live consolidation is `not_encoded`. | agentmemory, managed-memory references, llm-wiki. | No manifest project has live consolidation scoring. | Run reviewable consolidation proposal generation with source refs, unsupported-claim flags, and audit transitions. | | Knowledge pages | Fixture knowledge_compilation passes; live knowledge_compilation is `not_encoded`. | llm-wiki, gbrain, GraphRAG, graphify. | llm-wiki and gbrain are `research_gate` `not_encoded` or `blocked`; GraphRAG is `blocked`; graphify has a tiny scored smoke `wrong_result`. | Encode live derived-page rebuild/lint scoring and run contained knowledge/RAG adapters only after setup proof. | | Operator debugging | Fixture operator_debugging_ux passes; live operator_debugging_ux is `not_encoded`. | qmd, claude-mem, OpenMemory. | qmd has debug strengths but operator_debugging_ux is `not_encoded`; claude-mem and OpenMemory UX are `not_encoded`. | Score trace hydration, stage attribution, raw-SQL avoidance, and repair-action clarity through live artifacts. | | Capture/write policy | Fixture capture_integration passes; live capture_integration is `not_encoded`. | agentmemory, claude-mem. | agentmemory capture is `blocked`; claude-mem capture is `not_encoded`. | Run live capture/write-policy jobs proving redaction, exclusion, evidence binding, and no secret leakage. | -| Production ops | Fixture production_ops has 4 pass and 2 blocked; live production_ops is `incomplete`; production adoption has provider/backfill/restore evidence. | ELF production gate, qmd, RAG/RAGFlow resource gates. | qmd live production_ops is `incomplete`; RAG/resource gates are `research_gate` `blocked`. | Rerun private-corpus and credentialed gates only when operator-owned manifest and credentials exist. | +| Production ops | Fixture production_ops has 4 pass and 2 blocked; live production_ops is `blocked`; production adoption has provider/backfill/restore evidence. | ELF production gate, qmd, RAG/RAGFlow resource gates. | qmd live production_ops is `blocked`; RAG/resource gates are `research_gate` `blocked`. | Rerun private-corpus and credentialed gates only when operator-owned manifest and credentials exist. | | Personalization | Fixture and live personalization pass. | mem0/OpenMemory, Letta. | mem0/OpenMemory and Letta personalization are `not_encoded`. | Encode scoped preference readback for mem0/OpenMemory and Letta before personalization superiority claims. | | Context trajectory | ELF has trace direction but no comparable staged trajectory scenario. | OpenViking. | OpenViking setup is pinned, same-corpus retrieval is `wrong_result`, and hierarchy trajectory is `not_encoded`. | Make OpenViking evidence-bearing retrieval pass, then score staged context trajectory outputs. | | Core-vs-archival memory | ELF core-block semantics exist in the service contract, but comparative benchmark coverage is not encoded here. | Letta. | Letta is `research_gate` `not_encoded` until contained export proof exists. | Add ELF core-block versus archival-search jobs; compare Letta only after contained export proof. | @@ -117,8 +118,8 @@ now explicit: | --- | --- | --- | --- | --- | | qmd deep retrieval/debug profile | New benchmark issue | yes | None after this matrix lands. | Stress profile plus trace-level retrieval-debug artifacts for qmd and ELF. | | agentmemory durable lifecycle adapter | `[ELF benchmark P0] Make external adapters lifecycle-durable and fail-typed` | yes | Durable local adapter path selection. | Update, delete, cold-start reload, work_resume, and capture/write-policy jobs. | -| mem0/OpenMemory local and UI coverage | New adapter repair issue | yes | Comparable local OSS path for UI/readback evidence. | Same-corpus fix plus memory_evolution, personalization, and OpenMemory inspection jobs. | -| memsearch source-of-truth and reindex coverage | New adapter repair issue | yes | Docker same-corpus retrieval and reindex correctness. | Canonical markdown store, rebuild/reindex, retrieval, update/delete/reload jobs. | +| mem0/OpenMemory history and UI coverage | New adapter repair issue | yes | Comparable local OSS path for history/UI/readback evidence. | Preference/entity history, deletion audit readback, personalization, OpenMemory inspection/export, and optional graph-context jobs. | +| memsearch source-of-truth real-world coverage | New adapter repair issue | yes | Real-world prompt adapter over the canonical Markdown store. | Source-of-truth rebuild/reload jobs and retrieval-debug jobs that preserve baseline reindex/update/delete evidence without converting it into suite pass claims. | | OpenViking context trajectory | New benchmark issue after evidence output fix | yes | Evidence-bearing same-corpus retrieval output. | Hierarchical expansion, staged trajectory, and resume/retrieval evidence jobs. | | claude-mem progressive disclosure | New adapter issue | yes | Durable repository path and progressive-disclosure output contract. | Work resume, operator debugging, capture/write-policy, and progressive disclosure jobs. | | RAGFlow evidence smoke | XY-885 | yes | Resource envelope accepted for tiny Docker smoke. | `reference.chunks` to benchmark evidence mapping. | diff --git a/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md b/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md index 12ee4bc1..c5f988bb 100644 --- a/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md +++ b/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md @@ -66,17 +66,18 @@ sweeps for ELF and qmd: | Adapter | Jobs | Pass | Wrong result | Incomplete | Blocked | Not encoded | Mean score | Evidence recall | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| ELF live service adapter | `38` | `18` | `5` | `1` | `2` | `12` | `0.514` | `41/75` | -| qmd live CLI adapter | `38` | `18` | `5` | `1` | `2` | `12` | `0.512` | `41/75` | +| ELF live service adapter | `38` | `18` | `5` | `0` | `2` | `13` | `0.525` | `41/77` | +| qmd live CLI adapter | `38` | `17` | `6` | `0` | `2` | `13` | `0.486` | `38/77` | Interpretation: -- This is a tie for the currently encoded live real-world sweep. +- This is a near tie for the currently encoded live real-world sweep, with ELF one + job ahead in this fresh run. - Both pass `trust_source_of_truth`, `work_resume`, `project_decisions`, `retrieval`, and `personalization`. - Both fail `memory_evolution` live conflict evidence with `wrong_result`. - Both leave consolidation, knowledge compilation, operator debugging, capture - integration, and parts of production operations as `not_encoded` or incomplete. + integration, and production-ops operator boundaries as `not_encoded` or `blocked`. ### Production Evidence @@ -108,8 +109,8 @@ Overall adapter statuses: | Status | Count | | --- | ---: | -| `pass` | `1` | -| `wrong_result` | `7` | +| `pass` | `3` | +| `wrong_result` | `5` | | `lifecycle_fail` | `1` | | `blocked` | `5` | | `not_encoded` | `7` | @@ -144,8 +145,8 @@ one misleading score. | ELF | `fixture_backed` plus `live_real_world`; live full sweep is `wrong_result`. | Evidence-linked memory service, strict provenance, rebuildable Qdrant, production backfill/restore proof. | Keep this as the core; do not weaken source-of-truth or typed failure semantics while adding product ergonomics. | | qmd | `live_real_world` plus `live_baseline_only`; targeted retrieval passes, full sweep is `wrong_result`. | Local retrieval-debug workflow, transparent CLI, weighted fusion, rerank, replayable commands. | Treat qmd as the retrieval-debug bar. ELF should match its introspection and local replay without becoming CLI-only. | | agentmemory | `live_baseline_only`; current status is `lifecycle_fail`. | Coding-agent continuity, hooks, MCP/REST packaging, viewer/console observability. | Borrow capture breadth and continuity UX, but require durable lifecycle proof before claims. | -| mem0/OpenMemory | `live_baseline_only`; current status is `wrong_result`. | Entity-scoped memory, lifecycle/history surfaces, hosted ecosystem, OpenMemory UI. | Add entity/preference history and UI readback patterns, while keeping hosted claims out of local OSS benchmarks. | -| memsearch | `live_baseline_only`; current status is `wrong_result` with source-of-truth gaps. | Markdown-first canonical store and local reindex clarity. | Borrow local inspectability and canonical-file ergonomics, not file-as-authority semantics. | +| mem0/OpenMemory | `live_baseline_only`; basic local smoke now passes, while entity/preference history, hosted ecosystem, graph memory, and OpenMemory UI remain untested locally. | Entity-scoped memory, lifecycle/history surfaces, hosted ecosystem, OpenMemory UI. | Add entity/preference history and UI readback patterns, while keeping hosted claims out of local OSS benchmarks. | +| memsearch | `live_baseline_only`; canonical Markdown reindex/reload smoke now passes, while real-world source-of-truth prompts remain unencoded. | Markdown-first canonical store and local reindex clarity. | Borrow local inspectability and canonical-file ergonomics, not file-as-authority semantics. | | OpenViking | `live_baseline_only` plus `research_gate`; current status is `wrong_result`. | Filesystem-like context model, hierarchy, staged context trajectory. | Add staged retrieval and trajectory scoring after same-corpus evidence output is correct. | | claude-mem | `live_baseline_only`; current status is `wrong_result`. | Progressive disclosure, automatic capture, local viewer workflow. | Borrow progressive disclosure and viewer comfort; benchmark capture and operator-debugging live paths. | | RAGFlow | `research_gate`; current status is `blocked`. | Full RAG application workflow with document/chunk/reference handles. | Use as a resource-aware RAG adapter benchmark, not as a current ELF competitor win/loss. | @@ -235,9 +236,10 @@ These are needed for broad credibility but should not block personal production scoring. 3. mem0/OpenMemory and memsearch coverage - - Current state: both are `wrong_result` or partially incomplete in local checks. - - Benchmark gate: fix same-corpus correctness first; only then score entity - history, UI readback, markdown store, and reindex workflows. + - Current state: both now pass the basic local OSS smoke, but their strongest + real-world scenarios remain unencoded. + - Benchmark gate: score mem0/OpenMemory entity history and UI readback, plus + memsearch source-of-truth and retrieval-debug workflows. ## What Not To Claim Yet diff --git a/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md b/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md new file mode 100644 index 00000000..368bbb86 --- /dev/null +++ b/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md @@ -0,0 +1,113 @@ +# First-Generation OSS Adapter Promotion Report - June 11, 2026 + +Goal: Promote first-generation OSS memory baselines into scenario-level adapter +evidence without converting live-baseline-only runs into real-world suite wins. +Read this when: You need the current XY-898 status for agentmemory, mem0/OpenMemory, +memsearch, and claude-mem scenario evidence. +Inputs: Fresh scoped Docker baseline run, updated external adapter manifest, and the +June 11 temporal/history competitor-gap report. +Outputs: Scenario judgments, ELF win/tie/loss/untested positions, and next adapter +gates. + +## Scope Boundary + +This is benchmark/report evidence only. No ELF retrieval, ranking, memory-quality, or +service behavior optimization is implemented here. + +The updated external adapter manifest now includes scenario-level judgments for the +first-generation OSS memory projects. These judgments are intentionally narrower than +suite passes: + +- `live_baseline_only` pass evidence proves the encoded Docker same-corpus or + lifecycle smoke for that project. +- It does not prove `real_world_job` suite parity unless a project adapter actually + executes real-world prompts and scoring. +- Hosted mem0 Platform behavior, OpenMemory UI, host-global hooks, and + operator-owned credentials remain out of scope for local OSS evidence. + +## Fresh Run + +| Command | Result | Runtime | Artifact | +| --- | --- | ---: | --- | +| `ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker` | fail with typed non-pass projects | 295.74 seconds | `tmp/live-baseline/live-baseline-report.json` | + +The aggregate failed because two projects remained typed non-pass, not because setup +collapsed: + +| Project | Status | Retrieval | Checks | Scenario meaning | +| --- | --- | --- | ---: | --- | +| ELF | `pass` | `retrieval_pass` | `8/8` pass | Baseline reference for same-class scenario comparisons; no ELF optimization change was made. | +| agentmemory | `lifecycle_fail` | `retrieval_pass` | `2/4` pass, `1` lifecycle_fail, `1` blocked | Same-corpus retrieval runs, but update supersession and durable cold-start are not proven through the in-memory mock. | +| mem0/OpenMemory | `pass` | `retrieval_pass` | `4/4` pass | Basic local OSS same-corpus, update, delete, and cold-start smoke passes. | +| memsearch | `pass` | `retrieval_pass` | `4/4` pass | Canonical Markdown reindex/update/delete/reload smoke passes. | +| claude-mem | `wrong_result` | `retrieval_wrong_result` | `4/5` pass | Durable repository lifecycle, detail hydration, and reload pass, but same-corpus retrieval misses expected evidence. | + +## Scenario Judgments + +| Project | Scenario | Status | ELF position | Evidence boundary | +| --- | --- | --- | --- | --- | +| agentmemory | basic same-corpus retrieval | `pass` | `untested` | Baseline retrieval passes through an in-memory mock; no durable continuity claim. | +| agentmemory | durable update/reload lifecycle | `lifecycle_fail` | `wins` | Update supersession fails and cold-start is blocked; ELF has broader encoded local lifecycle proof. | +| agentmemory | work-resume capture continuity | `blocked` | `untested` | Needs a durable local session/capture path before fair scoring. | +| mem0/OpenMemory | basic local lifecycle | `pass` | `ties` | ELF and mem0 both pass the encoded local lifecycle smoke; mem0 is no longer a basic-smoke failure. | +| mem0/OpenMemory | preference/entity history | `not_encoded` | `untested` | History, correction chains, entity scope, and deletion audit are not scored. | +| mem0/OpenMemory | OpenMemory UI/export readback | `not_encoded` | `untested` | Local OSS UI/export readback is not executed; hosted behavior remains out of scope. | +| memsearch | canonical Markdown reindex/reload | `pass` | `untested` | Baseline reindex/update/delete/reload passes over the canonical file store; ELF has no directly comparable canonical Markdown source-store scenario in this run. | +| memsearch | TTL/expiry lifecycle | `unsupported` | `untested` | The encoded CLI path has reindex/delete but no TTL/expiry behavior; unsupported competitor evidence does not create an ELF win/loss without a comparable scenario artifact. | +| memsearch | real-world prompt adapter | `not_encoded` | `untested` | No memsearch real_world_job prompt adapter is encoded. | +| claude-mem | same-corpus retrieval | `wrong_result` | `wins` | The durable repository path runs but misses expected retrieval evidence. | +| claude-mem | repository lifecycle reload | `pass` | `ties` | Update, delete, and cold-start reload pass over Docker-local SQLite. | +| claude-mem | progressive-disclosure detail hydration | `pass` | `untested` | Search-to-detail/source hydration passes, but ELF has no directly comparable claude-mem-style progressive-disclosure scenario here. | +| claude-mem | hook capture viewer workflow | `not_encoded` | `untested` | Hooks, viewer, timeline, and observations are not executed. | + +Summary: 13 scenario judgments: 5 `pass`, 1 `wrong_result`, 1 `lifecycle_fail`, +1 `blocked`, 1 `unsupported`, and 4 `not_encoded`. ELF positions are 2 `wins`, +2 `ties`, 0 `loses`, and 9 `untested`. + +## Manifest And Report Changes + +The external adapter manifest is now +`real-world-memory-project-adapters-2026-06-11` and includes `scenarios[]` records +with: + +- `scenario_id` +- optional `suite_id` +- typed scenario `status` +- `elf_position`: `wins`, `ties`, `loses`, or `untested` +- evidence text plus optional command/artifact pointers + +`real_world_job_benchmark` now preserves these fields in generated reports and +renders an **Adapter Scenario Judgments** table. This makes the report input capable +of saying whether ELF wins, ties, loses, or remains untested per scenario without +changing the real-world suite status rules. + +## Claim Boundaries + +Allowed: + +- mem0/OpenMemory passes the current basic local OSS lifecycle smoke. +- memsearch passes the current canonical Markdown reindex/reload smoke. +- agentmemory remains non-pass for durable lifecycle because the current adapter uses + an in-memory mock and cannot prove cold-start recovery. +- claude-mem remains wrong-result for same-corpus retrieval while preserving useful + passed evidence for repository lifecycle and detail hydration. + +Not allowed: + +- Do not claim hosted OpenMemory behavior from local OSS evidence. +- Do not claim mem0/OpenMemory history, UI/export, hosted, or graph-memory parity. +- Do not claim memsearch source-of-truth real-world suite parity from baseline smoke. +- Do not claim claude-mem hook/viewer/capture parity from repository-only checks. +- Do not collapse `wrong_result`, `lifecycle_fail`, `blocked`, `unsupported`, + `not_encoded`, and `incomplete` into one generic failure bucket. + +## Next Gates + +- agentmemory: select a durable local KV/index/session path before work-resume and + capture jobs. +- mem0/OpenMemory: encode preference/entity history, deletion audit, UI/export + readback, and optional graph memory for local OSS only. +- memsearch: encode real-world source-of-truth and retrieval-debug prompt jobs over + the canonical Markdown store. +- claude-mem: fix or explain same-corpus retrieval misses, then encode hook capture, + viewer/operator, and progressive-disclosure jobs. diff --git a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md index 9daa9eb6..ea0636aa 100644 --- a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md +++ b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md @@ -6,7 +6,8 @@ Read this when: You need to answer whether ELF has enough empirical evidence to claim a win, tie, loss, or non-claim against tracked memory, RAG, graph, and agent-continuity projects. Inputs: Fresh local runs of `cargo make real-world-memory` and -`cargo make real-world-memory-live-adapters` on commit `286af8b`, plus +`cargo make real-world-memory-live-adapters` in the current XY-898 lane after +adapter-report consistency repairs, plus `apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json`, `2026-06-11-competitor-strength-evidence-matrix.md`, and `2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md`. @@ -23,9 +24,10 @@ What is proven today: - ELF has a strong fixture-backed real-world benchmark contract: 38 jobs, 36 pass, 2 blocked operator boundaries, and no wrong results in the fixture aggregate. -- ELF and qmd have comparable full-suite live real-world sweeps. They are effectively - tied on pass/fail shape: each has 38 jobs, 18 pass, 5 wrong_result, 2 blocked, and - 13 not_encoded. +- ELF and qmd have comparable full-suite live real-world sweeps. The latest generated + artifacts are close but no longer identical: ELF has 38 jobs with 18 pass, + 5 wrong_result, 2 blocked, and 13 not_encoded, while qmd has 17 pass, + 6 wrong_result, 2 blocked, and 13 not_encoded. - ELF is ahead on production-operation evidence among tracked systems because it has checked-in provider synthetic, stress, backfill, backup/restore, and Qdrant rebuild evidence. @@ -39,12 +41,13 @@ production," but the competitiveness objective remains open. ## Fresh Runs -These commands were run from an isolated report worktree based on `origin/main`: +These commands were run in the current XY-898 lane after adapter-report consistency +repairs: | Command | Result | Runtime | | --- | --- | ---: | -| `cargo make real-world-memory` | pass | 42.38 seconds | -| `cargo make real-world-memory-live-adapters` | pass | 121.93 seconds | +| `cargo make real-world-memory` | pass | 11.91 seconds | +| `cargo make real-world-memory-live-adapters` | pass | 121.51 seconds | The live adapter run emitted repeated Qdrant client/server compatibility warnings, but the command completed successfully and produced ELF and qmd JSON/Markdown reports. @@ -82,33 +85,36 @@ live adapter or competitor runtime can complete those jobs. | Adapter | Jobs | Pass | Wrong result | Blocked | Not encoded | Mean score | Mean latency | Evidence recall | Evidence coverage | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| ELF live service adapter | `38` | `18` | `5` | `2` | `13` | `0.525` | `5.100 ms` | `41/77` | `48/84` | -| qmd live CLI adapter | `38` | `18` | `5` | `2` | `13` | `0.512` | `719.758 ms` | `41/77` | `48/84` | +| ELF live service adapter | `38` | `18` | `5` | `2` | `13` | `0.525` | `6.761 ms` | `41/77` | `48/84` | +| qmd live CLI adapter | `38` | `17` | `6` | `2` | `13` | `0.486` | `842.057 ms` | `38/77` | `45/84` | -This supports a narrow tie on the currently encoded live real-world suite shape. It -does not support a broad ELF-over-qmd claim because qmd remains the stronger -retrieval-debug UX reference and its deep profile is still not encoded. +This supports a near tie on the currently encoded live real-world suite shape, with +ELF one job ahead in this fresh run. It does not support a broad ELF-over-qmd claim +because qmd remains the stronger retrieval-debug UX reference and its deep profile is +still not encoded. ### Live Suite Breakdown -ELF and qmd had the same suite status shape: - -| Suite | Jobs | Status breakdown | -| --- | ---: | --- | -| `trust_source_of_truth` | `1` | `pass:1` | -| `work_resume` | `5` | `pass:5` | -| `retrieval` | `5` | `pass:5` | -| `project_decisions` | `5` | `pass:5` | -| `personalization` | `1` | `pass:1` | -| `memory_evolution` | `6` | `pass:1`, `wrong_result:5` | -| `capture_integration` | `2` | `not_encoded:2` | -| `consolidation` | `4` | `not_encoded:4` | -| `knowledge_compilation` | `2` | `not_encoded:2` | -| `operator_debugging_ux` | `1` | `not_encoded:1` | -| `production_ops` | `6` | `blocked:2`, `not_encoded:4` | - -The five live wrong results are all memory-evolution jobs. The live adapters retrieve -current evidence but do not yet provide the required historical conflict evidence +ELF and qmd have the same status shape outside `memory_evolution`. The difference is +`memory-evolution-delete-ttl-001`: ELF passes that job while qmd reports +`wrong_result`, leaving ELF at five memory-evolution wrong results and qmd at six. + +| Suite | Jobs | ELF breakdown | qmd breakdown | +| --- | ---: | --- | --- | +| `trust_source_of_truth` | `1` | `pass:1` | `pass:1` | +| `work_resume` | `5` | `pass:5` | `pass:5` | +| `retrieval` | `5` | `pass:5` | `pass:5` | +| `project_decisions` | `5` | `pass:5` | `pass:5` | +| `personalization` | `1` | `pass:1` | `pass:1` | +| `memory_evolution` | `6` | `pass:1`, `wrong_result:5` | `wrong_result:6` | +| `capture_integration` | `2` | `not_encoded:2` | `not_encoded:2` | +| `consolidation` | `4` | `not_encoded:4` | `not_encoded:4` | +| `knowledge_compilation` | `2` | `not_encoded:2` | `not_encoded:2` | +| `operator_debugging_ux` | `1` | `not_encoded:1` | `not_encoded:1` | +| `production_ops` | `6` | `blocked:2`, `not_encoded:4` | `blocked:2`, `not_encoded:4` | + +The live wrong results are all memory-evolution jobs. The live adapters retrieve +current evidence but do not yet provide all required historical conflict evidence links for current-vs-historical reasoning. ## External Adapter Ledger @@ -124,8 +130,8 @@ The checked-in manifest records 21 adapter records across 17 unique project name | Overall status | Adapter records | | --- | ---: | -| `pass` | `1` | -| `wrong_result` | `7` | +| `pass` | `3` | +| `wrong_result` | `5` | | `lifecycle_fail` | `1` | | `blocked` | `5` | | `not_encoded` | `7` | @@ -139,10 +145,10 @@ when ELF is included. | Project | Best current evidence | Current measured state | Strongest unproven scenario | Next measurement before claim | | --- | --- | --- | --- | --- | | ELF | `fixture_backed` plus `live_real_world` | Fixture aggregate passes except 2 blocked operator boundaries; live full sweep is `wrong_result`. | Full live memory evolution, live consolidation, live knowledge pages, live capture, live production ops. | Memory-evolution diagnostic report, then live operator/capture/consolidation reports. | -| qmd | `live_real_world` plus `live_baseline_only` | Same live sweep shape as ELF; same-corpus baseline passes. | Deep retrieval-debug ergonomics and trace replay. | qmd/ELF deep retrieval-debug profile with expansion, fusion, rerank, and dropped-candidate traces. | +| qmd | `live_real_world` plus `live_baseline_only` | Same-corpus baseline passes; current live sweep is one memory-evolution job behind ELF. | Deep retrieval-debug ergonomics and trace replay. | qmd/ELF deep retrieval-debug profile with expansion, fusion, rerank, and dropped-candidate traces. | | agentmemory | `live_baseline_only` | `lifecycle_fail`. | Durable coding-agent continuity and capture hooks. | Durable lifecycle and work-resume/capture adapter report. | -| mem0/OpenMemory | `live_baseline_only` | `wrong_result`. | Entity history, lifecycle UI, OpenMemory inspection. | Same-corpus repair first, then entity-history and UI-readback report. | -| memsearch | `live_baseline_only` | `wrong_result`; source-of-truth is `incomplete`. | Markdown canonical store and local reindex clarity. | Reindex/update/delete/reload plus source-of-truth report. | +| mem0/OpenMemory | `live_baseline_only` | Basic local smoke now passes; history/UI/hosted/graph behavior remains `not_encoded`. | Entity history, lifecycle UI, OpenMemory inspection. | Entity-history, deletion-audit, and UI/export readback report. | +| memsearch | `live_baseline_only` | Basic canonical Markdown reindex/reload smoke now passes; real-world prompt coverage remains `not_encoded`. | Markdown canonical store and local reindex clarity. | Source-of-truth and retrieval-debug real-world adapter report. | | OpenViking | `live_baseline_only` plus `research_gate` | Same-corpus retrieval is `wrong_result`; trajectory is `not_encoded`. | Hierarchical staged context trajectory. | Evidence-bearing retrieval fix, then staged trajectory report. | | claude-mem | `live_baseline_only` | `wrong_result`. | Progressive disclosure and automatic capture review. | Work-resume, operator-debugging, and capture/write-policy report. | | RAGFlow | `research_gate` | `blocked`. | RAG app workflow with document/chunk references. | Tiny Docker evidence-smoke with `reference.chunks` mapped to evidence ids. | diff --git a/docs/guide/benchmarking/index.md b/docs/guide/benchmarking/index.md index 20de8c2d..c4b979d2 100644 --- a/docs/guide/benchmarking/index.md +++ b/docs/guide/benchmarking/index.md @@ -70,6 +70,11 @@ cleanup, use `docs/guide/single_user_production.md`. records Graphiti/Zep and Letta claim boundaries, and turns qmd, mem0/OpenMemory, Graphiti/Zep, Letta, and adjacent project strengths into benchmark-gated ELF optimization directions. +- `2026-06-11-first-generation-oss-adapter-promotion-report.md`: XY-898 + first-generation OSS adapter promotion report that updates agentmemory, + mem0/OpenMemory, memsearch, and claude-mem with fresh scenario-level baseline + evidence and ELF win/tie/loss/untested positions without converting baseline-only + evidence into real-world suite wins. - `2026-06-11-graph-rag-scored-smoke-adapter-report.md`: XY-900 graph/RAG scored-smoke adapter report that promotes RAGFlow, LightRAG, GraphRAG, Graphiti/Zep, and graphify smoke contracts into scored or typed non-pass diff --git a/docs/guide/benchmarking/live_baseline_benchmark.md b/docs/guide/benchmarking/live_baseline_benchmark.md index 30377951..ad839597 100644 --- a/docs/guide/benchmarking/live_baseline_benchmark.md +++ b/docs/guide/benchmarking/live_baseline_benchmark.md @@ -375,6 +375,9 @@ leave real-world suites as `not_encoded`, `blocked`, `incomplete`, `wrong_result scoring. The same manifest can also contain `research_gate` records for future adapter packs; those records provide source/setup/runtime/resource/retry guidance but are not live-baseline evidence. +The manifest may also include scenario judgments with an ELF position of `wins`, +`ties`, `loses`, or `untested`; these are dimension-level report inputs and do not +upgrade live-baseline-only evidence into real-world suite pass evidence. The full live real-world adapter sweep for ELF and qmd is separate from the same-corpus live baseline: diff --git a/docs/research/2026-06-11-measurement-coverage-audit.json b/docs/research/2026-06-11-measurement-coverage-audit.json index 0019110a..3f6c7965 100644 --- a/docs/research/2026-06-11-measurement-coverage-audit.json +++ b/docs/research/2026-06-11-measurement-coverage-audit.json @@ -1,20 +1,20 @@ { - "schema": "elf.benchmark_measurement_coverage_audit/v1", + "schema": "elf.benchmark_measurement_coverage_audit/v2", "run_id": "2026-06-11-measurement-coverage-audit", - "commit": "286af8b", + "source_revision": "current XY-898 lane after adapter-report consistency repairs", "created_at": "2026-06-11", "scope": "ELF memory-system competitiveness measurement coverage, external competitor comparison evidence, and next report directions", "commands": [ { "command": "cargo make real-world-memory", "status": "pass", - "runtime_seconds": 42.38, + "runtime_seconds": 11.91, "artifact": "tmp/real-world-memory/real-world-memory-report.json" }, { "command": "cargo make real-world-memory-live-adapters", "status": "pass", - "runtime_seconds": 121.93, + "runtime_seconds": 121.51, "artifact": "tmp/real-world-memory/live-adapters/" } ], @@ -45,7 +45,7 @@ "blocked": 2, "not_encoded": 13, "mean_score": 0.525, - "mean_latency_ms": 5.1, + "mean_latency_ms": 6.761, "expected_evidence_total": 77, "expected_evidence_matched": 41, "evidence_required_count": 84, @@ -55,35 +55,93 @@ "adapter": "qmd live CLI adapter", "job_count": 38, "encoded_suite_count": 11, - "pass": 18, - "wrong_result": 5, + "pass": 17, + "wrong_result": 6, "blocked": 2, "not_encoded": 13, - "mean_score": 0.512, - "mean_latency_ms": 719.758, + "mean_score": 0.486, + "mean_latency_ms": 842.057, "expected_evidence_total": 77, - "expected_evidence_matched": 41, + "expected_evidence_matched": 38, "evidence_required_count": 84, - "evidence_covered_count": 48 + "evidence_covered_count": 45 } ], - "live_suite_breakdown": [ - {"suite": "trust_source_of_truth", "jobs": 1, "status_counts": {"pass": 1}}, - {"suite": "work_resume", "jobs": 5, "status_counts": {"pass": 5}}, - {"suite": "retrieval", "jobs": 5, "status_counts": {"pass": 5}}, - {"suite": "project_decisions", "jobs": 5, "status_counts": {"pass": 5}}, - {"suite": "personalization", "jobs": 1, "status_counts": {"pass": 1}}, - {"suite": "memory_evolution", "jobs": 6, "status_counts": {"pass": 1, "wrong_result": 5}}, - {"suite": "capture_integration", "jobs": 2, "status_counts": {"not_encoded": 2}}, - {"suite": "consolidation", "jobs": 4, "status_counts": {"not_encoded": 4}}, - {"suite": "knowledge_compilation", "jobs": 2, "status_counts": {"not_encoded": 2}}, - {"suite": "operator_debugging_ux", "jobs": 1, "status_counts": {"not_encoded": 1}}, - {"suite": "production_ops", "jobs": 6, "status_counts": {"blocked": 2, "not_encoded": 4}} - ], + "live_suite_breakdown": { + "delta": "ELF passes memory-evolution-delete-ttl-001 while qmd reports wrong_result; other suite status shapes match.", + "suites": [ + { + "suite": "trust_source_of_truth", + "jobs": 1, + "elf_status_counts": {"pass": 1}, + "qmd_status_counts": {"pass": 1} + }, + { + "suite": "work_resume", + "jobs": 5, + "elf_status_counts": {"pass": 5}, + "qmd_status_counts": {"pass": 5} + }, + { + "suite": "retrieval", + "jobs": 5, + "elf_status_counts": {"pass": 5}, + "qmd_status_counts": {"pass": 5} + }, + { + "suite": "project_decisions", + "jobs": 5, + "elf_status_counts": {"pass": 5}, + "qmd_status_counts": {"pass": 5} + }, + { + "suite": "personalization", + "jobs": 1, + "elf_status_counts": {"pass": 1}, + "qmd_status_counts": {"pass": 1} + }, + { + "suite": "memory_evolution", + "jobs": 6, + "elf_status_counts": {"pass": 1, "wrong_result": 5}, + "qmd_status_counts": {"wrong_result": 6} + }, + { + "suite": "capture_integration", + "jobs": 2, + "elf_status_counts": {"not_encoded": 2}, + "qmd_status_counts": {"not_encoded": 2} + }, + { + "suite": "consolidation", + "jobs": 4, + "elf_status_counts": {"not_encoded": 4}, + "qmd_status_counts": {"not_encoded": 4} + }, + { + "suite": "knowledge_compilation", + "jobs": 2, + "elf_status_counts": {"not_encoded": 2}, + "qmd_status_counts": {"not_encoded": 2} + }, + { + "suite": "operator_debugging_ux", + "jobs": 1, + "elf_status_counts": {"not_encoded": 1}, + "qmd_status_counts": {"not_encoded": 1} + }, + { + "suite": "production_ops", + "jobs": 6, + "elf_status_counts": {"blocked": 2, "not_encoded": 4}, + "qmd_status_counts": {"blocked": 2, "not_encoded": 4} + } + ] + }, "adapter_ledger": { "adapter_records": 21, "unique_project_names": 17, - "external_project_count_note": "At audit commit 286af8b, the generated report field external_project_count counted non-ELF adapter records, not unique external project names; XY-900 later repaired the runner to report unique non-ELF project names.", + "external_project_count_note": "The generated report field external_project_count reports unique non-ELF project names after the XY-900 runner repair; the manifest has 16 external projects and 17 total project names including ELF.", "evidence_class_counts": { "fixture_backed": 1, "live_baseline_only": 6, @@ -91,8 +149,8 @@ "research_gate": 11 }, "overall_status_counts": { - "pass": 1, - "wrong_result": 7, + "pass": 3, + "wrong_result": 5, "lifecycle_fail": 1, "blocked": 5, "not_encoded": 7 @@ -100,7 +158,7 @@ "xy900_update_note": "XY-900 promotes graphify from research_gate/blocked to a tiny scored live_real_world wrong_result smoke; broad graph/RAG quality remains unproven." }, "claim_boundary": { - "elf_vs_qmd": "tie_on_current_encoded_live_real_world_shape_not_overall_win", + "elf_vs_qmd": "near_tie_on_current_encoded_live_real_world_shape_not_overall_win", "elf_personal_production": "credible_with_bounded_caveats", "broad_competitor_superiority": "not_proven", "major_unmeasured_strengths": [ diff --git a/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json b/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json index 893caf9b..a5ed566f 100644 --- a/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json +++ b/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json @@ -29,11 +29,11 @@ "research_gate": 11 }, "overall_status_counts": { - "pass": 1, - "wrong_result": 7, "lifecycle_fail": 1, "blocked": 5, - "not_encoded": 7 + "not_encoded": 7, + "pass": 3, + "wrong_result": 5 } }, "state_taxonomy": [ @@ -117,7 +117,7 @@ "unsupported_or_blocked_status": { "state": "not_encoded", "typed_reason": "deep_profile_and_non_retrieval_suites_not_encoded", - "details": "The full live sweep passes targeted retrieval suites but keeps memory_evolution wrong_result and several broader suites not_encoded or incomplete." + "details": "The full live sweep passes targeted retrieval suites but keeps memory_evolution wrong_result and several broader suites not_encoded or blocked." }, "benchmark_before_claim": "Run qmd deep retrieval/debug profile and full-suite live real-world replay with trace-level diagnostics before claiming ELF wins, ties, or loses on retrieval debugging.", "borrow_if_stronger": "Borrow transparent local knobs for query rewriting, weighted fusion, rerank explanation, and command-line replay." @@ -149,17 +149,17 @@ "supporting_evidence_classes": [ "live_baseline_only" ], - "measured_status": "wrong_result", + "measured_status": "pass", "proof": { - "command": "ELF_BASELINE_PROJECTS=mem0 cargo make baseline-live-docker", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "unsupported_or_blocked_status": { "state": "not_encoded", - "typed_reason": "openmemory_ui_and_hosted_claims_not_encoded", - "details": "Local OSS setup is represented, but hosted/OpenMemory UI parity and real-world personalization coverage are not encoded." + "typed_reason": "history_ui_hosted_graph_claims_not_encoded", + "details": "Basic local OSS same-corpus/update/delete/reload smoke now passes, but hosted/OpenMemory UI parity, entity/preference history, deletion-audit readback, optional graph memory, and real-world personalization coverage are not encoded." }, - "benchmark_before_claim": "Fix the local adapter's same-corpus result, then encode memory_evolution, personalization, OpenMemory UI readback, and optional graph-context jobs.", + "benchmark_before_claim": "Encode memory_evolution preference/entity history, deletion audit readback, personalization, OpenMemory UI/export readback, and optional graph-context jobs.", "borrow_if_stronger": "Borrow entity-scoped memory history, lifecycle surfaces, async update ergonomics, and OpenMemory-style inspection UX." }, { @@ -169,17 +169,17 @@ "supporting_evidence_classes": [ "live_baseline_only" ], - "measured_status": "wrong_result", + "measured_status": "pass", "proof": { - "command": "ELF_BASELINE_PROJECTS=memsearch cargo make baseline-live-docker", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "unsupported_or_blocked_status": { - "state": "incomplete", - "typed_reason": "source_of_truth_and_reindex_real_world_jobs_incomplete", - "details": "Same-corpus retrieval is wrong_result and source-of-truth plus real-world reindex behavior is not yet cleanly scored." + "state": "not_encoded", + "typed_reason": "source_of_truth_and_reindex_real_world_jobs_not_encoded", + "details": "Basic canonical Markdown same-corpus/reindex/update/delete/reload smoke now passes, but source-of-truth, retrieval-debug, and memory-evolution real-world prompt adapters are not encoded." }, - "benchmark_before_claim": "Fix Docker same-corpus retrieval and reindex/update/delete reload evidence, then score source-of-truth and retrieval-debug real-world jobs.", + "benchmark_before_claim": "Score source-of-truth and retrieval-debug real-world jobs over the canonical Markdown store; keep TTL/expiry unsupported unless a comparable path exists.", "borrow_if_stronger": "Borrow the canonical markdown-store ergonomics, local reindex clarity, and user-inspectable source files." }, { @@ -457,18 +457,18 @@ "scenario": "source-of-truth", "current_elf_evidence": "ELF fixture-backed trust_source_of_truth passes and ELF live_real_world trust_source_of_truth passes.", "strongest_competitor_or_reference": "memsearch", - "current_competitor_evidence": "memsearch has live_baseline_only canonical store evidence but trust_source_of_truth is incomplete and retrieval is wrong_result.", - "current_state": "ELF has stronger measured source-of-truth evidence; memsearch remains a local-store ergonomics reference.", - "next_measurement": "Fix memsearch same-corpus retrieval/reindex evidence, then run source-of-truth rebuild and reload jobs before any win/loss claim." + "current_competitor_evidence": "memsearch has live_baseline_only canonical store evidence and now passes same-corpus retrieval, reindex/update/delete, and cold-start reload smoke, but trust_source_of_truth real-world prompts are not_encoded.", + "current_state": "ELF has stronger measured real-world source-of-truth evidence; memsearch now ties the local canonical-store reindex/reload smoke and remains a local-store ergonomics reference.", + "next_measurement": "Run memsearch source-of-truth rebuild and reload real_world_job prompts before any suite-level win/loss claim." }, { "scenario_id": "temporal_current_historical", "scenario": "temporal/current-vs-historical memory", "current_elf_evidence": "ELF fixture-backed memory_evolution passes, but ELF live_real_world memory_evolution is wrong_result.", "strongest_competitor_or_reference": "Graphiti/Zep, mem0/OpenMemory", - "current_competitor_evidence": "Graphiti/Zep is research_gate blocked; mem0/OpenMemory is live_baseline_only wrong_result.", + "current_competitor_evidence": "Graphiti/Zep is research_gate blocked; mem0/OpenMemory now passes basic live_baseline_only local lifecycle smoke but preference/entity history, deletion audit, UI/export, and graph-memory scenarios are not_encoded.", "current_state": "No project has a comparable live pass for current-vs-historical evidence; ELF cannot claim live superiority yet.", - "next_measurement": "Fix ELF/qmd live memory_evolution evidence links and run XY-888 Graphiti/Zep temporal graph adapter." + "next_measurement": "Fix ELF/qmd live memory_evolution evidence links, encode mem0/OpenMemory history and UI/export jobs, and run XY-888 Graphiti/Zep temporal graph adapter." }, { "scenario_id": "consolidation", @@ -509,9 +509,9 @@ { "scenario_id": "production_ops", "scenario": "production ops", - "current_elf_evidence": "ELF production runbooks and fixture production_ops cover restore, Qdrant rebuild, backfill resume, resource envelope, and typed private/credential blockers; live_real_world production_ops is incomplete.", + "current_elf_evidence": "ELF production runbooks and fixture production_ops cover restore, Qdrant rebuild, backfill resume, resource envelope, and typed private/credential blockers; live_real_world production_ops is blocked.", "strongest_competitor_or_reference": "ELF production gate, qmd, RAG/RAGFlow resource gates", - "current_competitor_evidence": "qmd live production_ops is incomplete; RAGFlow/GraphRAG/LightRAG resource gates are research_gate blocked.", + "current_competitor_evidence": "qmd live production_ops is blocked; RAGFlow/GraphRAG/LightRAG resource gates are research_gate blocked.", "current_state": "ELF has the strongest checked-in production evidence, but private corpus and credentialed gates remain blocked.", "next_measurement": "Rerun private-corpus and credentialed production-ops gates only when operator-owned manifest and credentials are supplied." }, @@ -568,18 +568,18 @@ "measurement": "Update, delete, cold-start reload, work_resume, and capture/write-policy jobs." }, { - "workstream": "mem0/OpenMemory local and UI coverage", + "workstream": "mem0/OpenMemory history and UI coverage", "issue_or_candidate": "new adapter repair issue", "parallelizable": true, - "blocked_by": "Comparable local OSS path for UI/readback evidence.", - "measurement": "Same-corpus fix plus memory_evolution, personalization, and OpenMemory inspection jobs." + "blocked_by": "Comparable local OSS path for history/UI/readback evidence.", + "measurement": "Preference/entity history, deletion audit readback, personalization, OpenMemory inspection/export, and optional graph-context jobs." }, { - "workstream": "memsearch source-of-truth and reindex coverage", + "workstream": "memsearch source-of-truth real-world coverage", "issue_or_candidate": "new adapter repair issue", "parallelizable": true, - "blocked_by": "Docker same-corpus retrieval and reindex correctness.", - "measurement": "Canonical markdown store, rebuild/reindex, retrieval, update/delete/reload jobs." + "blocked_by": "Real-world prompt adapter over the canonical Markdown store.", + "measurement": "Source-of-truth rebuild/reload jobs and retrieval-debug jobs that preserve baseline reindex/update/delete evidence without converting it into suite pass claims." }, { "workstream": "OpenViking context trajectory", diff --git a/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json b/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json new file mode 100644 index 00000000..81e9179c --- /dev/null +++ b/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json @@ -0,0 +1,207 @@ +{ + "schema": "elf.first_generation_oss_adapter_promotion_report/v1", + "report_id": "xy-898-first-generation-oss-adapter-promotion-2026-06-11", + "authority": "XY-898", + "date": "2026-06-11", + "scope": "Scenario-level adapter evidence for agentmemory, mem0/OpenMemory, memsearch, and claude-mem without ELF optimization changes.", + "source_inputs": [ + "docs/guide/benchmarking/2026-06-11-temporal-history-competitor-gap-report.md", + "docs/research/2026-06-11-temporal-history-competitor-gap-report.json", + "docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md", + "docs/spec/real_world_agent_memory_benchmark_v1.md", + "docs/guide/benchmarking/live_baseline_benchmark.md", + "apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json", + "tmp/live-baseline/live-baseline-report.json" + ], + "fresh_run": { + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "run_id": "live-baseline-20260611061612", + "status": "fail", + "runtime_seconds": 295.74, + "artifact": "tmp/live-baseline/live-baseline-report.json", + "summary": { + "total": 5, + "pass": 3, + "wrong_result": 1, + "lifecycle_fail": 1, + "incomplete": 0, + "blocked": 0, + "not_encoded": 0 + }, + "full_check_summary": { + "total": 25, + "pass": 22, + "wrong_result": 1, + "lifecycle_fail": 1, + "blocked": 1, + "incomplete": 0, + "not_encoded": 0 + } + }, + "projects": [ + { + "project": "ELF", + "status": "pass", + "retrieval_status": "retrieval_pass", + "check_summary": { + "total": 8, + "pass": 8 + }, + "evidence": "ELF passed 8/8 encoded local lifecycle and retrieval checks in the comparable baseline run and serves only as the reference row for same-class scenario positions." + }, + { + "project": "agentmemory", + "status": "lifecycle_fail", + "retrieval_status": "retrieval_pass", + "check_summary": { + "total": 4, + "pass": 2, + "lifecycle_fail": 1, + "blocked": 1 + }, + "evidence": "Same-corpus retrieval and delete suppression pass through the in-memory mock, but update supersession fails and cold-start recovery is blocked." + }, + { + "project": "mem0/OpenMemory", + "status": "pass", + "retrieval_status": "retrieval_pass", + "check_summary": { + "total": 4, + "pass": 4 + }, + "evidence": "Local OSS mem0 passes same-corpus retrieval, update, delete, and cold-start reload. This does not include OpenMemory UI, hosted Platform, entity history, or graph memory." + }, + { + "project": "memsearch", + "status": "pass", + "retrieval_status": "retrieval_pass", + "check_summary": { + "total": 4, + "pass": 4 + }, + "evidence": "memsearch passes same-corpus retrieval, update reindex, delete suppression, and cold-start reload over the canonical Markdown corpus." + }, + { + "project": "claude-mem", + "status": "wrong_result", + "retrieval_status": "retrieval_wrong_result", + "check_summary": { + "total": 5, + "pass": 4, + "wrong_result": 1 + }, + "evidence": "claude-mem passes update, delete, progressive detail/source hydration, and cold-start reload over Docker-local SQLite, but same-corpus retrieval misses expected evidence." + } + ], + "scenario_summary": { + "count": 13, + "status_counts": { + "pass": 5, + "wrong_result": 1, + "lifecycle_fail": 1, + "blocked": 1, + "unsupported": 1, + "not_encoded": 4 + }, + "elf_position_counts": { + "wins": 2, + "ties": 2, + "loses": 0, + "untested": 9 + } + }, + "scenario_judgments": [ + { + "project": "agentmemory", + "scenario_id": "basic_same_corpus_retrieval", + "status": "pass", + "elf_position": "untested" + }, + { + "project": "agentmemory", + "scenario_id": "durable_update_reload_lifecycle", + "status": "lifecycle_fail", + "elf_position": "wins" + }, + { + "project": "agentmemory", + "scenario_id": "work_resume_capture_continuity", + "status": "blocked", + "elf_position": "untested" + }, + { + "project": "mem0/OpenMemory", + "scenario_id": "basic_local_lifecycle", + "status": "pass", + "elf_position": "ties" + }, + { + "project": "mem0/OpenMemory", + "scenario_id": "preference_entity_history", + "status": "not_encoded", + "elf_position": "untested" + }, + { + "project": "mem0/OpenMemory", + "scenario_id": "openmemory_ui_export_readback", + "status": "not_encoded", + "elf_position": "untested" + }, + { + "project": "memsearch", + "scenario_id": "canonical_markdown_reindex_reload", + "status": "pass", + "elf_position": "untested" + }, + { + "project": "memsearch", + "scenario_id": "ttl_expiry_lifecycle", + "status": "unsupported", + "elf_position": "untested" + }, + { + "project": "memsearch", + "scenario_id": "real_world_prompt_adapter", + "status": "not_encoded", + "elf_position": "untested" + }, + { + "project": "claude-mem", + "scenario_id": "same_corpus_retrieval", + "status": "wrong_result", + "elf_position": "wins" + }, + { + "project": "claude-mem", + "scenario_id": "repository_lifecycle_reload", + "status": "pass", + "elf_position": "ties" + }, + { + "project": "claude-mem", + "scenario_id": "progressive_disclosure_detail_hydration", + "status": "pass", + "elf_position": "untested" + }, + { + "project": "claude-mem", + "scenario_id": "hook_capture_viewer_workflow", + "status": "not_encoded", + "elf_position": "untested" + } + ], + "claim_boundaries": { + "allowed": [ + "mem0/OpenMemory passes the current basic local OSS lifecycle smoke.", + "memsearch passes the current canonical Markdown reindex/reload smoke.", + "agentmemory remains lifecycle_fail for durable update/reload because the current adapter is in-memory.", + "claude-mem remains wrong_result for same-corpus retrieval while preserving passed repository lifecycle and detail hydration evidence." + ], + "not_allowed": [ + "Do not claim hosted OpenMemory behavior from local OSS evidence.", + "Do not claim mem0/OpenMemory history, UI/export, hosted, or graph-memory parity.", + "Do not claim memsearch source-of-truth real-world suite parity from baseline smoke.", + "Do not claim claude-mem hook/viewer/capture parity from repository-only checks." + ] + } +} diff --git a/docs/spec/real_world_agent_memory_benchmark_v1.md b/docs/spec/real_world_agent_memory_benchmark_v1.md index bb0a4b82..115acc3f 100644 --- a/docs/spec/real_world_agent_memory_benchmark_v1.md +++ b/docs/spec/real_world_agent_memory_benchmark_v1.md @@ -174,6 +174,11 @@ Each `adapters[]` record MUST include: and `evidence`. - `suites`: array of real-world suite coverage records with `suite_id`, `status`, and `evidence`. +- `scenarios`: optional array of scenario judgment records with `scenario_id`, + optional `suite_id`, `status`, `elf_position`, `evidence`, and optional `command` + and `artifact`. `elf_position` MUST be one of `wins`, `ties`, `loses`, or + `untested`. Scenario judgments are report inputs for dimension-level comparison; + they MUST NOT convert live-baseline-only evidence into real-world suite pass claims. - `evidence`: array of evidence pointers with `kind`, `ref`, and `status`. - `notes`: optional bounded explanatory strings. - `follow_up`: optional `title` and `reason`.