From 92bb79ccfef1c340a7f916ef329897a3f4aa7dbb Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 10:31:09 +0800 Subject: [PATCH 1/6] {"schema":"decodex/commit/1","summary":"Promote first-generation OSS memory baselines into scenario evidence","authority":"XY-898"} --- .../memory_projects_manifest.json | 173 ++++++++++++--- .../src/bin/real_world_job_benchmark.rs | 165 ++++++++++++++- .../tests/real_world_job_benchmark.rs | 63 +++++- ...-11-competitor-strength-evidence-matrix.md | 12 +- ...on-direction-from-competitor-benchmarks.md | 4 +- ...generation-oss-adapter-promotion-report.md | 112 ++++++++++ .../2026-06-11-measurement-coverage-audit.md | 4 +- docs/guide/benchmarking/index.md | 5 + .../benchmarking/live_baseline_benchmark.md | 3 + ...-11-xy-897-competitor-strength-matrix.json | 50 ++--- ...irst-generation-oss-adapter-promotion.json | 197 ++++++++++++++++++ .../real_world_agent_memory_benchmark_v1.md | 5 + 12 files changed, 725 insertions(+), 68 deletions(-) create mode 100644 docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md create mode 100644 docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index 152b1f15..b1e3347b 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -1,6 +1,6 @@ { "schema": "elf.real_world_external_adapter_manifest/v1", - "manifest_id": "real-world-memory-project-adapters-2026-06-10", + "manifest_id": "real-world-memory-project-adapters-2026-06-11", "docker_isolation": { "default": true, "compose_file": "docker-compose.baseline.yml", @@ -544,6 +544,34 @@ "evidence": "Durable update/supersede/delete history is not proven by the in-memory adapter." } ], + "scenarios": [ + { + "scenario_id": "basic_same_corpus_retrieval", + "suite_id": "retrieval", + "status": "pass", + "elf_position": "untested", + "evidence": "Fresh scoped baseline run live-baseline-20260611015125 reports agentmemory retrieval_pass with 3/3 same-corpus retrieval checks through mem::remember and mem::search. This is live-baseline-only evidence through an in-memory mock, not a real_world_job suite pass.", + "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "durable_update_reload_lifecycle", + "suite_id": "memory_evolution", + "status": "lifecycle_fail", + "elf_position": "wins", + "evidence": "The same run reports agentmemory update_replaces_note_text as lifecycle_fail and cold_start_recovery_search as blocked because the harness uses an in-memory SDK/KV mock. ELF has broader encoded local lifecycle coverage in the June 11 ELF+mem0 baseline, so this scenario is an ELF baseline win only at the local lifecycle-smoke evidence class.", + "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "work_resume_capture_continuity", + "suite_id": "work_resume", + "status": "blocked", + "elf_position": "untested", + "evidence": "agentmemory's relevant strength is durable coding-agent continuity and capture, but the Docker harness has not proven a persistent session/capture path. Keep work_resume and capture claims blocked until a durable local adapter path exists.", + "artifact": "apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json" + } + ], "evidence": [ { "kind": "guide", @@ -571,7 +599,7 @@ "evidence_class": "live_baseline_only", "docker_default": true, "host_global_installs_required": false, - "overall_status": "wrong_result", + "overall_status": "pass", "setup": { "status": "pass", "evidence": "The live-baseline Docker runner can install mem0 and configure local FastEmbed/Qdrant paths.", @@ -579,13 +607,13 @@ "artifact": "tmp/live-baseline/mem0.log" }, "run": { - "status": "wrong_result", - "evidence": "The Docker runner exercises local OSS mem0 with FastEmbed, Qdrant path storage, Memory.update, Memory.delete, and cold-start reload; same-corpus retrieval remains typed wrong_result or incomplete when evidence is missed.", + "status": "pass", + "evidence": "Fresh scoped baseline run live-baseline-20260611015125 exercises local OSS mem0 with FastEmbed, Qdrant path storage, Memory.update, Memory.delete, and cold-start reload; mem0 passed 4/4 encoded checks.", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "result": { - "status": "wrong_result", - "evidence": "No real_world_job mem0/OpenMemory adapter is encoded; local same-corpus evidence must not be upgraded to suite coverage.", + "status": "pass", + "evidence": "The local OSS mem0 baseline now passes basic same-corpus/update/delete/reload smoke. No real_world_job mem0/OpenMemory adapter, OpenMemory UI, hosted Platform, entity-history, or graph-memory behavior is encoded.", "artifact": "docs/guide/research/comparison_external_projects.md" }, "capabilities": [ @@ -596,13 +624,13 @@ }, { "capability": "same_corpus_retrieval", - "status": "wrong_result", - "evidence": "The checked-in smoke evidence did not prove a correct same-corpus result for mem0." + "status": "pass", + "evidence": "Fresh scoped baseline run live-baseline-20260611015125 reports mem0 retrieval_pass with 3/3 same-corpus retrieval checks." }, { "capability": "local_lifecycle_update_delete_reload", - "status": "real", - "evidence": "The Docker runner exercises public Memory.update, Memory.delete, and a new Memory.from_config over the same local Qdrant/history paths; any miss is reported as lifecycle_fail instead of pass." + "status": "pass", + "evidence": "The Docker runner exercises public Memory.update, Memory.delete, and a new Memory.from_config over the same local Qdrant/history paths; the fresh scoped run reports 4/4 encoded checks passing." }, { "capability": "openmemory_ui_readback", @@ -623,8 +651,8 @@ "suites": [ { "suite_id": "memory_evolution", - "status": "wrong_result", - "evidence": "Local lifecycle checks are encoded in the Docker baseline, but real_world_job memory-evolution prompts are not executed and missed local evidence must remain typed non-pass." + "status": "not_encoded", + "evidence": "Basic local lifecycle checks now pass in Docker, but real_world_job memory-evolution prompts, preference history, deletion audit readback, and entity history are not encoded for mem0/OpenMemory." }, { "suite_id": "personalization", @@ -637,6 +665,33 @@ "evidence": "OpenMemory inspection is not encoded in this runner." } ], + "scenarios": [ + { + "scenario_id": "basic_local_lifecycle", + "suite_id": "memory_evolution", + "status": "pass", + "elf_position": "ties", + "evidence": "The June 11 ELF+mem0 baseline passed 12/12 combined checks, and fresh scoped run live-baseline-20260611015125 confirms mem0 passed same-corpus retrieval, update, delete, and cold-start reload. This is a basic local lifecycle tie at the encoded smoke surface, not a claim about OpenMemory UI, hosted behavior, entity history, or graph memory.", + "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "preference_entity_history", + "suite_id": "personalization", + "status": "not_encoded", + "elf_position": "untested", + "evidence": "mem0/OpenMemory's strongest next comparison is preference and entity-scoped history. The current local OSS Docker baseline does not inspect memory history events, correction chains, or entity-scoped readback under real_world_job scoring.", + "artifact": "docs/guide/benchmarking/2026-06-11-temporal-history-competitor-gap-report.md" + }, + { + "scenario_id": "openmemory_ui_export_readback", + "suite_id": "operator_debugging_ux", + "status": "not_encoded", + "elf_position": "untested", + "evidence": "OpenMemory UI/export readback is not exercised by the local OSS Docker baseline and hosted Platform behavior remains out of scope for local OSS evidence.", + "artifact": "docs/guide/benchmarking/2026-06-11-temporal-history-competitor-gap-report.md" + } + ], "evidence": [ { "kind": "runner", @@ -655,7 +710,7 @@ "evidence_class": "live_baseline_only", "docker_default": true, "host_global_installs_required": false, - "overall_status": "wrong_result", + "overall_status": "pass", "setup": { "status": "pass", "evidence": "The live-baseline Docker runner can install memsearch and run its CLI path.", @@ -663,13 +718,13 @@ "artifact": "tmp/live-baseline/memsearch.log" }, "run": { - "status": "wrong_result", - "evidence": "The Docker runner indexes a per-adapter corpus copy, rewrites and deletes files, reruns memsearch index, and records wrong_result or lifecycle_fail when expected evidence is missed.", + "status": "pass", + "evidence": "Fresh scoped baseline run live-baseline-20260611015125 indexes a per-adapter corpus copy, rewrites and deletes files, reruns memsearch index, and reports 4/4 encoded checks passing.", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "result": { - "status": "wrong_result", - "evidence": "No real_world_job memsearch adapter is encoded; Markdown-first behavior remains a design reference.", + "status": "pass", + "evidence": "memsearch now passes the local same-corpus/reindex/update/delete/reload smoke. No real_world_job memsearch prompt adapter is encoded, so Markdown-first behavior remains baseline scenario evidence rather than suite pass evidence.", "artifact": "docs/guide/research/comparison_external_projects.md" }, "capabilities": [ @@ -680,13 +735,13 @@ }, { "capability": "same_corpus_retrieval", - "status": "wrong_result", - "evidence": "The checked-in smoke evidence did not prove correct same-corpus retrieval." + "status": "pass", + "evidence": "Fresh scoped baseline run live-baseline-20260611015125 reports memsearch retrieval_pass with 3/3 same-corpus retrieval checks." }, { "capability": "reindex_update_delete_reload", - "status": "real", - "evidence": "The runner rewrites auth-memory.md, deletes a second corpus file, reruns memsearch index, and starts fresh memsearch search processes for update/delete/cold-start checks." + "status": "pass", + "evidence": "The runner rewrites auth-memory.md, deletes a second corpus file, reruns memsearch index, and starts fresh memsearch search processes; the fresh scoped run reports update, delete, and cold-start reload passing." }, { "capability": "real_world_job_adapter", @@ -697,18 +752,45 @@ "suites": [ { "suite_id": "trust_source_of_truth", - "status": "incomplete", - "evidence": "The Markdown-first source model is relevant, but no real_world_job source-of-truth run is encoded." + "status": "not_encoded", + "evidence": "The Markdown-first source model passed the local reindex/reload smoke, but no real_world_job source-of-truth prompt run is encoded." }, { "suite_id": "retrieval", - "status": "wrong_result", - "evidence": "The Docker same-corpus check reaches memsearch search, but current evidence is not a clean retrieval pass and no job-level run is encoded." + "status": "not_encoded", + "evidence": "The Docker same-corpus check now passes, but no job-level real_world retrieval run is encoded for memsearch." }, { "suite_id": "memory_evolution", - "status": "wrong_result", - "evidence": "Update/delete reindex semantics are exercised in Docker; misses remain typed wrong_result or lifecycle_fail and do not become suite passes." + "status": "not_encoded", + "evidence": "Update/delete reindex semantics pass in Docker, but memory_evolution real_world_job prompts are not encoded for memsearch." + } + ], + "scenarios": [ + { + "scenario_id": "canonical_markdown_reindex_reload", + "suite_id": "trust_source_of_truth", + "status": "pass", + "elf_position": "ties", + "evidence": "Fresh scoped baseline run live-baseline-20260611015125 reports memsearch passed same-corpus retrieval, update reindex, delete suppression, and cold-start reload over a canonical Markdown corpus. This is a local source-of-truth/reindex smoke tie at the scenario level, not a real_world_job suite pass.", + "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "ttl_expiry_lifecycle", + "suite_id": "memory_evolution", + "status": "unsupported", + "elf_position": "wins", + "evidence": "The encoded memsearch CLI path supports reindex/delete but no TTL or expiry behavior. ELF has encoded TTL/tombstone fixture and live delete/TTL evidence, so ELF wins this specific lifecycle-smoke scenario while broader memsearch real_world jobs remain untested.", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "real_world_prompt_adapter", + "suite_id": "retrieval", + "status": "not_encoded", + "elf_position": "untested", + "evidence": "No memsearch adapter currently executes real_world_job prompts and answer scoring; baseline retrieval/reindex evidence must stay separate from suite pass claims.", + "artifact": "apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json" } ], "evidence": [ @@ -890,6 +972,43 @@ "evidence": "claude-mem hooks are not executed by this runner." } ], + "scenarios": [ + { + "scenario_id": "same_corpus_retrieval", + "suite_id": "retrieval", + "status": "wrong_result", + "elf_position": "wins", + "evidence": "Fresh scoped baseline run live-baseline-20260611015125 reports claude-mem same_corpus_retrieval as wrong_result with 0/3 expected query checks passing, while its durable repository setup completed. ELF has encoded local same-corpus retrieval passes, so this is an ELF baseline win for the narrow retrieval smoke scenario.", + "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "repository_lifecycle_reload", + "suite_id": "memory_evolution", + "status": "pass", + "elf_position": "ties", + "evidence": "The same run reports claude-mem update, delete, and cold-start reload checks passing over a durable Docker-local SQLite repository. This is a local lifecycle-smoke tie, not a hook-driven work-resume or full progressive-disclosure job pass.", + "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "progressive_disclosure_detail_hydration", + "suite_id": "operator_debugging_ux", + "status": "pass", + "elf_position": "untested", + "evidence": "claude-mem passed the repository-level search-to-detail/source hydration check, which is a useful progressive-disclosure signal. ELF does not have a directly comparable claude-mem-style progressive-disclosure scenario in this baseline, so the ELF position remains untested rather than a loss claim.", + "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "artifact": "tmp/live-baseline/live-baseline-report.json" + }, + { + "scenario_id": "hook_capture_viewer_workflow", + "suite_id": "capture_integration", + "status": "not_encoded", + "elf_position": "untested", + "evidence": "The Docker baseline uses repository classes only. claude-mem hooks, viewer, timeline, and observation workflows are not executed by the runner.", + "artifact": "apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json" + } + ], "evidence": [ { "kind": "runner", diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark.rs b/apps/elf-eval/src/bin/real_world_job_benchmark.rs index e987986b..2c988134 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark.rs @@ -638,6 +638,15 @@ enum AdapterCoverageStatus { NotEncoded, } +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +enum ElfScenarioPosition { + Wins, + Ties, + Loses, + Untested, +} + #[derive(Debug, Deserialize)] struct ExternalAdapterManifest { schema: String, @@ -685,6 +694,8 @@ struct ExternalAdapterReport { #[serde(default)] suites: Vec, #[serde(default)] + scenarios: Vec, + #[serde(default)] evidence: Vec, #[serde(skip_serializing_if = "Option::is_none")] execution_metadata: Option, @@ -718,6 +729,20 @@ struct AdapterSuiteCoverage { evidence: String, } +#[derive(Clone, Debug, Deserialize, Serialize)] +struct AdapterScenarioJudgment { + scenario_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + suite_id: Option, + status: AdapterCoverageStatus, + elf_position: ElfScenarioPosition, + evidence: String, + #[serde(skip_serializing_if = "Option::is_none")] + command: Option, + #[serde(skip_serializing_if = "Option::is_none")] + artifact: Option, +} + #[derive(Clone, Debug, Deserialize, Serialize)] struct AdapterEvidencePointer { kind: String, @@ -760,6 +785,10 @@ struct ExternalAdapterSummary { overall_status_counts: AdapterStatusCounts, capability_status_counts: AdapterStatusCounts, suite_status_counts: AdapterStatusCounts, + #[serde(default)] + scenario_status_counts: AdapterStatusCounts, + #[serde(default)] + scenario_position_counts: ScenarioPositionCounts, } #[derive(Clone, Debug, Default, Deserialize, Serialize)] @@ -775,6 +804,14 @@ struct AdapterStatusCounts { not_encoded: usize, } +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +struct ScenarioPositionCounts { + wins: usize, + ties: usize, + loses: usize, + untested: usize, +} + #[derive(Clone, Debug, Default, Deserialize, Serialize)] struct CaptureIntegrationReport { #[serde(default)] @@ -3763,6 +3800,7 @@ fn validate_external_adapter(path: &Path, adapter: &ExternalAdapterReport) -> Re validate_adapter_execution(path, adapter)?; validate_adapter_capabilities(path, adapter)?; validate_adapter_suites(path, adapter)?; + validate_adapter_scenarios(path, adapter)?; validate_adapter_evidence(path, adapter)?; validate_adapter_execution_metadata(path, adapter)?; @@ -3833,6 +3871,36 @@ fn validate_adapter_suites(path: &Path, adapter: &ExternalAdapterReport) -> Resu Ok(()) } +fn validate_adapter_scenarios(path: &Path, adapter: &ExternalAdapterReport) -> Result<()> { + for scenario in &adapter.scenarios { + if scenario.scenario_id.trim().is_empty() + || scenario.evidence.trim().is_empty() + || scenario.command.as_deref().is_some_and(str::is_empty) + || scenario.artifact.as_deref().is_some_and(str::is_empty) + { + return Err(eyre::eyre!( + "{} adapter {} has incomplete scenario judgment.", + path.display(), + adapter.adapter_id + )); + } + + if let Some(suite_id) = &scenario.suite_id + && !SUITES.contains(&suite_id.as_str()) + { + return Err(eyre::eyre!( + "{} adapter {} scenario {} references unknown suite {}.", + path.display(), + adapter.adapter_id, + scenario.scenario_id, + suite_id + )); + } + } + + Ok(()) +} + fn validate_adapter_evidence(path: &Path, adapter: &ExternalAdapterReport) -> Result<()> { for evidence in &adapter.evidence { if evidence.kind.trim().is_empty() || evidence.reference.trim().is_empty() { @@ -3915,6 +3983,13 @@ fn accumulate_adapter_summary( for suite in &adapter.suites { increment_adapter_status_count(&mut summary.suite_status_counts, suite.status); } + for scenario in &adapter.scenarios { + increment_adapter_status_count(&mut summary.scenario_status_counts, scenario.status); + increment_scenario_position_count( + &mut summary.scenario_position_counts, + scenario.elf_position, + ); + } } fn increment_adapter_status_count(counts: &mut AdapterStatusCounts, status: AdapterCoverageStatus) { @@ -3931,6 +4006,18 @@ fn increment_adapter_status_count(counts: &mut AdapterStatusCounts, status: Adap } } +fn increment_scenario_position_count( + counts: &mut ScenarioPositionCounts, + position: ElfScenarioPosition, +) { + match position { + ElfScenarioPosition::Wins => counts.wins += 1, + ElfScenarioPosition::Ties => counts.ties += 1, + ElfScenarioPosition::Loses => counts.loses += 1, + ElfScenarioPosition::Untested => counts.untested += 1, + } +} + fn capture_integration_report(jobs: &[RealWorldJob]) -> CaptureIntegrationReport { let mut report = CaptureIntegrationReport::default(); @@ -4088,9 +4175,17 @@ fn render_markdown_external_adapters(out: &mut String, report: &RealWorldReport) adapter_status_counts_display(&summary.capability_status_counts) )); out.push_str(&format!( - "- Real-world suite statuses: `{}`\n\n", + "- Real-world suite statuses: `{}`\n", adapter_status_counts_display(&summary.suite_status_counts) )); + out.push_str(&format!( + "- Scenario coverage statuses: `{}`\n", + adapter_status_counts_display(&summary.scenario_status_counts) + )); + out.push_str(&format!( + "- ELF scenario positions: `{}`\n\n", + scenario_position_counts_display(&summary.scenario_position_counts) + )); out.push_str("| Project | Adapter | Evidence Class | Overall | Setup | Run | Result | Docker | Suites | Evidence |\n"); out.push_str("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n"); @@ -4126,11 +4221,40 @@ fn render_markdown_external_adapters(out: &mut String, report: &RealWorldReport) } } + render_markdown_adapter_scenarios(out, report.external_adapters.adapters.as_slice()); render_markdown_adapter_execution_metadata(out, report.external_adapters.adapters.as_slice()); out.push('\n'); } +fn render_markdown_adapter_scenarios(out: &mut String, adapters: &[ExternalAdapterReport]) { + if !adapters.iter().any(|adapter| !adapter.scenarios.is_empty()) { + return; + } + + out.push_str("\n### Adapter Scenario Judgments\n\n"); + out.push_str("| Adapter | Scenario | Suite | Status | ELF Position | Evidence |\n"); + out.push_str("| --- | --- | --- | --- | --- | --- |\n"); + + for adapter in adapters { + for scenario in &adapter.scenarios { + out.push_str(&format!( + "| `{}` | `{}` | {} | `{}` | `{}` | {} |\n", + md_inline(adapter.adapter_id.as_str()), + md_inline(scenario.scenario_id.as_str()), + scenario + .suite_id + .as_deref() + .map(|suite| format!("`{}`", md_inline(suite))) + .unwrap_or_else(|| "`none`".to_string()), + adapter_status_str(scenario.status), + scenario_position_str(scenario.elf_position), + adapter_scenario_evidence_cell(scenario) + )); + } + } +} + fn render_markdown_adapter_execution_metadata( out: &mut String, adapters: &[ExternalAdapterReport], @@ -4769,6 +4893,15 @@ fn adapter_status_str(status: AdapterCoverageStatus) -> &'static str { } } +fn scenario_position_str(position: ElfScenarioPosition) -> &'static str { + match position { + ElfScenarioPosition::Wins => "wins", + ElfScenarioPosition::Ties => "ties", + ElfScenarioPosition::Loses => "loses", + ElfScenarioPosition::Untested => "untested", + } +} + fn adapter_status_counts_display(counts: &AdapterStatusCounts) -> String { [ ("real", counts.real), @@ -4788,6 +4921,20 @@ fn adapter_status_counts_display(counts: &AdapterStatusCounts) -> String { .join(", ") } +fn scenario_position_counts_display(counts: &ScenarioPositionCounts) -> String { + [ + ("wins", counts.wins), + ("ties", counts.ties), + ("loses", counts.loses), + ("untested", counts.untested), + ] + .into_iter() + .filter(|(_, count)| *count > 0) + .map(|(position, count)| format!("{position}={count}")) + .collect::>() + .join(", ") +} + fn adapter_suite_cell(suites: &[AdapterSuiteCoverage]) -> String { if suites.is_empty() { return "`none`".to_string(); @@ -4823,6 +4970,22 @@ fn adapter_evidence_cell(adapter: &ExternalAdapterReport) -> String { format!("setup: `{}`
result: `{}`", md_inline(setup), md_inline(result)) } +fn adapter_scenario_evidence_cell(scenario: &AdapterScenarioJudgment) -> String { + let evidence = md_cell(scenario.evidence.as_str()); + let command = scenario + .command + .as_deref() + .map(|command| format!("
command: `{}`", md_inline(command))) + .unwrap_or_default(); + let artifact = scenario + .artifact + .as_deref() + .map(|artifact| format!("
artifact: `{}`", md_inline(artifact))) + .unwrap_or_default(); + + format!("{evidence}{command}{artifact}") +} + fn adapter_sources_cell(sources: &[AdapterSource]) -> String { if sources.is_empty() { return "`none`".to_string(); diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index b8f14a81..38ad3066 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -191,7 +191,7 @@ fn assert_external_adapter_manifest_summary(report: &Value) { ); assert_eq!( report.pointer("/external_adapters/manifest_id").and_then(Value::as_str), - Some("real-world-memory-project-adapters-2026-06-10") + Some("real-world-memory-project-adapters-2026-06-11") ); assert_eq!( report.pointer("/external_adapters/docker_isolation/default").and_then(Value::as_bool), @@ -233,13 +233,13 @@ fn assert_external_adapter_manifest_summary(report: &Value) { report .pointer("/external_adapters/summary/overall_status_counts/pass") .and_then(Value::as_u64), - Some(1) + Some(3) ); assert_eq!( report .pointer("/external_adapters/summary/overall_status_counts/wrong_result") .and_then(Value::as_u64), - Some(6) + Some(4) ); assert_eq!( report @@ -283,6 +283,35 @@ fn assert_external_adapter_manifest_summary(report: &Value) { .and_then(Value::as_u64), Some(11) ); + + assert_external_adapter_manifest_scenario_summary(report); +} + +fn assert_external_adapter_manifest_scenario_summary(report: &Value) { + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/pass") + .and_then(Value::as_u64), + Some(5) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/not_encoded") + .and_then(Value::as_u64), + Some(4) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/wins") + .and_then(Value::as_u64), + Some(3) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/ties") + .and_then(Value::as_u64), + Some(3) + ); } fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { @@ -388,13 +417,32 @@ fn assert_first_generation_adapter_records(mem0: &Value, memsearch: &Value, clau mem0.pointer("/capabilities/2/capability").and_then(Value::as_str), Some("local_lifecycle_update_delete_reload") ); - assert_eq!(mem0.pointer("/capabilities/2/status").and_then(Value::as_str), Some("real")); + assert_eq!(mem0.pointer("/capabilities/2/status").and_then(Value::as_str), Some("pass")); assert_eq!(mem0.pointer("/capabilities/4/status").and_then(Value::as_str), Some("not_encoded")); + assert_eq!(mem0.pointer("/scenarios/0/status").and_then(Value::as_str), Some("pass")); + assert_eq!(mem0.pointer("/scenarios/0/elf_position").and_then(Value::as_str), Some("ties")); + assert_eq!( + mem0.pointer("/scenarios/2/scenario_id").and_then(Value::as_str), + Some("openmemory_ui_export_readback") + ); + assert_eq!(mem0.pointer("/scenarios/2/status").and_then(Value::as_str), Some("not_encoded")); assert_eq!( memsearch.pointer("/capabilities/2/capability").and_then(Value::as_str), Some("reindex_update_delete_reload") ); - assert_eq!(memsearch.pointer("/capabilities/2/status").and_then(Value::as_str), Some("real")); + assert_eq!(memsearch.pointer("/capabilities/2/status").and_then(Value::as_str), Some("pass")); + assert_eq!( + memsearch.pointer("/scenarios/0/scenario_id").and_then(Value::as_str), + Some("canonical_markdown_reindex_reload") + ); + assert_eq!( + memsearch.pointer("/scenarios/1/status").and_then(Value::as_str), + Some("unsupported") + ); + assert_eq!( + memsearch.pointer("/scenarios/1/elf_position").and_then(Value::as_str), + Some("wins") + ); assert_eq!(claude_mem.pointer("/capabilities/1/status").and_then(Value::as_str), Some("real")); assert_eq!( claude_mem.pointer("/capabilities/3/capability").and_then(Value::as_str), @@ -404,6 +452,11 @@ fn assert_first_generation_adapter_records(mem0: &Value, memsearch: &Value, clau claude_mem.pointer("/capabilities/4/status").and_then(Value::as_str), Some("not_encoded") ); + assert_eq!( + claude_mem.pointer("/scenarios/0/status").and_then(Value::as_str), + Some("wrong_result") + ); + assert_eq!(claude_mem.pointer("/scenarios/1/status").and_then(Value::as_str), Some("pass")); } fn assert_graphiti_zep_adapter(adapter: &Value) { diff --git a/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md b/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md index 1802eaf5..4f773094 100644 --- a/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md +++ b/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md @@ -74,8 +74,8 @@ lifecycle-fail -> `lifecycle_fail`, and not-encoded -> `not_encoded`. | ELF | Evidence-linked source-of-truth memory service with real-world fixtures and live retrieval sweeps. | `live_real_world`; supporting `fixture_backed`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/elf-report.md`. Fixture contract: `cargo make real-world-memory`, `tmp/real-world-memory/real-world-memory-report.json`. | `blocked`: private manifest and provider credentials; broader live suites remain `wrong_result`, `incomplete`, or `not_encoded`. | Full-suite live pass plus separate private-corpus and credentialed production-ops proof. | Keep borrowing qmd debug knobs, OpenViking staged trajectory, mem0 history, Letta core memory, and graph/RAG navigation. | | qmd | Local retrieval-debug workflow with transparent CLI indexing, querying, expansion, fusion, and rerank ergonomics. | `live_real_world`; supporting `live_baseline_only` and `research_gate`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/qmd-report.md`; targeted retrieval suites pass. | `not_encoded`: deep profile and non-retrieval live behavior are not encoded; memory_evolution is `wrong_result`. | qmd deep retrieval/debug profile plus full-suite live replay with trace-level diagnostics. | Weighted fusion, rerank explanation, local debug knobs, and command-line replay. | | agentmemory | Coding-agent continuity, MCP/REST packaging, viewer workflow, and durable cross-agent memory lifecycle. | `live_baseline_only`. | `lifecycle_fail`: `ELF_BASELINE_PROJECTS=agentmemory cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `blocked`: durable cold-start and real-world adapter coverage are missing. | Durable local adapter with update, delete, cold-start reload, work_resume, capture/write-policy, and lifecycle-staleness jobs. | Cross-agent hooks, packaging, continuity scenarios, and viewer affordances. | -| mem0/OpenMemory | Memory lifecycle, personalization, hosted/OpenMemory UI ergonomics, and optional graph memory. | `live_baseline_only`. | `wrong_result`: `ELF_BASELINE_PROJECTS=mem0 cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `not_encoded`: OpenMemory UI, hosted claims, and real-world personalization coverage are not encoded. | Fix local same-corpus result, then encode memory_evolution, personalization, UI readback, and optional graph-context jobs. | Entity-scoped history, lifecycle surfaces, async update ergonomics, and OpenMemory inspection UX. | -| memsearch | Markdown-first canonical store with rebuildable local index and practical hybrid retrieval. | `live_baseline_only`. | `wrong_result`: `ELF_BASELINE_PROJECTS=memsearch cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `incomplete`: source-of-truth and real-world reindex behavior are not cleanly scored. | Fix Docker same-corpus retrieval and reindex/update/delete reload evidence, then score source-of-truth and retrieval-debug jobs. | Canonical markdown store, local reindex clarity, and user-inspectable source files. | +| mem0/OpenMemory | Memory lifecycle, personalization, hosted/OpenMemory UI ergonomics, and optional graph memory. | `live_baseline_only`. | `pass`: fresh scoped run `ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`, with mem0 `4/4` local checks passing. | `not_encoded`: OpenMemory UI, hosted claims, entity/preference history, graph memory, and real-world personalization coverage are not encoded. | Encode memory_evolution preference/entity history, deletion audit readback, personalization, UI/export readback, and optional graph-context jobs. | Entity-scoped history, lifecycle surfaces, async update ergonomics, and OpenMemory inspection UX. | +| memsearch | Markdown-first canonical store with rebuildable local index and practical hybrid retrieval. | `live_baseline_only`. | `pass`: fresh scoped run `ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`, with memsearch `4/4` local checks passing. | `not_encoded`: real-world source-of-truth, retrieval, and memory-evolution prompt adapters are not encoded; TTL/expiry is unsupported by the current CLI path. | Score source-of-truth and retrieval-debug real-world jobs over the canonical Markdown store; keep TTL/expiry as unsupported unless a comparable path exists. | Canonical markdown store, local reindex clarity, and user-inspectable source files. | | OpenViking | Filesystem-like context trajectory, hierarchical retrieval, and staged context loading. | `live_baseline_only`; supporting `research_gate`. | `wrong_result`: `ELF_BASELINE_PROJECTS=OpenViking cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `not_encoded`: hierarchical context trajectory is not encoded; same-corpus output still misses expected evidence. | Make evidence-bearing same-corpus output pass, then score staged trajectory and hierarchy expansion. | `viking://`-style context model, trajectory readback, and staged retrieval planning. | | claude-mem | Progressive disclosure, automatic capture loop, repository-local lifecycle, and local viewer workflow. | `live_baseline_only`. | `wrong_result`: `ELF_BASELINE_PROJECTS=claude-mem cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `not_encoded`: progressive-disclosure real-world jobs are not encoded. | Durable repository-backed work_resume, operator_debugging_ux, capture/write-policy, and progressive-disclosure jobs. | Progressive disclosure, automatic capture review loops, and local viewer/operator comfort. | | RAGFlow | Full RAG application workflow with document, chunk, and reference evidence handles. | `research_gate`. | `blocked`: `ELF_RAGFLOW_SMOKE_START=1 ELF_RAGFLOW_SMOKE_ACCEPT_RESOURCE_ENVELOPE=1 cargo make ragflow-docker-smoke`, `tmp/real-world-memory/ragflow-smoke/ragflow-smoke.json`. | `blocked`: Docker resource envelope and adapter output mapping still need proof. | XY-885 tiny Docker evidence-smoke adapter mapping `reference.chunks` to scored evidence. | Document/chunk references, resource-envelope reporting, and RAG app evidence handles. | @@ -96,8 +96,8 @@ lifecycle-fail -> `lifecycle_fail`, and not-encoded -> `not_encoded`. | Retrieval/debug | Fixture retrieval passes; live retrieval passes. | qmd. | qmd live retrieval passes and live baseline passes, but full-suite live status is `wrong_result`. | Run qmd deep profile and ELF/qmd trace-level replay with expansion, fusion, rerank, and candidate-drop diagnostics. | | Work resume | Fixture and live work_resume pass. | agentmemory, claude-mem, OpenViking. | agentmemory `lifecycle_fail`, claude-mem `wrong_result`, OpenViking work_resume `not_encoded`. | Encode durable work_resume adapters or keep each blocked with lifecycle/setup evidence. | | Project decisions | Fixture and live project_decisions pass. | qmd, Letta. | qmd live project_decisions pass; Letta is `research_gate` `not_encoded`. | Add Letta core/archival decision jobs only after a contained export path exists. | -| Source-of-truth | Fixture and live trust_source_of_truth pass. | memsearch. | memsearch canonical-store evidence exists, but source-of-truth is `incomplete` and retrieval is `wrong_result`. | Fix memsearch reindex/retrieval evidence and score source-of-truth rebuild/reload jobs. | -| Temporal/current-vs-historical memory | Fixture memory_evolution passes; live memory_evolution is `wrong_result`. | Graphiti/Zep, mem0/OpenMemory. | Graphiti/Zep is `research_gate` `blocked`; mem0/OpenMemory is `wrong_result`. | Fix ELF/qmd live memory_evolution evidence links and run XY-888. | +| Source-of-truth | Fixture and live trust_source_of_truth pass. | memsearch. | memsearch canonical-store, reindex, delete, and reload smoke now passes, but source-of-truth real_world_job prompts are `not_encoded`. | Score memsearch source-of-truth rebuild/reload jobs before any suite-level win/loss claim. | +| Temporal/current-vs-historical memory | Fixture memory_evolution passes; live memory_evolution is `wrong_result`. | Graphiti/Zep, mem0/OpenMemory. | Graphiti/Zep is `research_gate` `blocked`; mem0/OpenMemory basic local lifecycle now passes, but preference/entity history, deletion audit, UI/export, and graph-memory scenarios are `not_encoded`. | Fix ELF/qmd live memory_evolution evidence links, encode mem0/OpenMemory history/UI jobs, and run XY-888. | | Consolidation | Fixture consolidation passes; live consolidation is `not_encoded`. | agentmemory, managed-memory references, llm-wiki. | No manifest project has live consolidation scoring. | Run reviewable consolidation proposal generation with source refs, unsupported-claim flags, and audit transitions. | | Knowledge pages | Fixture knowledge_compilation passes; live knowledge_compilation is `not_encoded`. | llm-wiki, gbrain, GraphRAG, graphify. | llm-wiki and gbrain are `research_gate` `not_encoded` or `blocked`; GraphRAG and graphify are `blocked`. | Encode live derived-page rebuild/lint scoring and run contained knowledge/RAG adapters only after setup proof. | | Operator debugging | Fixture operator_debugging_ux passes; live operator_debugging_ux is `not_encoded`. | qmd, claude-mem, OpenMemory. | qmd has debug strengths but operator_debugging_ux is `not_encoded`; claude-mem and OpenMemory UX are `not_encoded`. | Score trace hydration, stage attribution, raw-SQL avoidance, and repair-action clarity through live artifacts. | @@ -117,8 +117,8 @@ now explicit: | --- | --- | --- | --- | --- | | qmd deep retrieval/debug profile | New benchmark issue | yes | None after this matrix lands. | Stress profile plus trace-level retrieval-debug artifacts for qmd and ELF. | | agentmemory durable lifecycle adapter | `[ELF benchmark P0] Make external adapters lifecycle-durable and fail-typed` | yes | Durable local adapter path selection. | Update, delete, cold-start reload, work_resume, and capture/write-policy jobs. | -| mem0/OpenMemory local and UI coverage | New adapter repair issue | yes | Comparable local OSS path for UI/readback evidence. | Same-corpus fix plus memory_evolution, personalization, and OpenMemory inspection jobs. | -| memsearch source-of-truth and reindex coverage | New adapter repair issue | yes | Docker same-corpus retrieval and reindex correctness. | Canonical markdown store, rebuild/reindex, retrieval, update/delete/reload jobs. | +| mem0/OpenMemory history and UI coverage | New adapter repair issue | yes | Comparable local OSS path for history/UI/readback evidence. | Preference/entity history, deletion audit readback, personalization, OpenMemory inspection/export, and optional graph-context jobs. | +| memsearch source-of-truth real-world coverage | New adapter repair issue | yes | Real-world prompt adapter over the canonical Markdown store. | Source-of-truth rebuild/reload jobs and retrieval-debug jobs that preserve baseline reindex/update/delete evidence without converting it into suite pass claims. | | OpenViking context trajectory | New benchmark issue after evidence output fix | yes | Evidence-bearing same-corpus retrieval output. | Hierarchical expansion, staged trajectory, and resume/retrieval evidence jobs. | | claude-mem progressive disclosure | New adapter issue | yes | Durable repository path and progressive-disclosure output contract. | Work resume, operator debugging, capture/write-policy, and progressive disclosure jobs. | | RAGFlow evidence smoke | XY-885 | yes | Resource envelope accepted for tiny Docker smoke. | `reference.chunks` to benchmark evidence mapping. | diff --git a/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md b/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md index d581b76c..e5a6738a 100644 --- a/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md +++ b/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md @@ -144,8 +144,8 @@ one misleading score. | ELF | `fixture_backed` plus `live_real_world`; live full sweep is `wrong_result`. | Evidence-linked memory service, strict provenance, rebuildable Qdrant, production backfill/restore proof. | Keep this as the core; do not weaken source-of-truth or typed failure semantics while adding product ergonomics. | | qmd | `live_real_world` plus `live_baseline_only`; targeted retrieval passes, full sweep is `wrong_result`. | Local retrieval-debug workflow, transparent CLI, weighted fusion, rerank, replayable commands. | Treat qmd as the retrieval-debug bar. ELF should match its introspection and local replay without becoming CLI-only. | | agentmemory | `live_baseline_only`; current status is `lifecycle_fail`. | Coding-agent continuity, hooks, MCP/REST packaging, viewer/console observability. | Borrow capture breadth and continuity UX, but require durable lifecycle proof before claims. | -| mem0/OpenMemory | `live_baseline_only`; current status is `wrong_result`. | Entity-scoped memory, lifecycle/history surfaces, hosted ecosystem, OpenMemory UI. | Add entity/preference history and UI readback patterns, while keeping hosted claims out of local OSS benchmarks. | -| memsearch | `live_baseline_only`; current status is `wrong_result` with source-of-truth gaps. | Markdown-first canonical store and local reindex clarity. | Borrow local inspectability and canonical-file ergonomics, not file-as-authority semantics. | +| mem0/OpenMemory | `live_baseline_only`; basic local smoke now passes, while entity/preference history, hosted ecosystem, graph memory, and OpenMemory UI remain untested locally. | Entity-scoped memory, lifecycle/history surfaces, hosted ecosystem, OpenMemory UI. | Add entity/preference history and UI readback patterns, while keeping hosted claims out of local OSS benchmarks. | +| memsearch | `live_baseline_only`; canonical Markdown reindex/reload smoke now passes, while real-world source-of-truth prompts remain unencoded. | Markdown-first canonical store and local reindex clarity. | Borrow local inspectability and canonical-file ergonomics, not file-as-authority semantics. | | OpenViking | `live_baseline_only` plus `research_gate`; current status is `wrong_result`. | Filesystem-like context model, hierarchy, staged context trajectory. | Add staged retrieval and trajectory scoring after same-corpus evidence output is correct. | | claude-mem | `live_baseline_only`; current status is `wrong_result`. | Progressive disclosure, automatic capture, local viewer workflow. | Borrow progressive disclosure and viewer comfort; benchmark capture and operator-debugging live paths. | | RAGFlow | `research_gate`; current status is `blocked`. | Full RAG application workflow with document/chunk/reference handles. | Use as a resource-aware RAG adapter benchmark, not as a current ELF competitor win/loss. | diff --git a/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md b/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md new file mode 100644 index 00000000..4e4e72b6 --- /dev/null +++ b/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md @@ -0,0 +1,112 @@ +# First-Generation OSS Adapter Promotion Report - June 11, 2026 + +Goal: Promote first-generation OSS memory baselines into scenario-level adapter +evidence without converting live-baseline-only runs into real-world suite wins. +Read this when: You need the current XY-898 status for agentmemory, mem0/OpenMemory, +memsearch, and claude-mem scenario evidence. +Inputs: Fresh scoped Docker baseline run, updated external adapter manifest, and the +June 11 temporal/history competitor-gap report. +Outputs: Scenario judgments, ELF win/tie/loss/untested positions, and next adapter +gates. + +## Scope Boundary + +This is benchmark/report evidence only. No ELF retrieval, ranking, memory-quality, or +service behavior optimization is implemented here. + +The updated external adapter manifest now includes scenario-level judgments for the +first-generation OSS memory projects. These judgments are intentionally narrower than +suite passes: + +- `live_baseline_only` pass evidence proves the encoded Docker same-corpus or + lifecycle smoke for that project. +- It does not prove `real_world_job` suite parity unless a project adapter actually + executes real-world prompts and scoring. +- Hosted mem0 Platform behavior, OpenMemory UI, host-global hooks, and + operator-owned credentials remain out of scope for local OSS evidence. + +## Fresh Run + +| Command | Result | Runtime | Artifact | +| --- | --- | ---: | --- | +| `ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker` | fail with typed non-pass projects | 244.42 seconds | `tmp/live-baseline/live-baseline-report.json` | + +The aggregate failed because two projects remained typed non-pass, not because setup +collapsed: + +| Project | Status | Retrieval | Checks | Scenario meaning | +| --- | --- | --- | ---: | --- | +| agentmemory | `lifecycle_fail` | `retrieval_pass` | `2/4` pass, `1` lifecycle_fail, `1` blocked | Same-corpus retrieval runs, but update supersession and durable cold-start are not proven through the in-memory mock. | +| mem0/OpenMemory | `pass` | `retrieval_pass` | `4/4` pass | Basic local OSS same-corpus, update, delete, and cold-start smoke passes. | +| memsearch | `pass` | `retrieval_pass` | `4/4` pass | Canonical Markdown reindex/update/delete/reload smoke passes. | +| claude-mem | `wrong_result` | `retrieval_wrong_result` | `4/5` pass | Durable repository lifecycle, detail hydration, and reload pass, but same-corpus retrieval misses expected evidence. | + +## Scenario Judgments + +| Project | Scenario | Status | ELF position | Evidence boundary | +| --- | --- | --- | --- | --- | +| agentmemory | basic same-corpus retrieval | `pass` | `untested` | Baseline retrieval passes through an in-memory mock; no durable continuity claim. | +| agentmemory | durable update/reload lifecycle | `lifecycle_fail` | `wins` | Update supersession fails and cold-start is blocked; ELF has broader encoded local lifecycle proof. | +| agentmemory | work-resume capture continuity | `blocked` | `untested` | Needs a durable local session/capture path before fair scoring. | +| mem0/OpenMemory | basic local lifecycle | `pass` | `ties` | ELF and mem0 both pass the encoded local lifecycle smoke; mem0 is no longer a basic-smoke failure. | +| mem0/OpenMemory | preference/entity history | `not_encoded` | `untested` | History, correction chains, entity scope, and deletion audit are not scored. | +| mem0/OpenMemory | OpenMemory UI/export readback | `not_encoded` | `untested` | Local OSS UI/export readback is not executed; hosted behavior remains out of scope. | +| memsearch | canonical Markdown reindex/reload | `pass` | `ties` | Baseline reindex/update/delete/reload passes over the canonical file store. | +| memsearch | TTL/expiry lifecycle | `unsupported` | `wins` | The encoded CLI path has reindex/delete but no TTL/expiry behavior. | +| memsearch | real-world prompt adapter | `not_encoded` | `untested` | No memsearch real_world_job prompt adapter is encoded. | +| claude-mem | same-corpus retrieval | `wrong_result` | `wins` | The durable repository path runs but misses expected retrieval evidence. | +| claude-mem | repository lifecycle reload | `pass` | `ties` | Update, delete, and cold-start reload pass over Docker-local SQLite. | +| claude-mem | progressive-disclosure detail hydration | `pass` | `untested` | Search-to-detail/source hydration passes, but ELF has no directly comparable claude-mem-style progressive-disclosure scenario here. | +| claude-mem | hook capture viewer workflow | `not_encoded` | `untested` | Hooks, viewer, timeline, and observations are not executed. | + +Summary: 13 scenario judgments: 5 `pass`, 1 `wrong_result`, 1 `lifecycle_fail`, +1 `blocked`, 1 `unsupported`, and 4 `not_encoded`. ELF positions are 3 `wins`, +3 `ties`, 0 `loses`, and 7 `untested`. + +## Manifest And Report Changes + +The external adapter manifest is now +`real-world-memory-project-adapters-2026-06-11` and includes `scenarios[]` records +with: + +- `scenario_id` +- optional `suite_id` +- typed scenario `status` +- `elf_position`: `wins`, `ties`, `loses`, or `untested` +- evidence text plus optional command/artifact pointers + +`real_world_job_benchmark` now preserves these fields in generated reports and +renders an **Adapter Scenario Judgments** table. This makes the report input capable +of saying whether ELF wins, ties, loses, or remains untested per scenario without +changing the real-world suite status rules. + +## Claim Boundaries + +Allowed: + +- mem0/OpenMemory passes the current basic local OSS lifecycle smoke. +- memsearch passes the current canonical Markdown reindex/reload smoke. +- agentmemory remains non-pass for durable lifecycle because the current adapter uses + an in-memory mock and cannot prove cold-start recovery. +- claude-mem remains wrong-result for same-corpus retrieval while preserving useful + passed evidence for repository lifecycle and detail hydration. + +Not allowed: + +- Do not claim hosted OpenMemory behavior from local OSS evidence. +- Do not claim mem0/OpenMemory history, UI/export, hosted, or graph-memory parity. +- Do not claim memsearch source-of-truth real-world suite parity from baseline smoke. +- Do not claim claude-mem hook/viewer/capture parity from repository-only checks. +- Do not collapse `wrong_result`, `lifecycle_fail`, `blocked`, `unsupported`, + `not_encoded`, and `incomplete` into one generic failure bucket. + +## Next Gates + +- agentmemory: select a durable local KV/index/session path before work-resume and + capture jobs. +- mem0/OpenMemory: encode preference/entity history, deletion audit, UI/export + readback, and optional graph memory for local OSS only. +- memsearch: encode real-world source-of-truth and retrieval-debug prompt jobs over + the canonical Markdown store. +- claude-mem: fix or explain same-corpus retrieval misses, then encode hook capture, + viewer/operator, and progressive-disclosure jobs. diff --git a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md index 862395b4..f3be7a56 100644 --- a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md +++ b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md @@ -142,8 +142,8 @@ Interpret the unique manifest project list as the project coverage count. | ELF | `fixture_backed` plus `live_real_world` | Fixture aggregate passes except 2 blocked operator boundaries; live full sweep is `wrong_result`. | Full live memory evolution, live consolidation, live knowledge pages, live capture, live production ops. | Memory-evolution diagnostic report, then live operator/capture/consolidation reports. | | qmd | `live_real_world` plus `live_baseline_only` | Same live sweep shape as ELF; same-corpus baseline passes. | Deep retrieval-debug ergonomics and trace replay. | qmd/ELF deep retrieval-debug profile with expansion, fusion, rerank, and dropped-candidate traces. | | agentmemory | `live_baseline_only` | `lifecycle_fail`. | Durable coding-agent continuity and capture hooks. | Durable lifecycle and work-resume/capture adapter report. | -| mem0/OpenMemory | `live_baseline_only` | `wrong_result`. | Entity history, lifecycle UI, OpenMemory inspection. | Same-corpus repair first, then entity-history and UI-readback report. | -| memsearch | `live_baseline_only` | `wrong_result`; source-of-truth is `incomplete`. | Markdown canonical store and local reindex clarity. | Reindex/update/delete/reload plus source-of-truth report. | +| mem0/OpenMemory | `live_baseline_only` | Basic local smoke now passes; history/UI/hosted/graph behavior remains `not_encoded`. | Entity history, lifecycle UI, OpenMemory inspection. | Entity-history, deletion-audit, and UI/export readback report. | +| memsearch | `live_baseline_only` | Basic canonical Markdown reindex/reload smoke now passes; real-world prompt coverage remains `not_encoded`. | Markdown canonical store and local reindex clarity. | Source-of-truth and retrieval-debug real-world adapter report. | | OpenViking | `live_baseline_only` plus `research_gate` | Same-corpus retrieval is `wrong_result`; trajectory is `not_encoded`. | Hierarchical staged context trajectory. | Evidence-bearing retrieval fix, then staged trajectory report. | | claude-mem | `live_baseline_only` | `wrong_result`. | Progressive disclosure and automatic capture review. | Work-resume, operator-debugging, and capture/write-policy report. | | RAGFlow | `research_gate` | `blocked`. | RAG app workflow with document/chunk references. | Tiny Docker evidence-smoke with `reference.chunks` mapped to evidence ids. | diff --git a/docs/guide/benchmarking/index.md b/docs/guide/benchmarking/index.md index e7b0cded..8bd36fa0 100644 --- a/docs/guide/benchmarking/index.md +++ b/docs/guide/benchmarking/index.md @@ -70,6 +70,11 @@ cleanup, use `docs/guide/single_user_production.md`. records Graphiti/Zep and Letta claim boundaries, and turns qmd, mem0/OpenMemory, Graphiti/Zep, Letta, and adjacent project strengths into benchmark-gated ELF optimization directions. +- `2026-06-11-first-generation-oss-adapter-promotion-report.md`: XY-898 + first-generation OSS adapter promotion report that updates agentmemory, + mem0/OpenMemory, memsearch, and claude-mem with fresh scenario-level baseline + evidence and ELF win/tie/loss/untested positions without converting baseline-only + evidence into real-world suite wins. - `real_world_agent_memory_benchmark.md`: operator overview for the v1 real-world agent memory benchmark contract, including suite taxonomy, typed report states, knowledge-compilation fixture tasks, and the production-ops fixture target. diff --git a/docs/guide/benchmarking/live_baseline_benchmark.md b/docs/guide/benchmarking/live_baseline_benchmark.md index 30377951..ad839597 100644 --- a/docs/guide/benchmarking/live_baseline_benchmark.md +++ b/docs/guide/benchmarking/live_baseline_benchmark.md @@ -375,6 +375,9 @@ leave real-world suites as `not_encoded`, `blocked`, `incomplete`, `wrong_result scoring. The same manifest can also contain `research_gate` records for future adapter packs; those records provide source/setup/runtime/resource/retry guidance but are not live-baseline evidence. +The manifest may also include scenario judgments with an ELF position of `wins`, +`ties`, `loses`, or `untested`; these are dimension-level report inputs and do not +upgrade live-baseline-only evidence into real-world suite pass evidence. The full live real-world adapter sweep for ELF and qmd is separate from the same-corpus live baseline: diff --git a/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json b/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json index b847ecc7..96d549a1 100644 --- a/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json +++ b/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json @@ -29,11 +29,11 @@ "research_gate": 12 }, "overall_status_counts": { - "pass": 1, - "wrong_result": 6, "lifecycle_fail": 1, "blocked": 6, - "not_encoded": 7 + "not_encoded": 7, + "pass": 3, + "wrong_result": 4 } }, "state_taxonomy": [ @@ -149,17 +149,17 @@ "supporting_evidence_classes": [ "live_baseline_only" ], - "measured_status": "wrong_result", + "measured_status": "pass", "proof": { - "command": "ELF_BASELINE_PROJECTS=mem0 cargo make baseline-live-docker", + "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "unsupported_or_blocked_status": { "state": "not_encoded", - "typed_reason": "openmemory_ui_and_hosted_claims_not_encoded", - "details": "Local OSS setup is represented, but hosted/OpenMemory UI parity and real-world personalization coverage are not encoded." + "typed_reason": "history_ui_hosted_graph_claims_not_encoded", + "details": "Basic local OSS same-corpus/update/delete/reload smoke now passes, but hosted/OpenMemory UI parity, entity/preference history, deletion-audit readback, optional graph memory, and real-world personalization coverage are not encoded." }, - "benchmark_before_claim": "Fix the local adapter's same-corpus result, then encode memory_evolution, personalization, OpenMemory UI readback, and optional graph-context jobs.", + "benchmark_before_claim": "Encode memory_evolution preference/entity history, deletion audit readback, personalization, OpenMemory UI/export readback, and optional graph-context jobs.", "borrow_if_stronger": "Borrow entity-scoped memory history, lifecycle surfaces, async update ergonomics, and OpenMemory-style inspection UX." }, { @@ -169,17 +169,17 @@ "supporting_evidence_classes": [ "live_baseline_only" ], - "measured_status": "wrong_result", + "measured_status": "pass", "proof": { - "command": "ELF_BASELINE_PROJECTS=memsearch cargo make baseline-live-docker", + "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "unsupported_or_blocked_status": { - "state": "incomplete", - "typed_reason": "source_of_truth_and_reindex_real_world_jobs_incomplete", - "details": "Same-corpus retrieval is wrong_result and source-of-truth plus real-world reindex behavior is not yet cleanly scored." + "state": "not_encoded", + "typed_reason": "source_of_truth_and_reindex_real_world_jobs_not_encoded", + "details": "Basic canonical Markdown same-corpus/reindex/update/delete/reload smoke now passes, but source-of-truth, retrieval-debug, and memory-evolution real-world prompt adapters are not encoded." }, - "benchmark_before_claim": "Fix Docker same-corpus retrieval and reindex/update/delete reload evidence, then score source-of-truth and retrieval-debug real-world jobs.", + "benchmark_before_claim": "Score source-of-truth and retrieval-debug real-world jobs over the canonical Markdown store; keep TTL/expiry unsupported unless a comparable path exists.", "borrow_if_stronger": "Borrow the canonical markdown-store ergonomics, local reindex clarity, and user-inspectable source files." }, { @@ -457,18 +457,18 @@ "scenario": "source-of-truth", "current_elf_evidence": "ELF fixture-backed trust_source_of_truth passes and ELF live_real_world trust_source_of_truth passes.", "strongest_competitor_or_reference": "memsearch", - "current_competitor_evidence": "memsearch has live_baseline_only canonical store evidence but trust_source_of_truth is incomplete and retrieval is wrong_result.", - "current_state": "ELF has stronger measured source-of-truth evidence; memsearch remains a local-store ergonomics reference.", - "next_measurement": "Fix memsearch same-corpus retrieval/reindex evidence, then run source-of-truth rebuild and reload jobs before any win/loss claim." + "current_competitor_evidence": "memsearch has live_baseline_only canonical store evidence and now passes same-corpus retrieval, reindex/update/delete, and cold-start reload smoke, but trust_source_of_truth real-world prompts are not_encoded.", + "current_state": "ELF has stronger measured real-world source-of-truth evidence; memsearch now ties the local canonical-store reindex/reload smoke and remains a local-store ergonomics reference.", + "next_measurement": "Run memsearch source-of-truth rebuild and reload real_world_job prompts before any suite-level win/loss claim." }, { "scenario_id": "temporal_current_historical", "scenario": "temporal/current-vs-historical memory", "current_elf_evidence": "ELF fixture-backed memory_evolution passes, but ELF live_real_world memory_evolution is wrong_result.", "strongest_competitor_or_reference": "Graphiti/Zep, mem0/OpenMemory", - "current_competitor_evidence": "Graphiti/Zep is research_gate blocked; mem0/OpenMemory is live_baseline_only wrong_result.", + "current_competitor_evidence": "Graphiti/Zep is research_gate blocked; mem0/OpenMemory now passes basic live_baseline_only local lifecycle smoke but preference/entity history, deletion audit, UI/export, and graph-memory scenarios are not_encoded.", "current_state": "No project has a comparable live pass for current-vs-historical evidence; ELF cannot claim live superiority yet.", - "next_measurement": "Fix ELF/qmd live memory_evolution evidence links and run XY-888 Graphiti/Zep temporal graph adapter." + "next_measurement": "Fix ELF/qmd live memory_evolution evidence links, encode mem0/OpenMemory history and UI/export jobs, and run XY-888 Graphiti/Zep temporal graph adapter." }, { "scenario_id": "consolidation", @@ -568,18 +568,18 @@ "measurement": "Update, delete, cold-start reload, work_resume, and capture/write-policy jobs." }, { - "workstream": "mem0/OpenMemory local and UI coverage", + "workstream": "mem0/OpenMemory history and UI coverage", "issue_or_candidate": "new adapter repair issue", "parallelizable": true, - "blocked_by": "Comparable local OSS path for UI/readback evidence.", - "measurement": "Same-corpus fix plus memory_evolution, personalization, and OpenMemory inspection jobs." + "blocked_by": "Comparable local OSS path for history/UI/readback evidence.", + "measurement": "Preference/entity history, deletion audit readback, personalization, OpenMemory inspection/export, and optional graph-context jobs." }, { - "workstream": "memsearch source-of-truth and reindex coverage", + "workstream": "memsearch source-of-truth real-world coverage", "issue_or_candidate": "new adapter repair issue", "parallelizable": true, - "blocked_by": "Docker same-corpus retrieval and reindex correctness.", - "measurement": "Canonical markdown store, rebuild/reindex, retrieval, update/delete/reload jobs." + "blocked_by": "Real-world prompt adapter over the canonical Markdown store.", + "measurement": "Source-of-truth rebuild/reload jobs and retrieval-debug jobs that preserve baseline reindex/update/delete evidence without converting it into suite pass claims." }, { "workstream": "OpenViking context trajectory", diff --git a/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json b/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json new file mode 100644 index 00000000..d28d07d8 --- /dev/null +++ b/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json @@ -0,0 +1,197 @@ +{ + "schema": "elf.first_generation_oss_adapter_promotion_report/v1", + "report_id": "xy-898-first-generation-oss-adapter-promotion-2026-06-11", + "authority": "XY-898", + "date": "2026-06-11", + "scope": "Scenario-level adapter evidence for agentmemory, mem0/OpenMemory, memsearch, and claude-mem without ELF optimization changes.", + "source_inputs": [ + "docs/guide/benchmarking/2026-06-11-temporal-history-competitor-gap-report.md", + "docs/research/2026-06-11-temporal-history-competitor-gap-report.json", + "docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md", + "docs/spec/real_world_agent_memory_benchmark_v1.md", + "docs/guide/benchmarking/live_baseline_benchmark.md", + "apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json", + "tmp/live-baseline/live-baseline-report.json" + ], + "fresh_run": { + "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "run_id": "live-baseline-20260611015125", + "status": "fail", + "runtime_seconds": 244.42, + "artifact": "tmp/live-baseline/live-baseline-report.json", + "summary": { + "total": 4, + "pass": 2, + "wrong_result": 1, + "lifecycle_fail": 1, + "incomplete": 0, + "blocked": 0, + "not_encoded": 0 + }, + "full_check_summary": { + "total": 17, + "pass": 14, + "wrong_result": 1, + "lifecycle_fail": 1, + "blocked": 1, + "incomplete": 0, + "not_encoded": 0 + } + }, + "projects": [ + { + "project": "agentmemory", + "status": "lifecycle_fail", + "retrieval_status": "retrieval_pass", + "check_summary": { + "total": 4, + "pass": 2, + "lifecycle_fail": 1, + "blocked": 1 + }, + "evidence": "Same-corpus retrieval and delete suppression pass through the in-memory mock, but update supersession fails and cold-start recovery is blocked." + }, + { + "project": "mem0/OpenMemory", + "status": "pass", + "retrieval_status": "retrieval_pass", + "check_summary": { + "total": 4, + "pass": 4 + }, + "evidence": "Local OSS mem0 passes same-corpus retrieval, update, delete, and cold-start reload. This does not include OpenMemory UI, hosted Platform, entity history, or graph memory." + }, + { + "project": "memsearch", + "status": "pass", + "retrieval_status": "retrieval_pass", + "check_summary": { + "total": 4, + "pass": 4 + }, + "evidence": "memsearch passes same-corpus retrieval, update reindex, delete suppression, and cold-start reload over the canonical Markdown corpus." + }, + { + "project": "claude-mem", + "status": "wrong_result", + "retrieval_status": "retrieval_wrong_result", + "check_summary": { + "total": 5, + "pass": 4, + "wrong_result": 1 + }, + "evidence": "claude-mem passes update, delete, progressive detail/source hydration, and cold-start reload over Docker-local SQLite, but same-corpus retrieval misses expected evidence." + } + ], + "scenario_summary": { + "count": 13, + "status_counts": { + "pass": 5, + "wrong_result": 1, + "lifecycle_fail": 1, + "blocked": 1, + "unsupported": 1, + "not_encoded": 4 + }, + "elf_position_counts": { + "wins": 3, + "ties": 3, + "loses": 0, + "untested": 7 + } + }, + "scenario_judgments": [ + { + "project": "agentmemory", + "scenario_id": "basic_same_corpus_retrieval", + "status": "pass", + "elf_position": "untested" + }, + { + "project": "agentmemory", + "scenario_id": "durable_update_reload_lifecycle", + "status": "lifecycle_fail", + "elf_position": "wins" + }, + { + "project": "agentmemory", + "scenario_id": "work_resume_capture_continuity", + "status": "blocked", + "elf_position": "untested" + }, + { + "project": "mem0/OpenMemory", + "scenario_id": "basic_local_lifecycle", + "status": "pass", + "elf_position": "ties" + }, + { + "project": "mem0/OpenMemory", + "scenario_id": "preference_entity_history", + "status": "not_encoded", + "elf_position": "untested" + }, + { + "project": "mem0/OpenMemory", + "scenario_id": "openmemory_ui_export_readback", + "status": "not_encoded", + "elf_position": "untested" + }, + { + "project": "memsearch", + "scenario_id": "canonical_markdown_reindex_reload", + "status": "pass", + "elf_position": "ties" + }, + { + "project": "memsearch", + "scenario_id": "ttl_expiry_lifecycle", + "status": "unsupported", + "elf_position": "wins" + }, + { + "project": "memsearch", + "scenario_id": "real_world_prompt_adapter", + "status": "not_encoded", + "elf_position": "untested" + }, + { + "project": "claude-mem", + "scenario_id": "same_corpus_retrieval", + "status": "wrong_result", + "elf_position": "wins" + }, + { + "project": "claude-mem", + "scenario_id": "repository_lifecycle_reload", + "status": "pass", + "elf_position": "ties" + }, + { + "project": "claude-mem", + "scenario_id": "progressive_disclosure_detail_hydration", + "status": "pass", + "elf_position": "untested" + }, + { + "project": "claude-mem", + "scenario_id": "hook_capture_viewer_workflow", + "status": "not_encoded", + "elf_position": "untested" + } + ], + "claim_boundaries": { + "allowed": [ + "mem0/OpenMemory passes the current basic local OSS lifecycle smoke.", + "memsearch passes the current canonical Markdown reindex/reload smoke.", + "agentmemory remains lifecycle_fail for durable update/reload because the current adapter is in-memory.", + "claude-mem remains wrong_result for same-corpus retrieval while preserving passed repository lifecycle and detail hydration evidence." + ], + "not_allowed": [ + "Do not claim hosted OpenMemory behavior from local OSS evidence.", + "Do not claim mem0/OpenMemory history, UI/export, hosted, or graph-memory parity.", + "Do not claim memsearch source-of-truth real-world suite parity from baseline smoke.", + "Do not claim claude-mem hook/viewer/capture parity from repository-only checks." + ] + } +} diff --git a/docs/spec/real_world_agent_memory_benchmark_v1.md b/docs/spec/real_world_agent_memory_benchmark_v1.md index bb0a4b82..115acc3f 100644 --- a/docs/spec/real_world_agent_memory_benchmark_v1.md +++ b/docs/spec/real_world_agent_memory_benchmark_v1.md @@ -174,6 +174,11 @@ Each `adapters[]` record MUST include: and `evidence`. - `suites`: array of real-world suite coverage records with `suite_id`, `status`, and `evidence`. +- `scenarios`: optional array of scenario judgment records with `scenario_id`, + optional `suite_id`, `status`, `elf_position`, `evidence`, and optional `command` + and `artifact`. `elf_position` MUST be one of `wins`, `ties`, `loses`, or + `untested`. Scenario judgments are report inputs for dimension-level comparison; + they MUST NOT convert live-baseline-only evidence into real-world suite pass claims. - `evidence`: array of evidence pointers with `kind`, `ref`, and `status`. - `notes`: optional bounded explanatory strings. - `follow_up`: optional `title` and `reason`. From 3230f2f6fc33891099cff8e981544e3c462ee4da Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 12:47:01 +0800 Subject: [PATCH 2/6] {"schema":"decodex/commit/1","summary":"Refresh first-generation OSS adapter evidence","authority":"XY-898"} --- .../memory_projects_manifest.json | 24 +++++++++---------- ...generation-oss-adapter-promotion-report.md | 2 +- ...irst-generation-oss-adapter-promotion.json | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index b1e3347b..b221969e 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -503,7 +503,7 @@ "result": { "status": "lifecycle_fail", "evidence": "agentmemory remains a reference for capture and continuity UX, but current Docker evidence is not a durable lifecycle pass.", - "artifact": "docs/guide/benchmarking/live_baseline_benchmark.md" + "artifact": "tmp/live-baseline/live-baseline-report.json" }, "capabilities": [ { @@ -550,7 +550,7 @@ "suite_id": "retrieval", "status": "pass", "elf_position": "untested", - "evidence": "Fresh scoped baseline run live-baseline-20260611015125 reports agentmemory retrieval_pass with 3/3 same-corpus retrieval checks through mem::remember and mem::search. This is live-baseline-only evidence through an in-memory mock, not a real_world_job suite pass.", + "evidence": "Fresh scoped baseline run live-baseline-20260611043440 reports agentmemory retrieval_pass with 3/3 same-corpus retrieval checks through mem::remember and mem::search. This is live-baseline-only evidence through an in-memory mock, not a real_world_job suite pass.", "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, @@ -608,13 +608,13 @@ }, "run": { "status": "pass", - "evidence": "Fresh scoped baseline run live-baseline-20260611015125 exercises local OSS mem0 with FastEmbed, Qdrant path storage, Memory.update, Memory.delete, and cold-start reload; mem0 passed 4/4 encoded checks.", + "evidence": "Fresh scoped baseline run live-baseline-20260611043440 exercises local OSS mem0 with FastEmbed, Qdrant path storage, Memory.update, Memory.delete, and cold-start reload; mem0 passed 4/4 encoded checks.", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "result": { "status": "pass", "evidence": "The local OSS mem0 baseline now passes basic same-corpus/update/delete/reload smoke. No real_world_job mem0/OpenMemory adapter, OpenMemory UI, hosted Platform, entity-history, or graph-memory behavior is encoded.", - "artifact": "docs/guide/research/comparison_external_projects.md" + "artifact": "tmp/live-baseline/live-baseline-report.json" }, "capabilities": [ { @@ -625,7 +625,7 @@ { "capability": "same_corpus_retrieval", "status": "pass", - "evidence": "Fresh scoped baseline run live-baseline-20260611015125 reports mem0 retrieval_pass with 3/3 same-corpus retrieval checks." + "evidence": "Fresh scoped baseline run live-baseline-20260611043440 reports mem0 retrieval_pass with 3/3 same-corpus retrieval checks." }, { "capability": "local_lifecycle_update_delete_reload", @@ -671,7 +671,7 @@ "suite_id": "memory_evolution", "status": "pass", "elf_position": "ties", - "evidence": "The June 11 ELF+mem0 baseline passed 12/12 combined checks, and fresh scoped run live-baseline-20260611015125 confirms mem0 passed same-corpus retrieval, update, delete, and cold-start reload. This is a basic local lifecycle tie at the encoded smoke surface, not a claim about OpenMemory UI, hosted behavior, entity history, or graph memory.", + "evidence": "The June 11 ELF+mem0 baseline passed 12/12 combined checks, and fresh scoped run live-baseline-20260611043440 confirms mem0 passed same-corpus retrieval, update, delete, and cold-start reload. This is a basic local lifecycle tie at the encoded smoke surface, not a claim about OpenMemory UI, hosted behavior, entity history, or graph memory.", "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, @@ -719,13 +719,13 @@ }, "run": { "status": "pass", - "evidence": "Fresh scoped baseline run live-baseline-20260611015125 indexes a per-adapter corpus copy, rewrites and deletes files, reruns memsearch index, and reports 4/4 encoded checks passing.", + "evidence": "Fresh scoped baseline run live-baseline-20260611043440 indexes a per-adapter corpus copy, rewrites and deletes files, reruns memsearch index, and reports 4/4 encoded checks passing.", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "result": { "status": "pass", "evidence": "memsearch now passes the local same-corpus/reindex/update/delete/reload smoke. No real_world_job memsearch prompt adapter is encoded, so Markdown-first behavior remains baseline scenario evidence rather than suite pass evidence.", - "artifact": "docs/guide/research/comparison_external_projects.md" + "artifact": "tmp/live-baseline/live-baseline-report.json" }, "capabilities": [ { @@ -736,7 +736,7 @@ { "capability": "same_corpus_retrieval", "status": "pass", - "evidence": "Fresh scoped baseline run live-baseline-20260611015125 reports memsearch retrieval_pass with 3/3 same-corpus retrieval checks." + "evidence": "Fresh scoped baseline run live-baseline-20260611043440 reports memsearch retrieval_pass with 3/3 same-corpus retrieval checks." }, { "capability": "reindex_update_delete_reload", @@ -772,7 +772,7 @@ "suite_id": "trust_source_of_truth", "status": "pass", "elf_position": "ties", - "evidence": "Fresh scoped baseline run live-baseline-20260611015125 reports memsearch passed same-corpus retrieval, update reindex, delete suppression, and cold-start reload over a canonical Markdown corpus. This is a local source-of-truth/reindex smoke tie at the scenario level, not a real_world_job suite pass.", + "evidence": "Fresh scoped baseline run live-baseline-20260611043440 reports memsearch passed same-corpus retrieval, update reindex, delete suppression, and cold-start reload over a canonical Markdown corpus. This is a local source-of-truth/reindex smoke tie at the scenario level, not a real_world_job suite pass.", "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, @@ -926,7 +926,7 @@ "result": { "status": "wrong_result", "evidence": "No real_world_job claude-mem adapter is encoded; progressive disclosure remains a design reference.", - "artifact": "docs/guide/research/comparison_external_projects.md" + "artifact": "tmp/live-baseline/live-baseline-report.json" }, "capabilities": [ { @@ -978,7 +978,7 @@ "suite_id": "retrieval", "status": "wrong_result", "elf_position": "wins", - "evidence": "Fresh scoped baseline run live-baseline-20260611015125 reports claude-mem same_corpus_retrieval as wrong_result with 0/3 expected query checks passing, while its durable repository setup completed. ELF has encoded local same-corpus retrieval passes, so this is an ELF baseline win for the narrow retrieval smoke scenario.", + "evidence": "Fresh scoped baseline run live-baseline-20260611043440 reports claude-mem same_corpus_retrieval as wrong_result with 0/3 expected query checks passing, while its durable repository setup completed. ELF has encoded local same-corpus retrieval passes, so this is an ELF baseline win for the narrow retrieval smoke scenario.", "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, diff --git a/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md b/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md index 4e4e72b6..3953d995 100644 --- a/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md +++ b/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md @@ -29,7 +29,7 @@ suite passes: | Command | Result | Runtime | Artifact | | --- | --- | ---: | --- | -| `ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker` | fail with typed non-pass projects | 244.42 seconds | `tmp/live-baseline/live-baseline-report.json` | +| `ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker` | fail with typed non-pass projects | 237.29 seconds | `tmp/live-baseline/live-baseline-report.json` | The aggregate failed because two projects remained typed non-pass, not because setup collapsed: diff --git a/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json b/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json index d28d07d8..770f7b5f 100644 --- a/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json +++ b/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json @@ -15,9 +15,9 @@ ], "fresh_run": { "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", - "run_id": "live-baseline-20260611015125", + "run_id": "live-baseline-20260611043440", "status": "fail", - "runtime_seconds": 244.42, + "runtime_seconds": 237.29, "artifact": "tmp/live-baseline/live-baseline-report.json", "summary": { "total": 4, From 2137d52484a455cd9cf0596faf90992f4bf65f84 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 13:09:04 +0800 Subject: [PATCH 3/6] {"schema":"decodex/commit/1","summary":"Align adapter scenario evidence boundaries","authority":"XY-898"} --- .../memory_projects_manifest.json | 40 ++++----- .../tests/real_world_job_benchmark.rs | 86 +++++++++++++++++-- ...generation-oss-adapter-promotion-report.md | 11 +-- ...irst-generation-oss-adapter-promotion.json | 34 +++++--- 4 files changed, 129 insertions(+), 42 deletions(-) diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index b221969e..1593ec35 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -550,8 +550,8 @@ "suite_id": "retrieval", "status": "pass", "elf_position": "untested", - "evidence": "Fresh scoped baseline run live-baseline-20260611043440 reports agentmemory retrieval_pass with 3/3 same-corpus retrieval checks through mem::remember and mem::search. This is live-baseline-only evidence through an in-memory mock, not a real_world_job suite pass.", - "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports agentmemory retrieval_pass with 3/3 same-corpus retrieval checks through mem::remember and mem::search. This is live-baseline-only evidence through an in-memory mock, not a real_world_job suite pass.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, { @@ -559,8 +559,8 @@ "suite_id": "memory_evolution", "status": "lifecycle_fail", "elf_position": "wins", - "evidence": "The same run reports agentmemory update_replaces_note_text as lifecycle_fail and cold_start_recovery_search as blocked because the harness uses an in-memory SDK/KV mock. ELF has broader encoded local lifecycle coverage in the June 11 ELF+mem0 baseline, so this scenario is an ELF baseline win only at the local lifecycle-smoke evidence class.", - "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports ELF passing 8/8 local lifecycle checks, while agentmemory update_replaces_note_text is lifecycle_fail and cold_start_recovery_search is blocked because the harness uses an in-memory SDK/KV mock. This is an ELF baseline win only at the local lifecycle-smoke evidence class.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, { @@ -608,7 +608,7 @@ }, "run": { "status": "pass", - "evidence": "Fresh scoped baseline run live-baseline-20260611043440 exercises local OSS mem0 with FastEmbed, Qdrant path storage, Memory.update, Memory.delete, and cold-start reload; mem0 passed 4/4 encoded checks.", + "evidence": "Fresh comparable baseline run live-baseline-20260611045504 exercises local OSS mem0 with FastEmbed, Qdrant path storage, Memory.update, Memory.delete, and cold-start reload; mem0 passed 4/4 encoded checks.", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "result": { @@ -625,7 +625,7 @@ { "capability": "same_corpus_retrieval", "status": "pass", - "evidence": "Fresh scoped baseline run live-baseline-20260611043440 reports mem0 retrieval_pass with 3/3 same-corpus retrieval checks." + "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports mem0 retrieval_pass with 3/3 same-corpus retrieval checks." }, { "capability": "local_lifecycle_update_delete_reload", @@ -671,8 +671,8 @@ "suite_id": "memory_evolution", "status": "pass", "elf_position": "ties", - "evidence": "The June 11 ELF+mem0 baseline passed 12/12 combined checks, and fresh scoped run live-baseline-20260611043440 confirms mem0 passed same-corpus retrieval, update, delete, and cold-start reload. This is a basic local lifecycle tie at the encoded smoke surface, not a claim about OpenMemory UI, hosted behavior, entity history, or graph memory.", - "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports ELF passing 8/8 local lifecycle checks and mem0 passing 4/4 same-corpus retrieval, update, delete, and cold-start reload checks. This is a basic local lifecycle tie at the encoded smoke surface, not a claim about OpenMemory UI, hosted behavior, entity history, or graph memory.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, { @@ -719,7 +719,7 @@ }, "run": { "status": "pass", - "evidence": "Fresh scoped baseline run live-baseline-20260611043440 indexes a per-adapter corpus copy, rewrites and deletes files, reruns memsearch index, and reports 4/4 encoded checks passing.", + "evidence": "Fresh comparable baseline run live-baseline-20260611045504 indexes a per-adapter corpus copy, rewrites and deletes files, reruns memsearch index, and reports memsearch 4/4 encoded checks passing.", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "result": { @@ -736,7 +736,7 @@ { "capability": "same_corpus_retrieval", "status": "pass", - "evidence": "Fresh scoped baseline run live-baseline-20260611043440 reports memsearch retrieval_pass with 3/3 same-corpus retrieval checks." + "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports memsearch retrieval_pass with 3/3 same-corpus retrieval checks." }, { "capability": "reindex_update_delete_reload", @@ -771,17 +771,17 @@ "scenario_id": "canonical_markdown_reindex_reload", "suite_id": "trust_source_of_truth", "status": "pass", - "elf_position": "ties", - "evidence": "Fresh scoped baseline run live-baseline-20260611043440 reports memsearch passed same-corpus retrieval, update reindex, delete suppression, and cold-start reload over a canonical Markdown corpus. This is a local source-of-truth/reindex smoke tie at the scenario level, not a real_world_job suite pass.", - "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "elf_position": "untested", + "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports memsearch passed same-corpus retrieval, update reindex, delete suppression, and cold-start reload over a canonical Markdown corpus. ELF has no directly comparable canonical Markdown source-store scenario in this baseline, so the ELF position remains untested.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, { "scenario_id": "ttl_expiry_lifecycle", "suite_id": "memory_evolution", "status": "unsupported", - "elf_position": "wins", - "evidence": "The encoded memsearch CLI path supports reindex/delete but no TTL or expiry behavior. ELF has encoded TTL/tombstone fixture and live delete/TTL evidence, so ELF wins this specific lifecycle-smoke scenario while broader memsearch real_world jobs remain untested.", + "elf_position": "untested", + "evidence": "The encoded memsearch CLI path supports reindex/delete but no TTL or expiry behavior. Unsupported TTL behavior is preserved as unsupported competitor evidence and does not create an ELF win/loss claim without a directly comparable scenario artifact.", "artifact": "tmp/live-baseline/live-baseline-report.json" }, { @@ -978,8 +978,8 @@ "suite_id": "retrieval", "status": "wrong_result", "elf_position": "wins", - "evidence": "Fresh scoped baseline run live-baseline-20260611043440 reports claude-mem same_corpus_retrieval as wrong_result with 0/3 expected query checks passing, while its durable repository setup completed. ELF has encoded local same-corpus retrieval passes, so this is an ELF baseline win for the narrow retrieval smoke scenario.", - "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports ELF retrieval_pass and claude-mem same_corpus_retrieval as wrong_result with 0/3 expected query checks passing, while its durable repository setup completed. This is an ELF baseline win for the narrow retrieval smoke scenario.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, { @@ -987,8 +987,8 @@ "suite_id": "memory_evolution", "status": "pass", "elf_position": "ties", - "evidence": "The same run reports claude-mem update, delete, and cold-start reload checks passing over a durable Docker-local SQLite repository. This is a local lifecycle-smoke tie, not a hook-driven work-resume or full progressive-disclosure job pass.", - "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports ELF passing local lifecycle checks and claude-mem update, delete, and cold-start reload checks passing over a durable Docker-local SQLite repository. This is a local lifecycle-smoke tie, not a hook-driven work-resume or full progressive-disclosure job pass.", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, { @@ -997,7 +997,7 @@ "status": "pass", "elf_position": "untested", "evidence": "claude-mem passed the repository-level search-to-detail/source hydration check, which is a useful progressive-disclosure signal. ELF does not have a directly comparable claude-mem-style progressive-disclosure scenario in this baseline, so the ELF position remains untested rather than a loss claim.", - "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, { diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 38ad3066..8501640b 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -288,6 +288,48 @@ fn assert_external_adapter_manifest_summary(report: &Value) { } fn assert_external_adapter_manifest_scenario_summary(report: &Value) { + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/real") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/mocked") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/unsupported") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/blocked") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/incomplete") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/wrong_result") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_status_counts/lifecycle_fail") + .and_then(Value::as_u64), + Some(1) + ); assert_eq!( report .pointer("/external_adapters/summary/scenario_status_counts/pass") @@ -304,13 +346,25 @@ fn assert_external_adapter_manifest_scenario_summary(report: &Value) { report .pointer("/external_adapters/summary/scenario_position_counts/wins") .and_then(Value::as_u64), - Some(3) + Some(2) ); assert_eq!( report .pointer("/external_adapters/summary/scenario_position_counts/ties") .and_then(Value::as_u64), - Some(3) + Some(2) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/loses") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/untested") + .and_then(Value::as_u64), + Some(9) ); } @@ -357,7 +411,7 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { Some("mocked") ); - assert_first_generation_adapter_records(mem0, memsearch, claude_mem); + assert_first_generation_adapter_records(agentmemory, mem0, memsearch, claude_mem); assert_eq!(openviking.pointer("/overall_status").and_then(Value::as_str), Some("wrong_result")); assert_eq!(ragflow.pointer("/evidence_class").and_then(Value::as_str), Some("research_gate")); @@ -412,7 +466,21 @@ fn assert_external_adapter_manifest_records(report: &Value) -> Result<()> { Ok(()) } -fn assert_first_generation_adapter_records(mem0: &Value, memsearch: &Value, claude_mem: &Value) { +fn assert_first_generation_adapter_records( + agentmemory: &Value, + mem0: &Value, + memsearch: &Value, + claude_mem: &Value, +) { + assert_eq!( + agentmemory.pointer("/scenarios/1/status").and_then(Value::as_str), + Some("lifecycle_fail") + ); + assert_eq!( + agentmemory.pointer("/scenarios/1/elf_position").and_then(Value::as_str), + Some("wins") + ); + assert_eq!(agentmemory.pointer("/scenarios/2/status").and_then(Value::as_str), Some("blocked")); assert_eq!( mem0.pointer("/capabilities/2/capability").and_then(Value::as_str), Some("local_lifecycle_update_delete_reload") @@ -435,13 +503,17 @@ fn assert_first_generation_adapter_records(mem0: &Value, memsearch: &Value, clau memsearch.pointer("/scenarios/0/scenario_id").and_then(Value::as_str), Some("canonical_markdown_reindex_reload") ); + assert_eq!( + memsearch.pointer("/scenarios/0/elf_position").and_then(Value::as_str), + Some("untested") + ); assert_eq!( memsearch.pointer("/scenarios/1/status").and_then(Value::as_str), Some("unsupported") ); assert_eq!( memsearch.pointer("/scenarios/1/elf_position").and_then(Value::as_str), - Some("wins") + Some("untested") ); assert_eq!(claude_mem.pointer("/capabilities/1/status").and_then(Value::as_str), Some("real")); assert_eq!( @@ -852,6 +924,10 @@ fn generated_json_report_renders_markdown() -> Result<()> { assert!(markdown.contains("agentmemory-style hook capture")); assert!(markdown.contains("xy844-current-worktree")); assert!(markdown.contains("Existing live-baseline reports remain valid")); + assert!(markdown.contains("### Adapter Scenario Judgments")); + assert!(markdown.contains("ELF scenario positions: `wins=2, ties=2, untested=9`")); + assert!(markdown.contains("| `claude_mem_live_baseline` | `same_corpus_retrieval`")); + assert!(markdown.contains("| `memsearch_live_baseline` | `ttl_expiry_lifecycle`")); Ok(()) } diff --git a/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md b/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md index 3953d995..47197b66 100644 --- a/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md +++ b/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md @@ -29,13 +29,14 @@ suite passes: | Command | Result | Runtime | Artifact | | --- | --- | ---: | --- | -| `ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker` | fail with typed non-pass projects | 237.29 seconds | `tmp/live-baseline/live-baseline-report.json` | +| `ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker` | fail with typed non-pass projects | 233.69 seconds | `tmp/live-baseline/live-baseline-report.json` | The aggregate failed because two projects remained typed non-pass, not because setup collapsed: | Project | Status | Retrieval | Checks | Scenario meaning | | --- | --- | --- | ---: | --- | +| ELF | `pass` | `retrieval_pass` | `8/8` pass | Baseline reference for same-class scenario comparisons; no ELF optimization change was made. | | agentmemory | `lifecycle_fail` | `retrieval_pass` | `2/4` pass, `1` lifecycle_fail, `1` blocked | Same-corpus retrieval runs, but update supersession and durable cold-start are not proven through the in-memory mock. | | mem0/OpenMemory | `pass` | `retrieval_pass` | `4/4` pass | Basic local OSS same-corpus, update, delete, and cold-start smoke passes. | | memsearch | `pass` | `retrieval_pass` | `4/4` pass | Canonical Markdown reindex/update/delete/reload smoke passes. | @@ -51,8 +52,8 @@ collapsed: | mem0/OpenMemory | basic local lifecycle | `pass` | `ties` | ELF and mem0 both pass the encoded local lifecycle smoke; mem0 is no longer a basic-smoke failure. | | mem0/OpenMemory | preference/entity history | `not_encoded` | `untested` | History, correction chains, entity scope, and deletion audit are not scored. | | mem0/OpenMemory | OpenMemory UI/export readback | `not_encoded` | `untested` | Local OSS UI/export readback is not executed; hosted behavior remains out of scope. | -| memsearch | canonical Markdown reindex/reload | `pass` | `ties` | Baseline reindex/update/delete/reload passes over the canonical file store. | -| memsearch | TTL/expiry lifecycle | `unsupported` | `wins` | The encoded CLI path has reindex/delete but no TTL/expiry behavior. | +| memsearch | canonical Markdown reindex/reload | `pass` | `untested` | Baseline reindex/update/delete/reload passes over the canonical file store; ELF has no directly comparable canonical Markdown source-store scenario in this run. | +| memsearch | TTL/expiry lifecycle | `unsupported` | `untested` | The encoded CLI path has reindex/delete but no TTL/expiry behavior; unsupported competitor evidence does not create an ELF win/loss without a comparable scenario artifact. | | memsearch | real-world prompt adapter | `not_encoded` | `untested` | No memsearch real_world_job prompt adapter is encoded. | | claude-mem | same-corpus retrieval | `wrong_result` | `wins` | The durable repository path runs but misses expected retrieval evidence. | | claude-mem | repository lifecycle reload | `pass` | `ties` | Update, delete, and cold-start reload pass over Docker-local SQLite. | @@ -60,8 +61,8 @@ collapsed: | claude-mem | hook capture viewer workflow | `not_encoded` | `untested` | Hooks, viewer, timeline, and observations are not executed. | Summary: 13 scenario judgments: 5 `pass`, 1 `wrong_result`, 1 `lifecycle_fail`, -1 `blocked`, 1 `unsupported`, and 4 `not_encoded`. ELF positions are 3 `wins`, -3 `ties`, 0 `loses`, and 7 `untested`. +1 `blocked`, 1 `unsupported`, and 4 `not_encoded`. ELF positions are 2 `wins`, +2 `ties`, 0 `loses`, and 9 `untested`. ## Manifest And Report Changes diff --git a/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json b/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json index 770f7b5f..4e9132c9 100644 --- a/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json +++ b/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json @@ -14,14 +14,14 @@ "tmp/live-baseline/live-baseline-report.json" ], "fresh_run": { - "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", - "run_id": "live-baseline-20260611043440", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "run_id": "live-baseline-20260611045504", "status": "fail", - "runtime_seconds": 237.29, + "runtime_seconds": 233.69, "artifact": "tmp/live-baseline/live-baseline-report.json", "summary": { - "total": 4, - "pass": 2, + "total": 5, + "pass": 3, "wrong_result": 1, "lifecycle_fail": 1, "incomplete": 0, @@ -29,8 +29,8 @@ "not_encoded": 0 }, "full_check_summary": { - "total": 17, - "pass": 14, + "total": 25, + "pass": 22, "wrong_result": 1, "lifecycle_fail": 1, "blocked": 1, @@ -39,6 +39,16 @@ } }, "projects": [ + { + "project": "ELF", + "status": "pass", + "retrieval_status": "retrieval_pass", + "check_summary": { + "total": 8, + "pass": 8 + }, + "evidence": "ELF passed 8/8 encoded local lifecycle and retrieval checks in the comparable baseline run and serves only as the reference row for same-class scenario positions." + }, { "project": "agentmemory", "status": "lifecycle_fail", @@ -94,10 +104,10 @@ "not_encoded": 4 }, "elf_position_counts": { - "wins": 3, - "ties": 3, + "wins": 2, + "ties": 2, "loses": 0, - "untested": 7 + "untested": 9 } }, "scenario_judgments": [ @@ -141,13 +151,13 @@ "project": "memsearch", "scenario_id": "canonical_markdown_reindex_reload", "status": "pass", - "elf_position": "ties" + "elf_position": "untested" }, { "project": "memsearch", "scenario_id": "ttl_expiry_lifecycle", "status": "unsupported", - "elf_position": "wins" + "elf_position": "untested" }, { "project": "memsearch", From a9a0a3a56763828e70ade4b0052903fa97fe407d Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 13:30:03 +0800 Subject: [PATCH 4/6] {"schema":"decodex/commit/1","summary":"Repair adapter report consistency","authority":"XY-898"} --- .../memory_projects_manifest.json | 20 +++++++------- .../src/bin/real_world_job_benchmark.rs | 8 +++++- .../tests/real_world_job_benchmark.rs | 12 ++++++--- ...-11-competitor-strength-evidence-matrix.md | 11 ++++---- .../2026-06-11-measurement-coverage-audit.md | 27 +++++++++---------- ...2026-06-11-measurement-coverage-audit.json | 20 +++++++------- ...-11-xy-897-competitor-strength-matrix.json | 4 +-- 7 files changed, 57 insertions(+), 45 deletions(-) diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index 1593ec35..8cec6b21 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -146,13 +146,13 @@ }, "run": { "status": "wrong_result", - "evidence": "ELF materializes 38 real_world_job adapter_response objects through ElfService, worker indexing, and search_raw before scoring; the full sweep includes typed wrong_result, incomplete, blocked, and not_encoded records.", + "evidence": "ELF materializes 38 real_world_job adapter_response objects through ElfService, worker indexing, and search_raw before scoring; the full sweep includes typed wrong_result, blocked, and not_encoded job records.", "command": "cargo make real-world-memory-live-adapters", "artifact": "tmp/real-world-memory/live-adapters/elf-report.json" }, "result": { "status": "wrong_result", - "evidence": "The full live sweep scores 38 jobs across all 11 encoded suites: 18 pass, 5 wrong_result, 1 incomplete, 2 blocked, and 12 not_encoded. This is not a full-suite live pass.", + "evidence": "The fresh full live sweep scores 38 jobs across all 11 encoded suites: 18 pass, 5 wrong_result, 0 incomplete, 2 blocked, and 13 not_encoded. This is not a full-suite live pass.", "command": "cargo make real-world-memory-live-adapters", "artifact": "tmp/real-world-memory/live-adapters/elf-report.md" }, @@ -180,7 +180,7 @@ { "capability": "full_suite_live_pass", "status": "wrong_result", - "evidence": "No full-suite live pass is claimed; generated reports preserve wrong_result, incomplete, blocked, and not_encoded outcomes." + "evidence": "No full-suite live pass is claimed; generated reports preserve wrong_result, blocked, and not_encoded job outcomes." }, { "capability": "typed_failure_reporting", @@ -236,8 +236,8 @@ }, { "suite_id": "production_ops", - "status": "incomplete", - "evidence": "The live adapter sweep does not run backup/restore, private corpus, provider credential, or backfill operations; existing production-ops credential and private-manifest boundaries remain blocked and the cold-start dependency fixture remains incomplete." + "status": "blocked", + "evidence": "The live adapter sweep does not run backup/restore, private corpus, provider credential, or backfill operations; existing production-ops credential and private-manifest boundaries remain blocked." }, { "suite_id": "personalization", @@ -359,13 +359,13 @@ }, "run": { "status": "wrong_result", - "evidence": "qmd materializes 38 real_world_job adapter_response objects through collection add, update, embed, and query --json before scoring; the full sweep includes typed wrong_result, incomplete, blocked, and not_encoded records.", + "evidence": "qmd materializes 38 real_world_job adapter_response objects through collection add, update, embed, and query --json before scoring; the full sweep includes typed wrong_result, blocked, and not_encoded job records.", "command": "cargo make real-world-memory-live-adapters", "artifact": "tmp/real-world-memory/live-adapters/qmd-report.json" }, "result": { "status": "wrong_result", - "evidence": "The full qmd live sweep scores 38 jobs across all 11 encoded suites: 18 pass, 5 wrong_result, 1 incomplete, 2 blocked, and 12 not_encoded. This is not a full-suite live pass.", + "evidence": "The fresh full qmd live sweep scores 38 jobs across all 11 encoded suites: 17 pass, 6 wrong_result, 0 incomplete, 2 blocked, and 13 not_encoded. This is not a full-suite live pass.", "command": "cargo make real-world-memory-live-adapters", "artifact": "tmp/real-world-memory/live-adapters/qmd-report.md" }, @@ -393,7 +393,7 @@ { "capability": "full_suite_live_pass", "status": "wrong_result", - "evidence": "No full-suite live pass is claimed; generated reports preserve wrong_result, incomplete, blocked, and not_encoded outcomes." + "evidence": "No full-suite live pass is claimed; generated reports preserve wrong_result, blocked, and not_encoded job outcomes." }, { "capability": "typed_failure_reporting", @@ -449,8 +449,8 @@ }, { "suite_id": "production_ops", - "status": "incomplete", - "evidence": "The qmd live adapter sweep does not run backup/restore, private corpus, provider credential, or backfill operations; existing production-ops credential and private-manifest boundaries remain blocked and the cold-start dependency fixture remains incomplete." + "status": "blocked", + "evidence": "The qmd live adapter sweep does not run backup/restore, private corpus, provider credential, or backfill operations; existing production-ops credential and private-manifest boundaries remain blocked." }, { "suite_id": "personalization", diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark.rs b/apps/elf-eval/src/bin/real_world_job_benchmark.rs index 2c988134..890d3468 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark.rs @@ -3950,9 +3950,15 @@ fn validate_adapter_execution_metadata(path: &Path, adapter: &ExternalAdapterRep } fn external_adapter_summary(adapters: &[ExternalAdapterReport]) -> ExternalAdapterSummary { + let external_project_count = adapters + .iter() + .filter(|adapter| adapter.project != "ELF") + .map(|adapter| adapter.project.as_str()) + .collect::>() + .len(); let mut summary = ExternalAdapterSummary { adapter_count: adapters.len(), - external_project_count: adapters.iter().filter(|adapter| adapter.project != "ELF").count(), + external_project_count, ..ExternalAdapterSummary::default() }; diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 8501640b..7ff86cf9 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -209,7 +209,7 @@ fn assert_external_adapter_manifest_summary(report: &Value) { ); assert_eq!( report.pointer("/external_adapters/summary/external_project_count").and_then(Value::as_u64), - Some(19) + Some(16) ); assert_eq!( report.pointer("/external_adapters/summary/fixture_backed_count").and_then(Value::as_u64), @@ -281,7 +281,13 @@ fn assert_external_adapter_manifest_summary(report: &Value) { report .pointer("/external_adapters/summary/suite_status_counts/blocked") .and_then(Value::as_u64), - Some(11) + Some(13) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/suite_status_counts/incomplete") + .and_then(Value::as_u64), + Some(0) ); assert_external_adapter_manifest_scenario_summary(report); @@ -593,7 +599,7 @@ fn assert_live_sweep_record(adapter: &Value) -> Result<()> { assert_eq!(full_pass.pointer("/status").and_then(Value::as_str), Some("wrong_result")); assert_eq!(work_resume.pointer("/status").and_then(Value::as_str), Some("pass")); assert_eq!(memory_evolution.pointer("/status").and_then(Value::as_str), Some("wrong_result")); - assert_eq!(production_ops.pointer("/status").and_then(Value::as_str), Some("incomplete")); + assert_eq!(production_ops.pointer("/status").and_then(Value::as_str), Some("blocked")); assert_eq!(consolidation.pointer("/status").and_then(Value::as_str), Some("not_encoded")); Ok(()) diff --git a/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md b/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md index 4f773094..7fd4a3de 100644 --- a/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md +++ b/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md @@ -26,8 +26,9 @@ is encoded and run at a comparable evidence class. Current boundary: - ELF and qmd have full-suite `live_real_world` sweeps, but neither has a full-suite - live pass. Each sweep produced 38 jobs with 18 pass, 5 wrong_result, 1 incomplete, - 2 blocked, and 12 not_encoded. + live pass. The fresh ELF sweep produced 38 jobs with 18 pass, 5 wrong_result, + 0 incomplete, 2 blocked, and 13 not_encoded; the fresh qmd sweep produced 17 pass, + 6 wrong_result, 0 incomplete, 2 blocked, and 13 not_encoded. - ELF fixture evidence is strong: `cargo make real-world-memory` reports 38 jobs across 11 suites with 36 pass and 2 blocked production-ops operator boundaries. That proves the fixture contract, not live-service parity. @@ -43,7 +44,7 @@ Current boundary: The current manifest has 21 adapter records across 17 projects. Evidence-class counts: 1 `fixture_backed`, 6 `live_baseline_only`, 2 `live_real_world`, and 12 -`research_gate`. Overall adapter-status counts: 1 `pass`, 6 `wrong_result`, 1 +`research_gate`. Overall adapter-status counts: 3 `pass`, 4 `wrong_result`, 1 `lifecycle_fail`, 6 `blocked`, and 7 `not_encoded`. ## State Taxonomy @@ -71,7 +72,7 @@ lifecycle-fail -> `lifecycle_fail`, and not-encoded -> `not_encoded`. | Project | Strongest user-facing scenario | Current evidence | Measured status and proof | Unsupported or blocked status | Required benchmark before ELF claim | Borrow if stronger | | --- | --- | --- | --- | --- | --- | --- | -| ELF | Evidence-linked source-of-truth memory service with real-world fixtures and live retrieval sweeps. | `live_real_world`; supporting `fixture_backed`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/elf-report.md`. Fixture contract: `cargo make real-world-memory`, `tmp/real-world-memory/real-world-memory-report.json`. | `blocked`: private manifest and provider credentials; broader live suites remain `wrong_result`, `incomplete`, or `not_encoded`. | Full-suite live pass plus separate private-corpus and credentialed production-ops proof. | Keep borrowing qmd debug knobs, OpenViking staged trajectory, mem0 history, Letta core memory, and graph/RAG navigation. | +| ELF | Evidence-linked source-of-truth memory service with real-world fixtures and live retrieval sweeps. | `live_real_world`; supporting `fixture_backed`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/elf-report.md`. Fixture contract: `cargo make real-world-memory`, `tmp/real-world-memory/real-world-memory-report.json`. | `blocked`: private manifest and provider credentials; broader live suites remain `wrong_result`, `blocked`, or `not_encoded`. | Full-suite live pass plus separate private-corpus and credentialed production-ops proof. | Keep borrowing qmd debug knobs, OpenViking staged trajectory, mem0 history, Letta core memory, and graph/RAG navigation. | | qmd | Local retrieval-debug workflow with transparent CLI indexing, querying, expansion, fusion, and rerank ergonomics. | `live_real_world`; supporting `live_baseline_only` and `research_gate`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/qmd-report.md`; targeted retrieval suites pass. | `not_encoded`: deep profile and non-retrieval live behavior are not encoded; memory_evolution is `wrong_result`. | qmd deep retrieval/debug profile plus full-suite live replay with trace-level diagnostics. | Weighted fusion, rerank explanation, local debug knobs, and command-line replay. | | agentmemory | Coding-agent continuity, MCP/REST packaging, viewer workflow, and durable cross-agent memory lifecycle. | `live_baseline_only`. | `lifecycle_fail`: `ELF_BASELINE_PROJECTS=agentmemory cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `blocked`: durable cold-start and real-world adapter coverage are missing. | Durable local adapter with update, delete, cold-start reload, work_resume, capture/write-policy, and lifecycle-staleness jobs. | Cross-agent hooks, packaging, continuity scenarios, and viewer affordances. | | mem0/OpenMemory | Memory lifecycle, personalization, hosted/OpenMemory UI ergonomics, and optional graph memory. | `live_baseline_only`. | `pass`: fresh scoped run `ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`, with mem0 `4/4` local checks passing. | `not_encoded`: OpenMemory UI, hosted claims, entity/preference history, graph memory, and real-world personalization coverage are not encoded. | Encode memory_evolution preference/entity history, deletion audit readback, personalization, UI/export readback, and optional graph-context jobs. | Entity-scoped history, lifecycle surfaces, async update ergonomics, and OpenMemory inspection UX. | @@ -102,7 +103,7 @@ lifecycle-fail -> `lifecycle_fail`, and not-encoded -> `not_encoded`. | Knowledge pages | Fixture knowledge_compilation passes; live knowledge_compilation is `not_encoded`. | llm-wiki, gbrain, GraphRAG, graphify. | llm-wiki and gbrain are `research_gate` `not_encoded` or `blocked`; GraphRAG and graphify are `blocked`. | Encode live derived-page rebuild/lint scoring and run contained knowledge/RAG adapters only after setup proof. | | Operator debugging | Fixture operator_debugging_ux passes; live operator_debugging_ux is `not_encoded`. | qmd, claude-mem, OpenMemory. | qmd has debug strengths but operator_debugging_ux is `not_encoded`; claude-mem and OpenMemory UX are `not_encoded`. | Score trace hydration, stage attribution, raw-SQL avoidance, and repair-action clarity through live artifacts. | | Capture/write policy | Fixture capture_integration passes; live capture_integration is `not_encoded`. | agentmemory, claude-mem. | agentmemory capture is `blocked`; claude-mem capture is `not_encoded`. | Run live capture/write-policy jobs proving redaction, exclusion, evidence binding, and no secret leakage. | -| Production ops | Fixture production_ops has 4 pass and 2 blocked; live production_ops is `incomplete`; production adoption has provider/backfill/restore evidence. | ELF production gate, qmd, RAG/RAGFlow resource gates. | qmd live production_ops is `incomplete`; RAG/resource gates are `research_gate` `blocked`. | Rerun private-corpus and credentialed gates only when operator-owned manifest and credentials exist. | +| Production ops | Fixture production_ops has 4 pass and 2 blocked; live production_ops is `blocked`; production adoption has provider/backfill/restore evidence. | ELF production gate, qmd, RAG/RAGFlow resource gates. | qmd live production_ops is `blocked`; RAG/resource gates are `research_gate` `blocked`. | Rerun private-corpus and credentialed gates only when operator-owned manifest and credentials exist. | | Personalization | Fixture and live personalization pass. | mem0/OpenMemory, Letta. | mem0/OpenMemory and Letta personalization are `not_encoded`. | Encode scoped preference readback for mem0/OpenMemory and Letta before personalization superiority claims. | | Context trajectory | ELF has trace direction but no comparable staged trajectory scenario. | OpenViking. | OpenViking setup is pinned, same-corpus retrieval is `wrong_result`, and hierarchy trajectory is `not_encoded`. | Make OpenViking evidence-bearing retrieval pass, then score staged context trajectory outputs. | | Core-vs-archival memory | ELF core-block semantics exist in the service contract, but comparative benchmark coverage is not encoded here. | Letta. | Letta is `research_gate` `not_encoded` until contained export proof exists. | Add ELF core-block versus archival-search jobs; compare Letta only after contained export proof. | diff --git a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md index f3be7a56..c367fee6 100644 --- a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md +++ b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md @@ -23,9 +23,10 @@ What is proven today: - ELF has a strong fixture-backed real-world benchmark contract: 38 jobs, 36 pass, 2 blocked operator boundaries, and no wrong results in the fixture aggregate. -- ELF and qmd have comparable full-suite live real-world sweeps. They are effectively - tied on pass/fail shape: each has 38 jobs, 18 pass, 5 wrong_result, 2 blocked, and - 13 not_encoded. +- ELF and qmd have comparable full-suite live real-world sweeps. The latest generated + artifacts are close but no longer identical: ELF has 38 jobs with 18 pass, + 5 wrong_result, 2 blocked, and 13 not_encoded, while qmd has 17 pass, + 6 wrong_result, 2 blocked, and 13 not_encoded. - ELF is ahead on production-operation evidence among tracked systems because it has checked-in provider synthetic, stress, backfill, backup/restore, and Qdrant rebuild evidence. @@ -82,8 +83,8 @@ live adapter or competitor runtime can complete those jobs. | Adapter | Jobs | Pass | Wrong result | Blocked | Not encoded | Mean score | Mean latency | Evidence recall | Evidence coverage | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| ELF live service adapter | `38` | `18` | `5` | `2` | `13` | `0.525` | `5.100 ms` | `41/77` | `48/84` | -| qmd live CLI adapter | `38` | `18` | `5` | `2` | `13` | `0.512` | `719.758 ms` | `41/77` | `48/84` | +| ELF live service adapter | `38` | `18` | `5` | `2` | `13` | `0.525` | `6.823 ms` | `41/77` | `48/84` | +| qmd live CLI adapter | `38` | `17` | `6` | `2` | `13` | `0.486` | `819.626 ms` | `38/77` | `45/84` | This supports a narrow tie on the currently encoded live real-world suite shape. It does not support a broad ELF-over-qmd claim because qmd remains the stronger @@ -124,16 +125,15 @@ The checked-in manifest records 21 adapter records across 17 unique project name | Overall status | Adapter records | | --- | ---: | -| `pass` | `1` | -| `wrong_result` | `6` | +| `pass` | `3` | +| `wrong_result` | `4` | | `lifecycle_fail` | `1` | | `blocked` | `6` | | `not_encoded` | `7` | -The generated JSON report also emits `external_project_count: 19`, while the unique -project-name count from the manifest is 17. The runner currently computes that field -as adapter records whose project is not `ELF`, not as unique external project names. -Interpret the unique manifest project list as the project coverage count. +The generated JSON report now emits `external_project_count` as the distinct non-ELF +project-name count. The manifest still has 21 adapter records across 17 unique project +names, of which 16 are external projects. ## Project Coverage @@ -214,9 +214,8 @@ Order these by decision value, not implementation convenience: - Output: Docker-contained artifacts mapped to evidence ids, or typed setup and resource blockers. -Before publishing the next aggregate report, clarify or rename the generated -`external_project_count` field so readers do not confuse non-ELF adapter records with -unique external projects. +Keep the generated `external_project_count` field aligned with unique non-ELF project +names so readers do not confuse adapter records with project coverage. ## Fail Criteria diff --git a/docs/research/2026-06-11-measurement-coverage-audit.json b/docs/research/2026-06-11-measurement-coverage-audit.json index b04d86ef..0fbae859 100644 --- a/docs/research/2026-06-11-measurement-coverage-audit.json +++ b/docs/research/2026-06-11-measurement-coverage-audit.json @@ -45,7 +45,7 @@ "blocked": 2, "not_encoded": 13, "mean_score": 0.525, - "mean_latency_ms": 5.1, + "mean_latency_ms": 6.823, "expected_evidence_total": 77, "expected_evidence_matched": 41, "evidence_required_count": 84, @@ -55,16 +55,16 @@ "adapter": "qmd live CLI adapter", "job_count": 38, "encoded_suite_count": 11, - "pass": 18, - "wrong_result": 5, + "pass": 17, + "wrong_result": 6, "blocked": 2, "not_encoded": 13, - "mean_score": 0.512, - "mean_latency_ms": 719.758, + "mean_score": 0.486, + "mean_latency_ms": 819.626, "expected_evidence_total": 77, - "expected_evidence_matched": 41, + "expected_evidence_matched": 38, "evidence_required_count": 84, - "evidence_covered_count": 48 + "evidence_covered_count": 45 } ], "live_suite_breakdown": [ @@ -83,7 +83,7 @@ "adapter_ledger": { "adapter_records": 21, "unique_project_names": 17, - "external_project_count_note": "The generated report field external_project_count currently counts non-ELF adapter records, not unique external project names.", + "external_project_count_note": "The generated report field external_project_count now reports distinct non-ELF project names; the manifest has 16 external projects and 17 total project names including ELF.", "evidence_class_counts": { "fixture_backed": 1, "live_baseline_only": 6, @@ -91,8 +91,8 @@ "research_gate": 12 }, "overall_status_counts": { - "pass": 1, - "wrong_result": 6, + "pass": 3, + "wrong_result": 4, "lifecycle_fail": 1, "blocked": 6, "not_encoded": 7 diff --git a/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json b/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json index 96d549a1..79367ade 100644 --- a/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json +++ b/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json @@ -509,9 +509,9 @@ { "scenario_id": "production_ops", "scenario": "production ops", - "current_elf_evidence": "ELF production runbooks and fixture production_ops cover restore, Qdrant rebuild, backfill resume, resource envelope, and typed private/credential blockers; live_real_world production_ops is incomplete.", + "current_elf_evidence": "ELF production runbooks and fixture production_ops cover restore, Qdrant rebuild, backfill resume, resource envelope, and typed private/credential blockers; live_real_world production_ops is blocked.", "strongest_competitor_or_reference": "ELF production gate, qmd, RAG/RAGFlow resource gates", - "current_competitor_evidence": "qmd live production_ops is incomplete; RAGFlow/GraphRAG/LightRAG resource gates are research_gate blocked.", + "current_competitor_evidence": "qmd live production_ops is blocked; RAGFlow/GraphRAG/LightRAG resource gates are research_gate blocked.", "current_state": "ELF has the strongest checked-in production evidence, but private corpus and credentialed gates remain blocked.", "next_measurement": "Rerun private-corpus and credentialed production-ops gates only when operator-owned manifest and credentials are supplied." }, From 41526596cf063db832ce7821262c397082cf6986 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 13:50:38 +0800 Subject: [PATCH 5/6] {"schema":"decodex/commit/1","summary":"Repair stale benchmark report prose","authority":"XY-898"} --- .../memory_projects_manifest.json | 4 ++-- ...on-direction-from-competitor-benchmarks.md | 20 ++++++++++--------- .../2026-06-11-measurement-coverage-audit.md | 3 ++- ...2026-06-11-measurement-coverage-audit.json | 2 +- ...-11-xy-897-competitor-strength-matrix.json | 2 +- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index 8cec6b21..9ae20f74 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -264,7 +264,7 @@ ], "notes": [ "This Docker-isolated live real_world_job record now covers the full encoded fixture corpus, not only the original three-suite representative slice.", - "The record is a full-suite sweep, not a full-suite pass; wrong_result, incomplete, blocked, and not_encoded states remain visible.", + "The record is a full-suite sweep, not a full-suite pass; wrong_result, blocked, and not_encoded states remain visible.", "This record does not prove private-corpus production quality or provider-backed production operations." ] }, @@ -477,7 +477,7 @@ ], "notes": [ "This qmd record is real-world job evidence and must not be conflated with the same-corpus qmd_live_baseline record.", - "The record is a full-suite sweep, not a full-suite pass; wrong_result, incomplete, blocked, and not_encoded states remain visible.", + "The record is a full-suite sweep, not a full-suite pass; wrong_result, blocked, and not_encoded states remain visible.", "This record does not prove broad RAG/graph adapter parity or private-corpus production quality." ] }, diff --git a/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md b/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md index e5a6738a..37685049 100644 --- a/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md +++ b/docs/guide/benchmarking/2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md @@ -66,17 +66,18 @@ sweeps for ELF and qmd: | Adapter | Jobs | Pass | Wrong result | Incomplete | Blocked | Not encoded | Mean score | Evidence recall | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| ELF live service adapter | `38` | `18` | `5` | `1` | `2` | `12` | `0.514` | `41/75` | -| qmd live CLI adapter | `38` | `18` | `5` | `1` | `2` | `12` | `0.512` | `41/75` | +| ELF live service adapter | `38` | `18` | `5` | `0` | `2` | `13` | `0.525` | `41/77` | +| qmd live CLI adapter | `38` | `17` | `6` | `0` | `2` | `13` | `0.486` | `38/77` | Interpretation: -- This is a tie for the currently encoded live real-world sweep. +- This is a near tie for the currently encoded live real-world sweep, with ELF one + job ahead in this fresh run. - Both pass `trust_source_of_truth`, `work_resume`, `project_decisions`, `retrieval`, and `personalization`. - Both fail `memory_evolution` live conflict evidence with `wrong_result`. - Both leave consolidation, knowledge compilation, operator debugging, capture - integration, and parts of production operations as `not_encoded` or incomplete. + integration, and production-ops operator boundaries as `not_encoded` or `blocked`. ### Production Evidence @@ -108,8 +109,8 @@ Overall adapter statuses: | Status | Count | | --- | ---: | -| `pass` | `1` | -| `wrong_result` | `6` | +| `pass` | `3` | +| `wrong_result` | `4` | | `lifecycle_fail` | `1` | | `blocked` | `6` | | `not_encoded` | `7` | @@ -235,9 +236,10 @@ These are needed for broad credibility but should not block personal production scoring. 3. mem0/OpenMemory and memsearch coverage - - Current state: both are `wrong_result` or partially incomplete in local checks. - - Benchmark gate: fix same-corpus correctness first; only then score entity - history, UI readback, markdown store, and reindex workflows. + - Current state: both now pass the basic local OSS smoke, but their strongest + real-world scenarios remain unencoded. + - Benchmark gate: score mem0/OpenMemory entity history and UI readback, plus + memsearch source-of-truth and retrieval-debug workflows. ## What Not To Claim Yet diff --git a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md index c367fee6..266df128 100644 --- a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md +++ b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md @@ -6,7 +6,8 @@ Read this when: You need to answer whether ELF has enough empirical evidence to claim a win, tie, loss, or non-claim against tracked memory, RAG, graph, and agent-continuity projects. Inputs: Fresh local runs of `cargo make real-world-memory` and -`cargo make real-world-memory-live-adapters` on commit `286af8b`, plus +`cargo make real-world-memory-live-adapters` in the current XY-898 lane after +adapter-report consistency repairs, plus `apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json`, `2026-06-11-competitor-strength-evidence-matrix.md`, and `2026-06-11-elf-iteration-direction-from-competitor-benchmarks.md`. diff --git a/docs/research/2026-06-11-measurement-coverage-audit.json b/docs/research/2026-06-11-measurement-coverage-audit.json index 0fbae859..d8527fcb 100644 --- a/docs/research/2026-06-11-measurement-coverage-audit.json +++ b/docs/research/2026-06-11-measurement-coverage-audit.json @@ -1,7 +1,7 @@ { "schema": "elf.benchmark_measurement_coverage_audit/v1", "run_id": "2026-06-11-measurement-coverage-audit", - "commit": "286af8b", + "source_revision": "current XY-898 lane after adapter-report consistency repairs", "created_at": "2026-06-11", "scope": "ELF memory-system competitiveness measurement coverage, external competitor comparison evidence, and next report directions", "commands": [ diff --git a/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json b/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json index 79367ade..17e48620 100644 --- a/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json +++ b/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json @@ -117,7 +117,7 @@ "unsupported_or_blocked_status": { "state": "not_encoded", "typed_reason": "deep_profile_and_non_retrieval_suites_not_encoded", - "details": "The full live sweep passes targeted retrieval suites but keeps memory_evolution wrong_result and several broader suites not_encoded or incomplete." + "details": "The full live sweep passes targeted retrieval suites but keeps memory_evolution wrong_result and several broader suites not_encoded or blocked." }, "benchmark_before_claim": "Run qmd deep retrieval/debug profile and full-suite live real-world replay with trace-level diagnostics before claiming ELF wins, ties, or loses on retrieval debugging.", "borrow_if_stronger": "Borrow transparent local knobs for query rewriting, weighted fusion, rerank explanation, and command-line replay." From c5d4a9c144f8f3c6f1f2a284793dbe0f059ce26b Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 14:56:28 +0800 Subject: [PATCH 6/6] {"schema":"decodex/commit/1","summary":"Promote first-generation adapter scenario evidence","authority":"XY-898"} --- .../memory_projects_manifest.json | 20 +- .../src/bin/real_world_job_benchmark.rs | 27 ++- .../tests/real_world_job_benchmark.rs | 203 ++++++++++++++++++ ...-11-competitor-strength-evidence-matrix.md | 4 +- ...generation-oss-adapter-promotion-report.md | 2 +- .../2026-06-11-measurement-coverage-audit.md | 58 ++--- ...2026-06-11-measurement-coverage-audit.json | 96 +++++++-- ...-11-xy-897-competitor-strength-matrix.json | 4 +- ...irst-generation-oss-adapter-promotion.json | 4 +- 9 files changed, 346 insertions(+), 72 deletions(-) diff --git a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json index 9ae20f74..f7713a47 100644 --- a/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json +++ b/apps/elf-eval/fixtures/real_world_external_adapters/memory_projects_manifest.json @@ -550,7 +550,7 @@ "suite_id": "retrieval", "status": "pass", "elf_position": "untested", - "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports agentmemory retrieval_pass with 3/3 same-corpus retrieval checks through mem::remember and mem::search. This is live-baseline-only evidence through an in-memory mock, not a real_world_job suite pass.", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports agentmemory retrieval_pass with 3/3 same-corpus retrieval checks through mem::remember and mem::search. This is live-baseline-only evidence through an in-memory mock, not a real_world_job suite pass.", "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, @@ -559,7 +559,7 @@ "suite_id": "memory_evolution", "status": "lifecycle_fail", "elf_position": "wins", - "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports ELF passing 8/8 local lifecycle checks, while agentmemory update_replaces_note_text is lifecycle_fail and cold_start_recovery_search is blocked because the harness uses an in-memory SDK/KV mock. This is an ELF baseline win only at the local lifecycle-smoke evidence class.", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports ELF passing 8/8 local lifecycle checks, while agentmemory update_replaces_note_text is lifecycle_fail and cold_start_recovery_search is blocked because the harness uses an in-memory SDK/KV mock. This is an ELF baseline win only at the local lifecycle-smoke evidence class.", "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, @@ -608,7 +608,7 @@ }, "run": { "status": "pass", - "evidence": "Fresh comparable baseline run live-baseline-20260611045504 exercises local OSS mem0 with FastEmbed, Qdrant path storage, Memory.update, Memory.delete, and cold-start reload; mem0 passed 4/4 encoded checks.", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 exercises local OSS mem0 with FastEmbed, Qdrant path storage, Memory.update, Memory.delete, and cold-start reload; mem0 passed 4/4 encoded checks.", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "result": { @@ -625,7 +625,7 @@ { "capability": "same_corpus_retrieval", "status": "pass", - "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports mem0 retrieval_pass with 3/3 same-corpus retrieval checks." + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports mem0 retrieval_pass with 3/3 same-corpus retrieval checks." }, { "capability": "local_lifecycle_update_delete_reload", @@ -671,7 +671,7 @@ "suite_id": "memory_evolution", "status": "pass", "elf_position": "ties", - "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports ELF passing 8/8 local lifecycle checks and mem0 passing 4/4 same-corpus retrieval, update, delete, and cold-start reload checks. This is a basic local lifecycle tie at the encoded smoke surface, not a claim about OpenMemory UI, hosted behavior, entity history, or graph memory.", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports ELF passing 8/8 local lifecycle checks and mem0 passing 4/4 same-corpus retrieval, update, delete, and cold-start reload checks. This is a basic local lifecycle tie at the encoded smoke surface, not a claim about OpenMemory UI, hosted behavior, entity history, or graph memory.", "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, @@ -719,7 +719,7 @@ }, "run": { "status": "pass", - "evidence": "Fresh comparable baseline run live-baseline-20260611045504 indexes a per-adapter corpus copy, rewrites and deletes files, reruns memsearch index, and reports memsearch 4/4 encoded checks passing.", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 indexes a per-adapter corpus copy, rewrites and deletes files, reruns memsearch index, and reports memsearch 4/4 encoded checks passing.", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "result": { @@ -736,7 +736,7 @@ { "capability": "same_corpus_retrieval", "status": "pass", - "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports memsearch retrieval_pass with 3/3 same-corpus retrieval checks." + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports memsearch retrieval_pass with 3/3 same-corpus retrieval checks." }, { "capability": "reindex_update_delete_reload", @@ -772,7 +772,7 @@ "suite_id": "trust_source_of_truth", "status": "pass", "elf_position": "untested", - "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports memsearch passed same-corpus retrieval, update reindex, delete suppression, and cold-start reload over a canonical Markdown corpus. ELF has no directly comparable canonical Markdown source-store scenario in this baseline, so the ELF position remains untested.", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports memsearch passed same-corpus retrieval, update reindex, delete suppression, and cold-start reload over a canonical Markdown corpus. ELF has no directly comparable canonical Markdown source-store scenario in this baseline, so the ELF position remains untested.", "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, @@ -978,7 +978,7 @@ "suite_id": "retrieval", "status": "wrong_result", "elf_position": "wins", - "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports ELF retrieval_pass and claude-mem same_corpus_retrieval as wrong_result with 0/3 expected query checks passing, while its durable repository setup completed. This is an ELF baseline win for the narrow retrieval smoke scenario.", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports ELF retrieval_pass and claude-mem same_corpus_retrieval as wrong_result with 0/3 expected query checks passing, while its durable repository setup completed. This is an ELF baseline win for the narrow retrieval smoke scenario.", "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, @@ -987,7 +987,7 @@ "suite_id": "memory_evolution", "status": "pass", "elf_position": "ties", - "evidence": "Fresh comparable baseline run live-baseline-20260611045504 reports ELF passing local lifecycle checks and claude-mem update, delete, and cold-start reload checks passing over a durable Docker-local SQLite repository. This is a local lifecycle-smoke tie, not a hook-driven work-resume or full progressive-disclosure job pass.", + "evidence": "Fresh comparable baseline run live-baseline-20260611061612 reports ELF passing local lifecycle checks and claude-mem update, delete, and cold-start reload checks passing over a durable Docker-local SQLite repository. This is a local lifecycle-smoke tie, not a hook-driven work-resume or full progressive-disclosure job pass.", "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark.rs b/apps/elf-eval/src/bin/real_world_job_benchmark.rs index 890d3468..e4adba21 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark.rs @@ -4184,14 +4184,19 @@ fn render_markdown_external_adapters(out: &mut String, report: &RealWorldReport) "- Real-world suite statuses: `{}`\n", adapter_status_counts_display(&summary.suite_status_counts) )); - out.push_str(&format!( - "- Scenario coverage statuses: `{}`\n", - adapter_status_counts_display(&summary.scenario_status_counts) - )); - out.push_str(&format!( - "- ELF scenario positions: `{}`\n\n", - scenario_position_counts_display(&summary.scenario_position_counts) - )); + + if has_adapter_scenarios(report.external_adapters.adapters.as_slice()) { + out.push_str(&format!( + "- Scenario coverage statuses: `{}`\n", + adapter_status_counts_display(&summary.scenario_status_counts) + )); + out.push_str(&format!( + "- ELF scenario positions: `{}`\n", + scenario_position_counts_display(&summary.scenario_position_counts) + )); + } + + out.push('\n'); out.push_str("| Project | Adapter | Evidence Class | Overall | Setup | Run | Result | Docker | Suites | Evidence |\n"); out.push_str("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n"); @@ -4234,7 +4239,7 @@ fn render_markdown_external_adapters(out: &mut String, report: &RealWorldReport) } fn render_markdown_adapter_scenarios(out: &mut String, adapters: &[ExternalAdapterReport]) { - if !adapters.iter().any(|adapter| !adapter.scenarios.is_empty()) { + if !has_adapter_scenarios(adapters) { return; } @@ -4261,6 +4266,10 @@ fn render_markdown_adapter_scenarios(out: &mut String, adapters: &[ExternalAdapt } } +fn has_adapter_scenarios(adapters: &[ExternalAdapterReport]) -> bool { + adapters.iter().any(|adapter| !adapter.scenarios.is_empty()) +} + fn render_markdown_adapter_execution_metadata( out: &mut String, adapters: &[ExternalAdapterReport], diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 7ff86cf9..33d5ad52 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -184,6 +184,76 @@ fn real_world_report_includes_external_adapter_coverage_manifest() -> Result<()> Ok(()) } +#[test] +fn external_adapter_run_summarizes_nonzero_scenario_losses() -> Result<()> { + let manifest_path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("fixtures") + .join("real_world_external_adapters") + .join("memory_projects_manifest.json"); + let mut manifest = serde_json::from_str::(&fs::read_to_string(manifest_path)?)?; + let adapters = manifest + .pointer_mut("/adapters") + .and_then(Value::as_array_mut) + .ok_or_else(|| eyre::eyre!("missing manifest adapters"))?; + let adapter = adapters + .iter_mut() + .find(|adapter| { + adapter.pointer("/adapter_id").and_then(Value::as_str) + == Some("agentmemory_live_baseline") + }) + .ok_or_else(|| eyre::eyre!("missing agentmemory adapter"))?; + + set_json_pointer(adapter, "/scenarios/0/elf_position", serde_json::json!("loses"))?; + + let temp_dir = + env::temp_dir().join(format!("elf-real-world-loss-manifest-test-{}", process::id())); + + fs::create_dir_all(&temp_dir)?; + + let manifest_path = temp_dir.join("memory_projects_manifest.json"); + + fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)?; + + let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) + .arg("run") + .arg("--fixtures") + .arg(fixture_dir()) + .arg("--external-adapter-manifest") + .arg(&manifest_path) + .output()?; + + assert!( + output.status.success(), + "real_world_job runner failed: {}", + String::from_utf8_lossy(&output.stderr), + ); + + let report = serde_json::from_slice::(&output.stdout)?; + + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/loses") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/external_adapters/summary/scenario_position_counts/untested") + .and_then(Value::as_u64), + Some(8) + ); + + let adapters = array_at(&report, "/external_adapters/adapters")?; + let agentmemory = find_by_field(adapters, "/adapter_id", "agentmemory_live_baseline")?; + + assert_eq!( + agentmemory.pointer("/scenarios/0/elf_position").and_then(Value::as_str), + Some("loses") + ); + + Ok(()) +} + fn assert_external_adapter_manifest_summary(report: &Value) { assert_eq!( report.pointer("/external_adapters/schema").and_then(Value::as_str), @@ -938,6 +1008,139 @@ fn generated_json_report_renders_markdown() -> Result<()> { Ok(()) } +#[test] +fn external_adapter_markdown_renders_nonzero_scenario_losses() -> Result<()> { + let mut report = run_json_report()?; + let adapters = report + .pointer_mut("/external_adapters/adapters") + .and_then(Value::as_array_mut) + .ok_or_else(|| eyre::eyre!("missing external adapter records"))?; + let adapter = adapters + .iter_mut() + .find(|adapter| { + adapter.pointer("/adapter_id").and_then(Value::as_str) + == Some("agentmemory_live_baseline") + }) + .ok_or_else(|| eyre::eyre!("missing agentmemory adapter"))?; + + set_json_pointer(adapter, "/scenarios/0/elf_position", serde_json::json!("loses"))?; + set_json_pointer( + &mut report, + "/external_adapters/summary/scenario_position_counts", + serde_json::json!({ + "wins": 2, + "ties": 2, + "loses": 1, + "untested": 8 + }), + )?; + + let temp_dir = + env::temp_dir().join(format!("elf-real-world-loss-scenario-test-{}", process::id())); + + fs::create_dir_all(&temp_dir)?; + + let report_path = temp_dir.join("report.json"); + let markdown_path = temp_dir.join("report.md"); + + fs::write(&report_path, serde_json::to_vec_pretty(&report)?)?; + + let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) + .arg("publish") + .arg("--report") + .arg(&report_path) + .arg("--out") + .arg(&markdown_path) + .output()?; + + assert!( + output.status.success(), + "real_world_job publisher failed: {}", + String::from_utf8_lossy(&output.stderr), + ); + + let markdown = fs::read_to_string(markdown_path)?; + + assert!(markdown.contains("ELF scenario positions: `wins=2, ties=2, loses=1, untested=8`")); + assert!(markdown.contains( + "| `agentmemory_live_baseline` | `basic_same_corpus_retrieval` | `retrieval` | `pass` | `loses` |" + )); + + Ok(()) +} + +#[test] +fn external_adapter_markdown_omits_scenario_summary_when_manifest_has_no_scenarios() -> Result<()> { + let mut report = run_json_report()?; + let adapters = report + .pointer_mut("/external_adapters/adapters") + .and_then(Value::as_array_mut) + .ok_or_else(|| eyre::eyre!("missing external adapter records"))?; + + for adapter in adapters { + set_json_pointer(adapter, "/scenarios", serde_json::json!([]))?; + } + + set_json_pointer( + &mut report, + "/external_adapters/summary/scenario_status_counts", + serde_json::json!({ + "real": 0, + "mocked": 0, + "unsupported": 0, + "blocked": 0, + "incomplete": 0, + "wrong_result": 0, + "lifecycle_fail": 0, + "pass": 0, + "not_encoded": 0 + }), + )?; + set_json_pointer( + &mut report, + "/external_adapters/summary/scenario_position_counts", + serde_json::json!({ + "wins": 0, + "ties": 0, + "loses": 0, + "untested": 0 + }), + )?; + + let temp_dir = + env::temp_dir().join(format!("elf-real-world-no-scenario-test-{}", process::id())); + + fs::create_dir_all(&temp_dir)?; + + let report_path = temp_dir.join("report.json"); + let markdown_path = temp_dir.join("report.md"); + + fs::write(&report_path, serde_json::to_vec_pretty(&report)?)?; + + let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) + .arg("publish") + .arg("--report") + .arg(&report_path) + .arg("--out") + .arg(&markdown_path) + .output()?; + + assert!( + output.status.success(), + "real_world_job publisher failed: {}", + String::from_utf8_lossy(&output.stderr), + ); + + let markdown = fs::read_to_string(markdown_path)?; + + assert!(markdown.contains("External Adapter Coverage")); + assert!(!markdown.contains("Scenario coverage statuses:")); + assert!(!markdown.contains("ELF scenario positions:")); + assert!(!markdown.contains("### Adapter Scenario Judgments")); + + Ok(()) +} + #[test] fn knowledge_json_report_renders_markdown_metrics() -> Result<()> { let report = run_json_report_from(knowledge_fixture_dir())?; diff --git a/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md b/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md index 7fd4a3de..db29481b 100644 --- a/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md +++ b/docs/guide/benchmarking/2026-06-11-competitor-strength-evidence-matrix.md @@ -75,8 +75,8 @@ lifecycle-fail -> `lifecycle_fail`, and not-encoded -> `not_encoded`. | ELF | Evidence-linked source-of-truth memory service with real-world fixtures and live retrieval sweeps. | `live_real_world`; supporting `fixture_backed`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/elf-report.md`. Fixture contract: `cargo make real-world-memory`, `tmp/real-world-memory/real-world-memory-report.json`. | `blocked`: private manifest and provider credentials; broader live suites remain `wrong_result`, `blocked`, or `not_encoded`. | Full-suite live pass plus separate private-corpus and credentialed production-ops proof. | Keep borrowing qmd debug knobs, OpenViking staged trajectory, mem0 history, Letta core memory, and graph/RAG navigation. | | qmd | Local retrieval-debug workflow with transparent CLI indexing, querying, expansion, fusion, and rerank ergonomics. | `live_real_world`; supporting `live_baseline_only` and `research_gate`. | `wrong_result` full live sweep: `cargo make real-world-memory-live-adapters`, `tmp/real-world-memory/live-adapters/qmd-report.md`; targeted retrieval suites pass. | `not_encoded`: deep profile and non-retrieval live behavior are not encoded; memory_evolution is `wrong_result`. | qmd deep retrieval/debug profile plus full-suite live replay with trace-level diagnostics. | Weighted fusion, rerank explanation, local debug knobs, and command-line replay. | | agentmemory | Coding-agent continuity, MCP/REST packaging, viewer workflow, and durable cross-agent memory lifecycle. | `live_baseline_only`. | `lifecycle_fail`: `ELF_BASELINE_PROJECTS=agentmemory cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `blocked`: durable cold-start and real-world adapter coverage are missing. | Durable local adapter with update, delete, cold-start reload, work_resume, capture/write-policy, and lifecycle-staleness jobs. | Cross-agent hooks, packaging, continuity scenarios, and viewer affordances. | -| mem0/OpenMemory | Memory lifecycle, personalization, hosted/OpenMemory UI ergonomics, and optional graph memory. | `live_baseline_only`. | `pass`: fresh scoped run `ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`, with mem0 `4/4` local checks passing. | `not_encoded`: OpenMemory UI, hosted claims, entity/preference history, graph memory, and real-world personalization coverage are not encoded. | Encode memory_evolution preference/entity history, deletion audit readback, personalization, UI/export readback, and optional graph-context jobs. | Entity-scoped history, lifecycle surfaces, async update ergonomics, and OpenMemory inspection UX. | -| memsearch | Markdown-first canonical store with rebuildable local index and practical hybrid retrieval. | `live_baseline_only`. | `pass`: fresh scoped run `ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`, with memsearch `4/4` local checks passing. | `not_encoded`: real-world source-of-truth, retrieval, and memory-evolution prompt adapters are not encoded; TTL/expiry is unsupported by the current CLI path. | Score source-of-truth and retrieval-debug real-world jobs over the canonical Markdown store; keep TTL/expiry as unsupported unless a comparable path exists. | Canonical markdown store, local reindex clarity, and user-inspectable source files. | +| mem0/OpenMemory | Memory lifecycle, personalization, hosted/OpenMemory UI ergonomics, and optional graph memory. | `live_baseline_only`. | `pass`: fresh scoped run `ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`, with mem0 `4/4` local checks passing. | `not_encoded`: OpenMemory UI, hosted claims, entity/preference history, graph memory, and real-world personalization coverage are not encoded. | Encode memory_evolution preference/entity history, deletion audit readback, personalization, UI/export readback, and optional graph-context jobs. | Entity-scoped history, lifecycle surfaces, async update ergonomics, and OpenMemory inspection UX. | +| memsearch | Markdown-first canonical store with rebuildable local index and practical hybrid retrieval. | `live_baseline_only`. | `pass`: fresh scoped run `ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`, with memsearch `4/4` local checks passing. | `not_encoded`: real-world source-of-truth, retrieval, and memory-evolution prompt adapters are not encoded; TTL/expiry is unsupported by the current CLI path. | Score source-of-truth and retrieval-debug real-world jobs over the canonical Markdown store; keep TTL/expiry as unsupported unless a comparable path exists. | Canonical markdown store, local reindex clarity, and user-inspectable source files. | | OpenViking | Filesystem-like context trajectory, hierarchical retrieval, and staged context loading. | `live_baseline_only`; supporting `research_gate`. | `wrong_result`: `ELF_BASELINE_PROJECTS=OpenViking cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `not_encoded`: hierarchical context trajectory is not encoded; same-corpus output still misses expected evidence. | Make evidence-bearing same-corpus output pass, then score staged trajectory and hierarchy expansion. | `viking://`-style context model, trajectory readback, and staged retrieval planning. | | claude-mem | Progressive disclosure, automatic capture loop, repository-local lifecycle, and local viewer workflow. | `live_baseline_only`. | `wrong_result`: `ELF_BASELINE_PROJECTS=claude-mem cargo make baseline-live-docker`, `tmp/live-baseline/live-baseline-report.json`. | `not_encoded`: progressive-disclosure real-world jobs are not encoded. | Durable repository-backed work_resume, operator_debugging_ux, capture/write-policy, and progressive-disclosure jobs. | Progressive disclosure, automatic capture review loops, and local viewer/operator comfort. | | RAGFlow | Full RAG application workflow with document, chunk, and reference evidence handles. | `research_gate`. | `blocked`: `ELF_RAGFLOW_SMOKE_START=1 ELF_RAGFLOW_SMOKE_ACCEPT_RESOURCE_ENVELOPE=1 cargo make ragflow-docker-smoke`, `tmp/real-world-memory/ragflow-smoke/ragflow-smoke.json`. | `blocked`: Docker resource envelope and adapter output mapping still need proof. | XY-885 tiny Docker evidence-smoke adapter mapping `reference.chunks` to scored evidence. | Document/chunk references, resource-envelope reporting, and RAG app evidence handles. | diff --git a/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md b/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md index 47197b66..368bbb86 100644 --- a/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md +++ b/docs/guide/benchmarking/2026-06-11-first-generation-oss-adapter-promotion-report.md @@ -29,7 +29,7 @@ suite passes: | Command | Result | Runtime | Artifact | | --- | --- | ---: | --- | -| `ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker` | fail with typed non-pass projects | 233.69 seconds | `tmp/live-baseline/live-baseline-report.json` | +| `ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker` | fail with typed non-pass projects | 295.74 seconds | `tmp/live-baseline/live-baseline-report.json` | The aggregate failed because two projects remained typed non-pass, not because setup collapsed: diff --git a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md index 266df128..933a00cc 100644 --- a/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md +++ b/docs/guide/benchmarking/2026-06-11-measurement-coverage-audit.md @@ -41,12 +41,13 @@ production," but the competitiveness objective remains open. ## Fresh Runs -These commands were run from an isolated report worktree based on `origin/main`: +These commands were run in the current XY-898 lane after adapter-report consistency +repairs: | Command | Result | Runtime | | --- | --- | ---: | -| `cargo make real-world-memory` | pass | 42.38 seconds | -| `cargo make real-world-memory-live-adapters` | pass | 121.93 seconds | +| `cargo make real-world-memory` | pass | 11.91 seconds | +| `cargo make real-world-memory-live-adapters` | pass | 121.51 seconds | The live adapter run emitted repeated Qdrant client/server compatibility warnings, but the command completed successfully and produced ELF and qmd JSON/Markdown reports. @@ -84,33 +85,36 @@ live adapter or competitor runtime can complete those jobs. | Adapter | Jobs | Pass | Wrong result | Blocked | Not encoded | Mean score | Mean latency | Evidence recall | Evidence coverage | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| ELF live service adapter | `38` | `18` | `5` | `2` | `13` | `0.525` | `6.823 ms` | `41/77` | `48/84` | -| qmd live CLI adapter | `38` | `17` | `6` | `2` | `13` | `0.486` | `819.626 ms` | `38/77` | `45/84` | +| ELF live service adapter | `38` | `18` | `5` | `2` | `13` | `0.525` | `6.761 ms` | `41/77` | `48/84` | +| qmd live CLI adapter | `38` | `17` | `6` | `2` | `13` | `0.486` | `842.057 ms` | `38/77` | `45/84` | -This supports a narrow tie on the currently encoded live real-world suite shape. It -does not support a broad ELF-over-qmd claim because qmd remains the stronger -retrieval-debug UX reference and its deep profile is still not encoded. +This supports a near tie on the currently encoded live real-world suite shape, with +ELF one job ahead in this fresh run. It does not support a broad ELF-over-qmd claim +because qmd remains the stronger retrieval-debug UX reference and its deep profile is +still not encoded. ### Live Suite Breakdown -ELF and qmd had the same suite status shape: - -| Suite | Jobs | Status breakdown | -| --- | ---: | --- | -| `trust_source_of_truth` | `1` | `pass:1` | -| `work_resume` | `5` | `pass:5` | -| `retrieval` | `5` | `pass:5` | -| `project_decisions` | `5` | `pass:5` | -| `personalization` | `1` | `pass:1` | -| `memory_evolution` | `6` | `pass:1`, `wrong_result:5` | -| `capture_integration` | `2` | `not_encoded:2` | -| `consolidation` | `4` | `not_encoded:4` | -| `knowledge_compilation` | `2` | `not_encoded:2` | -| `operator_debugging_ux` | `1` | `not_encoded:1` | -| `production_ops` | `6` | `blocked:2`, `not_encoded:4` | - -The five live wrong results are all memory-evolution jobs. The live adapters retrieve -current evidence but do not yet provide the required historical conflict evidence +ELF and qmd have the same status shape outside `memory_evolution`. The difference is +`memory-evolution-delete-ttl-001`: ELF passes that job while qmd reports +`wrong_result`, leaving ELF at five memory-evolution wrong results and qmd at six. + +| Suite | Jobs | ELF breakdown | qmd breakdown | +| --- | ---: | --- | --- | +| `trust_source_of_truth` | `1` | `pass:1` | `pass:1` | +| `work_resume` | `5` | `pass:5` | `pass:5` | +| `retrieval` | `5` | `pass:5` | `pass:5` | +| `project_decisions` | `5` | `pass:5` | `pass:5` | +| `personalization` | `1` | `pass:1` | `pass:1` | +| `memory_evolution` | `6` | `pass:1`, `wrong_result:5` | `wrong_result:6` | +| `capture_integration` | `2` | `not_encoded:2` | `not_encoded:2` | +| `consolidation` | `4` | `not_encoded:4` | `not_encoded:4` | +| `knowledge_compilation` | `2` | `not_encoded:2` | `not_encoded:2` | +| `operator_debugging_ux` | `1` | `not_encoded:1` | `not_encoded:1` | +| `production_ops` | `6` | `blocked:2`, `not_encoded:4` | `blocked:2`, `not_encoded:4` | + +The live wrong results are all memory-evolution jobs. The live adapters retrieve +current evidence but do not yet provide all required historical conflict evidence links for current-vs-historical reasoning. ## External Adapter Ledger @@ -141,7 +145,7 @@ names, of which 16 are external projects. | Project | Best current evidence | Current measured state | Strongest unproven scenario | Next measurement before claim | | --- | --- | --- | --- | --- | | ELF | `fixture_backed` plus `live_real_world` | Fixture aggregate passes except 2 blocked operator boundaries; live full sweep is `wrong_result`. | Full live memory evolution, live consolidation, live knowledge pages, live capture, live production ops. | Memory-evolution diagnostic report, then live operator/capture/consolidation reports. | -| qmd | `live_real_world` plus `live_baseline_only` | Same live sweep shape as ELF; same-corpus baseline passes. | Deep retrieval-debug ergonomics and trace replay. | qmd/ELF deep retrieval-debug profile with expansion, fusion, rerank, and dropped-candidate traces. | +| qmd | `live_real_world` plus `live_baseline_only` | Same-corpus baseline passes; current live sweep is one memory-evolution job behind ELF. | Deep retrieval-debug ergonomics and trace replay. | qmd/ELF deep retrieval-debug profile with expansion, fusion, rerank, and dropped-candidate traces. | | agentmemory | `live_baseline_only` | `lifecycle_fail`. | Durable coding-agent continuity and capture hooks. | Durable lifecycle and work-resume/capture adapter report. | | mem0/OpenMemory | `live_baseline_only` | Basic local smoke now passes; history/UI/hosted/graph behavior remains `not_encoded`. | Entity history, lifecycle UI, OpenMemory inspection. | Entity-history, deletion-audit, and UI/export readback report. | | memsearch | `live_baseline_only` | Basic canonical Markdown reindex/reload smoke now passes; real-world prompt coverage remains `not_encoded`. | Markdown canonical store and local reindex clarity. | Source-of-truth and retrieval-debug real-world adapter report. | diff --git a/docs/research/2026-06-11-measurement-coverage-audit.json b/docs/research/2026-06-11-measurement-coverage-audit.json index d8527fcb..cdb9d553 100644 --- a/docs/research/2026-06-11-measurement-coverage-audit.json +++ b/docs/research/2026-06-11-measurement-coverage-audit.json @@ -1,5 +1,5 @@ { - "schema": "elf.benchmark_measurement_coverage_audit/v1", + "schema": "elf.benchmark_measurement_coverage_audit/v2", "run_id": "2026-06-11-measurement-coverage-audit", "source_revision": "current XY-898 lane after adapter-report consistency repairs", "created_at": "2026-06-11", @@ -8,13 +8,13 @@ { "command": "cargo make real-world-memory", "status": "pass", - "runtime_seconds": 42.38, + "runtime_seconds": 11.91, "artifact": "tmp/real-world-memory/real-world-memory-report.json" }, { "command": "cargo make real-world-memory-live-adapters", "status": "pass", - "runtime_seconds": 121.93, + "runtime_seconds": 121.51, "artifact": "tmp/real-world-memory/live-adapters/" } ], @@ -45,7 +45,7 @@ "blocked": 2, "not_encoded": 13, "mean_score": 0.525, - "mean_latency_ms": 6.823, + "mean_latency_ms": 6.761, "expected_evidence_total": 77, "expected_evidence_matched": 41, "evidence_required_count": 84, @@ -60,26 +60,84 @@ "blocked": 2, "not_encoded": 13, "mean_score": 0.486, - "mean_latency_ms": 819.626, + "mean_latency_ms": 842.057, "expected_evidence_total": 77, "expected_evidence_matched": 38, "evidence_required_count": 84, "evidence_covered_count": 45 } ], - "live_suite_breakdown": [ - {"suite": "trust_source_of_truth", "jobs": 1, "status_counts": {"pass": 1}}, - {"suite": "work_resume", "jobs": 5, "status_counts": {"pass": 5}}, - {"suite": "retrieval", "jobs": 5, "status_counts": {"pass": 5}}, - {"suite": "project_decisions", "jobs": 5, "status_counts": {"pass": 5}}, - {"suite": "personalization", "jobs": 1, "status_counts": {"pass": 1}}, - {"suite": "memory_evolution", "jobs": 6, "status_counts": {"pass": 1, "wrong_result": 5}}, - {"suite": "capture_integration", "jobs": 2, "status_counts": {"not_encoded": 2}}, - {"suite": "consolidation", "jobs": 4, "status_counts": {"not_encoded": 4}}, - {"suite": "knowledge_compilation", "jobs": 2, "status_counts": {"not_encoded": 2}}, - {"suite": "operator_debugging_ux", "jobs": 1, "status_counts": {"not_encoded": 1}}, - {"suite": "production_ops", "jobs": 6, "status_counts": {"blocked": 2, "not_encoded": 4}} - ], + "live_suite_breakdown": { + "delta": "ELF passes memory-evolution-delete-ttl-001 while qmd reports wrong_result; other suite status shapes match.", + "suites": [ + { + "suite": "trust_source_of_truth", + "jobs": 1, + "elf_status_counts": {"pass": 1}, + "qmd_status_counts": {"pass": 1} + }, + { + "suite": "work_resume", + "jobs": 5, + "elf_status_counts": {"pass": 5}, + "qmd_status_counts": {"pass": 5} + }, + { + "suite": "retrieval", + "jobs": 5, + "elf_status_counts": {"pass": 5}, + "qmd_status_counts": {"pass": 5} + }, + { + "suite": "project_decisions", + "jobs": 5, + "elf_status_counts": {"pass": 5}, + "qmd_status_counts": {"pass": 5} + }, + { + "suite": "personalization", + "jobs": 1, + "elf_status_counts": {"pass": 1}, + "qmd_status_counts": {"pass": 1} + }, + { + "suite": "memory_evolution", + "jobs": 6, + "elf_status_counts": {"pass": 1, "wrong_result": 5}, + "qmd_status_counts": {"wrong_result": 6} + }, + { + "suite": "capture_integration", + "jobs": 2, + "elf_status_counts": {"not_encoded": 2}, + "qmd_status_counts": {"not_encoded": 2} + }, + { + "suite": "consolidation", + "jobs": 4, + "elf_status_counts": {"not_encoded": 4}, + "qmd_status_counts": {"not_encoded": 4} + }, + { + "suite": "knowledge_compilation", + "jobs": 2, + "elf_status_counts": {"not_encoded": 2}, + "qmd_status_counts": {"not_encoded": 2} + }, + { + "suite": "operator_debugging_ux", + "jobs": 1, + "elf_status_counts": {"not_encoded": 1}, + "qmd_status_counts": {"not_encoded": 1} + }, + { + "suite": "production_ops", + "jobs": 6, + "elf_status_counts": {"blocked": 2, "not_encoded": 4}, + "qmd_status_counts": {"blocked": 2, "not_encoded": 4} + } + ] + }, "adapter_ledger": { "adapter_records": 21, "unique_project_names": 17, @@ -99,7 +157,7 @@ } }, "claim_boundary": { - "elf_vs_qmd": "tie_on_current_encoded_live_real_world_shape_not_overall_win", + "elf_vs_qmd": "near_tie_on_current_encoded_live_real_world_shape_not_overall_win", "elf_personal_production": "credible_with_bounded_caveats", "broad_competitor_superiority": "not_proven", "major_unmeasured_strengths": [ diff --git a/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json b/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json index 17e48620..a9ef5fba 100644 --- a/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json +++ b/docs/research/2026-06-11-xy-897-competitor-strength-matrix.json @@ -151,7 +151,7 @@ ], "measured_status": "pass", "proof": { - "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "unsupported_or_blocked_status": { @@ -171,7 +171,7 @@ ], "measured_status": "pass", "proof": { - "command": "ELF_BASELINE_PROJECTS=agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", + "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", "artifact": "tmp/live-baseline/live-baseline-report.json" }, "unsupported_or_blocked_status": { diff --git a/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json b/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json index 4e9132c9..81e9179c 100644 --- a/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json +++ b/docs/research/2026-06-11-xy-898-first-generation-oss-adapter-promotion.json @@ -15,9 +15,9 @@ ], "fresh_run": { "command": "ELF_BASELINE_PROJECTS=ELF,agentmemory,mem0,memsearch,claude-mem cargo make baseline-live-docker", - "run_id": "live-baseline-20260611045504", + "run_id": "live-baseline-20260611061612", "status": "fail", - "runtime_seconds": 233.69, + "runtime_seconds": 295.74, "artifact": "tmp/live-baseline/live-baseline-report.json", "summary": { "total": 5,