From 71df08bcf01670d735b47be36a03f846fbf0fcc5 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Sat, 27 Jun 2026 19:44:27 +0800 Subject: [PATCH] {"schema":"decodex/commit/1","summary":"Add Work Continuity real-world benchmark suite and gates","authority":"XY-1118"} --- Makefile.toml | 52 + .../decision_rationale_readback.json | 151 ++ .../explicit_next_step_precision.json | 160 +++ .../handoff_source_ref_coverage.json | 151 ++ .../inferred_next_step_labeling.json | 160 +++ .../janitor_false_promotion_guard.json | 158 +++ .../redaction_sensitive_marker.json | 156 +++ .../rejected_option_suppression.json | 159 +++ .../reset_resume_session_readback.json | 151 ++ .../src/bin/real_world_job_benchmark.rs | 1229 +++++++++++++++-- .../tests/real_world_job_benchmark.rs | 393 +++++- docs/log.md | 5 + .../real_world_agent_memory_benchmark.md | 32 +- .../real_world_agent_memory_benchmark_v1.md | 35 +- 14 files changed, 2890 insertions(+), 102 deletions(-) create mode 100644 apps/elf-eval/fixtures/real_world_memory/work_continuity/decision_rationale_readback.json create mode 100644 apps/elf-eval/fixtures/real_world_memory/work_continuity/explicit_next_step_precision.json create mode 100644 apps/elf-eval/fixtures/real_world_memory/work_continuity/handoff_source_ref_coverage.json create mode 100644 apps/elf-eval/fixtures/real_world_memory/work_continuity/inferred_next_step_labeling.json create mode 100644 apps/elf-eval/fixtures/real_world_memory/work_continuity/janitor_false_promotion_guard.json create mode 100644 apps/elf-eval/fixtures/real_world_memory/work_continuity/redaction_sensitive_marker.json create mode 100644 apps/elf-eval/fixtures/real_world_memory/work_continuity/rejected_option_suppression.json create mode 100644 apps/elf-eval/fixtures/real_world_memory/work_continuity/reset_resume_session_readback.json diff --git a/Makefile.toml b/Makefile.toml index bccf5b03..f00c200a 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -82,6 +82,9 @@ # | real-world-memory-summary | composite | | # | real-world-memory-summary-json | command | | # | real-world-memory-summary-report | command | | +# | real-world-memory-work-continuity | composite | | +# | real-world-memory-work-continuity-json | command | | +# | real-world-memory-work-continuity-report | command | | [tasks.baseline-backfill-100k-docker] workspace = false @@ -1225,6 +1228,55 @@ args = [ "tmp/real-world-memory/scheduled/report.md", ] +[tasks.real-world-memory-work-continuity] +workspace = false +dependencies = [ + "real-world-memory-work-continuity-report", +] + +[tasks.real-world-memory-work-continuity-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/work_continuity", + "--out", + "tmp/real-world-memory/work-continuity/report.json", + "--run-id", + "real-world-memory-work-continuity", + "--adapter-id", + "fixture_work_continuity", + "--adapter-name", + "ELF Work Continuity fixture", +] + +[tasks.real-world-memory-work-continuity-report] +workspace = false +dependencies = [ + "real-world-memory-work-continuity-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/work-continuity/report.json", + "--out", + "tmp/real-world-memory/work-continuity/report.md", +] + [tasks.real-world-memory-service-native-dreaming] workspace = false command = "bash" diff --git a/apps/elf-eval/fixtures/real_world_memory/work_continuity/decision_rationale_readback.json b/apps/elf-eval/fixtures/real_world_memory/work_continuity/decision_rationale_readback.json new file mode 100644 index 00000000..ea885f28 --- /dev/null +++ b/apps/elf-eval/fixtures/real_world_memory/work_continuity/decision_rationale_readback.json @@ -0,0 +1,151 @@ +{ + "schema": "elf.real_world_job/v1", + "job_id": "work-continuity-decision-rationale-001", + "suite": "work_continuity", + "title": "Decision rationale survives Work Journal readback", + "corpus": { + "corpus_id": "real-world-memory-work-continuity-2026-06-27", + "profile": "synthetic", + "items": [ + { + "evidence_id": "wj-decision-rationale-source", + "kind": "decision_rationale", + "text": "Decision rationale: keep Work Journal entries source-adjacent because current facts require Memory Authority promotion or accepted Dreaming Review.", + "source_ref": { + "schema": "source_ref/v1", + "resolver": "real_world_job_fixture/v1", + "ref": { + "fixture": "decision_rationale_readback", + "evidence_id": "wj-decision-rationale-source" + }, + "locator": { + "quote": "source-adjacent" + } + }, + "created_at": "2026-06-27T00:02:00Z" + } + ], + "adapter_response": { + "adapter_id": "fixture_work_continuity", + "answer": { + "content": "The rationale is to keep Work Journal entries source-adjacent unless they are promoted through Memory Authority or accepted Dreaming Review.", + "claims": [ + { + "claim_id": "decision_rationale_recall", + "text": "The decision rationale for Work Journal is source-adjacent capture until explicit promotion.", + "evidence_ids": ["wj-decision-rationale-source"], + "confidence": "high" + } + ], + "evidence_ids": ["wj-decision-rationale-source"], + "work_journal_readbacks": [ + { + "readback_id": "wj-readback-decision-rationale", + "contract_schema": "elf.work_journal/v1", + "generated_at": "2026-06-27T00:03:00Z", + "session_id": "work-continuity-session", + "tenant_id": "fixture-tenant", + "project_id": "elf", + "agent_id": "xy-1118-fixture-agent", + "read_profile": "private_plus_project", + "items": [ + { + "entry_id": "wj-decision-entry", + "family": "decision_rationale", + "title": "Work Journal authority boundary", + "body": "Keep entries source-adjacent until accepted promotion.", + "source_refs": ["wj-decision-rationale-source"] + } + ], + "where_stopped": { + "reset_resume_entry_ids": [], + "decision_rationale_evidence_ids": ["wj-decision-rationale-source"], + "current_explicit_next_step_ids": [], + "labeled_inferred_next_step_ids": [], + "handoff_source_refs": ["wj-decision-rationale-source"], + "journal_only_authority_claims": [] + }, + "promotion_boundary": { + "journal_entry_authority": "source_adjacent_only", + "memory_promotion_required": true, + "accepted_refs": [] + }, + "janitor_candidates": [] + } + ], + "latency_ms": 1.0, + "cost": { + "currency": "USD", + "amount": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + } + }, + "prompt": { + "role": "user", + "content": "Why is the Work Journal not current fact authority?", + "job_mode": "work_continuity", + "constraints": ["cite_evidence", "recall_decision_rationale"] + }, + "expected_answer": { + "must_include": [ + { + "claim_id": "decision_rationale_recall", + "text": "The decision rationale for Work Journal is source-adjacent capture until explicit promotion." + } + ], + "must_not_include": ["Work Journal alone is the Memory Authority"], + "evidence_links": { + "decision_rationale_recall": ["wj-decision-rationale-source"] + }, + "answer_type": "work_journal_readback", + "accepted_alternates": [], + "requires_caveat": false, + "requires_refusal": false + }, + "required_evidence": [ + { + "evidence_id": "wj-decision-rationale-source", + "claim_id": "decision_rationale_recall", + "requirement": "cite", + "quote": "source-adjacent" + } + ], + "work_continuity": { + "required_decision_rationale_evidence_ids": ["wj-decision-rationale-source"] + }, + "scoring_rubric": { + "dimensions": { + "answer_correctness": { + "weight": 0.35, + "max_points": 1.0, + "criteria": "Recalls the stored rationale." + }, + "evidence_grounding": { + "weight": 0.3, + "max_points": 1.0, + "criteria": "Cites the rationale evidence." + }, + "trap_avoidance": { + "weight": 0.2, + "max_points": 1.0, + "criteria": "Does not promote journal-only content." + }, + "lifecycle_behavior": { + "weight": 0.15, + "max_points": 1.0, + "criteria": "Preserves promotion boundary." + } + }, + "pass_threshold": 0.85, + "hard_fail_rules": ["journal-only authority claim"] + }, + "allowed_uncertainty": { + "can_answer_unknown": false, + "acceptable_phrases": [], + "fallback_action": "state_blocker" + }, + "tags": ["synthetic", "work_continuity", "decision_rationale"] +} diff --git a/apps/elf-eval/fixtures/real_world_memory/work_continuity/explicit_next_step_precision.json b/apps/elf-eval/fixtures/real_world_memory/work_continuity/explicit_next_step_precision.json new file mode 100644 index 00000000..a84c6382 --- /dev/null +++ b/apps/elf-eval/fixtures/real_world_memory/work_continuity/explicit_next_step_precision.json @@ -0,0 +1,160 @@ +{ + "schema": "elf.real_world_job/v1", + "job_id": "work-continuity-explicit-next-step-001", + "suite": "work_continuity", + "title": "Explicit next steps are returned as instructions", + "corpus": { + "corpus_id": "real-world-memory-work-continuity-2026-06-27", + "profile": "synthetic", + "items": [ + { + "evidence_id": "wj-explicit-step-source", + "kind": "next_step", + "text": "Explicit next step: run cargo make real-world-memory-work-continuity after adding the fixtures.", + "source_ref": { + "schema": "source_ref/v1", + "resolver": "real_world_job_fixture/v1", + "ref": { + "fixture": "explicit_next_step_precision", + "evidence_id": "wj-explicit-step-source" + }, + "locator": { + "quote": "Explicit next step" + } + }, + "created_at": "2026-06-27T00:06:00Z" + } + ], + "adapter_response": { + "adapter_id": "fixture_work_continuity", + "answer": { + "content": "The explicit next step is to run cargo make real-world-memory-work-continuity after adding the fixtures.", + "claims": [ + { + "claim_id": "explicit_next_step", + "text": "The explicit next step is run cargo make real-world-memory-work-continuity.", + "evidence_ids": ["wj-explicit-step-source"], + "confidence": "high" + } + ], + "evidence_ids": ["wj-explicit-step-source"], + "work_journal_readbacks": [ + { + "readback_id": "wj-readback-explicit-next-step", + "contract_schema": "elf.work_journal/v1", + "generated_at": "2026-06-27T00:07:00Z", + "session_id": "work-continuity-session", + "tenant_id": "fixture-tenant", + "project_id": "elf", + "agent_id": "xy-1118-fixture-agent", + "read_profile": "private_plus_project", + "items": [ + { + "entry_id": "wj-explicit-step-entry", + "family": "next_step", + "title": "Run Work Continuity slice", + "body": "The next step was explicitly recorded by the operator.", + "source_refs": ["wj-explicit-step-source"], + "explicit_next_steps": [ + { + "step_id": "wj-explicit-run-slice", + "text": "Run cargo make real-world-memory-work-continuity.", + "label": "explicit", + "instruction": true, + "evidence_refs": ["wj-explicit-step-source"] + } + ] + } + ], + "where_stopped": { + "reset_resume_entry_ids": [], + "decision_rationale_evidence_ids": [], + "current_explicit_next_step_ids": ["wj-explicit-run-slice"], + "labeled_inferred_next_step_ids": [], + "handoff_source_refs": ["wj-explicit-step-source"], + "journal_only_authority_claims": [] + }, + "promotion_boundary": { + "journal_entry_authority": "source_adjacent_only", + "memory_promotion_required": true, + "accepted_refs": [] + }, + "janitor_candidates": [] + } + ], + "latency_ms": 1.0, + "cost": { + "currency": "USD", + "amount": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + } + }, + "prompt": { + "role": "user", + "content": "What is the explicit next step?", + "job_mode": "work_continuity", + "constraints": ["cite_evidence", "label_explicit_next_steps"] + }, + "expected_answer": { + "must_include": [ + { + "claim_id": "explicit_next_step", + "text": "The explicit next step is run cargo make real-world-memory-work-continuity." + } + ], + "must_not_include": ["This step was inferred"], + "evidence_links": { + "explicit_next_step": ["wj-explicit-step-source"] + }, + "answer_type": "work_journal_readback", + "accepted_alternates": [], + "requires_caveat": false, + "requires_refusal": false + }, + "required_evidence": [ + { + "evidence_id": "wj-explicit-step-source", + "claim_id": "explicit_next_step", + "requirement": "cite", + "quote": "Explicit next step" + } + ], + "work_continuity": { + "required_explicit_next_step_ids": ["wj-explicit-run-slice"] + }, + "scoring_rubric": { + "dimensions": { + "answer_correctness": { + "weight": 0.35, + "max_points": 1.0, + "criteria": "Returns only correct explicit next steps." + }, + "evidence_grounding": { + "weight": 0.3, + "max_points": 1.0, + "criteria": "Cites the explicit step evidence." + }, + "trap_avoidance": { + "weight": 0.2, + "max_points": 1.0, + "criteria": "Does not mislabel explicit steps." + }, + "lifecycle_behavior": { + "weight": 0.15, + "max_points": 1.0, + "criteria": "Keeps source ref attached." + } + }, + "pass_threshold": 0.85, + "hard_fail_rules": ["missing explicit next step"] + }, + "allowed_uncertainty": { + "can_answer_unknown": false, + "acceptable_phrases": [], + "fallback_action": "state_blocker" + }, + "tags": ["synthetic", "work_continuity", "explicit_next_step"] +} diff --git a/apps/elf-eval/fixtures/real_world_memory/work_continuity/handoff_source_ref_coverage.json b/apps/elf-eval/fixtures/real_world_memory/work_continuity/handoff_source_ref_coverage.json new file mode 100644 index 00000000..56733d88 --- /dev/null +++ b/apps/elf-eval/fixtures/real_world_memory/work_continuity/handoff_source_ref_coverage.json @@ -0,0 +1,151 @@ +{ + "schema": "elf.real_world_job/v1", + "job_id": "work-continuity-handoff-source-ref-001", + "suite": "work_continuity", + "title": "Handoff readback preserves source refs", + "corpus": { + "corpus_id": "real-world-memory-work-continuity-2026-06-27", + "profile": "synthetic", + "items": [ + { + "evidence_id": "wj-handoff-source-ref", + "kind": "handoff", + "text": "Handoff source ref: the Work Journal handoff cites wj-handoff-source-ref so the next agent can inspect the originating entry.", + "source_ref": { + "schema": "source_ref/v1", + "resolver": "real_world_job_fixture/v1", + "ref": { + "fixture": "handoff_source_ref_coverage", + "evidence_id": "wj-handoff-source-ref" + }, + "locator": { + "quote": "Handoff source ref" + } + }, + "created_at": "2026-06-27T00:10:00Z" + } + ], + "adapter_response": { + "adapter_id": "fixture_work_continuity", + "answer": { + "content": "The handoff includes source ref wj-handoff-source-ref for the next agent to inspect.", + "claims": [ + { + "claim_id": "handoff_source_ref_present", + "text": "The Work Journal handoff preserves source ref wj-handoff-source-ref.", + "evidence_ids": ["wj-handoff-source-ref"], + "confidence": "high" + } + ], + "evidence_ids": ["wj-handoff-source-ref"], + "work_journal_readbacks": [ + { + "readback_id": "wj-readback-handoff-source-ref", + "contract_schema": "elf.work_journal/v1", + "generated_at": "2026-06-27T00:11:00Z", + "session_id": "work-continuity-session", + "tenant_id": "fixture-tenant", + "project_id": "elf", + "agent_id": "xy-1118-fixture-agent", + "read_profile": "private_plus_project", + "items": [ + { + "entry_id": "wj-handoff-entry", + "family": "handoff", + "title": "Source-linked handoff", + "body": "The handoff has a stable source ref.", + "source_refs": ["wj-handoff-source-ref"] + } + ], + "where_stopped": { + "reset_resume_entry_ids": [], + "decision_rationale_evidence_ids": [], + "current_explicit_next_step_ids": [], + "labeled_inferred_next_step_ids": [], + "handoff_source_refs": ["wj-handoff-source-ref"], + "journal_only_authority_claims": [] + }, + "promotion_boundary": { + "journal_entry_authority": "source_adjacent_only", + "memory_promotion_required": true, + "accepted_refs": [] + }, + "janitor_candidates": [] + } + ], + "latency_ms": 1.0, + "cost": { + "currency": "USD", + "amount": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + } + }, + "prompt": { + "role": "user", + "content": "Does the handoff carry a source ref?", + "job_mode": "work_continuity", + "constraints": ["cite_evidence", "preserve_handoff_source_refs"] + }, + "expected_answer": { + "must_include": [ + { + "claim_id": "handoff_source_ref_present", + "text": "The Work Journal handoff preserves source ref wj-handoff-source-ref." + } + ], + "must_not_include": ["handoff has no source ref"], + "evidence_links": { + "handoff_source_ref_present": ["wj-handoff-source-ref"] + }, + "answer_type": "work_journal_readback", + "accepted_alternates": [], + "requires_caveat": false, + "requires_refusal": false + }, + "required_evidence": [ + { + "evidence_id": "wj-handoff-source-ref", + "claim_id": "handoff_source_ref_present", + "requirement": "cite", + "quote": "Handoff source ref" + } + ], + "work_continuity": { + "required_handoff_source_ref_ids": ["wj-handoff-source-ref"] + }, + "scoring_rubric": { + "dimensions": { + "answer_correctness": { + "weight": 0.35, + "max_points": 1.0, + "criteria": "Reports handoff source refs." + }, + "evidence_grounding": { + "weight": 0.3, + "max_points": 1.0, + "criteria": "Cites the handoff evidence." + }, + "trap_avoidance": { + "weight": 0.2, + "max_points": 1.0, + "criteria": "Does not omit source refs." + }, + "lifecycle_behavior": { + "weight": 0.15, + "max_points": 1.0, + "criteria": "Keeps handoff source-adjacent." + } + }, + "pass_threshold": 0.85, + "hard_fail_rules": ["missing handoff source ref"] + }, + "allowed_uncertainty": { + "can_answer_unknown": false, + "acceptable_phrases": [], + "fallback_action": "state_blocker" + }, + "tags": ["synthetic", "work_continuity", "handoff_source_ref"] +} diff --git a/apps/elf-eval/fixtures/real_world_memory/work_continuity/inferred_next_step_labeling.json b/apps/elf-eval/fixtures/real_world_memory/work_continuity/inferred_next_step_labeling.json new file mode 100644 index 00000000..44bc4eec --- /dev/null +++ b/apps/elf-eval/fixtures/real_world_memory/work_continuity/inferred_next_step_labeling.json @@ -0,0 +1,160 @@ +{ + "schema": "elf.real_world_job/v1", + "job_id": "work-continuity-inferred-next-step-001", + "suite": "work_continuity", + "title": "Inferred next steps are labeled and not treated as instructions", + "corpus": { + "corpus_id": "real-world-memory-work-continuity-2026-06-27", + "profile": "synthetic", + "items": [ + { + "evidence_id": "wj-inferred-step-source", + "kind": "inferred_next_step", + "text": "Inferred next step candidate: compare Work Continuity rates against the Work Journal spec, but this was not an operator instruction.", + "source_ref": { + "schema": "source_ref/v1", + "resolver": "real_world_job_fixture/v1", + "ref": { + "fixture": "inferred_next_step_labeling", + "evidence_id": "wj-inferred-step-source" + }, + "locator": { + "quote": "Inferred next step candidate" + } + }, + "created_at": "2026-06-27T00:08:00Z" + } + ], + "adapter_response": { + "adapter_id": "fixture_work_continuity", + "answer": { + "content": "The comparison task is labeled as inferred and is not an operator instruction.", + "claims": [ + { + "claim_id": "inferred_next_step_label", + "text": "The comparison task is an inferred next-step candidate, not an instruction.", + "evidence_ids": ["wj-inferred-step-source"], + "confidence": "high" + } + ], + "evidence_ids": ["wj-inferred-step-source"], + "work_journal_readbacks": [ + { + "readback_id": "wj-readback-inferred-next-step", + "contract_schema": "elf.work_journal/v1", + "generated_at": "2026-06-27T00:09:00Z", + "session_id": "work-continuity-session", + "tenant_id": "fixture-tenant", + "project_id": "elf", + "agent_id": "xy-1118-fixture-agent", + "read_profile": "private_plus_project", + "items": [ + { + "entry_id": "wj-inferred-step-entry", + "family": "inferred_next_step", + "title": "Compare benchmark rates", + "body": "The possible comparison was inferred from context.", + "source_refs": ["wj-inferred-step-source"], + "inferred_next_steps": [ + { + "step_id": "wj-inferred-compare-rates", + "text": "Compare Work Continuity rates against the Work Journal spec.", + "label": "inferred", + "instruction": false, + "evidence_refs": ["wj-inferred-step-source"] + } + ] + } + ], + "where_stopped": { + "reset_resume_entry_ids": [], + "decision_rationale_evidence_ids": [], + "current_explicit_next_step_ids": [], + "labeled_inferred_next_step_ids": ["wj-inferred-compare-rates"], + "handoff_source_refs": ["wj-inferred-step-source"], + "journal_only_authority_claims": [] + }, + "promotion_boundary": { + "journal_entry_authority": "source_adjacent_only", + "memory_promotion_required": true, + "accepted_refs": [] + }, + "janitor_candidates": [] + } + ], + "latency_ms": 1.0, + "cost": { + "currency": "USD", + "amount": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + } + }, + "prompt": { + "role": "user", + "content": "Was the comparison task explicit or inferred?", + "job_mode": "work_continuity", + "constraints": ["cite_evidence", "label_inferred_next_steps"] + }, + "expected_answer": { + "must_include": [ + { + "claim_id": "inferred_next_step_label", + "text": "The comparison task is an inferred next-step candidate, not an instruction." + } + ], + "must_not_include": ["The operator instructed the agent to compare Work Continuity rates"], + "evidence_links": { + "inferred_next_step_label": ["wj-inferred-step-source"] + }, + "answer_type": "work_journal_readback", + "accepted_alternates": [], + "requires_caveat": false, + "requires_refusal": false + }, + "required_evidence": [ + { + "evidence_id": "wj-inferred-step-source", + "claim_id": "inferred_next_step_label", + "requirement": "cite", + "quote": "Inferred next step candidate" + } + ], + "work_continuity": { + "required_inferred_next_step_ids": ["wj-inferred-compare-rates"] + }, + "scoring_rubric": { + "dimensions": { + "answer_correctness": { + "weight": 0.35, + "max_points": 1.0, + "criteria": "Labels inferred next steps correctly." + }, + "evidence_grounding": { + "weight": 0.3, + "max_points": 1.0, + "criteria": "Cites the inferred-step evidence." + }, + "trap_avoidance": { + "weight": 0.2, + "max_points": 1.0, + "criteria": "Does not treat inferred work as instruction." + }, + "lifecycle_behavior": { + "weight": 0.15, + "max_points": 1.0, + "criteria": "Keeps inferred status visible." + } + }, + "pass_threshold": 0.85, + "hard_fail_rules": ["inferred-step-as-instruction"] + }, + "allowed_uncertainty": { + "can_answer_unknown": false, + "acceptable_phrases": [], + "fallback_action": "state_blocker" + }, + "tags": ["synthetic", "work_continuity", "inferred_next_step"] +} diff --git a/apps/elf-eval/fixtures/real_world_memory/work_continuity/janitor_false_promotion_guard.json b/apps/elf-eval/fixtures/real_world_memory/work_continuity/janitor_false_promotion_guard.json new file mode 100644 index 00000000..5172ed5b --- /dev/null +++ b/apps/elf-eval/fixtures/real_world_memory/work_continuity/janitor_false_promotion_guard.json @@ -0,0 +1,158 @@ +{ + "schema": "elf.real_world_job/v1", + "job_id": "work-continuity-janitor-false-promotion-001", + "suite": "work_continuity", + "title": "Janitor candidates are not falsely promoted to memory", + "corpus": { + "corpus_id": "real-world-memory-work-continuity-2026-06-27", + "profile": "synthetic", + "items": [ + { + "evidence_id": "wj-janitor-candidate-source", + "kind": "janitor_candidate", + "text": "Janitor candidate: entry wj-janitor-candidate needs review and must not be promoted to Memory Authority automatically.", + "source_ref": { + "schema": "source_ref/v1", + "resolver": "real_world_job_fixture/v1", + "ref": { + "fixture": "janitor_false_promotion_guard", + "evidence_id": "wj-janitor-candidate-source" + }, + "locator": { + "quote": "must not be promoted" + } + }, + "created_at": "2026-06-27T00:14:00Z" + } + ], + "adapter_response": { + "adapter_id": "fixture_work_continuity", + "answer": { + "content": "The janitor candidate remains review-required and source-adjacent.", + "claims": [ + { + "claim_id": "janitor_no_false_promotion", + "text": "The Work Journal janitor candidate was not promoted to memory without review.", + "evidence_ids": ["wj-janitor-candidate-source"], + "confidence": "high" + } + ], + "evidence_ids": ["wj-janitor-candidate-source"], + "work_journal_readbacks": [ + { + "readback_id": "wj-readback-janitor", + "contract_schema": "elf.work_journal/v1", + "generated_at": "2026-06-27T00:15:00Z", + "session_id": "work-continuity-session", + "tenant_id": "fixture-tenant", + "project_id": "elf", + "agent_id": "xy-1118-fixture-agent", + "read_profile": "private_plus_project", + "items": [ + { + "entry_id": "wj-janitor-entry", + "family": "janitor", + "title": "Review-only janitor candidate", + "body": "The candidate requires review before any memory promotion.", + "source_refs": ["wj-janitor-candidate-source"] + } + ], + "where_stopped": { + "reset_resume_entry_ids": [], + "decision_rationale_evidence_ids": [], + "current_explicit_next_step_ids": [], + "labeled_inferred_next_step_ids": [], + "handoff_source_refs": ["wj-janitor-candidate-source"], + "journal_only_authority_claims": [] + }, + "promotion_boundary": { + "journal_entry_authority": "source_adjacent_only", + "memory_promotion_required": true, + "accepted_refs": [] + }, + "janitor_candidates": [ + { + "candidate_id": "wj-janitor-candidate", + "evidence_refs": ["wj-janitor-candidate-source"], + "review_required": true, + "promoted_to_memory": false + } + ] + } + ], + "latency_ms": 1.0, + "cost": { + "currency": "USD", + "amount": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + } + }, + "prompt": { + "role": "user", + "content": "Did the janitor candidate become authoritative memory automatically?", + "job_mode": "work_continuity", + "constraints": ["cite_evidence", "prevent_false_promotion"] + }, + "expected_answer": { + "must_include": [ + { + "claim_id": "janitor_no_false_promotion", + "text": "The Work Journal janitor candidate was not promoted to memory without review." + } + ], + "must_not_include": ["was promoted to memory without review"], + "evidence_links": { + "janitor_no_false_promotion": ["wj-janitor-candidate-source"] + }, + "answer_type": "work_journal_readback", + "accepted_alternates": [], + "requires_caveat": false, + "requires_refusal": false + }, + "required_evidence": [ + { + "evidence_id": "wj-janitor-candidate-source", + "claim_id": "janitor_no_false_promotion", + "requirement": "cite", + "quote": "must not be promoted" + } + ], + "work_continuity": { + "required_janitor_candidate_ids": ["wj-janitor-candidate"] + }, + "scoring_rubric": { + "dimensions": { + "answer_correctness": { + "weight": 0.35, + "max_points": 1.0, + "criteria": "Reports review-required janitor candidates." + }, + "evidence_grounding": { + "weight": 0.3, + "max_points": 1.0, + "criteria": "Cites the janitor candidate evidence." + }, + "trap_avoidance": { + "weight": 0.2, + "max_points": 1.0, + "criteria": "Does not false-promote journal entries." + }, + "lifecycle_behavior": { + "weight": 0.15, + "max_points": 1.0, + "criteria": "Preserves review boundary." + } + }, + "pass_threshold": 0.85, + "hard_fail_rules": ["janitor false promotion"] + }, + "allowed_uncertainty": { + "can_answer_unknown": false, + "acceptable_phrases": [], + "fallback_action": "state_blocker" + }, + "tags": ["synthetic", "work_continuity", "janitor"] +} diff --git a/apps/elf-eval/fixtures/real_world_memory/work_continuity/redaction_sensitive_marker.json b/apps/elf-eval/fixtures/real_world_memory/work_continuity/redaction_sensitive_marker.json new file mode 100644 index 00000000..bf8e59fa --- /dev/null +++ b/apps/elf-eval/fixtures/real_world_memory/work_continuity/redaction_sensitive_marker.json @@ -0,0 +1,156 @@ +{ + "schema": "elf.real_world_job/v1", + "job_id": "work-continuity-redaction-001", + "suite": "work_continuity", + "title": "Sensitive markers are redacted from Work Journal readback", + "corpus": { + "corpus_id": "real-world-memory-work-continuity-2026-06-27", + "profile": "synthetic", + "items": [ + { + "evidence_id": "wj-redaction-source", + "kind": "redaction_audit", + "text": "Redaction audit: marker secret-demo-token was detected and persisted only as REDACTED in the Work Journal entry.", + "source_ref": { + "schema": "source_ref/v1", + "resolver": "real_world_job_fixture/v1", + "ref": { + "fixture": "redaction_sensitive_marker", + "evidence_id": "wj-redaction-source" + }, + "locator": { + "quote": "secret-demo-token" + } + }, + "created_at": "2026-06-27T00:12:00Z" + } + ], + "adapter_response": { + "adapter_id": "fixture_work_continuity", + "answer": { + "content": "The Work Journal redaction audit shows marker secret-demo-token was redacted and no sensitive marker persisted.", + "claims": [ + { + "claim_id": "redaction_applied", + "text": "The sensitive marker secret-demo-token was redacted from the Work Journal readback.", + "evidence_ids": ["wj-redaction-source"], + "confidence": "high" + } + ], + "evidence_ids": ["wj-redaction-source"], + "work_journal_readbacks": [ + { + "readback_id": "wj-readback-redaction", + "contract_schema": "elf.work_journal/v1", + "generated_at": "2026-06-27T00:13:00Z", + "session_id": "work-continuity-session", + "tenant_id": "fixture-tenant", + "project_id": "elf", + "agent_id": "xy-1118-fixture-agent", + "read_profile": "private_plus_project", + "items": [ + { + "entry_id": "wj-redaction-entry", + "family": "redaction", + "title": "Redacted command output", + "body": "The command output contained a sensitive marker that was redacted before readback.", + "source_refs": ["wj-redaction-source"], + "redaction_audit": { + "required_marker_ids": ["secret-demo-token"], + "redacted_marker_ids": ["secret-demo-token"], + "persisted_sensitive_marker_ids": [] + } + } + ], + "where_stopped": { + "reset_resume_entry_ids": [], + "decision_rationale_evidence_ids": [], + "current_explicit_next_step_ids": [], + "labeled_inferred_next_step_ids": [], + "handoff_source_refs": ["wj-redaction-source"], + "journal_only_authority_claims": [] + }, + "promotion_boundary": { + "journal_entry_authority": "source_adjacent_only", + "memory_promotion_required": true, + "accepted_refs": [] + }, + "janitor_candidates": [] + } + ], + "latency_ms": 1.0, + "cost": { + "currency": "USD", + "amount": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + } + }, + "prompt": { + "role": "user", + "content": "Was the sensitive marker redacted?", + "job_mode": "work_continuity", + "constraints": ["cite_evidence", "preserve_redaction_audit"] + }, + "expected_answer": { + "must_include": [ + { + "claim_id": "redaction_applied", + "text": "The sensitive marker secret-demo-token was redacted from the Work Journal readback." + } + ], + "must_not_include": ["persisted sensitive marker"], + "evidence_links": { + "redaction_applied": ["wj-redaction-source"] + }, + "answer_type": "work_journal_readback", + "accepted_alternates": [], + "requires_caveat": false, + "requires_refusal": false + }, + "required_evidence": [ + { + "evidence_id": "wj-redaction-source", + "claim_id": "redaction_applied", + "requirement": "cite", + "quote": "secret-demo-token" + } + ], + "work_continuity": { + "required_redaction_marker_ids": ["secret-demo-token"] + }, + "scoring_rubric": { + "dimensions": { + "answer_correctness": { + "weight": 0.35, + "max_points": 1.0, + "criteria": "Reports redaction state." + }, + "evidence_grounding": { + "weight": 0.3, + "max_points": 1.0, + "criteria": "Cites the redaction audit." + }, + "trap_avoidance": { + "weight": 0.2, + "max_points": 1.0, + "criteria": "Does not persist sensitive markers." + }, + "lifecycle_behavior": { + "weight": 0.15, + "max_points": 1.0, + "criteria": "Keeps audit source-adjacent." + } + }, + "pass_threshold": 0.85, + "hard_fail_rules": ["sensitive-marker persistence"] + }, + "allowed_uncertainty": { + "can_answer_unknown": false, + "acceptable_phrases": [], + "fallback_action": "state_blocker" + }, + "tags": ["synthetic", "work_continuity", "redaction"] +} diff --git a/apps/elf-eval/fixtures/real_world_memory/work_continuity/rejected_option_suppression.json b/apps/elf-eval/fixtures/real_world_memory/work_continuity/rejected_option_suppression.json new file mode 100644 index 00000000..1ad959fa --- /dev/null +++ b/apps/elf-eval/fixtures/real_world_memory/work_continuity/rejected_option_suppression.json @@ -0,0 +1,159 @@ +{ + "schema": "elf.real_world_job/v1", + "job_id": "work-continuity-rejected-option-001", + "suite": "work_continuity", + "title": "Rejected options remain suppressed on resume", + "corpus": { + "corpus_id": "real-world-memory-work-continuity-2026-06-27", + "profile": "synthetic", + "items": [ + { + "evidence_id": "wj-rejected-option-source", + "kind": "rejected_option", + "text": "Rejected option: do not rewrite Work Journal entries into durable Memory Notes during janitor cleanup.", + "source_ref": { + "schema": "source_ref/v1", + "resolver": "real_world_job_fixture/v1", + "ref": { + "fixture": "rejected_option_suppression", + "evidence_id": "wj-rejected-option-source" + }, + "locator": { + "quote": "Rejected option" + } + }, + "created_at": "2026-06-27T00:04:00Z" + } + ], + "adapter_response": { + "adapter_id": "fixture_work_continuity", + "answer": { + "content": "The rejected janitor rewrite option remains suppressed and is not presented as a current instruction.", + "claims": [ + { + "claim_id": "rejected_option_suppressed", + "text": "The janitor rewrite option was rejected and must not be resurrected as current work.", + "evidence_ids": ["wj-rejected-option-source"], + "confidence": "high" + } + ], + "evidence_ids": ["wj-rejected-option-source"], + "work_journal_readbacks": [ + { + "readback_id": "wj-readback-rejected-option", + "contract_schema": "elf.work_journal/v1", + "generated_at": "2026-06-27T00:05:00Z", + "session_id": "work-continuity-session", + "tenant_id": "fixture-tenant", + "project_id": "elf", + "agent_id": "xy-1118-fixture-agent", + "read_profile": "private_plus_project", + "items": [ + { + "entry_id": "wj-rejected-entry", + "family": "rejected_option", + "title": "Rejected janitor rewrite", + "body": "The janitor rewrite option remains rejected.", + "source_refs": ["wj-rejected-option-source"], + "rejected_options": [ + { + "option_id": "wj-reject-autopromote", + "text": "Rewrite Work Journal entries into durable Memory Notes during janitor cleanup.", + "evidence_refs": ["wj-rejected-option-source"], + "resurrected_as_current": false + } + ] + } + ], + "where_stopped": { + "reset_resume_entry_ids": [], + "decision_rationale_evidence_ids": [], + "current_explicit_next_step_ids": [], + "labeled_inferred_next_step_ids": [], + "handoff_source_refs": ["wj-rejected-option-source"], + "journal_only_authority_claims": [] + }, + "promotion_boundary": { + "journal_entry_authority": "source_adjacent_only", + "memory_promotion_required": true, + "accepted_refs": [] + }, + "janitor_candidates": [] + } + ], + "latency_ms": 1.0, + "cost": { + "currency": "USD", + "amount": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + } + }, + "prompt": { + "role": "user", + "content": "Should the rejected janitor rewrite become a current next step?", + "job_mode": "work_continuity", + "constraints": ["cite_evidence", "suppress_rejected_options"] + }, + "expected_answer": { + "must_include": [ + { + "claim_id": "rejected_option_suppressed", + "text": "The janitor rewrite option was rejected and must not be resurrected as current work." + } + ], + "must_not_include": ["Rewrite Work Journal entries into durable Memory Notes during janitor cleanup."], + "evidence_links": { + "rejected_option_suppressed": ["wj-rejected-option-source"] + }, + "answer_type": "work_journal_readback", + "accepted_alternates": [], + "requires_caveat": false, + "requires_refusal": false + }, + "required_evidence": [ + { + "evidence_id": "wj-rejected-option-source", + "claim_id": "rejected_option_suppressed", + "requirement": "cite", + "quote": "Rejected option" + } + ], + "work_continuity": { + "required_rejected_option_ids": ["wj-reject-autopromote"] + }, + "scoring_rubric": { + "dimensions": { + "answer_correctness": { + "weight": 0.35, + "max_points": 1.0, + "criteria": "Suppresses rejected options." + }, + "evidence_grounding": { + "weight": 0.3, + "max_points": 1.0, + "criteria": "Cites the rejected option evidence." + }, + "trap_avoidance": { + "weight": 0.2, + "max_points": 1.0, + "criteria": "Does not resurrect rejected work." + }, + "lifecycle_behavior": { + "weight": 0.15, + "max_points": 1.0, + "criteria": "Preserves source-adjacent state." + } + }, + "pass_threshold": 0.85, + "hard_fail_rules": ["rejected-option resurrection"] + }, + "allowed_uncertainty": { + "can_answer_unknown": false, + "acceptable_phrases": [], + "fallback_action": "state_blocker" + }, + "tags": ["synthetic", "work_continuity", "rejected_option"] +} diff --git a/apps/elf-eval/fixtures/real_world_memory/work_continuity/reset_resume_session_readback.json b/apps/elf-eval/fixtures/real_world_memory/work_continuity/reset_resume_session_readback.json new file mode 100644 index 00000000..04afbfb4 --- /dev/null +++ b/apps/elf-eval/fixtures/real_world_memory/work_continuity/reset_resume_session_readback.json @@ -0,0 +1,151 @@ +{ + "schema": "elf.real_world_job/v1", + "job_id": "work-continuity-reset-resume-001", + "suite": "work_continuity", + "title": "Reset and resume reads the last Work Journal checkpoint", + "corpus": { + "corpus_id": "real-world-memory-work-continuity-2026-06-27", + "profile": "synthetic", + "items": [ + { + "evidence_id": "wj-reset-resume-checkpoint", + "kind": "work_journal_entry", + "text": "Work Journal reset checkpoint: after reset, resume from entry wj-reset-entry and rerun cargo make real-world-memory-work-continuity before review.", + "source_ref": { + "schema": "source_ref/v1", + "resolver": "real_world_job_fixture/v1", + "ref": { + "fixture": "reset_resume_session_readback", + "evidence_id": "wj-reset-resume-checkpoint" + }, + "locator": { + "quote": "resume from entry wj-reset-entry" + } + }, + "created_at": "2026-06-27T00:00:00Z" + } + ], + "adapter_response": { + "adapter_id": "fixture_work_continuity", + "answer": { + "content": "After reset, resume from Work Journal entry wj-reset-entry and rerun cargo make real-world-memory-work-continuity before review.", + "claims": [ + { + "claim_id": "reset_resume_readback", + "text": "Reset/resume readback returns entry wj-reset-entry as the current stopping point.", + "evidence_ids": ["wj-reset-resume-checkpoint"], + "confidence": "high" + } + ], + "evidence_ids": ["wj-reset-resume-checkpoint"], + "work_journal_readbacks": [ + { + "readback_id": "wj-readback-reset-resume", + "contract_schema": "elf.work_journal/v1", + "generated_at": "2026-06-27T00:01:00Z", + "session_id": "work-continuity-session", + "tenant_id": "fixture-tenant", + "project_id": "elf", + "agent_id": "xy-1118-fixture-agent", + "read_profile": "private_plus_project", + "items": [ + { + "entry_id": "wj-reset-entry", + "family": "handoff", + "title": "Reset resume checkpoint", + "body": "Resume from this Work Journal checkpoint after reset.", + "source_refs": ["wj-reset-resume-checkpoint"] + } + ], + "where_stopped": { + "reset_resume_entry_ids": ["wj-reset-entry"], + "decision_rationale_evidence_ids": [], + "current_explicit_next_step_ids": [], + "labeled_inferred_next_step_ids": [], + "handoff_source_refs": ["wj-reset-resume-checkpoint"], + "journal_only_authority_claims": [] + }, + "promotion_boundary": { + "journal_entry_authority": "source_adjacent_only", + "memory_promotion_required": true, + "accepted_refs": [] + }, + "janitor_candidates": [] + } + ], + "latency_ms": 1.0, + "cost": { + "currency": "USD", + "amount": 0.0, + "input_tokens": 0, + "output_tokens": 0 + } + } + } + }, + "prompt": { + "role": "user", + "content": "Where did the session stop after reset?", + "job_mode": "work_continuity", + "constraints": ["cite_evidence", "use_work_journal_readback"] + }, + "expected_answer": { + "must_include": [ + { + "claim_id": "reset_resume_readback", + "text": "Reset/resume readback returns entry wj-reset-entry as the current stopping point." + } + ], + "must_not_include": ["journal entry is current fact authority"], + "evidence_links": { + "reset_resume_readback": ["wj-reset-resume-checkpoint"] + }, + "answer_type": "work_journal_readback", + "accepted_alternates": [], + "requires_caveat": false, + "requires_refusal": false + }, + "required_evidence": [ + { + "evidence_id": "wj-reset-resume-checkpoint", + "claim_id": "reset_resume_readback", + "requirement": "cite", + "quote": "resume from entry wj-reset-entry" + } + ], + "work_continuity": { + "required_reset_resume_entry_ids": ["wj-reset-entry"] + }, + "scoring_rubric": { + "dimensions": { + "answer_correctness": { + "weight": 0.35, + "max_points": 1.0, + "criteria": "Returns the reset/resume Work Journal entry." + }, + "evidence_grounding": { + "weight": 0.3, + "max_points": 1.0, + "criteria": "Cites the Work Journal checkpoint source ref." + }, + "trap_avoidance": { + "weight": 0.2, + "max_points": 1.0, + "criteria": "Does not claim journal-only authority." + }, + "lifecycle_behavior": { + "weight": 0.15, + "max_points": 1.0, + "criteria": "Keeps the entry source-adjacent." + } + }, + "pass_threshold": 0.85, + "hard_fail_rules": ["journal-only authority claim"] + }, + "allowed_uncertainty": { + "can_answer_unknown": false, + "acceptable_phrases": [], + "fallback_action": "state_blocker" + }, + "tags": ["synthetic", "work_continuity", "reset_resume"] +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark.rs b/apps/elf-eval/src/bin/real_world_job_benchmark.rs index c2c08696..b694ebb9 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark.rs @@ -59,6 +59,7 @@ const SUITES: &[&str] = &[ "source_library", "operator_debugging_ux", "capture_integration", + "work_continuity", "production_ops", "personalization", "core_archival_memory", @@ -171,6 +172,7 @@ struct RealWorldJob { memory_summary: Option, proactive_brief: Option, scheduled_memory: Option, + work_continuity: Option, } #[derive(Debug, Deserialize)] @@ -396,6 +398,26 @@ struct ScheduledMemoryExpectation { required_task_kinds: Vec, } +#[derive(Debug, Deserialize)] +struct WorkContinuityExpectation { + #[serde(default)] + required_reset_resume_entry_ids: Vec, + #[serde(default)] + required_decision_rationale_evidence_ids: Vec, + #[serde(default)] + required_rejected_option_ids: Vec, + #[serde(default)] + required_explicit_next_step_ids: Vec, + #[serde(default)] + required_inferred_next_step_ids: Vec, + #[serde(default)] + required_handoff_source_ref_ids: Vec, + #[serde(default)] + required_redaction_marker_ids: Vec, + #[serde(default)] + required_janitor_candidate_ids: Vec, +} + #[derive(Debug, Deserialize)] struct ScoringRubric { #[serde(default)] @@ -442,6 +464,8 @@ struct ProducedAnswer { proactive_briefs: Vec, #[serde(default)] scheduled_tasks: Vec, + #[serde(default)] + work_journal_readbacks: Vec, #[serde(skip_serializing_if = "Option::is_none")] latency_ms: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -684,6 +708,116 @@ struct ScheduledMemoryTraceStage { evidence_refs: Vec, } +struct WorkContinuityObserved<'a> { + reset_resume_entry_ids: BTreeSet<&'a str>, + decision_rationale_evidence_ids: BTreeSet<&'a str>, + rejected_options: Vec<&'a WorkJournalRejectedOptionArtifact>, + explicit_next_steps: Vec<&'a WorkJournalNextStepArtifact>, + inferred_next_steps: Vec<&'a WorkJournalNextStepArtifact>, + handoff_source_refs: BTreeSet<&'a str>, + redacted_marker_ids: BTreeSet<&'a str>, + janitor_candidates: Vec<&'a WorkJournalJanitorCandidateArtifact>, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct WorkJournalReadbackArtifact { + readback_id: String, + contract_schema: String, + generated_at: String, + session_id: String, + tenant_id: String, + project_id: String, + agent_id: String, + read_profile: String, + #[serde(default)] + items: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + where_stopped: Option, + promotion_boundary: WorkJournalPromotionBoundaryArtifact, + #[serde(default)] + janitor_candidates: Vec, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +struct WorkJournalEntryArtifact { + entry_id: String, + family: String, + title: String, + body: String, + #[serde(default)] + source_refs: Vec, + #[serde(default)] + redaction_audit: WorkJournalRedactionAuditArtifact, + #[serde(default)] + explicit_next_steps: Vec, + #[serde(default)] + inferred_next_steps: Vec, + #[serde(default)] + rejected_options: Vec, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +struct WorkJournalRedactionAuditArtifact { + #[serde(default)] + required_marker_ids: Vec, + #[serde(default)] + redacted_marker_ids: Vec, + #[serde(default)] + persisted_sensitive_marker_ids: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct WorkJournalNextStepArtifact { + step_id: String, + text: String, + label: String, + instruction: bool, + #[serde(default)] + evidence_refs: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct WorkJournalRejectedOptionArtifact { + option_id: String, + text: String, + #[serde(default)] + evidence_refs: Vec, + resurrected_as_current: bool, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +struct WorkJournalWhereStoppedArtifact { + #[serde(default)] + reset_resume_entry_ids: Vec, + #[serde(default)] + decision_rationale_evidence_ids: Vec, + #[serde(default)] + current_explicit_next_step_ids: Vec, + #[serde(default)] + labeled_inferred_next_step_ids: Vec, + #[serde(default)] + handoff_source_refs: Vec, + #[serde(default)] + journal_only_authority_claims: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct WorkJournalPromotionBoundaryArtifact { + journal_entry_authority: String, + memory_promotion_required: bool, + #[serde(default)] + accepted_refs: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct WorkJournalJanitorCandidateArtifact { + candidate_id: String, + #[serde(default)] + evidence_refs: Vec, + review_required: bool, + promoted_to_memory: bool, +} + #[derive(Clone, Debug, Deserialize)] struct ConsolidationFixture { #[serde(default)] @@ -1269,6 +1403,8 @@ struct ReportSummary { #[serde(skip_serializing_if = "Option::is_none")] scheduled_memory: Option, #[serde(skip_serializing_if = "Option::is_none")] + work_continuity: Option, + #[serde(skip_serializing_if = "Option::is_none")] knowledge: Option, } @@ -1382,6 +1518,42 @@ struct ScheduledMemorySummaryReport { source_trace_tombstone_count: usize, } +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +struct WorkContinuitySummaryReport { + job_count: usize, + readback_count: usize, + entry_count: usize, + reset_resume_required_count: usize, + reset_resume_success_count: usize, + reset_resume_success_rate: f64, + decision_rationale_required_count: usize, + decision_rationale_recalled_count: usize, + decision_rationale_recall_rate: f64, + rejected_option_required_count: usize, + rejected_option_suppressed_count: usize, + rejected_option_resurrection_count: usize, + rejected_option_suppression_rate: f64, + explicit_next_step_required_count: usize, + explicit_next_step_returned_count: usize, + explicit_next_step_correct_count: usize, + explicit_next_step_precision: f64, + inferred_next_step_required_count: usize, + inferred_next_step_labeled_count: usize, + inferred_step_instruction_count: usize, + inferred_next_step_labeling_rate: f64, + handoff_source_ref_required_count: usize, + handoff_source_ref_covered_count: usize, + handoff_source_ref_coverage: f64, + redaction_required_count: usize, + redaction_applied_count: usize, + sensitive_marker_persistence_count: usize, + redaction_rate: f64, + janitor_candidate_count: usize, + janitor_false_promotion_count: usize, + janitor_false_promotion_rate: f64, + journal_only_authority_claim_count: usize, +} + #[derive(Clone, Debug, Default, Deserialize, Serialize)] struct KnowledgeSummary { job_count: usize, @@ -1465,6 +1637,8 @@ struct JobReport { proactive_brief: Option, #[serde(skip_serializing_if = "Option::is_none")] scheduled_memory: Option, + #[serde(skip_serializing_if = "Option::is_none")] + work_continuity: Option, trap_ids_used: Vec, dimension_scores: Vec, reason: String, @@ -1691,6 +1865,41 @@ struct ScheduledMemoryJobMetrics { source_trace_tombstone_count: usize, } +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +struct WorkContinuityJobMetrics { + readback_count: usize, + entry_count: usize, + reset_resume_required_count: usize, + reset_resume_success_count: usize, + reset_resume_success_rate: f64, + decision_rationale_required_count: usize, + decision_rationale_recalled_count: usize, + decision_rationale_recall_rate: f64, + rejected_option_required_count: usize, + rejected_option_suppressed_count: usize, + rejected_option_resurrection_count: usize, + rejected_option_suppression_rate: f64, + explicit_next_step_required_count: usize, + explicit_next_step_returned_count: usize, + explicit_next_step_correct_count: usize, + explicit_next_step_precision: f64, + inferred_next_step_required_count: usize, + inferred_next_step_labeled_count: usize, + inferred_step_instruction_count: usize, + inferred_next_step_labeling_rate: f64, + handoff_source_ref_required_count: usize, + handoff_source_ref_covered_count: usize, + handoff_source_ref_coverage: f64, + redaction_required_count: usize, + redaction_applied_count: usize, + sensitive_marker_persistence_count: usize, + redaction_rate: f64, + janitor_candidate_count: usize, + janitor_false_promotion_count: usize, + janitor_false_promotion_rate: f64, + journal_only_authority_claim_count: usize, +} + #[derive(Clone, Debug, Default, Deserialize, Serialize)] struct EvolutionSummary { stale_answer_count: usize, @@ -1759,6 +1968,7 @@ struct JobScoring { memory_summary: Option, proactive_brief: Option, scheduled_memory: Option, + work_continuity: Option, } #[derive(Debug, Default)] @@ -1802,6 +2012,19 @@ struct FailureCounts { scheduled_memory_unsupported_current_outputs: usize, scheduled_memory_tombstone_violations: usize, scheduled_memory_missing_trace: usize, + work_continuity_reset_resume_missing: usize, + work_continuity_decision_rationale_missing: usize, + work_continuity_rejected_option_unsuppressed: usize, + work_continuity_rejected_option_resurrection: usize, + work_continuity_explicit_next_step_missing: usize, + work_continuity_explicit_next_step_extra: usize, + work_continuity_inferred_step_unlabeled: usize, + work_continuity_inferred_step_as_instruction: usize, + work_continuity_handoff_source_ref_missing: usize, + work_continuity_redaction_missing: usize, + work_continuity_sensitive_marker_persistence: usize, + work_continuity_janitor_false_promotion: usize, + work_continuity_journal_only_authority_claim: usize, untraced_page_sections: usize, missed_stale_findings: usize, rebuild_failures: usize, @@ -1932,6 +2155,7 @@ fn validate_job(job: &RealWorldJob, path: &Path) -> Result<()> { validate_memory_summary_expectation(job, path)?; validate_proactive_brief_expectation(job, path)?; validate_scheduled_memory_expectation(job, path)?; + validate_work_continuity_expectation(job, path)?; validate_trace_explainability(job, path)?; Ok(()) @@ -2217,6 +2441,9 @@ fn validate_adapter_response(job: &RealWorldJob, path: &Path) -> Result<()> { for task in &adapter_response.answer.scheduled_tasks { validate_scheduled_memory_artifact(task, path, &evidence_ids)?; } + for readback in &adapter_response.answer.work_journal_readbacks { + validate_work_journal_readback_artifact(readback, path, &evidence_ids)?; + } if job.suite == "memory_summary" && adapter_response.answer.memory_summaries.is_empty() @@ -2245,6 +2472,15 @@ fn validate_adapter_response(job: &RealWorldJob, path: &Path) -> Result<()> { path.display() )); } + if job.suite == "work_continuity" + && adapter_response.answer.work_journal_readbacks.is_empty() + && job.encoding.status.is_none() + { + return Err(eyre::eyre!( + "{} work_continuity jobs must provide adapter_response.answer.work_journal_readbacks.", + path.display() + )); + } Ok(()) } @@ -2749,6 +2985,161 @@ fn validate_scheduled_memory_trace( Ok(()) } +fn validate_work_journal_readback_artifact( + readback: &WorkJournalReadbackArtifact, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + if readback.readback_id.trim().is_empty() + || readback.contract_schema != "elf.work_journal/v1" + || readback.generated_at.trim().is_empty() + || readback.session_id.trim().is_empty() + || readback.tenant_id.trim().is_empty() + || readback.project_id.trim().is_empty() + || readback.agent_id.trim().is_empty() + || readback.read_profile.trim().is_empty() + || readback.items.is_empty() + { + return Err(eyre::eyre!("{} has an incomplete Work Journal readback.", path.display())); + } + + validate_optional_rfc3339(&readback.generated_at, path, readback.readback_id.as_str())?; + + if readback.promotion_boundary.journal_entry_authority.trim().is_empty() { + return Err(eyre::eyre!( + "{} Work Journal readback {} has an incomplete promotion boundary.", + path.display(), + readback.readback_id + )); + } + + for accepted_ref in &readback.promotion_boundary.accepted_refs { + if accepted_ref.trim().is_empty() { + return Err(eyre::eyre!( + "{} Work Journal readback {} has an empty accepted ref.", + path.display(), + readback.readback_id + )); + } + } + for item in &readback.items { + validate_work_journal_entry(item, path, evidence_ids)?; + } + + if let Some(where_stopped) = &readback.where_stopped { + validate_work_journal_where_stopped(where_stopped, path, evidence_ids)?; + } + + for candidate in &readback.janitor_candidates { + if candidate.candidate_id.trim().is_empty() { + return Err(eyre::eyre!( + "{} Work Journal readback {} has an empty janitor candidate id.", + path.display(), + readback.readback_id + )); + } + + for evidence_ref in &candidate.evidence_refs { + ensure_known_evidence(path, evidence_ids, evidence_ref)?; + } + } + + Ok(()) +} + +fn validate_work_journal_entry( + entry: &WorkJournalEntryArtifact, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + if entry.entry_id.trim().is_empty() + || entry.family.trim().is_empty() + || entry.title.trim().is_empty() + || entry.body.trim().is_empty() + || entry.source_refs.is_empty() + { + return Err(eyre::eyre!("{} has an incomplete Work Journal entry.", path.display())); + } + + for source_ref in &entry.source_refs { + ensure_known_evidence(path, evidence_ids, source_ref)?; + } + for marker_id in entry + .redaction_audit + .required_marker_ids + .iter() + .chain(entry.redaction_audit.redacted_marker_ids.iter()) + .chain(entry.redaction_audit.persisted_sensitive_marker_ids.iter()) + { + if marker_id.trim().is_empty() { + return Err(eyre::eyre!( + "{} Work Journal entry {} has an empty redaction marker id.", + path.display(), + entry.entry_id + )); + } + } + for step in entry.explicit_next_steps.iter().chain(entry.inferred_next_steps.iter()) { + validate_work_journal_next_step(step, path, evidence_ids)?; + } + for option in &entry.rejected_options { + if option.option_id.trim().is_empty() || option.text.trim().is_empty() { + return Err(eyre::eyre!( + "{} Work Journal entry {} has an incomplete rejected option.", + path.display(), + entry.entry_id + )); + } + + for evidence_ref in &option.evidence_refs { + ensure_known_evidence(path, evidence_ids, evidence_ref)?; + } + } + + Ok(()) +} + +fn validate_work_journal_next_step( + step: &WorkJournalNextStepArtifact, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + if step.step_id.trim().is_empty() || step.text.trim().is_empty() || step.label.trim().is_empty() + { + return Err(eyre::eyre!("{} has an incomplete Work Journal next step.", path.display())); + } + + for evidence_ref in &step.evidence_refs { + ensure_known_evidence(path, evidence_ids, evidence_ref)?; + } + + Ok(()) +} + +fn validate_work_journal_where_stopped( + where_stopped: &WorkJournalWhereStoppedArtifact, + path: &Path, + evidence_ids: &BTreeSet, +) -> Result<()> { + for evidence_ref in where_stopped + .decision_rationale_evidence_ids + .iter() + .chain(where_stopped.handoff_source_refs.iter()) + { + ensure_known_evidence(path, evidence_ids, evidence_ref)?; + } + for claim in &where_stopped.journal_only_authority_claims { + if claim.trim().is_empty() { + return Err(eyre::eyre!( + "{} has an empty Work Journal journal-only authority claim.", + path.display() + )); + } + } + + Ok(()) +} + fn validate_optional_summary_time(path: &Path, value: Option<&str>, id: &str) -> Result<()> { if let Some(value) = value { validate_optional_rfc3339(value, path, id)?; @@ -3062,6 +3453,46 @@ fn validate_scheduled_memory_expectation(job: &RealWorldJob, path: &Path) -> Res Ok(()) } +fn validate_work_continuity_expectation(job: &RealWorldJob, path: &Path) -> Result<()> { + let Some(work_continuity) = &job.work_continuity else { + if job.suite == "work_continuity" && job.encoding.status.is_none() { + return Err(eyre::eyre!( + "{} work_continuity jobs must provide work_continuity expectations.", + path.display() + )); + } + + return Ok(()); + }; + let evidence_ids = corpus_evidence_ids(job); + + for value in work_continuity + .required_reset_resume_entry_ids + .iter() + .chain(work_continuity.required_rejected_option_ids.iter()) + .chain(work_continuity.required_explicit_next_step_ids.iter()) + .chain(work_continuity.required_inferred_next_step_ids.iter()) + .chain(work_continuity.required_redaction_marker_ids.iter()) + .chain(work_continuity.required_janitor_candidate_ids.iter()) + { + if value.trim().is_empty() { + return Err(eyre::eyre!( + "{} work_continuity expectations contain an empty required id.", + path.display() + )); + } + } + for evidence_ref in work_continuity + .required_decision_rationale_evidence_ids + .iter() + .chain(work_continuity.required_handoff_source_ref_ids.iter()) + { + ensure_known_evidence(path, &evidence_ids, evidence_ref)?; + } + + Ok(()) +} + fn validate_evolution_conflict( path: &Path, evidence_ids: &BTreeSet, @@ -3333,6 +3764,7 @@ fn score_job(job: &RealWorldJob) -> JobScoring { let memory_summary = memory_summary_metrics(job, answer); let proactive_brief = proactive_brief_metrics(job, answer); let scheduled_memory = scheduled_memory_metrics(job, answer); + let work_continuity = work_continuity_metrics(job, answer); let mut unsupported_claims = unsupported_claims(job, answer); unsupported_claims.extend(unsupported_page_claims(answer)); @@ -3380,6 +3812,7 @@ fn score_job(job: &RealWorldJob) -> JobScoring { apply_memory_summary_failure_counts(&mut counts, memory_summary.as_ref()); apply_proactive_brief_failure_counts(&mut counts, proactive_brief.as_ref()); apply_scheduled_memory_failure_counts(&mut counts, scheduled_memory.as_ref()); + apply_work_continuity_failure_counts(&mut counts, work_continuity.as_ref()); let dimension_scores = dimension_scores(job, &counts); let normalized_score = normalized_score(&dimension_scores); @@ -3414,6 +3847,7 @@ fn score_job(job: &RealWorldJob) -> JobScoring { memory_summary, proactive_brief, scheduled_memory, + work_continuity, } } @@ -3477,6 +3911,46 @@ fn apply_scheduled_memory_failure_counts( counts.source_mutations += metrics.source_mutation_count; } +fn apply_work_continuity_failure_counts( + counts: &mut FailureCounts, + metrics: Option<&WorkContinuityJobMetrics>, +) { + let Some(metrics) = metrics else { + return; + }; + + counts.work_continuity_reset_resume_missing = + metrics.reset_resume_required_count.saturating_sub(metrics.reset_resume_success_count); + counts.work_continuity_decision_rationale_missing = metrics + .decision_rationale_required_count + .saturating_sub(metrics.decision_rationale_recalled_count); + counts.work_continuity_rejected_option_unsuppressed = metrics + .rejected_option_required_count + .saturating_sub(metrics.rejected_option_suppressed_count); + counts.work_continuity_rejected_option_resurrection = + metrics.rejected_option_resurrection_count; + counts.work_continuity_explicit_next_step_missing = metrics + .explicit_next_step_required_count + .saturating_sub(metrics.explicit_next_step_correct_count); + counts.work_continuity_explicit_next_step_extra = metrics + .explicit_next_step_returned_count + .saturating_sub(metrics.explicit_next_step_correct_count); + counts.work_continuity_inferred_step_unlabeled = metrics + .inferred_next_step_required_count + .saturating_sub(metrics.inferred_next_step_labeled_count); + counts.work_continuity_inferred_step_as_instruction = metrics.inferred_step_instruction_count; + counts.work_continuity_handoff_source_ref_missing = metrics + .handoff_source_ref_required_count + .saturating_sub(metrics.handoff_source_ref_covered_count); + counts.work_continuity_redaction_missing = + metrics.redaction_required_count.saturating_sub(metrics.redaction_applied_count); + counts.work_continuity_sensitive_marker_persistence = + metrics.sensitive_marker_persistence_count; + counts.work_continuity_janitor_false_promotion = metrics.janitor_false_promotion_count; + counts.work_continuity_journal_only_authority_claim = + metrics.journal_only_authority_claim_count; +} + fn score_declared_job( job: &RealWorldJob, status: TypedStatus, @@ -3503,6 +3977,7 @@ fn score_declared_job( memory_summary: None, proactive_brief: None, scheduled_memory: None, + work_continuity: None, } } @@ -3541,6 +4016,7 @@ fn wrong_result_count(counts: &FailureCounts) -> usize { + counts.scheduled_memory_unsupported_current_outputs + counts.scheduled_memory_tombstone_violations + counts.scheduled_memory_missing_trace + + work_continuity_wrong_result_count(counts) + counts.untraced_page_sections + counts.missed_stale_findings + counts.rebuild_failures @@ -3597,6 +4073,7 @@ fn synthetic_answer(job: &RealWorldJob) -> &ProducedAnswer { memory_summaries: Vec::new(), proactive_briefs: Vec::new(), scheduled_tasks: Vec::new(), + work_journal_readbacks: Vec::new(), latency_ms: None, cost: None, trace_explainability: None, @@ -3619,6 +4096,27 @@ fn produced_evidence_ids(answer: &ProducedAnswer) -> BTreeSet { evidence.extend(output.evidence_refs.iter().cloned()); } } + for readback in &answer.work_journal_readbacks { + for entry in &readback.items { + evidence.extend(entry.source_refs.iter().cloned()); + + for step in entry.explicit_next_steps.iter().chain(entry.inferred_next_steps.iter()) { + evidence.extend(step.evidence_refs.iter().cloned()); + } + for option in &entry.rejected_options { + evidence.extend(option.evidence_refs.iter().cloned()); + } + } + + if let Some(where_stopped) = &readback.where_stopped { + evidence.extend(where_stopped.decision_rationale_evidence_ids.iter().cloned()); + evidence.extend(where_stopped.handoff_source_refs.iter().cloned()); + } + + for candidate in &readback.janitor_candidates { + evidence.extend(candidate.evidence_refs.iter().cloned()); + } + } evidence } @@ -4712,6 +5210,281 @@ fn scheduled_unsupported_claim_report( } } +fn work_continuity_metrics( + job: &RealWorldJob, + answer: &ProducedAnswer, +) -> Option { + if job.work_continuity.is_none() && answer.work_journal_readbacks.is_empty() { + return None; + } + + let expectation = job.work_continuity.as_ref(); + let observed = work_continuity_observed(answer); + let mut metrics = initial_work_continuity_metrics(expectation, answer); + + if let Some(expected) = expectation { + apply_expected_work_continuity_counts(&mut metrics, expected, &observed); + } + + apply_observed_work_continuity_counts(&mut metrics, answer, &observed); + apply_work_continuity_rates(&mut metrics); + + Some(metrics) +} + +fn work_continuity_observed(answer: &ProducedAnswer) -> WorkContinuityObserved<'_> { + WorkContinuityObserved { + reset_resume_entry_ids: work_journal_reset_resume_entry_ids(answer), + decision_rationale_evidence_ids: work_journal_decision_rationale_evidence_ids(answer), + rejected_options: work_journal_rejected_options(answer), + explicit_next_steps: work_journal_explicit_next_steps(answer), + inferred_next_steps: work_journal_inferred_next_steps(answer), + handoff_source_refs: work_journal_handoff_source_refs(answer), + redacted_marker_ids: work_journal_redacted_marker_ids(answer), + janitor_candidates: work_journal_janitor_candidates(answer), + } +} + +fn initial_work_continuity_metrics( + expectation: Option<&WorkContinuityExpectation>, + answer: &ProducedAnswer, +) -> WorkContinuityJobMetrics { + WorkContinuityJobMetrics { + readback_count: answer.work_journal_readbacks.len(), + entry_count: answer + .work_journal_readbacks + .iter() + .map(|readback| readback.items.len()) + .sum(), + reset_resume_required_count: expectation + .map_or(0, |expected| expected.required_reset_resume_entry_ids.len()), + decision_rationale_required_count: expectation + .map_or(0, |expected| expected.required_decision_rationale_evidence_ids.len()), + rejected_option_required_count: expectation + .map_or(0, |expected| expected.required_rejected_option_ids.len()), + explicit_next_step_required_count: expectation + .map_or(0, |expected| expected.required_explicit_next_step_ids.len()), + inferred_next_step_required_count: expectation + .map_or(0, |expected| expected.required_inferred_next_step_ids.len()), + handoff_source_ref_required_count: expectation + .map_or(0, |expected| expected.required_handoff_source_ref_ids.len()), + redaction_required_count: expectation + .map_or(0, |expected| expected.required_redaction_marker_ids.len()), + janitor_candidate_count: expectation + .map_or(0, |expected| expected.required_janitor_candidate_ids.len()), + ..WorkContinuityJobMetrics::default() + } +} + +fn apply_expected_work_continuity_counts( + metrics: &mut WorkContinuityJobMetrics, + expected: &WorkContinuityExpectation, + observed: &WorkContinuityObserved<'_>, +) { + metrics.reset_resume_success_count = expected + .required_reset_resume_entry_ids + .iter() + .filter(|entry_id| observed.reset_resume_entry_ids.contains(entry_id.as_str())) + .count(); + metrics.decision_rationale_recalled_count = expected + .required_decision_rationale_evidence_ids + .iter() + .filter(|evidence_id| { + observed.decision_rationale_evidence_ids.contains(evidence_id.as_str()) + }) + .count(); + metrics.rejected_option_suppressed_count = expected + .required_rejected_option_ids + .iter() + .filter(|option_id| { + observed + .rejected_options + .iter() + .any(|option| option.option_id == **option_id && !option.resurrected_as_current) + }) + .count(); + metrics.explicit_next_step_correct_count = expected + .required_explicit_next_step_ids + .iter() + .filter(|step_id| { + observed.explicit_next_steps.iter().any(|step| { + step.step_id == **step_id && step.label == "explicit" && step.instruction + }) + }) + .count(); + metrics.inferred_next_step_labeled_count = expected + .required_inferred_next_step_ids + .iter() + .filter(|step_id| { + observed.inferred_next_steps.iter().any(|step| { + step.step_id == **step_id && step.label == "inferred" && !step.instruction + }) + }) + .count(); + metrics.handoff_source_ref_covered_count = expected + .required_handoff_source_ref_ids + .iter() + .filter(|source_ref| observed.handoff_source_refs.contains(source_ref.as_str())) + .count(); + metrics.redaction_applied_count = expected + .required_redaction_marker_ids + .iter() + .filter(|marker_id| observed.redacted_marker_ids.contains(marker_id.as_str())) + .count(); +} + +fn apply_observed_work_continuity_counts( + metrics: &mut WorkContinuityJobMetrics, + answer: &ProducedAnswer, + observed: &WorkContinuityObserved<'_>, +) { + metrics.janitor_candidate_count = + metrics.janitor_candidate_count.max(observed.janitor_candidates.len()); + metrics.janitor_false_promotion_count = observed + .janitor_candidates + .iter() + .filter(|candidate| candidate.promoted_to_memory || !candidate.review_required) + .count(); + metrics.explicit_next_step_returned_count = observed.explicit_next_steps.len(); + metrics.rejected_option_resurrection_count = + observed.rejected_options.iter().filter(|option| option.resurrected_as_current).count(); + metrics.inferred_step_instruction_count = + observed.inferred_next_steps.iter().filter(|step| step.instruction).count(); + metrics.sensitive_marker_persistence_count = answer + .work_journal_readbacks + .iter() + .flat_map(|readback| readback.items.iter()) + .map(|entry| entry.redaction_audit.persisted_sensitive_marker_ids.len()) + .sum(); + metrics.journal_only_authority_claim_count = + answer.work_journal_readbacks.iter().map(work_journal_authority_claim_count).sum(); +} + +fn apply_work_continuity_rates(metrics: &mut WorkContinuityJobMetrics) { + metrics.reset_resume_success_rate = + ratio(metrics.reset_resume_success_count, metrics.reset_resume_required_count); + metrics.decision_rationale_recall_rate = + ratio(metrics.decision_rationale_recalled_count, metrics.decision_rationale_required_count); + metrics.rejected_option_suppression_rate = + ratio(metrics.rejected_option_suppressed_count, metrics.rejected_option_required_count); + metrics.explicit_next_step_precision = ratio_or( + metrics.explicit_next_step_correct_count, + metrics.explicit_next_step_returned_count, + usize::from(metrics.explicit_next_step_required_count == 0) as f64, + ); + metrics.inferred_next_step_labeling_rate = + ratio(metrics.inferred_next_step_labeled_count, metrics.inferred_next_step_required_count); + metrics.handoff_source_ref_coverage = + ratio(metrics.handoff_source_ref_covered_count, metrics.handoff_source_ref_required_count); + metrics.redaction_rate = + ratio(metrics.redaction_applied_count, metrics.redaction_required_count); + metrics.janitor_false_promotion_rate = + ratio(metrics.janitor_false_promotion_count, metrics.janitor_candidate_count); +} + +fn work_journal_reset_resume_entry_ids(answer: &ProducedAnswer) -> BTreeSet<&str> { + answer + .work_journal_readbacks + .iter() + .filter_map(|readback| readback.where_stopped.as_ref()) + .flat_map(|where_stopped| where_stopped.reset_resume_entry_ids.iter().map(String::as_str)) + .collect() +} + +fn work_journal_decision_rationale_evidence_ids(answer: &ProducedAnswer) -> BTreeSet<&str> { + answer + .work_journal_readbacks + .iter() + .filter_map(|readback| readback.where_stopped.as_ref()) + .flat_map(|where_stopped| { + where_stopped.decision_rationale_evidence_ids.iter().map(String::as_str) + }) + .collect() +} + +fn work_journal_rejected_options( + answer: &ProducedAnswer, +) -> Vec<&WorkJournalRejectedOptionArtifact> { + answer + .work_journal_readbacks + .iter() + .flat_map(|readback| readback.items.iter()) + .flat_map(|entry| entry.rejected_options.iter()) + .collect() +} + +fn work_journal_explicit_next_steps(answer: &ProducedAnswer) -> Vec<&WorkJournalNextStepArtifact> { + answer + .work_journal_readbacks + .iter() + .flat_map(|readback| readback.items.iter()) + .flat_map(|entry| entry.explicit_next_steps.iter()) + .collect() +} + +fn work_journal_inferred_next_steps(answer: &ProducedAnswer) -> Vec<&WorkJournalNextStepArtifact> { + answer + .work_journal_readbacks + .iter() + .flat_map(|readback| readback.items.iter()) + .flat_map(|entry| entry.inferred_next_steps.iter()) + .collect() +} + +fn work_journal_handoff_source_refs(answer: &ProducedAnswer) -> BTreeSet<&str> { + let mut refs = answer + .work_journal_readbacks + .iter() + .flat_map(|readback| readback.items.iter()) + .flat_map(|entry| entry.source_refs.iter().map(String::as_str)) + .collect::>(); + + for source_ref in answer + .work_journal_readbacks + .iter() + .filter_map(|readback| readback.where_stopped.as_ref()) + .flat_map(|where_stopped| where_stopped.handoff_source_refs.iter().map(String::as_str)) + { + refs.insert(source_ref); + } + + refs +} + +fn work_journal_redacted_marker_ids(answer: &ProducedAnswer) -> BTreeSet<&str> { + answer + .work_journal_readbacks + .iter() + .flat_map(|readback| readback.items.iter()) + .flat_map(|entry| entry.redaction_audit.redacted_marker_ids.iter().map(String::as_str)) + .collect() +} + +fn work_journal_janitor_candidates( + answer: &ProducedAnswer, +) -> Vec<&WorkJournalJanitorCandidateArtifact> { + answer + .work_journal_readbacks + .iter() + .flat_map(|readback| readback.janitor_candidates.iter()) + .collect() +} + +fn work_journal_authority_claim_count(readback: &WorkJournalReadbackArtifact) -> usize { + let boundary_claim_count = + usize::from(readback.promotion_boundary.journal_entry_authority != "source_adjacent_only"); + let missing_promotion_boundary_count = usize::from( + !readback.promotion_boundary.memory_promotion_required + && !readback.promotion_boundary.accepted_refs.is_empty(), + ); + let where_stopped_claim_count = readback + .where_stopped + .as_ref() + .map_or(0, |where_stopped| where_stopped.journal_only_authority_claims.len()); + + boundary_claim_count + missing_promotion_boundary_count + where_stopped_claim_count +} + fn hard_fail_hits( job: &RealWorldJob, unsupported_claims: &[UnsupportedClaimReport], @@ -4735,6 +5508,23 @@ fn hard_fail_hits( hits.push("missing required refusal".to_string()); } + if let Some(work_continuity) = work_continuity_metrics(job, produced_answer(job)) { + if work_continuity.sensitive_marker_persistence_count > 0 { + hits.push("sensitive-marker persistence in Work Journal output".to_string()); + } + if work_continuity.rejected_option_resurrection_count > 0 { + hits.push("rejected-option resurrection in Work Journal readback".to_string()); + } + if work_continuity.inferred_step_instruction_count > 0 { + hits.push("inferred Work Journal next step surfaced as an instruction".to_string()); + } + if work_continuity.journal_only_authority_claim_count > 0 { + hits.push("journal-only Work Journal content claimed as current authority".to_string()); + } + if work_continuity.janitor_false_promotion_count > 0 { + hits.push("janitor Work Journal candidate promoted without review".to_string()); + } + } if let Some(consolidation) = consolidation_job_report(job) { if consolidation.source_mutation_count > 0 { hits.push( @@ -4796,6 +5586,16 @@ fn dimension_score(dimension_id: &str, max_points: f64, counts: &FailureCounts) || counts.scheduled_memory_unsupported_current_outputs > 0 || counts.scheduled_memory_tombstone_violations > 0 || counts.scheduled_memory_missing_trace > 0 + || counts.work_continuity_reset_resume_missing > 0 + || counts.work_continuity_decision_rationale_missing > 0 + || counts.work_continuity_rejected_option_unsuppressed > 0 + || counts.work_continuity_rejected_option_resurrection > 0 + || counts.work_continuity_explicit_next_step_missing > 0 + || counts.work_continuity_explicit_next_step_extra > 0 + || counts.work_continuity_inferred_step_unlabeled > 0 + || counts.work_continuity_inferred_step_as_instruction > 0 + || counts.work_continuity_janitor_false_promotion > 0 + || counts.work_continuity_journal_only_authority_claim > 0 || counts.page_usefulness_failures > 0, "evidence_grounding" => counts.missing_evidence > 0 @@ -4805,6 +5605,10 @@ fn dimension_score(dimension_id: &str, max_points: f64, counts: &FailureCounts) || counts.proactive_brief_untraced_suggestions > 0 || counts.scheduled_memory_untraced_outputs > 0 || counts.scheduled_memory_missing_trace > 0 + || counts.work_continuity_decision_rationale_missing > 0 + || counts.work_continuity_handoff_source_ref_missing > 0 + || counts.work_continuity_redaction_missing > 0 + || counts.work_continuity_sensitive_marker_persistence > 0 || counts.untraced_page_sections > 0, "trap_avoidance" => counts.trap_uses > 0 @@ -4813,12 +5617,15 @@ fn dimension_score(dimension_id: &str, max_points: f64, counts: &FailureCounts) || counts.proactive_brief_tombstone_violations > 0 || counts.scheduled_memory_invalid_current_outputs > 0 || counts.scheduled_memory_tombstone_violations > 0 + || counts.work_continuity_rejected_option_resurrection > 0 + || counts.work_continuity_sensitive_marker_persistence > 0 || counts.missed_stale_findings > 0, "uncertainty_handling" => counts.unsupported_claims > 0 || counts.memory_summary_unsupported_current_entries > 0 || counts.proactive_brief_unsupported_current_suggestions > 0 - || counts.scheduled_memory_unsupported_current_outputs > 0, + || counts.scheduled_memory_unsupported_current_outputs > 0 + || counts.work_continuity_journal_only_authority_claim > 0, "lifecycle_behavior" => counts.stale_answers > 0 || counts.conflict_detection_missing > 0 @@ -4839,6 +5646,10 @@ fn dimension_score(dimension_id: &str, max_points: f64, counts: &FailureCounts) || counts.scheduled_memory_unsupported_current_outputs > 0 || counts.scheduled_memory_tombstone_violations > 0 || counts.scheduled_memory_missing_trace > 0 + || counts.work_continuity_reset_resume_missing > 0 + || counts.work_continuity_inferred_step_as_instruction > 0 + || counts.work_continuity_janitor_false_promotion > 0 + || counts.work_continuity_journal_only_authority_claim > 0 || counts.rebuild_failures > 0, "source_immutability" => counts.source_mutations > 0, "proposal_usefulness" => counts.proposal_usefulness_failures > 0, @@ -4850,7 +5661,8 @@ fn dimension_score(dimension_id: &str, max_points: f64, counts: &FailureCounts) || counts.operator_debug_missing > 0 || counts.operator_debug_raw_sql > 0 || counts.operator_debug_trace_gaps > 0 - || counts.scheduled_memory_missing_trace > 0, + || counts.scheduled_memory_missing_trace > 0 + || counts.work_continuity_reset_resume_missing > 0, "trace_readback" => counts.scheduled_memory_missing_trace > 0, "latency_resource" => counts.latency_violations > 0, "personalization_fit" | "ownership_correctness" => @@ -4974,12 +5786,29 @@ fn wrong_result_signal_count(counts: &FailureCounts) -> usize { + counts.scheduled_memory_unsupported_current_outputs + counts.scheduled_memory_tombstone_violations + counts.scheduled_memory_missing_trace + + work_continuity_wrong_result_count(counts) + counts.untraced_page_sections + counts.missed_stale_findings + counts.rebuild_failures + counts.page_usefulness_failures } +fn work_continuity_wrong_result_count(counts: &FailureCounts) -> usize { + counts.work_continuity_reset_resume_missing + + counts.work_continuity_decision_rationale_missing + + counts.work_continuity_rejected_option_unsuppressed + + counts.work_continuity_rejected_option_resurrection + + counts.work_continuity_explicit_next_step_missing + + counts.work_continuity_explicit_next_step_extra + + counts.work_continuity_inferred_step_unlabeled + + counts.work_continuity_inferred_step_as_instruction + + counts.work_continuity_handoff_source_ref_missing + + counts.work_continuity_redaction_missing + + counts.work_continuity_sensitive_marker_persistence + + counts.work_continuity_janitor_false_promotion + + counts.work_continuity_journal_only_authority_claim +} + fn job_report(job: &RealWorldJob, scoring: JobScoring) -> JobReport { let answer = produced_answer(job); let metrics = job_metrics(job, answer); @@ -5030,6 +5859,7 @@ fn job_report(job: &RealWorldJob, scoring: JobScoring) -> JobReport { memory_summary: scoring.memory_summary, proactive_brief: scoring.proactive_brief, scheduled_memory: scoring.scheduled_memory, + work_continuity: scoring.work_continuity, trap_ids_used: scoring.trap_ids_used, dimension_scores: scoring.dimension_scores, reason: scoring.reason, @@ -5534,6 +6364,7 @@ fn report_summary(jobs: &[JobReport], suites: &[SuiteReport]) -> ReportSummary { memory_summary: memory_summary_summary(jobs), proactive_brief: proactive_brief_summary(jobs), scheduled_memory: scheduled_memory_summary(jobs), + work_continuity: work_continuity_summary(jobs), knowledge: knowledge_summary(jobs), ..ReportSummary::default() }; @@ -6294,6 +7125,113 @@ fn scheduled_memory_summary(jobs: &[JobReport]) -> Option Option { + let work_jobs = jobs.iter().filter_map(|job| job.work_continuity.as_ref()).collect::>(); + + if work_jobs.is_empty() { + return None; + } + + let reset_resume_required_count = + work_jobs.iter().map(|metrics| metrics.reset_resume_required_count).sum(); + let reset_resume_success_count = + work_jobs.iter().map(|metrics| metrics.reset_resume_success_count).sum(); + let decision_rationale_required_count = + work_jobs.iter().map(|metrics| metrics.decision_rationale_required_count).sum(); + let decision_rationale_recalled_count = + work_jobs.iter().map(|metrics| metrics.decision_rationale_recalled_count).sum(); + let rejected_option_required_count = + work_jobs.iter().map(|metrics| metrics.rejected_option_required_count).sum(); + let rejected_option_suppressed_count = + work_jobs.iter().map(|metrics| metrics.rejected_option_suppressed_count).sum(); + let explicit_next_step_returned_count = + work_jobs.iter().map(|metrics| metrics.explicit_next_step_returned_count).sum(); + let explicit_next_step_correct_count = + work_jobs.iter().map(|metrics| metrics.explicit_next_step_correct_count).sum(); + let inferred_next_step_required_count = + work_jobs.iter().map(|metrics| metrics.inferred_next_step_required_count).sum(); + let inferred_next_step_labeled_count = + work_jobs.iter().map(|metrics| metrics.inferred_next_step_labeled_count).sum(); + let handoff_source_ref_required_count = + work_jobs.iter().map(|metrics| metrics.handoff_source_ref_required_count).sum(); + let handoff_source_ref_covered_count = + work_jobs.iter().map(|metrics| metrics.handoff_source_ref_covered_count).sum(); + let redaction_required_count = + work_jobs.iter().map(|metrics| metrics.redaction_required_count).sum(); + let redaction_applied_count = + work_jobs.iter().map(|metrics| metrics.redaction_applied_count).sum(); + let janitor_candidate_count = + work_jobs.iter().map(|metrics| metrics.janitor_candidate_count).sum(); + let janitor_false_promotion_count = + work_jobs.iter().map(|metrics| metrics.janitor_false_promotion_count).sum(); + + Some(WorkContinuitySummaryReport { + job_count: work_jobs.len(), + readback_count: work_jobs.iter().map(|metrics| metrics.readback_count).sum(), + entry_count: work_jobs.iter().map(|metrics| metrics.entry_count).sum(), + reset_resume_required_count, + reset_resume_success_count, + reset_resume_success_rate: ratio(reset_resume_success_count, reset_resume_required_count), + decision_rationale_required_count, + decision_rationale_recalled_count, + decision_rationale_recall_rate: ratio( + decision_rationale_recalled_count, + decision_rationale_required_count, + ), + rejected_option_required_count, + rejected_option_suppressed_count, + rejected_option_resurrection_count: work_jobs + .iter() + .map(|metrics| metrics.rejected_option_resurrection_count) + .sum(), + rejected_option_suppression_rate: ratio( + rejected_option_suppressed_count, + rejected_option_required_count, + ), + explicit_next_step_required_count: work_jobs + .iter() + .map(|metrics| metrics.explicit_next_step_required_count) + .sum(), + explicit_next_step_returned_count, + explicit_next_step_correct_count, + explicit_next_step_precision: ratio_or( + explicit_next_step_correct_count, + explicit_next_step_returned_count, + 1.0, + ), + inferred_next_step_required_count, + inferred_next_step_labeled_count, + inferred_step_instruction_count: work_jobs + .iter() + .map(|metrics| metrics.inferred_step_instruction_count) + .sum(), + inferred_next_step_labeling_rate: ratio( + inferred_next_step_labeled_count, + inferred_next_step_required_count, + ), + handoff_source_ref_required_count, + handoff_source_ref_covered_count, + handoff_source_ref_coverage: ratio( + handoff_source_ref_covered_count, + handoff_source_ref_required_count, + ), + redaction_required_count, + redaction_applied_count, + sensitive_marker_persistence_count: work_jobs + .iter() + .map(|metrics| metrics.sensitive_marker_persistence_count) + .sum(), + redaction_rate: ratio(redaction_applied_count, redaction_required_count), + janitor_candidate_count, + janitor_false_promotion_count, + janitor_false_promotion_rate: ratio(janitor_false_promotion_count, janitor_candidate_count), + journal_only_authority_claim_count: work_jobs + .iter() + .map(|metrics| metrics.journal_only_authority_claim_count) + .sum(), + }) +} + fn knowledge_summary(jobs: &[JobReport]) -> Option { let knowledge_jobs = jobs.iter().filter_map(|job| job.knowledge.as_ref()).collect::>(); @@ -7033,6 +7971,7 @@ fn render_markdown(report: &RealWorldReport, report_path: &Path) -> String { render_markdown_memory_summary(&mut out, report); render_markdown_proactive_brief(&mut out, report); render_markdown_scheduled_memory(&mut out, report); + render_markdown_work_continuity(&mut out, report); render_markdown_knowledge(&mut out, report); render_markdown_unsupported_claims(&mut out, report); render_markdown_follow_ups(&mut out, report); @@ -7485,103 +8424,152 @@ fn render_markdown_header(out: &mut String, report: &RealWorldReport, report_pat fn render_markdown_optional_summary_metrics(out: &mut String, summary: &ReportSummary) { if let Some(knowledge) = &summary.knowledge { - out.push_str(&format!( - "- Knowledge citation coverage: `{:.3}`\n", - knowledge.citation_coverage - )); - out.push_str(&format!( - "- Stale claim detection: `{:.3}`\n", - knowledge.stale_claim_detection - )); - out.push_str(&format!("- Rebuild determinism: `{:.3}`\n", knowledge.rebuild_determinism)); - out.push_str(&format!( - "- Backlinks: `{}` total, `{:.3}` page coverage\n", - knowledge.backlink_count, knowledge.backlink_coverage - )); - out.push_str(&format!( - "- Version diff coverage: `{:.3}`\n", - knowledge.version_diff_coverage - )); - out.push_str(&format!("- Page usefulness: `{:.3}`\n", knowledge.page_usefulness)); - out.push_str(&format!( - "- Unsupported summary count: `{}`\n", - knowledge.unsupported_summary_count - )); + render_markdown_knowledge_summary_metrics(out, knowledge); } if let Some(memory_summary) = &summary.memory_summary { - out.push_str(&format!( - "- Memory summary entries: `{}` across `{}` artifact(s)\n", - memory_summary.entry_count, memory_summary.summary_count - )); - out.push_str(&format!( - "- Memory summary source-ref coverage: `{}/{}` (`{:.3}`)\n", - memory_summary.source_ref_entry_count, - memory_summary.source_ref_required_count, - memory_summary.source_ref_coverage - )); - out.push_str(&format!( - "- Memory summary invalid top-of-mind count: `{}`\n", - memory_summary.invalid_top_of_mind_count - )); - out.push_str(&format!( - "- Memory summary unsupported derived entries: `{}`\n", - memory_summary.unsupported_derived_entry_count - )); - out.push_str(&format!( - "- Memory summary unsupported current entries: `{}`\n", - memory_summary.unsupported_current_entry_count - )); + render_markdown_memory_summary_metrics(out, memory_summary); } if let Some(proactive) = &summary.proactive_brief { - out.push_str(&format!( - "- Proactive brief suggestions: `{}` across `{}` artifact(s)\n", - proactive.suggestion_count, proactive.brief_count - )); - out.push_str(&format!( - "- Proactive evidence-ref coverage: `{}/{}` (`{:.3}`)\n", - proactive.evidence_ref_suggestion_count, - proactive.evidence_ref_required_count, - proactive.evidence_ref_coverage - )); - out.push_str(&format!( - "- Proactive freshness/action rationale coverage: `{:.3}` / `{:.3}`\n", - proactive.freshness_coverage, proactive.action_rationale_coverage - )); - out.push_str(&format!( - "- Proactive stale/currentness violations: `{}` invalid current, `{}` tombstone violation(s)\n", - proactive.invalid_current_suggestion_count, proactive.tombstone_violation_count - )); - out.push_str(&format!( - "- Proactive rejected/deferred suggestions: `{}` rejected, `{}` deferred\n", - proactive.rejected_count, proactive.deferred_count - )); + render_markdown_proactive_summary_metrics(out, proactive); } if let Some(scheduled) = &summary.scheduled_memory { - out.push_str(&format!( - "- Scheduled memory outputs: `{}` across `{}` task run(s)\n", - scheduled.output_count, scheduled.task_run_count - )); - out.push_str(&format!( - "- Scheduled memory evidence-ref coverage: `{}/{}` (`{:.3}`)\n", - scheduled.evidence_ref_output_count, - scheduled.evidence_ref_required_count, - scheduled.evidence_ref_coverage - )); - out.push_str(&format!( - "- Scheduled memory freshness/action/trace coverage: `{:.3}` / `{:.3}` / `{:.3}`\n", - scheduled.freshness_coverage, - scheduled.action_rationale_coverage, - scheduled.trace_coverage - )); - out.push_str(&format!( - "- Scheduled memory stale/currentness violations: `{}` invalid current, `{}` tombstone violation(s)\n", - scheduled.invalid_current_output_count, scheduled.tombstone_violation_count - )); - out.push_str(&format!( - "- Scheduled memory source mutations: `{}`\n", - scheduled.source_mutation_count - )); + render_markdown_scheduled_summary_metrics(out, scheduled); } + if let Some(work_continuity) = &summary.work_continuity { + render_markdown_work_continuity_summary_metrics(out, work_continuity); + } +} + +fn render_markdown_knowledge_summary_metrics(out: &mut String, knowledge: &KnowledgeSummary) { + out.push_str(&format!("- Knowledge citation coverage: `{:.3}`\n", knowledge.citation_coverage)); + out.push_str(&format!("- Stale claim detection: `{:.3}`\n", knowledge.stale_claim_detection)); + out.push_str(&format!("- Rebuild determinism: `{:.3}`\n", knowledge.rebuild_determinism)); + out.push_str(&format!( + "- Backlinks: `{}` total, `{:.3}` page coverage\n", + knowledge.backlink_count, knowledge.backlink_coverage + )); + out.push_str(&format!("- Version diff coverage: `{:.3}`\n", knowledge.version_diff_coverage)); + out.push_str(&format!("- Page usefulness: `{:.3}`\n", knowledge.page_usefulness)); + out.push_str(&format!( + "- Unsupported summary count: `{}`\n", + knowledge.unsupported_summary_count + )); +} + +fn render_markdown_memory_summary_metrics(out: &mut String, memory_summary: &MemorySummaryReport) { + out.push_str(&format!( + "- Memory summary entries: `{}` across `{}` artifact(s)\n", + memory_summary.entry_count, memory_summary.summary_count + )); + out.push_str(&format!( + "- Memory summary source-ref coverage: `{}/{}` (`{:.3}`)\n", + memory_summary.source_ref_entry_count, + memory_summary.source_ref_required_count, + memory_summary.source_ref_coverage + )); + out.push_str(&format!( + "- Memory summary invalid top-of-mind count: `{}`\n", + memory_summary.invalid_top_of_mind_count + )); + out.push_str(&format!( + "- Memory summary unsupported derived entries: `{}`\n", + memory_summary.unsupported_derived_entry_count + )); + out.push_str(&format!( + "- Memory summary unsupported current entries: `{}`\n", + memory_summary.unsupported_current_entry_count + )); +} + +fn render_markdown_proactive_summary_metrics( + out: &mut String, + proactive: &ProactiveBriefSummaryReport, +) { + out.push_str(&format!( + "- Proactive brief suggestions: `{}` across `{}` artifact(s)\n", + proactive.suggestion_count, proactive.brief_count + )); + out.push_str(&format!( + "- Proactive evidence-ref coverage: `{}/{}` (`{:.3}`)\n", + proactive.evidence_ref_suggestion_count, + proactive.evidence_ref_required_count, + proactive.evidence_ref_coverage + )); + out.push_str(&format!( + "- Proactive freshness/action rationale coverage: `{:.3}` / `{:.3}`\n", + proactive.freshness_coverage, proactive.action_rationale_coverage + )); + out.push_str(&format!( + "- Proactive stale/currentness violations: `{}` invalid current, `{}` tombstone violation(s)\n", + proactive.invalid_current_suggestion_count, proactive.tombstone_violation_count + )); + out.push_str(&format!( + "- Proactive rejected/deferred suggestions: `{}` rejected, `{}` deferred\n", + proactive.rejected_count, proactive.deferred_count + )); +} + +fn render_markdown_scheduled_summary_metrics( + out: &mut String, + scheduled: &ScheduledMemorySummaryReport, +) { + out.push_str(&format!( + "- Scheduled memory outputs: `{}` across `{}` task run(s)\n", + scheduled.output_count, scheduled.task_run_count + )); + out.push_str(&format!( + "- Scheduled memory evidence-ref coverage: `{}/{}` (`{:.3}`)\n", + scheduled.evidence_ref_output_count, + scheduled.evidence_ref_required_count, + scheduled.evidence_ref_coverage + )); + out.push_str(&format!( + "- Scheduled memory freshness/action/trace coverage: `{:.3}` / `{:.3}` / `{:.3}`\n", + scheduled.freshness_coverage, scheduled.action_rationale_coverage, scheduled.trace_coverage + )); + out.push_str(&format!( + "- Scheduled memory stale/currentness violations: `{}` invalid current, `{}` tombstone violation(s)\n", + scheduled.invalid_current_output_count, scheduled.tombstone_violation_count + )); + out.push_str(&format!( + "- Scheduled memory source mutations: `{}`\n", + scheduled.source_mutation_count + )); +} + +fn render_markdown_work_continuity_summary_metrics( + out: &mut String, + work_continuity: &WorkContinuitySummaryReport, +) { + out.push_str(&format!( + "- Work continuity readbacks: `{}` entries across `{}` artifact(s)\n", + work_continuity.entry_count, work_continuity.readback_count + )); + out.push_str(&format!( + "- Work continuity reset/resume and rationale recall: `{:.3}` / `{:.3}`\n", + work_continuity.reset_resume_success_rate, work_continuity.decision_rationale_recall_rate + )); + out.push_str(&format!( + "- Work continuity rejected-option suppression and explicit next-step precision: `{:.3}` / `{:.3}`\n", + work_continuity.rejected_option_suppression_rate, + work_continuity.explicit_next_step_precision + )); + out.push_str(&format!( + "- Work continuity inferred-step labeling and handoff source-ref coverage: `{:.3}` / `{:.3}`\n", + work_continuity.inferred_next_step_labeling_rate, + work_continuity.handoff_source_ref_coverage + )); + out.push_str(&format!( + "- Work continuity redaction and janitor false-promotion rates: `{:.3}` / `{:.3}`\n", + work_continuity.redaction_rate, work_continuity.janitor_false_promotion_rate + )); + out.push_str(&format!( + "- Work continuity hard-fail markers: `{}` sensitive persistence, `{}` rejected resurrection, `{}` inferred instructions, `{}` journal-only authority claim(s)\n", + work_continuity.sensitive_marker_persistence_count, + work_continuity.rejected_option_resurrection_count, + work_continuity.inferred_step_instruction_count, + work_continuity.journal_only_authority_claim_count + )); } fn render_markdown_quality_summary(out: &mut String, report: &RealWorldReport) { @@ -8140,6 +9128,62 @@ fn render_markdown_scheduled_memory(out: &mut String, report: &RealWorldReport) out.push('\n'); } +fn render_markdown_work_continuity(out: &mut String, report: &RealWorldReport) { + let work_jobs = + report.jobs.iter().filter(|job| job.work_continuity.is_some()).collect::>(); + + if work_jobs.is_empty() { + return; + } + + out.push_str("## Work Continuity Metrics\n\n"); + out.push_str("| Job | Readbacks | Entries | Reset/Resume | Decision Rationale | Rejected Suppression | Explicit Precision | Inferred Labeling | Handoff Sources | Redaction | Janitor False Promotion | Sensitive Persistence | Journal Authority Claims |\n"); + out.push_str( + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n", + ); + + for job in work_jobs { + let Some(metrics) = &job.work_continuity else { + continue; + }; + + out.push_str(&format!( + "| {} | {} | {} | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | `{}/{}` (`{:.3}`) | {} | {} |\n", + md_cell(job.job_id.as_str()), + metrics.readback_count, + metrics.entry_count, + metrics.reset_resume_success_count, + metrics.reset_resume_required_count, + metrics.reset_resume_success_rate, + metrics.decision_rationale_recalled_count, + metrics.decision_rationale_required_count, + metrics.decision_rationale_recall_rate, + metrics.rejected_option_suppressed_count, + metrics.rejected_option_required_count, + metrics.rejected_option_suppression_rate, + metrics.explicit_next_step_correct_count, + metrics.explicit_next_step_returned_count, + metrics.explicit_next_step_precision, + metrics.inferred_next_step_labeled_count, + metrics.inferred_next_step_required_count, + metrics.inferred_next_step_labeling_rate, + metrics.handoff_source_ref_covered_count, + metrics.handoff_source_ref_required_count, + metrics.handoff_source_ref_coverage, + metrics.redaction_applied_count, + metrics.redaction_required_count, + metrics.redaction_rate, + metrics.janitor_false_promotion_count, + metrics.janitor_candidate_count, + metrics.janitor_false_promotion_rate, + metrics.sensitive_marker_persistence_count, + metrics.journal_only_authority_claim_count + )); + } + + out.push('\n'); +} + fn render_markdown_unsupported_claims(out: &mut String, report: &RealWorldReport) { out.push_str("## Unsupported Claims\n\n"); @@ -8226,6 +9270,7 @@ fn render_markdown_semantics(out: &mut String, report: &RealWorldReport) { out.push_str("For `memory_summary` jobs, summary artifacts are derived review surfaces. Top-of-mind entries must be current, included or downgraded entries must carry source refs, and derived project-profile entries must either cite sources or be explicitly flagged as unsupported.\n\n"); out.push_str("For `proactive_brief` jobs, brief artifacts are fixture-scored derived outputs, not scheduled UI behavior. Every suggestion must carry evidence refs, freshness/currentness metadata, and an action rationale; stale, superseded, or tombstoned sources must not be presented as current recommendations.\n\n"); out.push_str("For `scheduled_memory` jobs, task artifacts are deterministic fixture-scored stand-ins for asynchronous work. Every output must carry evidence refs, freshness/currentness metadata, action rationale, and execution trace/readback evidence; scheduled tasks must not mutate source notes silently or claim hosted scheduler/private-provider parity from fixture-only output.\n\n"); + out.push_str("For `work_continuity` jobs, Work Journal entries are source-adjacent readback artifacts, not current fact authority. Reset/resume, decisions, rejected options, next steps, handoff refs, redactions, and janitor candidates must preserve source refs and promotion boundaries; sensitive marker persistence, rejected-option resurrection, inferred next steps treated as instructions, and journal-only authority claims are hard fails.\n\n"); out.push_str("## Suites With `not_encoded` Status\n\n"); if report.not_encoded_suites.is_empty() { diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index 248c3ba5..4015ab59 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -81,6 +81,10 @@ fn scheduled_memory_fixture_dir() -> PathBuf { real_world_memory_fixture_dir().join("scheduled_memory") } +fn work_continuity_fixture_dir() -> PathBuf { + real_world_memory_fixture_dir().join("work_continuity") +} + fn knowledge_fixture_dir() -> PathBuf { real_world_memory_fixture_dir().join("knowledge") } @@ -2907,7 +2911,7 @@ fn assert_live_sweep_record(adapter: &Value, production_ops_status: &str) -> Res fn runner_discovers_nested_fixture_layout() -> Result<()> { let report = run_json_report_from(fixture_root())?; - assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(73)); + assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(81)); Ok(()) } @@ -7551,6 +7555,327 @@ fn scheduled_memory_fixture_fails_source_mutation() -> Result<()> { Ok(()) } +#[test] +fn work_continuity_fixtures_score_required_metrics() -> Result<()> { + let report = run_json_report_from(work_continuity_fixture_dir())?; + + assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(8)); + assert_eq!(report.pointer("/summary/pass").and_then(Value::as_u64), Some(8)); + assert_eq!(report.pointer("/summary/wrong_result").and_then(Value::as_u64), Some(0)); + + assert_work_continuity_summary_counts(&report); + + let suites = array_at(&report, "/suites")?; + let work_continuity = find_by_field(suites, "/suite_id", "work_continuity")?; + + assert_eq!(work_continuity.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(work_continuity.pointer("/encoded_job_count").and_then(Value::as_u64), Some(8)); + + Ok(()) +} + +fn assert_work_continuity_summary_counts(report: &Value) { + for (field, expected) in [ + ("readback_count", 8), + ("entry_count", 8), + ("reset_resume_required_count", 1), + ("reset_resume_success_count", 1), + ("decision_rationale_required_count", 1), + ("decision_rationale_recalled_count", 1), + ("rejected_option_required_count", 1), + ("rejected_option_suppressed_count", 1), + ("rejected_option_resurrection_count", 0), + ("explicit_next_step_required_count", 1), + ("explicit_next_step_returned_count", 1), + ("explicit_next_step_correct_count", 1), + ("inferred_next_step_required_count", 1), + ("inferred_next_step_labeled_count", 1), + ("inferred_step_instruction_count", 0), + ("handoff_source_ref_required_count", 1), + ("handoff_source_ref_covered_count", 1), + ("redaction_required_count", 1), + ("redaction_applied_count", 1), + ("sensitive_marker_persistence_count", 0), + ("janitor_candidate_count", 1), + ("janitor_false_promotion_count", 0), + ("journal_only_authority_claim_count", 0), + ] { + assert_work_continuity_summary_u64(report, field, expected); + } + for (field, expected) in [ + ("reset_resume_success_rate", 1.0), + ("decision_rationale_recall_rate", 1.0), + ("rejected_option_suppression_rate", 1.0), + ("explicit_next_step_precision", 1.0), + ("inferred_next_step_labeling_rate", 1.0), + ("handoff_source_ref_coverage", 1.0), + ("redaction_rate", 1.0), + ("janitor_false_promotion_rate", 0.0), + ] { + assert_work_continuity_summary_f64(report, field, expected); + } +} + +fn assert_work_continuity_summary_u64(report: &Value, field: &str, expected: u64) { + assert_eq!( + report.pointer(&format!("/summary/work_continuity/{field}")).and_then(Value::as_u64), + Some(expected), + "unexpected Work Continuity summary field {field}", + ); +} + +fn assert_work_continuity_summary_f64(report: &Value, field: &str, expected: f64) { + assert_eq!( + report.pointer(&format!("/summary/work_continuity/{field}")).and_then(Value::as_f64), + Some(expected), + "unexpected Work Continuity summary field {field}", + ); +} + +#[test] +fn work_continuity_markdown_renders_required_metrics() -> Result<()> { + let report = run_json_report_from(work_continuity_fixture_dir())?; + let temp_dir = + env::temp_dir().join(format!("elf-real-world-work-continuity-test-{}", process::id())); + + fs::create_dir_all(&temp_dir)?; + + let report_path = temp_dir.join("work-continuity-report.json"); + let markdown_path = temp_dir.join("work-continuity-report.md"); + + fs::write(&report_path, serde_json::to_vec_pretty(&report)?)?; + + let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) + .arg("publish") + .arg("--report") + .arg(&report_path) + .arg("--out") + .arg(&markdown_path) + .output()?; + + assert!( + output.status.success(), + "real_world_job publisher failed: {}", + String::from_utf8_lossy(&output.stderr), + ); + + let markdown = fs::read_to_string(markdown_path)?; + + assert!(markdown.contains("Work Continuity Metrics")); + assert!(markdown.contains("work-continuity-redaction-001")); + assert!(markdown.contains("work-continuity-janitor-false-promotion-001")); + assert!(markdown.contains("Janitor False Promotion")); + assert!(markdown.contains("Sensitive Persistence")); + assert!(markdown.contains("Journal Authority Claims")); + assert!(markdown.contains("| work-continuity-reset-resume-001 | 1 | 1 | `1/1` (`1.000`)")); + assert!(markdown.contains( + "| work-continuity-explicit-next-step-001 | 1 | 1 | `0/0` (`0.000`) | `0/0` (`0.000`) | `0/0` (`0.000`) | `1/1` (`1.000`)" + )); + assert!(markdown.contains( + "| work-continuity-handoff-source-ref-001 | 1 | 1 | `0/0` (`0.000`) | `0/0` (`0.000`) | `0/0` (`0.000`) | `0/0` (`1.000`) | `0/0` (`0.000`) | `1/1` (`1.000`)" + )); + assert!(markdown.contains( + "| work-continuity-redaction-001 | 1 | 1 | `0/0` (`0.000`) | `0/0` (`0.000`) | `0/0` (`0.000`) | `0/0` (`1.000`) | `0/0` (`0.000`) | `0/0` (`0.000`) | `1/1` (`1.000`)" + )); + assert!(markdown.contains( + "| work-continuity-janitor-false-promotion-001 | 1 | 1 | `0/0` (`0.000`) | `0/0` (`0.000`) | `0/0` (`0.000`) | `0/0` (`1.000`) | `0/0` (`0.000`) | `0/0` (`0.000`) | `0/0` (`0.000`) | `0/1` (`0.000`)" + )); + + Ok(()) +} + +#[test] +fn work_continuity_fixture_fails_sensitive_marker_persistence() -> Result<()> { + let report = run_work_continuity_mutation( + "redaction_sensitive_marker.json", + "sensitive_marker_persistence.json", + |fixture| { + fixture["corpus"]["adapter_response"]["answer"]["work_journal_readbacks"][0]["items"] + [0]["redaction_audit"]["persisted_sensitive_marker_ids"] = + serde_json::json!(["secret-demo-token"]); + }, + )?; + let job = single_work_continuity_job(&report, "work-continuity-redaction-001")?; + + assert_work_continuity_wrong_result(job, "sensitive_marker_persistence_count", 1); + + Ok(()) +} + +#[test] +fn work_continuity_fixture_fails_rejected_option_resurrection() -> Result<()> { + let report = run_work_continuity_mutation( + "rejected_option_suppression.json", + "rejected_option_resurrection.json", + |fixture| { + fixture["corpus"]["adapter_response"]["answer"]["work_journal_readbacks"][0]["items"] + [0]["rejected_options"][0]["resurrected_as_current"] = Value::Bool(true); + }, + )?; + let job = single_work_continuity_job(&report, "work-continuity-rejected-option-001")?; + + assert_work_continuity_wrong_result(job, "rejected_option_resurrection_count", 1); + + Ok(()) +} + +#[test] +fn work_continuity_fixture_fails_inferred_step_instruction() -> Result<()> { + let report = run_work_continuity_mutation( + "inferred_next_step_labeling.json", + "inferred_step_instruction.json", + |fixture| { + fixture["corpus"]["adapter_response"]["answer"]["work_journal_readbacks"][0]["items"] + [0]["inferred_next_steps"][0]["instruction"] = Value::Bool(true); + }, + )?; + let job = single_work_continuity_job(&report, "work-continuity-inferred-next-step-001")?; + + assert_work_continuity_wrong_result(job, "inferred_step_instruction_count", 1); + + Ok(()) +} + +#[test] +fn work_continuity_fixture_fails_journal_only_authority_claim() -> Result<()> { + let report = run_work_continuity_mutation( + "handoff_source_ref_coverage.json", + "journal_only_authority_claim.json", + |fixture| { + fixture["corpus"]["adapter_response"]["answer"]["work_journal_readbacks"][0]["where_stopped"] + ["journal_only_authority_claims"] = serde_json::json!(["wj-handoff-source-ref"]); + }, + )?; + let job = single_work_continuity_job(&report, "work-continuity-handoff-source-ref-001")?; + + assert_work_continuity_wrong_result(job, "journal_only_authority_claim_count", 1); + + Ok(()) +} + +#[test] +fn work_continuity_fixture_fails_janitor_promotion_or_missing_review() -> Result<()> { + let promoted = run_work_continuity_mutation( + "janitor_false_promotion_guard.json", + "janitor_promoted.json", + |fixture| { + fixture["corpus"]["adapter_response"]["answer"]["work_journal_readbacks"][0]["janitor_candidates"] + [0]["promoted_to_memory"] = Value::Bool(true); + }, + )?; + let promoted_job = + single_work_continuity_job(&promoted, "work-continuity-janitor-false-promotion-001")?; + + assert_work_continuity_wrong_result(promoted_job, "janitor_false_promotion_count", 1); + assert_hard_fail_hit(promoted_job, "janitor Work Journal candidate promoted without review"); + + let missing_review = run_work_continuity_mutation( + "janitor_false_promotion_guard.json", + "janitor_missing_review_required.json", + |fixture| { + fixture["corpus"]["adapter_response"]["answer"]["work_journal_readbacks"][0]["janitor_candidates"] + [0]["review_required"] = Value::Bool(false); + }, + )?; + let missing_review_job = + single_work_continuity_job(&missing_review, "work-continuity-janitor-false-promotion-001")?; + + assert_work_continuity_wrong_result(missing_review_job, "janitor_false_promotion_count", 1); + assert_hard_fail_hit( + missing_review_job, + "janitor Work Journal candidate promoted without review", + ); + + let extra_bad_candidate = run_work_continuity_mutation( + "janitor_false_promotion_guard.json", + "janitor_extra_bad_candidate.json", + |fixture| { + fixture["corpus"]["adapter_response"]["answer"]["work_journal_readbacks"][0]["janitor_candidates"] = serde_json::json!([ + { + "candidate_id": "wj-janitor-candidate", + "evidence_refs": ["wj-janitor-candidate-source"], + "review_required": true, + "promoted_to_memory": false + }, + { + "candidate_id": "wj-extra-janitor-candidate", + "evidence_refs": ["wj-janitor-candidate-source"], + "review_required": true, + "promoted_to_memory": true + } + ]); + }, + )?; + let extra_bad_candidate_job = single_work_continuity_job( + &extra_bad_candidate, + "work-continuity-janitor-false-promotion-001", + )?; + + assert_work_continuity_wrong_result( + extra_bad_candidate_job, + "janitor_false_promotion_count", + 1, + ); + assert_hard_fail_hit( + extra_bad_candidate_job, + "janitor Work Journal candidate promoted without review", + ); + + assert_eq!( + extra_bad_candidate_job + .pointer("/work_continuity/janitor_candidate_count") + .and_then(Value::as_u64), + Some(2) + ); + + Ok(()) +} + +fn run_work_continuity_mutation( + fixture_name: &str, + output_name: &str, + mutate: impl FnOnce(&mut Value), +) -> Result { + let fixture_path = work_continuity_fixture_dir().join(fixture_name); + let temp_dir = + env::temp_dir().join(format!("elf-work-continuity-{output_name}-{}", process::id())); + let mut fixture = load_json(&fixture_path)?; + + mutate(&mut fixture); + + if temp_dir.exists() { + fs::remove_dir_all(&temp_dir)?; + } + + fs::create_dir_all(&temp_dir)?; + fs::write(temp_dir.join(output_name), serde_json::to_vec_pretty(&fixture)?)?; + + run_json_report_from(temp_dir) +} + +fn single_work_continuity_job<'a>(report: &'a Value, job_id: &str) -> Result<&'a Value> { + let jobs = array_at(report, "/jobs")?; + + find_by_field(jobs, "/job_id", job_id) +} + +fn assert_work_continuity_wrong_result(job: &Value, metric_name: &str, expected: u64) { + assert_eq!(job.pointer("/status").and_then(Value::as_str), Some("wrong_result")); + assert_eq!( + job.pointer(&format!("/work_continuity/{metric_name}")).and_then(Value::as_u64), + Some(expected) + ); +} + +fn assert_hard_fail_hit(job: &Value, expected_hit: &str) { + let hits = job.pointer("/hard_fail_hits").and_then(Value::as_array).expect("hard_fail_hits"); + + assert!( + hits.iter().filter_map(Value::as_str).any(|hit| hit == expected_hit), + "missing hard_fail_hits marker: {expected_hit}" + ); +} + #[test] fn production_ops_fixtures_report_bounded_typed_states() -> Result<()> { let report = run_json_report_from(production_ops_fixture_dir())?; @@ -7890,9 +8215,9 @@ fn assert_root_knowledge_summary(report: &Value) { } fn assert_root_aggregate_summary(report: &Value) -> Result<()> { - assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(73)); - assert_eq!(report.pointer("/summary/encoded_suite_count").and_then(Value::as_u64), Some(18)); - assert_eq!(report.pointer("/summary/pass").and_then(Value::as_u64), Some(66)); + assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(81)); + assert_eq!(report.pointer("/summary/encoded_suite_count").and_then(Value::as_u64), Some(19)); + assert_eq!(report.pointer("/summary/pass").and_then(Value::as_u64), Some(74)); assert_eq!(report.pointer("/summary/wrong_result").and_then(Value::as_u64), Some(0)); assert_eq!(report.pointer("/summary/incomplete").and_then(Value::as_u64), Some(0)); assert_eq!(report.pointer("/summary/blocked").and_then(Value::as_u64), Some(7)); @@ -7935,11 +8260,11 @@ fn assert_root_aggregate_summary(report: &Value) -> Result<()> { ); assert_eq!( report.pointer("/summary/evidence_required_count").and_then(Value::as_u64), - Some(165) + Some(173) ); assert_eq!( report.pointer("/summary/evidence_covered_count").and_then(Value::as_u64), - Some(165) + Some(173) ); assert_eq!(report.pointer("/summary/evidence_coverage").and_then(Value::as_f64), Some(1.0)); assert_eq!(report.pointer("/summary/source_ref_coverage").and_then(Value::as_f64), Some(1.0)); @@ -7985,6 +8310,7 @@ fn assert_root_aggregate_summary(report: &Value) -> Result<()> { assert_root_knowledge_summary(report); assert_root_proactive_brief_summary(report); assert_root_scheduled_memory_summary(report); + assert_root_work_continuity_summary(report); Ok(()) } @@ -8117,6 +8443,55 @@ fn assert_root_scheduled_memory_summary(report: &Value) { ); } +fn assert_root_work_continuity_summary(report: &Value) { + assert_eq!( + report.pointer("/summary/work_continuity/job_count").and_then(Value::as_u64), + Some(8) + ); + assert_eq!( + report + .pointer("/summary/work_continuity/reset_resume_success_rate") + .and_then(Value::as_f64), + Some(1.0) + ); + assert_eq!( + report + .pointer("/summary/work_continuity/decision_rationale_recall_rate") + .and_then(Value::as_f64), + Some(1.0) + ); + assert_eq!( + report + .pointer("/summary/work_continuity/rejected_option_suppression_rate") + .and_then(Value::as_f64), + Some(1.0) + ); + assert_eq!( + report + .pointer("/summary/work_continuity/inferred_step_instruction_count") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/summary/work_continuity/sensitive_marker_persistence_count") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/summary/work_continuity/janitor_false_promotion_count") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/summary/work_continuity/journal_only_authority_claim_count") + .and_then(Value::as_u64), + Some(0) + ); +} + fn assert_root_aggregate_suites(report: &Value) -> Result<()> { let suites = array_at(report, "/suites")?; @@ -8134,6 +8509,7 @@ fn assert_root_aggregate_suites(report: &Value) -> Result<()> { "memory_evolution", "adversarial_quality", "core_archival_memory", + "work_continuity", ] { let suite = find_by_field(suites, "/suite_id", suite_id)?; @@ -8191,6 +8567,11 @@ fn assert_root_aggregate_suites(report: &Value) -> Result<()> { assert_eq!(context_trajectory.pointer("/status").and_then(Value::as_str), Some("blocked")); assert_eq!(context_trajectory.pointer("/encoded_job_count").and_then(Value::as_u64), Some(3)); + let work_continuity = find_by_field(suites, "/suite_id", "work_continuity")?; + + assert_eq!(work_continuity.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(work_continuity.pointer("/encoded_job_count").and_then(Value::as_u64), Some(8)); + Ok(()) } diff --git a/docs/log.md b/docs/log.md index c6e1cced..c7e08500 100644 --- a/docs/log.md +++ b/docs/log.md @@ -135,3 +135,8 @@ logs. - Tightened the Work Journal promotion-boundary contract so accepted references must resolve to storage-backed Memory Authority or Dreaming Review evidence instead of granting authority from JSON shape alone. +- Added the XY-1118 Work Continuity benchmark slice, documenting the + `work_continuity` real-world suite, `cargo make real-world-memory-work-continuity`, + Work Journal oracle fields, report rates, and hard-fail counters for redaction, + rejected-option, inferred-step, journal-authority, and janitor false-promotion + boundaries. diff --git a/docs/runbook/benchmarking/real_world_agent_memory_benchmark.md b/docs/runbook/benchmarking/real_world_agent_memory_benchmark.md index 064f0213..94c16659 100644 --- a/docs/runbook/benchmarking/real_world_agent_memory_benchmark.md +++ b/docs/runbook/benchmarking/real_world_agent_memory_benchmark.md @@ -63,6 +63,7 @@ compile knowledge, and state honest uncertainty. | --- | --- | --- | | Trust/source-of-truth | Provenance, rebuildability, and derived-index boundaries. | Restore a note after index rebuild and cite authoritative source evidence. | | Work resume | Resuming agent work without repeating completed steps. | Identify the next action after a retained lane failure. | +| Work continuity | Source-adjacent Work Journal reset/resume, rationale, rejected-option, next-step, handoff, redaction, and janitor boundaries. | Resume a session from Work Journal readback without promoting journal-only content to current memory authority. | | Project decisions | Current decisions, rationale, reversals, and caveats. | Explain why a benchmark gate uses typed failures. | | Retrieval | Task-relevant search with decoys and alternates. | Answer a task query while avoiding near-duplicate project evidence. | | Memory evolution | Update, delete, expiry, contradiction, and history behavior. | Report what superseded an old fact and suppress deleted memory. | @@ -174,6 +175,10 @@ including the retrieval-quality slice below. The suite currently encodes: Postgres-held chunk embeddings before answering. - `work_resume`: stale worktree resume, Decodex/Linear lane status, failed command recovery, PR review blocker recovery, and exact next-action extraction. +- `work_continuity`: Work Journal reset/resume readback, decision-rationale recall, + rejected-option suppression, explicit next-step precision, inferred next-step + labeling, handoff source-ref coverage, redaction, and janitor false-promotion + guards while keeping journal-only content source-adjacent. - `project_decisions`: accepted durable decisions, superseded/reversed decisions, old-versus-current validation gates, tradeoff rationale, and bounded caveat or uncertainty handling. @@ -224,7 +229,10 @@ redaction leak count, capture/integration behavior classes, Qdrant rebuild case/pass counts, expected evidence recall, irrelevant context ratio, latency/cost, answer-type plus caveat/refusal/uncertainty flags, trace explainability counters, production-ops -blocked/wrong-result job states, and +blocked/wrong-result job states, Work Continuity rates for reset/resume, +decision-rationale recall, rejected-option suppression, explicit next-step precision, +inferred next-step labeling, handoff source-ref coverage, redaction, and janitor +false-promotion, Work Continuity hard-fail counters, and private-corpus redaction policy. The fixtures include negative traps for stale blockers, unsupported prior claims, stale deleted facts, stale historical facts, cross-project preference leakage, private/redacted text leakage, obsolete retrieval @@ -233,6 +241,21 @@ claims, overconfident unsupported decision answers, distractor context, index-only restore claims, private-corpus pass claims without a manifest, checked-in credential leakage, and adversarial stale or unsupported scoreboard claims. +Current checked-in Work Continuity increment: + +```sh +cargo make real-world-memory-work-continuity +``` + +This parses `apps/elf-eval/fixtures/real_world_memory/work_continuity/`, writes +`tmp/real-world-memory/work-continuity/report.json`, and renders +`tmp/real-world-memory/work-continuity/report.md`. The slice scores eight +fixture-backed jobs for reset/resume readback, decision rationale, rejected-option +suppression, explicit next-step precision, inferred next-step labeling, handoff +source-ref coverage, redaction, and janitor false-promotion prevention. It does not +claim diary-product behavior, current-fact authority from journal-only content, +provider-backed private-corpus quality, or competitor-wide superiority. + Current checked-in adversarial quality increment: ```sh @@ -366,8 +389,8 @@ The public quality scoreboard renders the existing manifest evidence bucket external adapter manifest is loaded, the scoreboard's typed non-pass count includes adapter coverage and scenario rows as well as fixture jobs. -Current fixture state: `cargo make real-world-memory-json` covers 73 jobs across 18 -suites, with 66 pass and 7 blocked. The adversarial quality slice contributes five +Current fixture state: `cargo make real-world-memory-json` covers 81 jobs across 19 +suites, with 74 pass and 7 blocked. The adversarial quality slice contributes five passing fixture-backed jobs that exercise stale fact suppression, unsupported-claim refusal, source-authority conflicts, private-span exclusion, and correction persistence. The P1 closeout fixture slice contributes four passing jobs for @@ -377,6 +400,9 @@ stale-core detection, archival fallback, and project-decision recovery. The `memory_summary` suite contributes one passing fixture-backed source-trace job for reviewable current, background, stale, superseded, tombstoned, and derived project-profile entries. The +`work_continuity` suite contributes eight passing fixture-backed Work Journal rows +for reset/resume, rationale, rejected-option, explicit/inferred next-step, handoff +source-ref, redaction, and janitor false-promotion boundaries. The `proactive_brief` suite contributes four passing source-linked proactive suggestions and one typed private-corpus refresh blocker tied to XY-930. The blocked jobs are production-ops operator boundaries, the private-corpus refresh blocker, the diff --git a/docs/spec/real_world_agent_memory_benchmark_v1.md b/docs/spec/real_world_agent_memory_benchmark_v1.md index 936028d4..8a415752 100644 --- a/docs/spec/real_world_agent_memory_benchmark_v1.md +++ b/docs/spec/real_world_agent_memory_benchmark_v1.md @@ -11,10 +11,15 @@ tags: - docs - spec source_refs: [] -code_refs: [] +code_refs: + - Makefile.toml + - apps/elf-eval/src/bin/real_world_job_benchmark.rs + - apps/elf-eval/fixtures/real_world_memory/ related: [] drift_watch: - docs/spec/real_world_agent_memory_benchmark_v1.md + - apps/elf-eval/src/bin/real_world_job_benchmark.rs + - apps/elf-eval/fixtures/real_world_memory/ --- # Real-World Agent Memory Benchmark v1 @@ -125,6 +130,7 @@ runner execution. "encoding": {}, "memory_evolution": {}, "memory_summary": {}, + "work_continuity": {}, "tags": [] } ``` @@ -149,6 +155,7 @@ runner execution. | `encoding` | object | Optional job-level limitation declaration. Only `not_encoded`, `blocked`, and `incomplete` statuses are allowed here. | | `memory_evolution` | object or null | Optional for most suites; used by `memory_evolution` jobs to report current evidence, historical evidence, stale traps, conflicts, update rationale, and temporal-validity limitations. | | `memory_summary` | object or null | Optional for most suites; used by `memory_summary` jobs to report reviewable summary/source-trace metrics defined in `system_memory_summary_v1.md`. | +| `work_continuity` | object or null | Optional for most suites; required for encoded `work_continuity` jobs. Records expected Work Journal reset/resume entries, decision-rationale evidence, rejected options, explicit/inferred next steps, handoff source refs, redaction markers, and janitor candidates. | | `tags` | array | Optional labels such as `private_corpus`, `public_proxy`, `provider_backed`, `synthetic`, `adapter_required`, or `no_live_claim`. | ### `corpus` @@ -620,6 +627,7 @@ Suite ids are stable public names. Each suite MUST contain at least one | --- | --- | --- | --- | --- | --- | | `trust_source_of_truth` | Verify authoritative storage, provenance, rebuild, and non-authoritative derived index handling. | Restore a note after Qdrant rebuild; identify whether a compiled page is derived; explain why a source ref supports a claim. | Source note/document ids, restore or rebuild trace, source_ref lineage, no hidden index-only evidence. | answer_correctness, evidence_grounding, trap_avoidance, lifecycle_behavior. | ELF, memsearch, OpenViking. | | `work_resume` | Help an agent resume real work without repeating completed steps or losing blockers. | Resume a retained lane; identify next command after a failed run; summarize what remains blocked. | Timeline events, issue/PR ids, run summaries, latest blocker evidence. | answer_correctness, workflow_helpfulness, uncertainty_handling, trap_avoidance. | agentmemory, claude-mem, OpenViking. | +| `work_continuity` | Verify source-adjacent Work Journal reset/resume, rationale, rejected-option, next-step, handoff, redaction, and janitor boundaries. | Resume a session from Work Journal readback; recall why a decision was made; suppress a rejected option; label inferred next steps as non-instructions; preserve handoff source refs; keep janitor candidates review-only. | Work Journal readback ids, entry ids, source refs, decision-rationale evidence ids, redaction marker ids, rejected option ids, explicit and inferred next-step ids, janitor candidate ids, promotion-boundary evidence. | answer_correctness, evidence_grounding, trap_avoidance, lifecycle_behavior, uncertainty_handling. | ELF, agentmemory, claude-mem, Always-On Memory Agent. | | `project_decisions` | Recover durable decisions, rationale, reversals, and current policy. | Explain why a design was chosen; distinguish old vs current validation gate; cite decision evidence. | Decision records, superseding events, accepted alternatives, current-policy timestamp. | answer_correctness, evidence_grounding, trap_avoidance, uncertainty_handling. | ELF, gbrain, llm-wiki, Letta. | | `retrieval` | Measure task-relevant retrieval quality beyond top-k keyword matching. | Answer a task query with expected evidence; find alternate phrasing; avoid near-duplicate project evidence. | Expected evidence ids, allowed alternates, decoy evidence ids, trace ids when available. | answer_correctness, evidence_grounding, trap_avoidance, latency_resource. | qmd, ELF, memsearch, OpenViking. | | `memory_evolution` | Verify updates, deletes, expiry, supersession, contradiction handling, and history. | Apply a new preference; suppress a deleted memory; explain what superseded an old fact. | Before/after memory versions, ingest decision rows or adapter history, current timeline event. | lifecycle_behavior, answer_correctness, evidence_grounding, trap_avoidance. | mem0, ELF, Graphiti/Zep, Letta. | @@ -700,6 +708,12 @@ Reports MUST include: - for encoded knowledge-compilation jobs with page artifacts: citation coverage, stale claim detection, rebuild determinism, page usefulness, backlink counts, unsupported summary count, and untraced section count; +- for encoded Work Continuity jobs with Work Journal readbacks: reset/resume success + rate, decision-rationale recall rate, rejected-option suppression rate, explicit + next-step precision, inferred next-step labeling rate, handoff source-ref coverage, + redaction rate, janitor false-promotion rate, and hard-fail counters for sensitive + marker persistence, rejected-option resurrection, inferred-step-as-instruction, and + journal-only authority claims; - explicit `not_encoded` suite list; - private-corpus redaction policy when private fixtures are used. - capture/integration coverage classes when any fixture declares `capture_behaviors`, @@ -736,6 +750,25 @@ unless it has source refs or explicit unsupported-claim flags. A derived entry w unsupported-claim flags MUST NOT pass when it is included as current memory instead of being excluded or downgraded for review. +Reports that encode `work_continuity` jobs MUST also include: + +- Work Journal readback artifact count and entry count; +- reset/resume required and successful entry counts; +- decision-rationale required and recalled evidence counts; +- rejected-option required, suppressed, and resurrected counts; +- explicit next-step required, returned, and correct counts; +- inferred next-step required, labeled, and instruction-violation counts; +- handoff source-ref required and covered counts; +- redaction required, applied, and persisted-sensitive-marker counts; +- janitor candidate and false-promotion counts; +- journal-only authority claim count. + +A `work_continuity` job MUST NOT pass when a sensitive marker persists after redaction, +a rejected option is resurrected as current work, an inferred next step is surfaced as +an instruction, or journal-only content is claimed as current authority. Janitor +candidates may be reported as review-required Work Journal evidence, but automatic +promotion to Memory Authority without accepted promotion evidence is a hard fail. + Consolidation suite reports MUST also include: - proposal usefulness score, or `null` when the job has no proposal payloads;