From 8f9cc688ad39c457471fb7c45a23710230d8966c Mon Sep 17 00:00:00 2001
From: David Souther <davidsouther@gmail.com>
Date: Mon, 6 Jul 2026 13:24:07 -0400
Subject: [PATCH 01/12] test: add failing feature test for report's
 statistical-rigor upgrade (paired-difference test + tool_call_collection)

---
 tests/report_statistics.rs | 274 +++++++++++++++++++++++++++++++++++++
 1 file changed, 274 insertions(+)
 create mode 100644 tests/report_statistics.rs

diff --git a/tests/report_statistics.rs b/tests/report_statistics.rs
new file mode 100644
index 00000000..fbe46616
--- /dev/null
+++ b/tests/report_statistics.rs
@@ -0,0 +1,274 @@
+//! Feature test for Feature F — `report`'s statistical-rigor upgrade.
+//! See `.ailly/developer/2026-07-06-A-ailly-evals/feature-f-report-stats/design.md`.
+//!
+//! Two independent capabilities land together in this feature-step:
+//!
+//! Story 1 (paired-difference + SEM, Closing Bell Task 5 — "Is this
+//! regression real?"): `compute_comparison` already buckets every paired
+//! assertion into Improved/Regressed/UnchangedPass/UnchangedFail
+//! (`src/knowledge/report.rs`). This feature-step folds those same paired
+//! outcomes into a two-tailed paired Student's t-test (mean difference,
+//! sample standard deviation, standard error of the mean, t-statistic,
+//! degrees of freedom, p-value, and a significance verdict at α = 0.05) so a
+//! developer gets an actual statistical answer instead of raw counts to
+//! eyeball.
+//!
+//! Story 2 (`tool_call_collection`): a suite author wants "these tools were
+//! all called, in any order" instead of `tool_call_order`'s rigid
+//! subsequence match — Anthropic's own guidance against over-rigid
+//! tool-call-order assertions, cited in the parent project's research.md.
+//! `tool_call_collection` reuses `extract_tool_uses`/`tool_use_name` (the
+//! same helpers `tool_call_order` already uses) as a multiset comparison:
+//! each named tool must appear at least as many times as requested,
+//! regardless of order or intervening calls, alongside (not replacing)
+//! `tool_call_order`.
+//!
+//! This file pins the happy path for both. Edge cases (insufficient pairs,
+//! a zero-variance/vacuous comparison, a perfect-separation comparison, an
+//! empty `tools` list) are Plan-phase red-green-refactor unit tests inside
+//! `src/knowledge/report.rs` and `src/knowledge/assertions.rs`, matching
+//! this repo's existing split between a feature test's public contract and
+//! a module's own unit-test coverage (see `tests/eval_judge.rs`'s own doc
+//! comment for the same pattern).
+
+use std::marker::PhantomData;
+
+use ailly_two::content::conversation::BindingMap;
+use ailly_two::content::conversation::Content;
+use ailly_two::content::conversation::ContentBlock;
+use ailly_two::content::conversation::Conversation;
+use ailly_two::content::conversation::Message;
+use ailly_two::content::conversation::Meta;
+use ailly_two::content::conversation::ModelId;
+use ailly_two::content::conversation::Role;
+use ailly_two::content::conversation::ToolUseId;
+use ailly_two::content::evaluation::Assertion;
+use ailly_two::knowledge::assertions::AssertionOutcome;
+use ailly_two::knowledge::assertions::EvaluationContext;
+use ailly_two::knowledge::eval::EvalReport;
+use ailly_two::knowledge::report::PairedDifferenceTest;
+use ailly_two::knowledge::report::compute_comparison;
+
+/// Reuses `tests/report_cmd.rs`'s exact ARM_A/ARM_B assertion split (3
+/// improved, 1 regressed, 4 unchanged-pass, 2 unchanged-fail — 10 paired
+/// assertions total, already proven and green in that file) so the
+/// paired-difference numbers asserted below cross-check against an
+/// already-verified fixture instead of an unvetted new one. Only the
+/// fields `compute_comparison` actually reads (`cases`) carry real data;
+/// `totals`/`per_class` are present-but-empty because `EvalReport`
+/// requires them to deserialize, not because this test exercises them.
+const ARM_A_JSON: &str = r#"{
+  "suite": "regression",
+  "run_id": "2026-05-20T10-00-00Z-baseline",
+  "totals": { "conversations_matched": 2, "assertions": { "passed": 5, "failed": 5, "deferred": 0, "malformed": 0 } },
+  "per_class": {},
+  "cases": [
+    { "name": "missing-fields", "matches": [ { "conversation": "missing-fields.yaml", "assertions": [
+      { "class": "must_call_tool",     "outcome": "fail" },
+      { "class": "text_contains",      "outcome": "fail" },
+      { "class": "text_equals",        "outcome": "fail" },
+      { "class": "must_not_call_tool", "outcome": "pass" },
+      { "class": "tool_call_count",    "outcome": "pass" }
+    ] } ] },
+    { "name": "over-limit", "matches": [ { "conversation": "over-limit.yaml", "assertions": [
+      { "class": "tool_call_order", "outcome": "pass" },
+      { "class": "text_contains",   "outcome": "fail" },
+      { "class": "text_equals",     "outcome": "fail" },
+      { "class": "judge",           "outcome": "pass" },
+      { "class": "latency_ms",      "outcome": "pass" }
+    ] } ] }
+  ]
+}"#;
+
+const ARM_B_JSON: &str = r#"{
+  "suite": "regression",
+  "run_id": "2026-05-27T14-43-31Z-target",
+  "totals": { "conversations_matched": 2, "assertions": { "passed": 7, "failed": 3, "deferred": 0, "malformed": 0 } },
+  "per_class": {},
+  "cases": [
+    { "name": "missing-fields", "matches": [ { "conversation": "missing-fields.yaml", "assertions": [
+      { "class": "must_call_tool",     "outcome": "pass" },
+      { "class": "text_contains",      "outcome": "pass" },
+      { "class": "text_equals",        "outcome": "pass" },
+      { "class": "must_not_call_tool", "outcome": "pass" },
+      { "class": "tool_call_count",    "outcome": "pass" }
+    ] } ] },
+    { "name": "over-limit", "matches": [ { "conversation": "over-limit.yaml", "assertions": [
+      { "class": "tool_call_order", "outcome": "fail" },
+      { "class": "text_contains",   "outcome": "fail" },
+      { "class": "text_equals",     "outcome": "fail" },
+      { "class": "judge",           "outcome": "pass" },
+      { "class": "latency_ms",      "outcome": "pass" }
+    ] } ] }
+  ]
+}"#;
+
+/// Given two runs whose paired assertions already bucket into 3 improved,
+/// 1 regressed, 4 unchanged-pass, 2 unchanged-fail (10 pairs; diffs
+/// = [+1,+1,+1,-1,0,0,0,0,0,0] using the "arm_b relative to arm_a" sign
+/// convention: pass=1, fail=0, diff = b - a):
+///
+/// When `compute_comparison` runs,
+///
+/// Then it reports the existing bucket totals unchanged (regression guard)
+/// AND a paired-difference test computed from the same 10 pairs: mean
+/// difference 0.2, sample std dev sqrt(0.4), SEM = sqrt(0.4)/sqrt(10) = 0.2
+/// exactly, t = mean/SEM = 1.0 exactly, df = 9, two-tailed p ≈ 0.3434 (not
+/// significant at α = 0.05 — a 30%-ish swing on 10 pairs is exactly the
+/// kind of small-sample noise this project's own research.md warns a raw
+/// bucket count can't distinguish from a real regression).
+#[test]
+fn compute_comparison_reports_paired_difference_test_with_standard_error() {
+    // Given
+    let arm_a: EvalReport = serde_json::from_str(ARM_A_JSON).expect("arm_a parses");
+    let arm_b: EvalReport = serde_json::from_str(ARM_B_JSON).expect("arm_b parses");
+
+    // When
+    let comparison = compute_comparison(&arm_a, &arm_b);
+
+    // Then: existing bucket counts are unchanged.
+    assert_eq!(comparison.totals.improved, 3, "3 improved (regression guard)");
+    assert_eq!(comparison.totals.regressed, 1, "1 regressed (regression guard)");
+    assert_eq!(
+        comparison.totals.unchanged_pass, 4,
+        "4 unchanged pass (regression guard)"
+    );
+    assert_eq!(
+        comparison.totals.unchanged_fail, 2,
+        "2 unchanged fail (regression guard)"
+    );
+
+    // Then: the new paired-difference test is computed from those same 10 pairs.
+    match comparison.paired_difference {
+        PairedDifferenceTest::Computed {
+            n,
+            mean_difference,
+            standard_error,
+            degrees_of_freedom,
+            t_statistic,
+            p_value,
+            significant,
+            ..
+        } => {
+            assert_eq!(n, 10, "10 paired assertions feed the test");
+            assert!(
+                (mean_difference - 0.2).abs() < 1e-9,
+                "mean difference: expected 0.2, got {mean_difference}"
+            );
+            assert!(
+                (standard_error - 0.2).abs() < 1e-9,
+                "SEM: expected 0.2, got {standard_error}"
+            );
+            assert_eq!(degrees_of_freedom, 9, "df = n - 1 = 9");
+            let t = t_statistic.expect("variance is nonzero here; t-statistic is defined");
+            assert!(
+                (t - 1.0).abs() < 1e-9,
+                "t = mean_difference / SEM = 1.0 exactly, got {t}"
+            );
+            assert!(
+                (p_value - 0.3434).abs() < 1e-3,
+                "two-tailed p for t=1.0, df=9 is ~0.3434 (must be a real Student's-t \
+                 computation: a one-tailed value would read ~0.172, a normal/z \
+                 approximation would read ~0.317, an off-by-one df=10 bug would read \
+                 ~0.341 — all outside this tolerance), got {p_value}"
+            );
+            assert!(
+                !significant,
+                "p ~0.34 does not clear the α = 0.05 significance bar"
+            );
+        }
+        other => panic!("expected PairedDifferenceTest::Computed, got {other:?}"),
+    }
+}
+
+/// Given an assistant turn that calls `lookup_claim_history` once and
+/// `lookup_policy` twice, in that order (the reverse order, and a
+/// different multiplicity, than a hand-written `tool_call_order` sequence
+/// would name),
+///
+/// When a `tool_call_collection` assertion requires the same multiset
+/// (`lookup_policy` twice, `lookup_claim_history` once) but lists it in the
+/// opposite order,
+///
+/// Then it passes — proving the check is order-insensitive, unlike
+/// `tool_call_order`. And when a second assertion requires a third
+/// `lookup_policy` call that was never made, it fails, naming the tool and
+/// both the required and observed counts.
+#[tokio::test]
+async fn tool_call_collection_passes_regardless_of_order_and_fails_on_missing_calls() {
+    // Given
+    let ctx = EvaluationContext::empty();
+    let conv = conversation_with(vec![assistant_blocks(vec![
+        tool_use("lookup_claim_history"),
+        tool_use("lookup_policy"),
+        tool_use("lookup_policy"),
+    ])]);
+
+    // When: requires the same multiset, listed in the opposite order.
+    let passing = Assertion::ToolCallCollection {
+        tools: vec![
+            String::from("lookup_policy"),
+            String::from("lookup_policy"),
+            String::from("lookup_claim_history"),
+        ],
+    };
+
+    // Then: passes — order-insensitive.
+    assert_eq!(passing.check(&conv, &ctx).await, AssertionOutcome::Pass);
+
+    // When: requires a third lookup_policy call that was never made.
+    let failing = Assertion::ToolCallCollection {
+        tools: vec![
+            String::from("lookup_policy"),
+            String::from("lookup_policy"),
+            String::from("lookup_policy"),
+        ],
+    };
+
+    // Then: fails, naming the tool and both the required (3) and observed
+    // (2) counts.
+    match failing.check(&conv, &ctx).await {
+        AssertionOutcome::Fail { reason } => {
+            assert!(reason.contains("lookup_policy"), "got {reason}");
+            assert!(
+                reason.contains('3'),
+                "reason should name the required count: {reason}"
+            );
+            assert!(
+                reason.contains('2'),
+                "reason should name the observed count: {reason}"
+            );
+        }
+        other => panic!("expected Fail, got {other:?}"),
+    }
+}
+
+fn conversation_with(session: Vec<Message>) -> Conversation {
+    Conversation {
+        meta: Meta {
+            model: ModelId::from("noop"),
+            debug: false,
+            assembly: None,
+            binding: BindingMap::new(),
+        },
+        session,
+    }
+}
+
+fn assistant_blocks(blocks: Vec<ContentBlock>) -> Message {
+    Message {
+        role: Role::Assistant,
+        body: Some(Content::Blocks(blocks)),
+        cache: false,
+        trace: None,
+        _phase: PhantomData,
+    }
+}
+
+fn tool_use(name: &str) -> ContentBlock {
+    ContentBlock::ToolUse {
+        id: ToolUseId::from("tool_1"),
+        name: String::from(name),
+        input: serde_yaml_ng::Value::Null,
+    }
+}

From 79bc406353da0399c9321ccf35f2764fcd532cd5 Mon Sep 17 00:00:00 2001
From: David Souther <davidsouther@gmail.com>
Date: Mon, 6 Jul 2026 13:33:28 -0400
Subject: [PATCH 02/12] feat(report): step 0 - API surface stubs for
 paired-difference test + tool_call_collection

Adds PairedDifferenceTest, PAIRED_DIFFERENCE_ALPHA, ComparisonReport.paired_difference,
Assertion::ToolCallCollection, and stub dispatch/class_tag wiring so the RED feature
test (tests/report_statistics.rs) compiles and fails at todo!()/assertion runtime
rather than at compile time. No behavior implemented yet.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 plan.md                     | 255 ------------------------------------
 src/content/evaluation.rs   |   3 +
 src/knowledge/assertions.rs |  12 ++
 src/knowledge/eval.rs       |   1 +
 src/knowledge/report.rs     |  59 +++++++++
 tests/report_statistics.rs  |  13 +-
 6 files changed, 85 insertions(+), 258 deletions(-)
 delete mode 100644 plan.md

diff --git a/plan.md b/plan.md
deleted file mode 100644
index 3b7a7001..00000000
--- a/plan.md
+++ /dev/null
@@ -1,255 +0,0 @@
-# Plan: Verify `ailly-skill-eval` Falsification Gate (Rust API only)
-
-**Feature test:** `tests/falsification_gate.rs` ::
-`falsification_gate_matches_the_documented_improved_and_regressed_formula`
-(committed RED in this worktree; confirmed failing with exactly three
-`E0599: no method named 'passes_falsification_gate' found for struct
-'ComparisonTotals'` errors, no other errors).
-
-**User story:** As a developer about to build a new `ailly-skill-eval` suite,
-I need `improved > 0 && regressed == 0` — the falsification gate that
-`SKILL.md` and `references/method.md` §6 document — to be one named, tested
-Rust function I can call, instead of a boolean I re-derive by hand from a
-`ComparisonTotals`.
-
-**Scope cut (per design.md and the coordinating brief):** this plan builds
-*only* `ComparisonTotals::passes_falsification_gate` in
-`src/knowledge/report.rs`. It does not touch `docs/developer/TASKS.md`, does
-not rewire `e2e/patterns-eval/ci.sh`'s Python heredoc or
-`tests/skill_forge_clean_comments_review.rs`, does not fix CI's
-`ANTHROPIC_API_KEY` secret, and does not run the live `patterns-eval`
-re-confirmation described in design.md's Specification step 2. See
-"Deferred items" at the end.
-
-## Steps checklist
-
-- [x] Step 0 — API surface stub (signature only, no logic)
-- [x] Step 1 — Unblock the "clears the gate" assertion (scenario 1); add `mod tests` + first unit test
-- [x] Step 2 — Unblock the "fails on regression" assertion (scenario 2); add second unit test
-- [x] Step 3 — Confirm the "fails as vacuous" assertion (scenario 3) needs no further generalization; add third unit test
-- [x] Step 4 — Doc comment, formatting, and whole-workspace lint/test confirmation
-
----
-
-## Step 0 — API surface stub
-
-Add the method's signature to `src/knowledge/report.rs`, next to the existing
-`ComparisonTotals` struct definition (there is no existing `impl
-ComparisonTotals` block; this creates the first one). No formula, no field
-reads — a body that compiles but does not yet satisfy any assertion:
-
-```rust
-impl ComparisonTotals {
-    pub fn passes_falsification_gate(&self) -> bool {
-        todo!()
-    }
-}
-```
-
-This is enough to turn the current compile-time `E0599` (method not found)
-into a runtime panic on the first assertion — i.e. it moves the failure from
-"won't build" to "builds, fails deliberately," which is the correct starting
-point for Step 1. No test assertion is satisfied yet.
-
-## Step 1 — Unblock scenario 1: "clears the gate"
-
-**Assertion unblocked:** `tests/falsification_gate.rs:106-109` —
-`assert!(clears.totals.passes_falsification_gate(), "improved > 0 &&
-regressed == 0 must clear the gate")`, where the fixture pair yields
-`improved: 1, regressed: 0`.
-
-**Happy-path test sketch** (already written, not new — this step targets the
-existing scenario 1 block): build a baseline/invocation `EvalReport` pair via
-the test's `fixture_report` helper where one assertion class flips
-`fail → pass` (judge) and the others stay stable (`script` fail/fail,
-`tokens` pass/pass); run `compute_comparison`; call
-`.totals.passes_falsification_gate()`; expect `true`.
-
-**Implementation outline:** replace the `todo!()` body with the simplest
-predicate that satisfies this one scenario without yet accounting for
-regression — i.e. a "fake it" pass keyed only on `self.improved`. This is
-intentionally under-specified relative to the documented two-conjunct gate;
-Step 2 supplies the second conjunct. Do not consult scenario 2 or 3's
-fixtures yet when writing this step's body.
-
-**Unit test (repo convention):** `src/knowledge/report.rs` has no
-`#[cfg(test)] mod tests` block yet, unlike its sibling modules
-(`src/knowledge/assertions.rs:1326`, `src/knowledge/eval.rs`). Add one in
-this step with a direct, fixture-free unit test constructing
-`ComparisonTotals { improved: 1, regressed: 0, ..Default::default() }` and
-asserting `.passes_falsification_gate()` — this exercises the predicate
-without going through `compute_comparison`/`EvalReport`, matching the "unit
-tests per module plus the one feature test" convention the feature test
-alone does not force.
-
-## Step 2 — Unblock scenario 2: "fails on regression"
-
-**Assertion unblocked:** `tests/falsification_gate.rs:130-133` —
-`assert!(!regressed.totals.passes_falsification_gate(), "a single regression
-must fail the gate even though improved > 0")`, where the fixture pair
-yields `improved: 1, regressed: 1`.
-
-**Happy-path test sketch** (already written — existing scenario 2 block):
-build a pair where `judge` improves (`fail → pass`) *and* `tokens` regresses
-(`pass → fail`) in the same comparison; run `compute_comparison`; call
-`.totals.passes_falsification_gate()`; expect `false`.
-
-**Implementation outline:** triangulate against Step 1's under-specified body
-— Step 1's predicate (keyed only on `improved`) would wrongly return `true`
-here, so this step forces the second conjunct into the body: the method must
-also read `self.regressed` and require it to be `0`. After this change,
-re-check that Step 1's scenario is still satisfied (it is, since `regressed
-== 0` holds there too). This step is where the implementation converges on
-the exact documented formula, `self.improved > 0 && self.regressed == 0`.
-
-**Unit test:** add a second case to Step 1's new `mod tests` block —
-`ComparisonTotals { improved: 1, regressed: 1, ..Default::default() }`
-asserting `!.passes_falsification_gate()` — pinning the same triangulation
-the feature test's scenario 2 pins, directly against the struct.
-
-## Step 3 — Confirm scenario 3: "fails as vacuous"
-
-**Assertion unblocked:** `tests/falsification_gate.rs:157-159` —
-`assert!(!vacuous.totals.passes_falsification_gate(), "improved == 0 must
-fail the gate even when regressed == 0 too")`, where the fixture pair yields
-`improved: 0, regressed: 0`.
-
-**Happy-path test sketch** (already written — existing scenario 3 block):
-build a pair where both arms fail the same checker (`script`
-`fail`/`fail`, no class changes at all); run `compute_comparison`; call
-`.totals.passes_falsification_gate()`; expect `false`.
-
-**Implementation outline:** no further code change is expected — the
-two-conjunct formula reached at the end of Step 2 already evaluates
-`0 > 0 && 0 == 0` to `false`. This step is a verification checkpoint, not a
-new generalization: run the full `falsification_gate` test and confirm all
-three scenarios pass together in one process (the earlier steps only reason
-about each scenario in isolation). If this step's assertion fails, that is a
-signal Step 2's generalization was wrong (e.g. it used `>=` instead of `>`,
-or dropped the `improved` conjunct entirely) and Step 2 must be revisited —
-do not patch scenario 3 with a special case.
-
-**Unit test:** add the third case to the same `mod tests` block —
-`ComparisonTotals::default()` (i.e. `improved: 0, regressed: 0`) asserting
-`!.passes_falsification_gate()` — completing the three-case unit-test set
-that mirrors the feature test's three scenarios directly against the struct.
-
-## Step 4 — Doc comment, formatting, and whole-workspace confirmation
-
-**Assertion unblocked:** none new — this step closes out the feature test
-as a whole and guards against regressions elsewhere in the workspace.
-
-**Happy-path test sketch:** re-run
-`cargo test --test falsification_gate --all-features` (or the project's
-canonical test task) and confirm it is green; then run the full workspace
-test/check/lint tasks to confirm nothing else broke.
-
-**Implementation outline:**
-1. Attach the doc comment from design.md's Specification verbatim (the
-   `/// The falsification gate documented in ... SKILL.md ... and
-   references/method.md §6: ...` comment) above the method, so the method's
-   own doc explains *why* the formula is what it is, not just what it
-   computes.
-2. Run `mise run check`, `mise run test`, and `mise run lint` (this repo's
-   canonical commands per `mise.toml`) from the worktree root — not ad hoc
-   `cargo` invocations — and confirm all three are clean, including the
-   existing `tests/report_cmd.rs` fixtures (which exercise `compute_comparison`
-   but not the new method) and `tests/skill_eval_guide.rs` (unaffected, reads
-   only the doc files on disk).
-3. Confirm `cargo fmt` produces no diff (or run `mise run format` and check
-   `git diff` is empty) so the new `impl` block matches the file's existing
-   style.
-
----
-
-## Deferred items (explicitly out of scope for this plan)
-
-- **`docs/developer/TASKS.md`** — not touched. It carries unrelated
-  uncommitted changes from the in-flight `docs/developer/2026-06-26-A-eval-static-doc`
-  session; design.md's own Open Artifact Decision 1 says not to append to it
-  until that session's edits have landed. *Needs a decision from:* the human
-  coordinator (sequencing between the two in-flight sessions).
-- **GitHub Actions `ANTHROPIC_API_KEY` secret (401 Unauthorized since
-  2026-06-02)** — not rotated or investigated further; requires
-  repo-secret-management access this plan does not exercise. *Needs a
-  decision from:* the human coordinator (someone with repo secrets access).
-- **`e2e/patterns-eval/ci.sh`'s Python heredoc** and
-  **`tests/skill_forge_clean_comments_review.rs`'s inline gate assertions**
-  — left exactly as-is; both already correctly re-derive the same formula by
-  hand today, and design.md's Open Artifact Decision 3 recommends deferring
-  their rewiring to the new API as a separate `TASKS.md` follow-up, not this
-  feature-step. *Needs a decision from:* the human coordinator (whether/when
-  to fold this refactor in).
-- **Live re-confirmation of the `patterns-eval` gate** (design.md
-  Specification step 2: clearing stale local run debris, running
-  `bash e2e/patterns-eval/ci.sh` with real credentials, and recording the
-  resulting bucket totals and verdict) — not attempted in this plan or its
-  implementation. It depends on unresolved Open Artifact Decisions
-  (where to record the result; whether to fix CI first) that design.md
-  explicitly leaves to human review. *Needs a decision from:* the human
-  coordinator (Open Artifact Decisions 1, 2, and 4 in design.md).
-
----
-
-## Resolved by the long-loop reviewer (2026-07-06)
-
-**1. Plan sizing and Step 0's API surface against actual code. Decided: keep
-the 5-step shape (Step 0–4), unchanged, plus the unit-test addition in item 2
-below.** Re-read cold against `src/knowledge/report.rs` on disk: the
-`ComparisonTotals` struct (5 `pub usize` fields, `Default`-derived, no
-existing `impl` block) matches Step 0's stub exactly, and a fresh
-`cargo test --test falsification_gate --all-features` run reproduces the
-plan's claimed RED state verbatim — exactly three `E0599` errors at lines
-107, 131, 158, no others. Five steps is within the 3–7 band. The
-fake-it-then-triangulate shape of Steps 1–2 is more ceremony than a
-one-line, four-source-corroborated formula strictly needs, but it is exactly
-the red-green-refactor discipline this feature-step's build instructions
-require ("write or adjust the relevant unit test first... implement the
-minimum to pass"), so it is right-sized for the process being followed, not
-oversized for the formula alone. No restructuring needed.
-
-**2. Repo convention gap: no unit tests inside `src/knowledge/report.rs`
-itself. Decided: add a `#[cfg(test)] mod tests` block to `report.rs` (folded
-into Steps 1–3, one case per step) with three direct, fixture-free unit
-tests against `ComparisonTotals` literals, alongside the existing
-fixture-driven feature test.** The plan as drafted only exercised the new
-method through `tests/falsification_gate.rs`'s `compute_comparison`-fixture
-path. `src/knowledge/assertions.rs:1326` and `src/knowledge/eval.rs` both
-carry `#[cfg(test)] mod tests` blocks; `report.rs` currently has none. The
-build instructions for this feature-step are explicit: "Write unit tests for
-new logic even where the feature test alone would not force it — this
-repo's convention is unit tests per module plus the one feature test." This
-is the conservative default (matching an already-established, repo-wide
-pattern) rather than a new convention being invented.
-
-**3. Design.md's five Open Artifact Decisions. Decided: all five stay
-deferred to the human coordinator, exactly as the plan's own "Deferred
-items" section and this feature-step's authoritative extra notes already
-state — no further action taken on any of them in this plan or its build.**
-Specifically: (OAD 1) where the Build-phase live-run result gets recorded —
-deferred, blocked on `docs/developer/2026-06-26-A-eval-static-doc` landing
-first, per the extra notes' explicit instruction not to touch
-`docs/developer/TASKS.md`. (OAD 2) fixing the GitHub Actions
-`ANTHROPIC_API_KEY` secret — deferred, per the extra notes' explicit
-instruction not to attempt this; it needs repo-secret-management access this
-session does not have. (OAD 3) wiring `passes_falsification_gate` into
-`ci.sh`'s Python heredoc and `tests/skill_forge_clean_comments_review.rs` —
-deferred, per the extra notes' explicit instruction to leave both call sites
-exactly as-is; this also matches design.md's own recommendation ("not in
-this feature-step... both call sites already work correctly today"). (OAD 4)
-cadence for re-running the live confirmation — deferred; design.md itself
-makes no recommendation and ties it to whoever resolves OAD 2, so it moves
-in lockstep with that decision. (OAD 5) cleaning up ~600 stale local
-`e2e/patterns-eval/runs`/`evals/reports` directories — deferred; this is
-listed in design.md as step 1 of the Build-phase live-run procedure (OAD
-Specification step 2), and the extra notes explicitly exclude running that
-live re-confirmation in this feature-step, so the cleanup step it belongs to
-does not apply here either. None of these five is a prerequisite for the
-Rust-API-only scope this plan builds (`ComparisonTotals::passes_falsification_gate`
-and its tests do not read `docs/developer/TASKS.md`, CI secrets, `ci.sh`, or
-local run directories), so none of them blocks this gate. No escalation
-triggered under the long-loop escalation rule (irreversible /
-out-of-recorded-scope / underdetermined): each of these five is already
-explicitly resolved by this feature-step's authoritative extra notes, so
-none is underdetermined, and none is being decided here beyond what those
-notes already state.
diff --git a/src/content/evaluation.rs b/src/content/evaluation.rs
index 20424b48..bb08042a 100644
--- a/src/content/evaluation.rs
+++ b/src/content/evaluation.rs
@@ -75,6 +75,9 @@ pub enum Assertion {
     ToolCallOrder {
         sequence: Vec<String>,
     },
+    ToolCallCollection {
+        tools: Vec<String>,
+    },
 
     TextContains {
         value: String,
diff --git a/src/knowledge/assertions.rs b/src/knowledge/assertions.rs
index cf8ecf7e..c397ca48 100644
--- a/src/knowledge/assertions.rs
+++ b/src/knowledge/assertions.rs
@@ -142,6 +142,9 @@ impl Assertion {
                 check_tool_call_count(conversation, tool.as_deref(), op, *value)
             }
             Assertion::ToolCallOrder { sequence } => check_tool_call_order(conversation, sequence),
+            Assertion::ToolCallCollection { tools } => {
+                check_tool_call_collection(conversation, tools)
+            }
 
             Assertion::JsonPath { path, op, value } => {
                 check_json_path(conversation, path, op, value)
@@ -1323,6 +1326,15 @@ fn check_tool_call_order(conversation: &Conversation, sequence: &[String]) -> As
     }
 }
 
+/// Order-insensitive multiset check: every name in `tools` must appear in
+/// the observed tool-use calls at least as many times as it is listed.
+/// Subset-of-multiset, not exact equality — extra calls (of any tool) are
+/// tolerated, and order carries no meaning.
+fn check_tool_call_collection(conversation: &Conversation, tools: &[String]) -> AssertionOutcome {
+    let _ = (conversation, tools);
+    todo!()
+}
+
 #[cfg(test)]
 mod tests {
     use std::collections::VecDeque;
diff --git a/src/knowledge/eval.rs b/src/knowledge/eval.rs
index e05ccc10..4d9421ef 100644
--- a/src/knowledge/eval.rs
+++ b/src/knowledge/eval.rs
@@ -126,6 +126,7 @@ pub(crate) fn class_tag(assertion: &Assertion) -> &'static str {
         Assertion::MustNotCallTool { .. } => "must_not_call_tool",
         Assertion::ToolCallCount { .. } => "tool_call_count",
         Assertion::ToolCallOrder { .. } => "tool_call_order",
+        Assertion::ToolCallCollection { .. } => "tool_call_collection",
         Assertion::TextContains { .. } => "text_contains",
         Assertion::TextNotContains { .. } => "text_not_contains",
         Assertion::TextMatches { .. } => "text_matches",
diff --git a/src/knowledge/report.rs b/src/knowledge/report.rs
index 7090a6f4..3cf4cbfd 100644
--- a/src/knowledge/report.rs
+++ b/src/knowledge/report.rs
@@ -10,6 +10,38 @@ use serde::Serialize;
 
 use crate::knowledge::eval::EvalReport;
 
+/// The α level used for the paired-difference test's significance verdict.
+/// See design.md Summary for why 0.05 (not a project-research-stated value)
+/// was chosen as the conservative default.
+pub const PAIRED_DIFFERENCE_ALPHA: f64 = 0.05;
+
+/// Two-tailed paired Student's t-test over every assertion pair
+/// `compute_comparison` already classifies as `Improved` (+1.0),
+/// `Regressed` (-1.0), or `Unchanged{Pass,Fail}` (0.0).
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(tag = "status", rename_all = "snake_case")]
+pub enum PairedDifferenceTest {
+    /// Fewer than 2 paired assertions exist between the two arms. A sample
+    /// variance (and therefore a standard error and a t-statistic) cannot
+    /// be estimated from 0 or 1 observations.
+    InsufficientPairs { n: usize },
+    Computed {
+        n: usize,
+        mean_difference: f64,
+        sample_std_dev: f64,
+        /// Standard error of the mean: `sample_std_dev / sqrt(n)`.
+        standard_error: f64,
+        degrees_of_freedom: usize,
+        /// `None` iff `sample_std_dev == 0.0` — see the zero-variance
+        /// convention below. `p_value`/`significant` stay well-defined by
+        /// convention even when the t-statistic itself is not a real
+        /// number (a 0/0 or x/0 limit).
+        t_statistic: Option<f64>,
+        p_value: f64,
+        significant: bool,
+    },
+}
+
 /// Top-level comparison report serialized to JSON.
 #[derive(Serialize, Deserialize, Debug)]
 pub struct ComparisonReport {
@@ -23,6 +55,7 @@ pub struct ComparisonReport {
     /// `arm_b`, so this is the gate's verdict on the totals as given, not a
     /// claim that this comparison *is* a baseline falsification run.
     pub falsification_gate: bool,
+    pub paired_difference: PairedDifferenceTest,
     pub cases: Vec<CaseComparison>,
 }
 
@@ -146,9 +179,34 @@ pub fn compute_comparison(arm_a: &EvalReport, arm_b: &EvalReport) -> ComparisonR
         falsification_gate: totals.passes_falsification_gate(),
         totals,
         cases,
+        paired_difference: todo!(),
     }
 }
 
+/// Regularized incomplete beta function `I_x(a, b)`, computed via Lentz's
+/// continued-fraction method with a Lanczos log-gamma approximation for the
+/// beta normalization constant. Used by [`paired_difference_p_value`] to
+/// derive the Student's-t two-tailed survival-function p-value.
+fn regularized_incomplete_beta(x: f64, a: f64, b: f64) -> f64 {
+    let _ = (x, a, b);
+    todo!()
+}
+
+/// Two-tailed p-value for a Student's-t statistic via the closed-form
+/// relationship to the regularized incomplete beta function:
+/// `p = I_x(df/2, 1/2)`, `x = df / (df + t^2)`.
+fn paired_difference_p_value(t: f64, df: usize) -> f64 {
+    let _ = (t, df);
+    todo!()
+}
+
+/// Fold a slice of per-pair diffs (`+1.0`/`-1.0`/`0.0`) into a
+/// [`PairedDifferenceTest`].
+fn compute_paired_difference(diffs: &[f64]) -> PairedDifferenceTest {
+    let _ = diffs;
+    todo!()
+}
+
 /// Render a single [`EvalReport`] as a markdown summary.
 ///
 /// Layer 1: suite, `run_id`, overall pass rate.
@@ -380,6 +438,7 @@ mod tests {
                 ..Default::default()
             },
             falsification_gate: true,
+            paired_difference: PairedDifferenceTest::InsufficientPairs { n: 0 },
             cases: vec![],
         };
         assert!(render_comparison_markdown(&passing, "arm-a", "arm-b").contains("PASS"));
diff --git a/tests/report_statistics.rs b/tests/report_statistics.rs
index fbe46616..5d24d706 100644
--- a/tests/report_statistics.rs
+++ b/tests/report_statistics.rs
@@ -1,5 +1,6 @@
 //! Feature test for Feature F — `report`'s statistical-rigor upgrade.
-//! See `.ailly/developer/2026-07-06-A-ailly-evals/feature-f-report-stats/design.md`.
+//! See `.ailly/developer/2026-07-06-A-ailly-evals/feature-f-report-stats/
+//! design.md`.
 //!
 //! Two independent capabilities land together in this feature-step:
 //!
@@ -127,8 +128,14 @@ fn compute_comparison_reports_paired_difference_test_with_standard_error() {
     let comparison = compute_comparison(&arm_a, &arm_b);
 
     // Then: existing bucket counts are unchanged.
-    assert_eq!(comparison.totals.improved, 3, "3 improved (regression guard)");
-    assert_eq!(comparison.totals.regressed, 1, "1 regressed (regression guard)");
+    assert_eq!(
+        comparison.totals.improved, 3,
+        "3 improved (regression guard)"
+    );
+    assert_eq!(
+        comparison.totals.regressed, 1,
+        "1 regressed (regression guard)"
+    );
     assert_eq!(
         comparison.totals.unchanged_pass, 4,
         "4 unchanged pass (regression guard)"

From a24fe99296b98c416a4cd648e854ea3f40885236 Mon Sep 17 00:00:00 2001
From: David Souther <davidsouther@gmail.com>
Date: Mon, 6 Jul 2026 13:36:42 -0400
Subject: [PATCH 03/12] feat(report): step 1 - regularized incomplete beta /
 two-tailed t p-value

Implements log_gamma (Lanczos approximation), incomplete_beta_continued_fraction
(Lentz's method), regularized_incomplete_beta, and paired_difference_p_value,
pinned by three unit tests: the standard printed two-tailed-5%-critical-value
table entry (t=2.262, df=9 -> p~=0.0500), t=0 -> p=1.0 for any df, and a
monotonicity spot-check. Not yet wired into compute_comparison (Step 2).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/knowledge/report.rs | 148 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 144 insertions(+), 4 deletions(-)

diff --git a/src/knowledge/report.rs b/src/knowledge/report.rs
index 3cf4cbfd..01a5902c 100644
--- a/src/knowledge/report.rs
+++ b/src/knowledge/report.rs
@@ -183,21 +183,124 @@ pub fn compute_comparison(arm_a: &EvalReport, arm_b: &EvalReport) -> ComparisonR
     }
 }
 
+/// Lanczos approximation of the natural log of the gamma function, in the
+/// classic Numerical Recipes coefficient form. Used to build the beta
+/// normalization constant in [`regularized_incomplete_beta`].
+fn log_gamma(xx: f64) -> f64 {
+    const COF: [f64; 6] = [
+        76.180_091_729_471_46,
+        -86.505_320_329_416_77,
+        24.014_098_240_830_91,
+        -1.231_739_572_450_155,
+        0.120_865_097_386_617_9e-2,
+        -0.539_523_938_495_3e-5,
+    ];
+    let x = xx;
+    let mut y = xx;
+    let tmp = x + 5.5;
+    let tmp = tmp - (x + 0.5) * tmp.ln();
+    let mut ser = 1.000_000_000_190_015;
+    for c in COF {
+        y += 1.0;
+        ser += c / y;
+    }
+    -tmp + (2.506_628_274_631_000_5 * ser / x).ln()
+}
+
+/// Lentz's continued-fraction evaluation of the incomplete beta function,
+/// used only inside its valid convergence domain (`x < (a+1)/(a+b+2)`) by
+/// [`regularized_incomplete_beta`], which flips to the complementary
+/// identity outside that domain.
+///
+/// Variable names (`a`, `b`, `c`, `d`, `h`, `m`) intentionally mirror the
+/// textbook Numerical Recipes `betacf` routine this implements, so the code
+/// can be checked line-by-line against that reference.
+#[allow(clippy::many_single_char_names)]
+fn incomplete_beta_continued_fraction(a: f64, b: f64, x: f64) -> f64 {
+    const MAX_ITERATIONS: u32 = 200;
+    const EPSILON: f64 = 1e-14;
+    const FP_MIN: f64 = 1e-300;
+
+    let qab = a + b;
+    let qap = a + 1.0;
+    let qam = a - 1.0;
+    let mut c = 1.0;
+    let mut d = 1.0 - qab * x / qap;
+    if d.abs() < FP_MIN {
+        d = FP_MIN;
+    }
+    d = 1.0 / d;
+    let mut h = d;
+
+    for m in 1..=MAX_ITERATIONS {
+        let m_f = f64::from(m);
+        let m2 = 2.0 * m_f;
+
+        let aa = m_f * (b - m_f) * x / ((qam + m2) * (a + m2));
+        d = 1.0 + aa * d;
+        if d.abs() < FP_MIN {
+            d = FP_MIN;
+        }
+        c = 1.0 + aa / c;
+        if c.abs() < FP_MIN {
+            c = FP_MIN;
+        }
+        d = 1.0 / d;
+        h *= d * c;
+
+        let aa = -(a + m_f) * (qab + m_f) * x / ((a + m2) * (qap + m2));
+        d = 1.0 + aa * d;
+        if d.abs() < FP_MIN {
+            d = FP_MIN;
+        }
+        c = 1.0 + aa / c;
+        if c.abs() < FP_MIN {
+            c = FP_MIN;
+        }
+        d = 1.0 / d;
+        let del = d * c;
+        h *= del;
+
+        if (del - 1.0).abs() < EPSILON {
+            break;
+        }
+    }
+
+    h
+}
+
 /// Regularized incomplete beta function `I_x(a, b)`, computed via Lentz's
 /// continued-fraction method with a Lanczos log-gamma approximation for the
 /// beta normalization constant. Used by [`paired_difference_p_value`] to
 /// derive the Student's-t two-tailed survival-function p-value.
 fn regularized_incomplete_beta(x: f64, a: f64, b: f64) -> f64 {
-    let _ = (x, a, b);
-    todo!()
+    if x <= 0.0 {
+        return 0.0;
+    }
+    if x >= 1.0 {
+        return 1.0;
+    }
+
+    let ln_beta_normalization = log_gamma(a + b) - log_gamma(a) - log_gamma(b);
+    let front = (ln_beta_normalization + a * x.ln() + b * (1.0 - x).ln()).exp();
+
+    if x < (a + 1.0) / (a + b + 2.0) {
+        front * incomplete_beta_continued_fraction(a, b, x) / a
+    } else {
+        1.0 - front * incomplete_beta_continued_fraction(b, a, 1.0 - x) / b
+    }
 }
 
 /// Two-tailed p-value for a Student's-t statistic via the closed-form
 /// relationship to the regularized incomplete beta function:
 /// `p = I_x(df/2, 1/2)`, `x = df / (df + t^2)`.
 fn paired_difference_p_value(t: f64, df: usize) -> f64 {
-    let _ = (t, df);
-    todo!()
+    // `df` is a paired-assertion count (realistically single-to-low-double
+    // digits per suite comparison), never near f64's 2^52 mantissa limit.
+    #[allow(clippy::cast_precision_loss)]
+    let df = df as f64;
+    let x = df / (df + t * t);
+    regularized_incomplete_beta(x, df / 2.0, 0.5)
 }
 
 /// Fold a slice of per-pair diffs (`+1.0`/`-1.0`/`0.0`) into a
@@ -475,4 +578,41 @@ mod tests {
         let totals = ComparisonTotals::default();
         assert!(!totals.passes_falsification_gate());
     }
+
+    /// Reference value cross-check, required by design.md Specification
+    /// item 1 before `paired_difference_p_value` is trusted: the standard
+    /// printed two-tailed-5%-critical-value table entry for `df = 9` is
+    /// `t = 2.262`. Independently re-verified for this plan against
+    /// `scipy.stats.t.sf` and a from-scratch continued-fraction port
+    /// (agreement to ~1e-8); this test pins the value at a looser `1e-3`
+    /// tolerance appropriate for this repo's own implementation.
+    #[test]
+    fn p_value_matches_standard_two_tailed_five_percent_critical_value() {
+        let p = paired_difference_p_value(2.262, 9);
+        assert!(
+            (p - 0.0500).abs() < 1e-3,
+            "expected p ~= 0.0500 for t=2.262, df=9, got {p}"
+        );
+    }
+
+    /// A t-statistic of exactly zero carries no evidence against the null,
+    /// regardless of sample size.
+    #[test]
+    fn zero_t_statistic_always_yields_p_one() {
+        assert!((paired_difference_p_value(0.0, 5) - 1.0).abs() < 1e-9);
+        assert!((paired_difference_p_value(0.0, 9) - 1.0).abs() < 1e-9);
+    }
+
+    /// Monotonicity spot-check guarding against a sign or formula
+    /// inversion that could still hit the two pinned reference values by
+    /// coincidence.
+    #[test]
+    fn p_value_strictly_decreases_as_t_grows() {
+        let p_small = paired_difference_p_value(1.0, 9);
+        let p_large = paired_difference_p_value(5.0, 9);
+        assert!(
+            p_large < p_small,
+            "expected p(t=5) < p(t=1), got p(t=5)={p_large}, p(t=1)={p_small}"
+        );
+    }
 }

From 46efcb203a8c03bfd0badeba4ae15c0f9c53689c Mon Sep 17 00:00:00 2001
From: David Souther <davidsouther@gmail.com>
Date: Mon, 6 Jul 2026 13:38:16 -0400
Subject: [PATCH 04/12] feat(report): step 2 - wire
 PairedDifferenceTest::Computed into compute_comparison

Collects the per-pair +1.0/-1.0/0.0 diff alongside the existing bucket-count
increments, then folds it through compute_paired_difference's general-case
branch (mean, sample std dev, SEM, t-statistic, df, p-value via Step 1's
paired_difference_p_value, significance at PAIRED_DIFFERENCE_ALPHA).
InsufficientPairs and zero-variance remain todo!() until Step 3.

Story 1's feature test (compute_comparison_reports_paired_difference_test_with_standard_error)
now passes; full lib test suite (216 tests) and tests/report_cmd.rs stay green.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/knowledge/report.rs | 64 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 57 insertions(+), 7 deletions(-)

diff --git a/src/knowledge/report.rs b/src/knowledge/report.rs
index 01a5902c..c0a51b9e 100644
--- a/src/knowledge/report.rs
+++ b/src/knowledge/report.rs
@@ -113,6 +113,7 @@ pub fn compute_comparison(arm_a: &EvalReport, arm_b: &EvalReport) -> ComparisonR
     let a_index = index_assertions(arm_a);
 
     let mut totals = ComparisonTotals::default();
+    let mut diffs: Vec<f64> = Vec::new();
     let mut case_map: BTreeMap<String, Vec<AssertionComparison>> = BTreeMap::new();
 
     for case in &arm_b.cases {
@@ -144,10 +145,22 @@ pub fn compute_comparison(arm_a: &EvalReport, arm_b: &EvalReport) -> ComparisonR
                 };
 
                 match change {
-                    "Improved" => totals.improved += 1,
-                    "Regressed" => totals.regressed += 1,
-                    "UnchangedPass" => totals.unchanged_pass += 1,
-                    _ => totals.unchanged_fail += 1,
+                    "Improved" => {
+                        totals.improved += 1;
+                        diffs.push(1.0);
+                    }
+                    "Regressed" => {
+                        totals.regressed += 1;
+                        diffs.push(-1.0);
+                    }
+                    "UnchangedPass" => {
+                        totals.unchanged_pass += 1;
+                        diffs.push(0.0);
+                    }
+                    _ => {
+                        totals.unchanged_fail += 1;
+                        diffs.push(0.0);
+                    }
                 }
                 totals.total_assertions += 1;
 
@@ -179,7 +192,7 @@ pub fn compute_comparison(arm_a: &EvalReport, arm_b: &EvalReport) -> ComparisonR
         falsification_gate: totals.passes_falsification_gate(),
         totals,
         cases,
-        paired_difference: todo!(),
+        paired_difference: compute_paired_difference(&diffs),
     }
 }
 
@@ -305,9 +318,46 @@ fn paired_difference_p_value(t: f64, df: usize) -> f64 {
 
 /// Fold a slice of per-pair diffs (`+1.0`/`-1.0`/`0.0`) into a
 /// [`PairedDifferenceTest`].
+///
+/// `n < 2` and the zero-variance edge cases are filled in by a later
+/// build step (design.md's fixed edge-case conventions); this general-case
+/// branch is correct for any sample with `n >= 2` and nonzero variance.
 fn compute_paired_difference(diffs: &[f64]) -> PairedDifferenceTest {
-    let _ = diffs;
-    todo!()
+    let n = diffs.len();
+    if n < 2 {
+        todo!("InsufficientPairs edge case, filled in by a later build step");
+    }
+
+    // `n` is a paired-assertion count (realistically single-to-low-double
+    // digits per suite comparison), never near f64's 2^52 mantissa limit.
+    #[allow(clippy::cast_precision_loss)]
+    let n_f64 = n as f64;
+    let mean_difference = diffs.iter().sum::<f64>() / n_f64;
+    let sum_squared_deviation: f64 = diffs
+        .iter()
+        .map(|diff| (diff - mean_difference).powi(2))
+        .sum();
+    let sample_std_dev = (sum_squared_deviation / (n_f64 - 1.0)).sqrt();
+
+    if sample_std_dev.abs() < f64::EPSILON {
+        todo!("zero-variance edge case, filled in by a later build step");
+    }
+
+    let standard_error = sample_std_dev / n_f64.sqrt();
+    let degrees_of_freedom = n - 1;
+    let t_statistic = mean_difference / standard_error;
+    let p_value = paired_difference_p_value(t_statistic, degrees_of_freedom);
+
+    PairedDifferenceTest::Computed {
+        n,
+        mean_difference,
+        sample_std_dev,
+        standard_error,
+        degrees_of_freedom,
+        t_statistic: Some(t_statistic),
+        p_value,
+        significant: p_value < PAIRED_DIFFERENCE_ALPHA,
+    }
 }
 
 /// Render a single [`EvalReport`] as a markdown summary.

From db8bf060fe7c5c9056849384d03e7ec60cd59ce3 Mon Sep 17 00:00:00 2001
From: David Souther <davidsouther@gmail.com>
Date: Mon, 6 Jul 2026 13:39:37 -0400
Subject: [PATCH 05/12] feat(report): step 3 - InsufficientPairs and
 zero-variance edge-case conventions

compute_paired_difference now returns InsufficientPairs { n } for n < 2, and
for n >= 2 with zero variance returns Computed with t_statistic: None and
the fixed limiting values design.md names: p_value=1.0/significant=false
when mean_difference is also zero (vacuous comparison), p_value=0.0/
significant=true otherwise (perfect unanimous shift). Pinned by three new
unit tests. Full lib suite (219 tests) green.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/knowledge/report.rs | 92 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 87 insertions(+), 5 deletions(-)

diff --git a/src/knowledge/report.rs b/src/knowledge/report.rs
index c0a51b9e..95a30415 100644
--- a/src/knowledge/report.rs
+++ b/src/knowledge/report.rs
@@ -319,13 +319,21 @@ fn paired_difference_p_value(t: f64, df: usize) -> f64 {
 /// Fold a slice of per-pair diffs (`+1.0`/`-1.0`/`0.0`) into a
 /// [`PairedDifferenceTest`].
 ///
-/// `n < 2` and the zero-variance edge cases are filled in by a later
-/// build step (design.md's fixed edge-case conventions); this general-case
-/// branch is correct for any sample with `n >= 2` and nonzero variance.
+/// Edge-case conventions, fixed by design.md's Specification (not
+/// reinvented here):
+/// - `n < 2` → [`PairedDifferenceTest::InsufficientPairs`]: a sample variance
+///   (and therefore a standard error and a t-statistic) cannot be estimated
+///   from 0 or 1 observations.
+/// - `n >= 2` and zero variance (every diff identical) → `t_statistic: None`
+///   (not `f64::INFINITY`/`NaN`, which `serde_json` cannot round-trip).
+///   `mean_difference == 0.0` is the vacuous "every pair unchanged" comparison
+///   (`p_value: 1.0, significant: false`); `mean_difference != 0.0` is a
+///   perfect unanimous shift, the strongest evidence a paired comparison can
+///   produce (`p_value: 0.0, significant: true`).
 fn compute_paired_difference(diffs: &[f64]) -> PairedDifferenceTest {
     let n = diffs.len();
     if n < 2 {
-        todo!("InsufficientPairs edge case, filled in by a later build step");
+        return PairedDifferenceTest::InsufficientPairs { n };
     }
 
     // `n` is a paired-assertion count (realistically single-to-low-double
@@ -340,7 +348,21 @@ fn compute_paired_difference(diffs: &[f64]) -> PairedDifferenceTest {
     let sample_std_dev = (sum_squared_deviation / (n_f64 - 1.0)).sqrt();
 
     if sample_std_dev.abs() < f64::EPSILON {
-        todo!("zero-variance edge case, filled in by a later build step");
+        let (p_value, significant) = if mean_difference.abs() < f64::EPSILON {
+            (1.0, false)
+        } else {
+            (0.0, true)
+        };
+        return PairedDifferenceTest::Computed {
+            n,
+            mean_difference,
+            sample_std_dev,
+            standard_error: 0.0,
+            degrees_of_freedom: n - 1,
+            t_statistic: None,
+            p_value,
+            significant,
+        };
     }
 
     let standard_error = sample_std_dev / n_f64.sqrt();
@@ -665,4 +687,64 @@ mod tests {
             "expected p(t=5) < p(t=1), got p(t=5)={p_large}, p(t=1)={p_small}"
         );
     }
+
+    /// Fewer than 2 paired assertions exist: a sample variance (and
+    /// therefore a standard error and a t-statistic) cannot be estimated
+    /// from 0 or 1 observations.
+    #[test]
+    fn fewer_than_two_diffs_yields_insufficient_pairs() {
+        match compute_paired_difference(&[]) {
+            PairedDifferenceTest::InsufficientPairs { n } => assert_eq!(n, 0),
+            other => panic!("expected InsufficientPairs, got {other:?}"),
+        }
+        match compute_paired_difference(&[1.0]) {
+            PairedDifferenceTest::InsufficientPairs { n } => assert_eq!(n, 1),
+            other => panic!("expected InsufficientPairs, got {other:?}"),
+        }
+    }
+
+    /// Every pair unchanged (the "vacuous comparison" Feature G's design
+    /// independently named): zero variance, zero mean difference. No
+    /// evidence of a difference either way.
+    #[test]
+    fn all_zero_diffs_yields_p_one_not_significant() {
+        match compute_paired_difference(&[0.0, 0.0, 0.0]) {
+            PairedDifferenceTest::Computed {
+                mean_difference,
+                t_statistic,
+                p_value,
+                significant,
+                ..
+            } => {
+                assert!((mean_difference - 0.0).abs() < 1e-9);
+                assert_eq!(t_statistic, None);
+                assert!((p_value - 1.0).abs() < 1e-9);
+                assert!(!significant);
+            }
+            other => panic!("expected Computed, got {other:?}"),
+        }
+    }
+
+    /// Every pair moved by the same nonzero amount: zero variance, nonzero
+    /// mean difference. The strongest evidence a paired comparison can
+    /// produce (a perfect, unanimous shift with zero within-pair
+    /// variance).
+    #[test]
+    fn all_equal_nonzero_diffs_yields_p_zero_significant() {
+        match compute_paired_difference(&[1.0, 1.0, 1.0]) {
+            PairedDifferenceTest::Computed {
+                mean_difference,
+                t_statistic,
+                p_value,
+                significant,
+                ..
+            } => {
+                assert!((mean_difference - 1.0).abs() < 1e-9);
+                assert_eq!(t_statistic, None);
+                assert!((p_value - 0.0).abs() < 1e-9);
+                assert!(significant);
+            }
+            other => panic!("expected Computed, got {other:?}"),
+        }
+    }
 }

From f415eb73e2e93f5adb37449f5d2ee32fff91e0f6 Mon Sep 17 00:00:00 2001
From: David Souther <davidsouther@gmail.com>
Date: Mon, 6 Jul 2026 13:40:36 -0400
Subject: [PATCH 06/12] feat(assertions): step 4 - check_tool_call_collection +
 dispatch wiring

Order-insensitive multiset check over extract_tool_uses/tool_use_name:
every name in tools must appear at least as many times as listed;
extra/intervening calls tolerated. Empty tools: [] is vacuously satisfied
(pinned by a new unit test; the feature test doesn't cover it). Fail reason
names every shortfall tool plus required/observed counts.

Both feature-test functions in tests/report_statistics.rs now pass (Story 1
and Story 2). Full lib suite (220 tests) green.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/knowledge/assertions.rs | 44 +++++++++++++++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/src/knowledge/assertions.rs b/src/knowledge/assertions.rs
index c397ca48..74dc981a 100644
--- a/src/knowledge/assertions.rs
+++ b/src/knowledge/assertions.rs
@@ -4,6 +4,7 @@
 //! `Deferred` until the orchestrator wires their collaborator. The dispatch
 //! table is a total `match` over `Assertion`.
 
+use std::collections::BTreeMap;
 use std::marker::PhantomData;
 use std::path::Path;
 use std::path::PathBuf;
@@ -1331,8 +1332,40 @@ fn check_tool_call_order(conversation: &Conversation, sequence: &[String]) -> As
 /// Subset-of-multiset, not exact equality — extra calls (of any tool) are
 /// tolerated, and order carries no meaning.
 fn check_tool_call_collection(conversation: &Conversation, tools: &[String]) -> AssertionOutcome {
-    let _ = (conversation, tools);
-    todo!()
+    let mut required: BTreeMap<&str, usize> = BTreeMap::new();
+    for tool in tools {
+        *required.entry(tool.as_str()).or_insert(0) += 1;
+    }
+
+    let calls = extract_tool_uses(conversation);
+    let mut observed: BTreeMap<&str, usize> = BTreeMap::new();
+    for block in &calls {
+        *observed.entry(tool_use_name(block)).or_insert(0) += 1;
+    }
+
+    let shortfalls: Vec<(&str, usize, usize)> = required
+        .iter()
+        .filter_map(|(&name, &need)| {
+            let have = observed.get(name).copied().unwrap_or(0);
+            (have < need).then_some((name, need, have))
+        })
+        .collect();
+
+    if shortfalls.is_empty() {
+        AssertionOutcome::Pass
+    } else {
+        let detail = shortfalls
+            .iter()
+            .map(|(name, need, have)| format!("{name:?} (need {need}, got {have})"))
+            .collect::<Vec<_>>()
+            .join(", ");
+        AssertionOutcome::Fail {
+            reason: format!(
+                "tool_call_collection: missing required call(s): {detail}; observed calls: {:?}",
+                calls.iter().map(|b| tool_use_name(b)).collect::<Vec<_>>(),
+            ),
+        }
+    }
 }
 
 #[cfg(test)]
@@ -1753,6 +1786,13 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn tool_call_collection_with_empty_tools_is_vacuously_satisfied() {
+        let conv = conversation_with(vec![assistant_blocks(vec![])]);
+        let assertion = Assertion::ToolCallCollection { tools: vec![] };
+        assert_eq!(assertion.check(&conv, &ctx()).await, AssertionOutcome::Pass);
+    }
+
     #[tokio::test]
     async fn json_path_returns_malformed_on_unparseable_path() {
         let conv = conversation_with(vec![assistant_text("ok")]);

From 3eb27312f9ea3164ecaf0218fbd983150f1b34e9 Mon Sep 17 00:00:00 2001
From: David Souther <davidsouther@gmail.com>
Date: Mon, 6 Jul 2026 13:42:19 -0400
Subject: [PATCH 07/12] feat(report): step 5 - markdown wiring + DESIGN.md
 tool_call_collection grammar

render_comparison_markdown gains a **Paired-difference test:** paragraph after
the existing Summary line, naming n, mean_difference, standard_error,
t_statistic (or "undefined" when None), degrees_of_freedom, p_value, and a
significant/not-significant verdict at PAIRED_DIFFERENCE_ALPHA; InsufficientPairs
renders a one-line "insufficient paired assertions" message naming n.
render_single_markdown is untouched (design's settled scope boundary) -
tests/report_cmd.rs's existing single-mode assertions stay green unchanged.

DESIGN.md's Assertion grammar gains the tool_call_collection line in the
Tool-call assertions group, immediately after tool_call_order.

Pinned by two new unit tests (facts-present, not exact-string). Full test
suite (222 lib tests + all integration test binaries) green.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 DESIGN.md               |  1 +
 src/knowledge/report.rs | 98 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+)

diff --git a/DESIGN.md b/DESIGN.md
index 930d13a6..b38cf94a 100644
--- a/DESIGN.md
+++ b/DESIGN.md
@@ -94,6 +94,7 @@ Assertion:
   | { type: "must_not_call_tool"; tool: string }
   | { type: "tool_call_count";    tool?: string; op: Op; value: number }
   | { type: "tool_call_order";    sequence: string[] }
+  | { type: "tool_call_collection"; tools: string[] }   # order-insensitive multiset: each name must appear at least as many times as listed; extra/intervening calls are ignored
 
   # ─── Text assertions (over the final assistant turn's text content)
   | { type: "text_contains";       value: string; case_sensitive?: bool }
diff --git a/src/knowledge/report.rs b/src/knowledge/report.rs
index 95a30415..3968138c 100644
--- a/src/knowledge/report.rs
+++ b/src/knowledge/report.rs
@@ -481,6 +481,37 @@ pub fn render_comparison_markdown(
     };
     let _ = write!(out, "**Falsification gate:** {gate}\n\n");
 
+    match &report.paired_difference {
+        PairedDifferenceTest::Computed {
+            n,
+            mean_difference,
+            standard_error,
+            degrees_of_freedom,
+            t_statistic,
+            p_value,
+            significant,
+            ..
+        } => {
+            let t_display =
+                t_statistic.map_or_else(|| "undefined".to_string(), |t| format!("{t:.2}"));
+            let verdict = if *significant {
+                format!("significant at α = {PAIRED_DIFFERENCE_ALPHA}")
+            } else {
+                format!("not significant at α = {PAIRED_DIFFERENCE_ALPHA}")
+            };
+            let _ = write!(
+                out,
+                "**Paired-difference test:** n={n}, mean Δ {mean_difference:.3} (SEM {standard_error:.3}), t({degrees_of_freedom}) = {t_display}, p = {p_value:.4} — {verdict}\n\n",
+            );
+        }
+        PairedDifferenceTest::InsufficientPairs { n } => {
+            let _ = write!(
+                out,
+                "**Paired-difference test:** insufficient paired assertions (n={n}) for a significance test\n\n",
+            );
+        }
+    }
+
     let _ = writeln!(out, "| Case | {label_a} | {label_b} |");
     out.push_str("|------|--------|--------|\n");
     for case in &report.cases {
@@ -747,4 +778,71 @@ mod tests {
             other => panic!("expected Computed, got {other:?}"),
         }
     }
+
+    fn comparison_report_with(paired_difference: PairedDifferenceTest) -> super::ComparisonReport {
+        super::ComparisonReport {
+            arm_a: super::ArmRef {
+                run_id: String::from("arm-a"),
+            },
+            arm_b: super::ArmRef {
+                run_id: String::from("arm-b"),
+            },
+            totals: super::ComparisonTotals::default(),
+            paired_difference,
+            cases: vec![],
+        }
+    }
+
+    /// Facts-present check (design.md's own resolved wording decision):
+    /// `n`, `mean_difference`, `standard_error`, `t_statistic`,
+    /// `degrees_of_freedom`, `p_value`, and a not-significant phrase
+    /// against α = 0.05 must all appear somewhere in the rendered output.
+    /// Sentence structure is not load-bearing.
+    #[test]
+    fn render_comparison_markdown_names_computed_paired_difference_facts() {
+        let report = comparison_report_with(PairedDifferenceTest::Computed {
+            n: 10,
+            mean_difference: 0.2,
+            sample_std_dev: 0.632_455_53,
+            standard_error: 0.2,
+            degrees_of_freedom: 9,
+            t_statistic: Some(1.0),
+            p_value: 0.3434,
+            significant: false,
+        });
+        let markdown = super::render_comparison_markdown(&report, "before", "after");
+
+        assert!(markdown.contains("10"), "should name n: {markdown}");
+        assert!(
+            markdown.contains("0.2"),
+            "should name mean_difference/standard_error: {markdown}"
+        );
+        assert!(markdown.contains('9'), "should name df: {markdown}");
+        assert!(
+            markdown.contains('1'),
+            "should name t_statistic: {markdown}"
+        );
+        assert!(
+            markdown.contains("0.3434") || markdown.contains("0.343"),
+            "should name p_value: {markdown}"
+        );
+        assert!(markdown.contains("0.05"), "should name α: {markdown}");
+        assert!(
+            markdown.to_lowercase().contains("not significant"),
+            "should say not significant: {markdown}"
+        );
+    }
+
+    /// `InsufficientPairs` names `n` and says "insufficient".
+    #[test]
+    fn render_comparison_markdown_names_insufficient_pairs() {
+        let report = comparison_report_with(PairedDifferenceTest::InsufficientPairs { n: 1 });
+        let markdown = super::render_comparison_markdown(&report, "before", "after");
+
+        assert!(markdown.contains('1'), "should name n: {markdown}");
+        assert!(
+            markdown.to_lowercase().contains("insufficient"),
+            "should say insufficient: {markdown}"
+        );
+    }
 }

From d3a4a3fc00d3a1dd54e566a9aaa495611f591155 Mon Sep 17 00:00:00 2001
From: David Souther <davidsouther@gmail.com>
Date: Mon, 6 Jul 2026 13:48:05 -0400
Subject: [PATCH 08/12] docs(tasks): defer ailly-skill-eval
 tool_call_collection recommendation

Adds the one Follow-up item design.md's Summary names as deferred to
TASKS.md for feature-f-report-stats: once tool_call_collection exists,
update skills/ailly-skill-eval/SKILL.md and references/method.md to
recommend it over tool_call_order by default for new suites.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 docs/developer/TASKS.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/developer/TASKS.md b/docs/developer/TASKS.md
index b9a1f738..d2017a08 100644
--- a/docs/developer/TASKS.md
+++ b/docs/developer/TASKS.md
@@ -37,6 +37,8 @@ Initial development queue to reach MVP for the three e2e projects under `e2e/`:
 
 - **live tool-definition wiring is completely missing (all providers)** — Confirmed via live reproduction 2026-07-07 while doing Feature C's (ailly-evals project) OpenAI live-confirmation pass: `CompletionRequest` ([src/engine/engine.rs:24-27](../../src/engine/engine.rs)) has no `tools` field at all, and `RigEngine::complete` ([src/engine/rig_engine.rs](../../src/engine/rig_engine.rs)) hardcodes `tools: Vec::new()` on every outgoing `rig::completion::CompletionRequest`, for every provider — not an OpenAI-specific or streaming-specific gap (this was previously named only as one bullet inside "engine deferred decisions" below, with no live evidence; promoting it here now that it has real reproduction). Consequence: no live `ailly run` has ever been able to make a model emit a genuine native tool call, so `must_call_tool`/`must_not_call_tool`/`tool_call_count`/`tool_call_order`/`tool_call_collection` assertions can only ever be exercised today against a conversation whose tool_use blocks were pre-filled by hand or mined from elsewhere (e.g. the judge-calibration project's mined conversations) — never against something `ailly run` itself produced live. The `ContentBlock::ToolUse`/`AssistantContent::ToolCall` mapping in `rig_engine.rs` already exists and is unit-tested, so it is reachable once a real tool schema is actually sent; the gap is entirely on the outbound request side. **Reproduction:** a live conversation with a system message describing a `get_weather` tool and a user message explicitly asking the model to use it, run against `gpt-4o-mini`, returns plain prose with no tool_use content block and no `tool_call` in the raw API response. Also confirmed while investigating: `RigEngine::complete` only ever calls `self.model.completion(...)` (non-streaming), never `.stream()` — `ailly_two` does not use Rig's streaming path today at all, so any future research framing this as a streaming-specific risk is wrong; the blocker is upstream of streaming entirely. **Trigger:** a harness needs a live model to actually emit a tool call (as opposed to grading a pre-filled one), or a live-run confidence claim about `tool_call_*` assertions is made without this caveat.
 
+- **ailly-skill-eval: recommend `tool_call_collection` by default** — Now that `tool_call_collection` exists ([src/content/evaluation.rs](../../src/content/evaluation.rs), [src/knowledge/assertions.rs](../../src/knowledge/assertions.rs)), update [skills/ailly-skill-eval/SKILL.md](../../skills/ailly-skill-eval/SKILL.md) and its `references/method.md` to recommend it over `tool_call_order` by default for new suites, per that design's own Alternatives section framing ("prefer the collection variant by default"; `tool_call_order` stays for genuine sequencing requirements). Named as deferred in `.ailly/developer/2026-07-06-A-ailly-evals/feature-f-report-stats/design.md`'s Summary rather than folded into that feature-step, because it touches a cross-cutting project skill the feature-step's own Specification does not otherwise read or write. **Trigger:** the next suite-authoring pass through `ailly-skill-eval`.
+
 ## Deferred
 
 - **content/conversation deferred decisions** — Five revisit-after items carried over from the conversation design doc: typed `ImageSource`, narrowing `tool_result.content`, closing `TraceEvent` into a named enum, promoting `Conversation` to an aggregate root, and lifting blank/filled `Message` into type-states. See [TASK-NOTES-conversation-deferred.md](TASK-NOTES-conversation-deferred.md) for trigger conditions per item.

From 0b5e06d79a9306378bbd3236d845f4af366adb81 Mon Sep 17 00:00:00 2001
From: David Souther <davidsouther@gmail.com>
Date: Mon, 6 Jul 2026 13:51:13 -0400
Subject: [PATCH 09/12] fix(report): clear clippy -D warnings on
 report_statistics test + unit tests

Independent re-verification found `cargo clippy --all-features --all-targets
-- -D warnings` actually failing (contradicting the build's own self-report)
with 8 errors, all in code this feature-step introduced: missing backticks
around identifiers in tests/report_statistics.rs doc comments
(clippy::doc_markdown), and `other => panic!(...)` wildcard matches against
PairedDifferenceTest's two variants (clippy::match_wildcard_for_single_variants)
in both the feature test and report.rs's own new unit tests. Replaced each
wildcard arm with an explicit, named binding on the specific unwanted variant.

Co-Authored-By: "Ailly <developer@ailly.dev>"
---
 src/knowledge/report.rs    | 16 ++++++++++++----
 tests/report_statistics.rs |  8 +++++---
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/src/knowledge/report.rs b/src/knowledge/report.rs
index 3968138c..fab95149 100644
--- a/src/knowledge/report.rs
+++ b/src/knowledge/report.rs
@@ -726,11 +726,15 @@ mod tests {
     fn fewer_than_two_diffs_yields_insufficient_pairs() {
         match compute_paired_difference(&[]) {
             PairedDifferenceTest::InsufficientPairs { n } => assert_eq!(n, 0),
-            other => panic!("expected InsufficientPairs, got {other:?}"),
+            computed @ PairedDifferenceTest::Computed { .. } => {
+                panic!("expected InsufficientPairs, got {computed:?}")
+            }
         }
         match compute_paired_difference(&[1.0]) {
             PairedDifferenceTest::InsufficientPairs { n } => assert_eq!(n, 1),
-            other => panic!("expected InsufficientPairs, got {other:?}"),
+            computed @ PairedDifferenceTest::Computed { .. } => {
+                panic!("expected InsufficientPairs, got {computed:?}")
+            }
         }
     }
 
@@ -752,7 +756,9 @@ mod tests {
                 assert!((p_value - 1.0).abs() < 1e-9);
                 assert!(!significant);
             }
-            other => panic!("expected Computed, got {other:?}"),
+            insufficient @ PairedDifferenceTest::InsufficientPairs { .. } => {
+                panic!("expected Computed, got {insufficient:?}")
+            }
         }
     }
 
@@ -775,7 +781,9 @@ mod tests {
                 assert!((p_value - 0.0).abs() < 1e-9);
                 assert!(significant);
             }
-            other => panic!("expected Computed, got {other:?}"),
+            insufficient @ PairedDifferenceTest::InsufficientPairs { .. } => {
+                panic!("expected Computed, got {insufficient:?}")
+            }
         }
     }
 
diff --git a/tests/report_statistics.rs b/tests/report_statistics.rs
index 5d24d706..729a0594 100644
--- a/tests/report_statistics.rs
+++ b/tests/report_statistics.rs
@@ -50,7 +50,7 @@ use ailly_two::knowledge::eval::EvalReport;
 use ailly_two::knowledge::report::PairedDifferenceTest;
 use ailly_two::knowledge::report::compute_comparison;
 
-/// Reuses `tests/report_cmd.rs`'s exact ARM_A/ARM_B assertion split (3
+/// Reuses `tests/report_cmd.rs`'s exact `ARM_A`/`ARM_B` assertion split (3
 /// improved, 1 regressed, 4 unchanged-pass, 2 unchanged-fail — 10 paired
 /// assertions total, already proven and green in that file) so the
 /// paired-difference numbers asserted below cross-check against an
@@ -106,7 +106,7 @@ const ARM_B_JSON: &str = r#"{
 
 /// Given two runs whose paired assertions already bucket into 3 improved,
 /// 1 regressed, 4 unchanged-pass, 2 unchanged-fail (10 pairs; diffs
-/// = [+1,+1,+1,-1,0,0,0,0,0,0] using the "arm_b relative to arm_a" sign
+/// = [+1,+1,+1,-1,0,0,0,0,0,0] using the "`arm_b` relative to `arm_a`" sign
 /// convention: pass=1, fail=0, diff = b - a):
 ///
 /// When `compute_comparison` runs,
@@ -184,7 +184,9 @@ fn compute_comparison_reports_paired_difference_test_with_standard_error() {
                 "p ~0.34 does not clear the α = 0.05 significance bar"
             );
         }
-        other => panic!("expected PairedDifferenceTest::Computed, got {other:?}"),
+        insufficient @ PairedDifferenceTest::InsufficientPairs { .. } => {
+            panic!("expected PairedDifferenceTest::Computed, got {insufficient:?}")
+        }
     }
 }
 

From ed3e7ce66c6a751b7ff2c0c86655c32ac8bed1b1 Mon Sep 17 00:00:00 2001
From: David Souther <davidsouther@gmail.com>
Date: Wed, 8 Jul 2026 14:15:40 -0400
Subject: [PATCH 10/12] docs(assertions): fix stale sync-family count in
 Assertion::check doc comment

Caught in independent pre-merge review: ToolCallCollection made this
13, not 12.

Co-Authored-By: "Ailly <developer@ailly.dev>"
---
 src/knowledge/assertions.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/knowledge/assertions.rs b/src/knowledge/assertions.rs
index 74dc981a..da9b1076 100644
--- a/src/knowledge/assertions.rs
+++ b/src/knowledge/assertions.rs
@@ -87,7 +87,7 @@ impl EvaluationContext<'_> {
 
 impl Assertion {
     /// Check this assertion against `conversation` in `ctx`. Total over every
-    /// variant. The 12 sync families ignore `ctx`; the 5 LLM/subprocess
+    /// variant. The 13 sync families ignore `ctx`; the 5 LLM/subprocess
     /// families return `Deferred` when their collaborator is `None`.
     ///
     /// Invariant: never panics, never performs I/O on its own, never

From fbc346f4d320df02584b663e209d5776f9ed2ae8 Mon Sep 17 00:00:00 2001
From: David Souther <davidsouther@gmail.com>
Date: Wed, 8 Jul 2026 14:20:07 -0400
Subject: [PATCH 11/12] docs(tasks): extract Feature F's remaining deferred
 decisions to TASKS.md

Per-case paired-difference granularity and a round-trip serde test for
Assertion::ToolCallCollection, both named as deferred-to-TASKS in the
feature's design.md Summary. The tool_call_collection guidance item
was already recorded during Build.

Co-Authored-By: "Ailly <developer@ailly.dev>"
---
 docs/developer/TASKS.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/developer/TASKS.md b/docs/developer/TASKS.md
index d2017a08..4b56badf 100644
--- a/docs/developer/TASKS.md
+++ b/docs/developer/TASKS.md
@@ -39,6 +39,9 @@ Initial development queue to reach MVP for the three e2e projects under `e2e/`:
 
 - **ailly-skill-eval: recommend `tool_call_collection` by default** — Now that `tool_call_collection` exists ([src/content/evaluation.rs](../../src/content/evaluation.rs), [src/knowledge/assertions.rs](../../src/knowledge/assertions.rs)), update [skills/ailly-skill-eval/SKILL.md](../../skills/ailly-skill-eval/SKILL.md) and its `references/method.md` to recommend it over `tool_call_order` by default for new suites, per that design's own Alternatives section framing ("prefer the collection variant by default"; `tool_call_order` stays for genuine sequencing requirements). Named as deferred in `.ailly/developer/2026-07-06-A-ailly-evals/feature-f-report-stats/design.md`'s Summary rather than folded into that feature-step, because it touches a cross-cutting project skill the feature-step's own Specification does not otherwise read or write. **Trigger:** the next suite-authoring pass through `ailly-skill-eval`.
 
+- **per-case paired-difference granularity** — `compute_comparison`'s new paired-difference test ([src/knowledge/report.rs](../../src/knowledge/report.rs)) reports one significance figure for the whole comparison, not one per `CaseComparison`. Not built now: a single case's `n` is typically a handful of assertions, too small for a test to have real power, and the parent brief's "wire it through, don't redesign the report UX" instruction argued for the minimal whole-comparison version first. **Trigger:** a real comparison's case-level detail turns out to need its own significance read. See `.ailly/developer/2026-07-06-A-ailly-evals/feature-f-report-stats/design.md` Summary.
+- **round-trip serde test for `Assertion::ToolCallCollection`** — [src/content/evaluation.rs](../../src/content/evaluation.rs)'s test module has a `regression_suite_parses_round_trips_and_exposes_typed_variants` fixture covering every other `Assertion` variant's parse/serialize round trip; `ToolCallCollection` was not added to it. Natural Plan-phase follow-on for whichever change next touches that fixture. See `.ailly/developer/2026-07-06-A-ailly-evals/feature-f-report-stats/design.md` Summary.
+
 ## Deferred
 
 - **content/conversation deferred decisions** — Five revisit-after items carried over from the conversation design doc: typed `ImageSource`, narrowing `tool_result.content`, closing `TraceEvent` into a named enum, promoting `Conversation` to an aggregate root, and lifting blank/filled `Message` into type-states. See [TASK-NOTES-conversation-deferred.md](TASK-NOTES-conversation-deferred.md) for trigger conditions per item.

From b5cf46398c806d074ef0db2ced3b20b0be981328 Mon Sep 17 00:00:00 2001
From: David Souther <davidsouther@gmail.com>
Date: Wed, 8 Jul 2026 14:30:58 -0400
Subject: [PATCH 12/12] fix(report): add missing falsification_gate field after
 rebase onto Feature G

comparison_report_with (a test helper added by Step 5) was written
before Feature G's ComparisonReport::falsification_gate field existed
on main_two; rebasing this branch onto Feature G's merge surfaced it
as a compile error under --all-targets, not caught by cargo test alone.

Co-Authored-By: "Ailly <developer@ailly.dev>"
---
 src/knowledge/report.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/knowledge/report.rs b/src/knowledge/report.rs
index fab95149..467bc89b 100644
--- a/src/knowledge/report.rs
+++ b/src/knowledge/report.rs
@@ -796,6 +796,7 @@ mod tests {
                 run_id: String::from("arm-b"),
             },
             totals: super::ComparisonTotals::default(),
+            falsification_gate: false,
             paired_difference,
             cases: vec![],
         }