feat: persist leftover observed Y and expected E (v2.12.20)

AGENTS.md

@@ -186,12 +186,13 @@ in the same spirit) -- never against real data, per the hard rule above.
 against a live local stack (`make up`) and self-skip without one -- see
 [README.md](README.md#local-product-stack-docker-compose).
 
-Period leftover pairs (ADR 0017 / 0018) are computed in
-`lineageweave/leftover_pairs.py` from the residual after a real
-GRM/GPCM score, never invented. Missing cells stay out of the
+Period leftover pairs (ADR 0017 / 0018 / 0048 / 0049 / 0177) are
+computed in `lineageweave/leftover_pairs.py` from the residual after a
+real GRM/GPCM score, never invented. Missing cells stay out of the
 Gabriel factorization. Closest and farthest post–criterion pairs
-persist to `report_leftover_pair` and sit above the member list so
-a click opens that post.
+persist to `report_leftover_pair` with observed `Y` and expected
+`E[Y|θ, item]` so residual reconciles to `Y − E`, and sit above the
+member list so a click opens that post.
 
 `frontend/` has its own toolchain (Node pinned via `frontend/mise.toml`,
 pnpm via Corepack -- do not add a second Node package manager or a

ARCHITECTURE.md

@@ -590,8 +590,9 @@ on those same fixed parameters (Kim, 2006 FIPC). After scoring,
 `information_polytomous` ranks the shared-bank items by Fisher
 information at the group's mean θ (Lord, 1980 max-info CAT). Rankings
 persist to `report_item_information`. After those IRT main effects,
-residual SVD leftover pairs (Jeon et al., 2021; ADR 0017) persist to
-`report_leftover_pair`. Results persist to
+residual SVD leftover pairs (Jeon et al., 2021; ADR 0017 / 0048 / 0177)
+persist to `report_leftover_pair` with observed `Y` and expected
+`E[Y|θ, item]`. Results persist to
 `report_period_score` / `report_member_score`.
 `GET /api/reports/{grouping}` lists the trend;
 `GET /api/reports/{grouping}/{period}` is ABAC-filtered;
@@ -603,7 +604,8 @@ bank as the dummy high/low band rows, so comparison-strip click
 through opens those DAG posts. Report members include the earliest
 open ticket title, status lookup label, and due date when one exists. The home page renders
 the actual mean θ, the FIPC delta, the CAT-selected item, leftover
-closest/farthest pairs above the member list, and the
+closest/farthest pairs (observed `Y` and expected `E` after IRT main
+effects plus leftover-map distance `d`) above the member list, and the
 PU / corp / thread comparison -- never a placeholder. TEPP is unchanged.
 
 ## Phase 6b: Knowledge Graph as a real Ontology + Semantic Layer

CHANGELOG.d/2.12.20-leftover-observed-expected.md

@@ -0,0 +1,8 @@
+## 2.12.20 — Leftover observed Y and expected E
+
+- Persist observed `Y` and expected `E[Y|θ, item]` on leftover
+  post–criterion pairs (ADR 0177). Residual stays `R = Y − E`. After
+  `make seed`, closest and farthest leftover pairs sit above the member
+  list with `Y` and `E` next to leftover-map distance `d`; click opens
+  that post. Omit the badge when either value is missing. Never invent
+  a leftover score or a theta.

CLAUDE.md

@@ -55,7 +55,9 @@ chip name contains `Corporate entity: Demo Corp` and the persisted
 mean θ. The period-report panel says Demo Corp is the opened grouping
 and to read its mean θ and member posts, then open a post. Those
 members land immediately under that next action, ahead of Other Corp
-and the week strip. Opening Public post names the next action: read
+and the week strip. Leftover closest/farthest pairs name observed `Y`
+and expected `E` after IRT main effects next to leftover-map distance
+`d`; click a pair to open that post. Opening Public post names the next action: read
 Event Lineage, Keyman, and evaluation on that post. The popup Event
 Lineage DAG marks that post current. After that current node, the
 popup names Keyman and evaluation as the next read. After landed

backend/app/report_ingestion.py

@@ -443,8 +443,9 @@ async def persist_period_report(
             """
             insert into report_leftover_pair (
                 grouping_kind, grouping_key, period_code, rubric_version,
-                pair_kind, post_id, criterion_code, leftover_distance, leftover_residual
-            ) values ($1,$2,$3,$4,$5,$6,$7,$8,$9)
+                pair_kind, post_id, criterion_code, leftover_distance, leftover_residual,
+                observed_response, expected_response
+            ) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
             """,
             grouping_kind,
             grouping_key,
@@ -455,6 +456,8 @@ async def persist_period_report(
             pair.criterion_code,
             pair.leftover_distance,
             pair.leftover_residual,
+            pair.observed_response,
+            pair.expected_response,
         )
 
 
@@ -602,7 +605,8 @@ async def fetch_period_reports(
     leftover = await conn.fetch(  # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli
         f"""
         select lp.grouping_key, lp.pair_kind, lp.post_id, lp.criterion_code,
-               lp.leftover_distance, lp.leftover_residual, p.post_title,
+               lp.leftover_distance, lp.leftover_residual,
+               lp.observed_response, lp.expected_response, p.post_title,
                p.visibility_code, p.corporate_entity_id,
                p.author_account_id, p.source_detail_state_code,
                ({_SOURCE_CONTEXT_PRESENT_SQL}) as has_real_source_context
@@ -702,6 +706,16 @@ async def fetch_period_reports(
                         "criterion_code": str(row["criterion_code"]),
                         "leftover_distance": float(row["leftover_distance"]),
                         "leftover_residual": float(row["leftover_residual"]),
+                        "observed_response": (
+                            None
+                            if row["observed_response"] is None
+                            else float(row["observed_response"])
+                        ),
+                        "expected_response": (
+                            None
+                            if row["expected_response"] is None
+                            else float(row["expected_response"])
+                        ),
                         "visibility_code": row["visibility_code"],
                         "corporate_entity_id": str(row["corporate_entity_id"]),
                         "author_account_id": str(row["author_account_id"]),

backend/tests/test_api.py

@@ -193,6 +193,11 @@
     / "migrations"
     / "0137_cross_post_customer_identity.sql"
 )
+_LEFTOVER_OBSERVED_EXPECTED_MIGRATION = (
+    Path(__file__).resolve().parents[2]
+    / "migrations"
+    / "0177_report_leftover_observed_expected.sql"
+)
 
 
 def _postgres_available() -> bool:
@@ -322,6 +327,7 @@ def seeded_db(demo_analyst_token):
             cur.execute(_CATALOG_UNRESOLVED_REASON_MIGRATION.read_text())
             cur.execute(_POST_ASK_HISTORY_MIGRATION.read_text())
             cur.execute(_CUSTOMER_IDENTITY_MIGRATION.read_text())
+            cur.execute(_LEFTOVER_OBSERVED_EXPECTED_MIGRATION.read_text())
             cur.execute(
                 "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values "
                 "('corporate_entity_level', 'group', 'Group'), "
@@ -5677,6 +5683,12 @@ def test_seed_period_report_surfaces_on_get_reports(client, demo_analyst_token,
     assert leftover_kinds <= {"closest", "farthest"}
     assert all(pair["post_title"] for pair in high_report.get("leftover_pairs", []))
     assert all(pair["leftover_distance"] >= 0 for pair in high_report.get("leftover_pairs", []))
+    for pair in high_report.get("leftover_pairs", []):
+        observed = pair.get("observed_response")
+        expected = pair.get("expected_response")
+        if observed is None or expected is None:
+            continue
+        assert abs(pair["leftover_residual"] - (observed - expected)) < 1e-6
 
     week3 = client.get(
         "/api/reports/process_unit/2026-W03",

docker/postgres-init/migrate.sh

@@ -18,7 +18,7 @@ for migration in /opt/lineageweave/migrations/*.sql; do
     migration_name=${migration##*/}
     case "$migration_name" in
         0012_*|0013_*|0014_*|0015_*|0016_*|0017_*|0018_*|0019_*|0020_*|0021_*|0022_*|0023_*|0024_*|0025_*|0026_*|0027_*|0028_*|0029_*|0030_*|0031_*|0032_*|0033_*|0034_*|0035_*|0036_*|0037_*|0038_*|0039_*|0040_*|0041_*|0042_*|0043_*|0044_*|0045_*|0046_*|0047_*|0048_*|0049_*|0050_*) ;;
-        0060_*|0100_*|0101_*|0102_*|0103_*|0104_*|0105_*|0106_*|0107_*|0108_*|0109_*|0110_*|0111_*|0112_*|0113_*|0114_*|0130_*|0133_*|0134_*|0136_*|0137_*|0138_*|0139_*|0173_*|0176_*) ;;
+        0060_*|0100_*|0101_*|0102_*|0103_*|0104_*|0105_*|0106_*|0107_*|0108_*|0109_*|0110_*|0111_*|0112_*|0113_*|0114_*|0130_*|0133_*|0134_*|0136_*|0137_*|0138_*|0139_*|0173_*|0176_*|0177_*) ;;
         *) continue ;;
     esac
     printf 'Applying %s\n' "$migration_name"

docs/adr/0003-fast-mlsirm-report-integration.md

@@ -103,10 +103,11 @@ than one large PR:
    `information_polytomous` (Lord, 1980 max-info). Persist the ranking
    (`report_item_information`) and show the rank-1 item on the Period
    reports panel. Do not reimplement an information function here.
-7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018): after
+7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018 / 0048 / 0049 / 0177): after
    IRT main effects, persist closest and farthest post–criterion pairs
-   from the residual leftover map. Do not fork LSIRM; do not invent a
-   leftover-pair API inside `fast-mlsirm` in this slice.
+   from the residual leftover map, naming observed `Y` and expected
+   `E[Y|θ, item]` so residual reconciles to `Y − E`. Do not fork LSIRM;
+   do not invent a leftover-pair API inside `fast-mlsirm` in this slice.
 
 **TEPP boundary.** [ARCHITECTURE.md](../../ARCHITECTURE.md) already
 assigns calibrated temporal/event measurement to

docs/adr/0048-persist-lsirm-leftover-pairs.md

@@ -2,6 +2,7 @@
 
 **Decision status:** Accepted
 **Date:** 2026-08-17
+**Amended by:** [ADR 0177](0177-leftover-observed-expected.md) (observed Y and expected E)
 
 ## Context
 
@@ -30,7 +31,9 @@ and one `farthest` observed cell per period report in
 `report_leftover_pair` (3NF, two-or-more-word `snake_case`).
 
 The biplot lives in `lineageweave/leftover_pairs.py` so leftover
-tests do not import `period_report` or `fast_mlsirm`.
+tests do not import `period_report` or `fast_mlsirm`. Each leftover
+row also names observed `Y` and expected `E[Y|θ, item]` so residual
+reconciles to `Y − E` (ADR 0177).
 
 Cascade the rows with `report_period_score`. A leftover post must
 also be a `report_member_score` row, and the leftover criterion

docs/adr/0049-leftover-pair-report-ui.md

@@ -2,6 +2,7 @@
 
 **Decision status:** Accepted
 **Date:** 2026-08-17
+**Amended by:** [ADR 0177](0177-leftover-observed-expected.md) (observed Y and expected E)
 
 ## Context
 

docs/adr/0177-leftover-observed-expected.md

@@ -0,0 +1,76 @@
+# ADR 0177 — Persist observed Y and expected E on leftover pairs
+
+**Decision status:** Accepted
+**Date:** 2026-08-24
+
+Amends [ADR 0048](0048-persist-lsirm-leftover-pairs.md) and
+[ADR 0049](0049-leftover-pair-report-ui.md).
+
+## Context
+
+ADR 0048 already persists leftover-map distance and leftover residual
+`R = Y − E[Y|θ, item]` on `report_leftover_pair`. ADR 0049 already
+renders closest and farthest pairs above the member list and opens the
+named post. Residual disclosure without naming `Y` and `E` leaves a
+reader unable to tell whether a leftover cell is a high observed
+response or a low expected category after IRT main effects.
+
+Jeon et al. (2021, eq. 3) leftover interaction is `−γ‖ξ_p − ζ_i‖`.
+Gabriel (1971) supplies the leftover-map coordinates from a residual
+biplot of `R`. `R` is not an invented leftover score: it is the
+observed category minus the already-fitted expected category. Those
+two inputs must travel with the pair row.
+
+This increment does not persist leftover-map coordinates, does not
+change leftover-map axis count, does not name complete-case coverage,
+and does not land Post quality on the leftover criterion.
+
+The unprotected-stack ADR for the same leftover-pair fact was 0163. This
+protected-main reconstruction uses **0177** so it does not collide
+with any current open-head ADR.
+
+## Decision
+
+Each leftover pair names:
+
+1. `observed_response` — the observed category `Y` for that
+   post–criterion cell;
+2. `expected_response` — `E[Y|θ, item]` from the already-fitted
+   GRM/GPCM main effects;
+3. `leftover_residual`, which must equal `Y − E` within `1e-6`.
+
+Migration `0177` is the single source of both columns on every
+install path, fresh or existing -- shipped migrations (`0001`/`0012`)
+are never edited after the fact. It adds them as nullable so older
+leftover rows keep distance and residual without fabricating `Y` or
+`E`. The pair button shows
+`Y {observed} · E {expected}` next to leftover-map distance `d` when
+both values are finite. The next action remains ADR 0049:
+`Open {post}, then read Post quality criterion {criterion}.` Omit the
+`Y` / `E` badge when either value is missing or non-finite. Do not
+invent a leftover score. Do not invent a theta.
+
+## Consequences
+
+`GET /api/reports/{grouping}/{period}` returns `observed_response`
+and `expected_response`. After `make seed`, closest and farthest
+leftover pairs sit above the member list with named `Y` and `E`;
+click opens that post. Hidden posts stay hidden.
+
+## Related
+
+Independent of leftover interaction-map persistence, leftover-criterion
+evaluation landing, leftover residual UI extraction, leftover-map
+complete-case coverage, leftover-map axis share, leftover pairs on the
+grouping comparison strip, and two-axis leftover-map distance.
+
+## References
+
+Gabriel, K. R. (1971). The biplot graphic display of matrices with
+application to principal component analysis. *Biometrika, 58*(3),
+453–467. https://doi.org/10.1093/biomet/58.3.453
+
+Jeon, M., Jin, I. H., Schweinberger, M., & Baugh, S. (2021). Mapping
+unobserved item–respondent interactions: A latent space item response
+model with interaction map. *Psychometrika, 86*(2), 378–403.
+https://doi.org/10.1007/s11336-021-09762-5

frontend/src/App.test.tsx

@@ -1186,6 +1186,8 @@ describe("App, authenticated", () => {
                     criterion_code: "sales_lead_specificity",
                     leftover_distance: 0.12,
                     leftover_residual: 0.4,
+                    observed_response: 2.4,
+                    expected_response: 2.0,
                   },
                   {
                     pair_kind: "farthest",
@@ -1194,6 +1196,8 @@ describe("App, authenticated", () => {
                     criterion_code: "general_sentiment_negative",
                     leftover_distance: 1.84,
                     leftover_residual: -1.1,
+                    observed_response: 0.9,
+                    expected_response: 2.0,
                   },
                 ],
                 members: [
@@ -5334,12 +5338,14 @@ describe("App, authenticated", () => {
       "Open Public post, then read Post quality criterion sales-lead.",
     );
     expect(closestPair).not.toHaveTextContent(/sat closest to after main effects/);
+    expect(closestPair).toHaveTextContent("Y 2.40 · E 2.00");
     expect(closestPair).toHaveTextContent("d 0.12");
     expect(farthestPair).toHaveTextContent("Farthest leftover: Specification revision requested · negative");
     expect(farthestPair).toHaveTextContent(
       "Open Specification revision requested, then read Post quality criterion negative.",
     );
     expect(farthestPair).not.toHaveTextContent(/sat farthest from after main effects/);
+    expect(farthestPair).toHaveTextContent("Y 0.90 · E 2.00");
     expect(farthestPair).toHaveTextContent("d 1.84");
     const memberButton = screen.getByRole("button", { name: /open report post: public post/i });
     expect(closestPair.compareDocumentPosition(memberButton) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy();

frontend/src/App.tsx

@@ -4198,6 +4198,8 @@ function ReportsPanel({
                     <LeftoverPairButton
                       pair={pair}
                       leftoverDistance={pair.leftover_distance}
+                      observedResponse={pair.observed_response}
+                      expectedResponse={pair.expected_response}
                       onOpen={onSelectPost}
                     />
                   </li>

frontend/src/api.ts

@@ -994,6 +994,8 @@ export interface LeftoverPair {
   criterion_code: string;
   leftover_distance: number;
   leftover_residual: number;
+  observed_response?: number | null;
+  expected_response?: number | null;
 }
 
 export interface PeriodGroupReport {

frontend/src/components/LeftoverPairButton.stories.tsx

@@ -25,13 +25,25 @@ const farthest: LeftoverPairOpen = {
   criterion_code: "general_sentiment_negative",
 };
 
-function LeftoverLanding({ pair, leftoverDistance }: { pair: LeftoverPairOpen; leftoverDistance: number }) {
+function LeftoverLanding({
+  pair,
+  leftoverDistance,
+  observedResponse,
+  expectedResponse,
+}: {
+  pair: LeftoverPairOpen;
+  leftoverDistance: number;
+  observedResponse?: number;
+  expectedResponse?: number;
+}) {
   const [focusCode, setFocusCode] = useState<string | null>(null);
   return (
     <section>
       <LeftoverPairButton
         pair={pair}
         leftoverDistance={leftoverDistance}
+        observedResponse={observedResponse}
+        expectedResponse={expectedResponse}
         onOpen={(_postId, options) => setFocusCode(options.focusCriterionCode)}
       />
       {focusCode ? (
@@ -64,11 +76,14 @@ export const ClosestPair: Story = {
     leftoverDistance: 0.12,
     onOpen: () => undefined,
   },
-  render: () => <LeftoverLanding pair={closest} leftoverDistance={0.12} />,
+  render: () => (
+    <LeftoverLanding pair={closest} leftoverDistance={0.12} observedResponse={2.4} expectedResponse={2.0} />
+  ),
   play: async ({ canvasElement }) => {
     const canvas = within(canvasElement);
     const button = canvas.getByRole("button", { name: /open leftover closest pair: public post/i });
     await expect(button).toHaveTextContent(leftoverPairNextAction(closest));
+    await expect(button).toHaveTextContent("Y 2.40 · E 2.00");
     await userEvent.click(button);
     const landed = canvas.getByRole("status");
     await expect(landed).toHaveAttribute("id", postQualityCriterionElementId(closest.criterion_code));

frontend/src/components/LeftoverPairButton.tsx

@@ -1,3 +1,4 @@
+import { formatLeftoverObservedExpected } from "../leftoverObservedExpected";
 import {
   leftoverPairAriaLabel,
   leftoverPairNextAction,
@@ -10,12 +11,17 @@ import {
 export function LeftoverPairButton({
   pair,
   leftoverDistance,
+  observedResponse,
+  expectedResponse,
   onOpen,
 }: {
   pair: LeftoverPairOpen;
   leftoverDistance: number;
+  observedResponse?: number | null;
+  expectedResponse?: number | null;
   onOpen: (postId: string, options: LeftoverPairOpenOptions) => void;
 }) {
+  const observedExpected = formatLeftoverObservedExpected(observedResponse, expectedResponse);
   return (
     <button
       type="button"
@@ -25,6 +31,7 @@ export function LeftoverPairButton({
     >
       <span className="ticket-title">{leftoverPairTitle(pair)}</span>
       <span className="post-badge">{leftoverPairNextAction(pair)}</span>
+      {observedExpected ? <span className="post-badge">{observedExpected}</span> : null}
       <span className="post-badge">d {leftoverDistance.toFixed(2)}</span>
     </button>
   );