From 74ed95d86306babe861cad386eeef887bd9c146f Mon Sep 17 00:00:00 2001 From: seonghobae Date: Mon, 24 Aug 2026 22:43:26 +0000 Subject: [PATCH 1/6] feat: name leftover-map reconstruction on leftover pairs (v2.12.31) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Persist two-axis Gabriel reconstruction R̂ = ξ_{1:2} · ζ_{1:2} on period leftover pairs so landed unexplained leftover U = R − R̂ stays auditable as U + R̂ = R (ADR 0183). Reconstruction is the same internal two-axis inner product already used for U; do not substitute the centered R̃ reconstruction claimed by leftover stack #564. After make seed, closest and farthest leftover pairs sit above the member list with signed R̂ next to leftover-map distance d; click opens that post. Missing or non-finite reconstruction omits the badge rather than inventing a leftover score. Signed reconstruction is stored, never clamped. Complete-case coverage (ADR 0168) stays independent. Independent of leftover stacks #485, #518 (landed), #521, #537, #539, #563, #564, and #579. --- AGENTS.md | 12 +-- ARCHITECTURE.md | 7 +- .../2.12.31-leftover-map-reconstruction.md | 8 ++ CHANGELOG.md | 6 ++ CLAUDE.md | 3 +- backend/app/report_ingestion.py | 12 ++- backend/tests/test_api.py | 11 ++- .../0003-fast-mlsirm-report-integration.md | 11 +-- docs/adr/0048-persist-lsirm-leftover-pairs.md | 8 +- docs/adr/0049-leftover-pair-report-ui.md | 18 ++-- docs/adr/0182-leftover-map-unexplained.md | 7 +- docs/adr/0183-leftover-map-reconstruction.md | 93 +++++++++++++++++++ frontend/src/App.test.tsx | 8 +- frontend/src/api.ts | 1 + .../components/LeftoverPairList.stories.tsx | 4 + .../src/components/LeftoverPairList.test.tsx | 39 ++++++++ frontend/src/components/LeftoverPairList.tsx | 22 ++++- frontend/src/i18n.test.ts | 28 ++++++ frontend/src/i18n.ts | 8 ++ .../src/leftoverMapReconstruction.test.ts | 17 ++++ frontend/src/leftoverMapReconstruction.ts | 16 ++++ lineageweave/leftover_pairs.py | 54 +++++++---- ...183_report_leftover_map_reconstruction.sql | 11 +++ ...183_report_leftover_map_reconstruction.sql | 4 + scripts/seed_demo_data.py | 6 +- tests/test_leftover_pairs.py | 22 ++++- tests/test_period_report.py | 10 +- tests/test_schema.py | 22 ++++- 28 files changed, 404 insertions(+), 64 deletions(-) create mode 100644 CHANGELOG.d/2.12.31-leftover-map-reconstruction.md create mode 100644 docs/adr/0183-leftover-map-reconstruction.md create mode 100644 frontend/src/leftoverMapReconstruction.test.ts create mode 100644 frontend/src/leftoverMapReconstruction.ts create mode 100644 migrations/0183_report_leftover_map_reconstruction.sql create mode 100644 migrations/rollback/0183_report_leftover_map_reconstruction.sql diff --git a/AGENTS.md b/AGENTS.md index 06999f370..e6445856f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -196,17 +196,17 @@ against a live local stack (`make up`) and self-skip without one -- see [README.md](README.md#local-product-stack-docker-compose). Period leftover pairs (ADR 0017 / 0018 / 0048 / 0049 / 0119 / 0162 / 0163 / -0164 / 0182) are computed in `lineageweave/leftover_pairs.py` from the +0164 / 0182 / 0183) are computed in `lineageweave/leftover_pairs.py` from the residual after a real GRM/GPCM score, never invented. Distances are Euclidean on the two-dimensional Gabriel leftover map; missing cells stay out of the factorization. Closest and farthest post–criterion pairs persist to `report_leftover_pair` with signed residual `R`, observed `Y`, and expected `E[Y|θ, item]` so `R = Y − E` remains auditable, -plus leftover-map rank so rank 0 is not read as structure, and -unexplained leftover `U = R − R̂` next to leftover-map distance `d` -after two-axis Gabriel reconstruction. They sit above the member -list so a click opens that post. Two-axis reconstruction `R̂` is not -persisted. Leftover-map axis share (ADR 0148) is Gabriel inertia of +plus leftover-map rank so rank 0 is not read as structure, +unexplained leftover `U = R − R̂`, and leftover-map reconstruction +`R̂ = ξ_{1:2} · ζ_{1:2}` next to leftover-map distance `d` so +`U + R̂ = R` stays auditable. They sit above the member +list so a click opens that post. Leftover-map axis share (ADR 0148) is Gabriel inertia of residual SVD axes 1 and 2 and persists to `report_leftover_map_axis`. Rank-0 residuals emit two zero-share axes; the shares are report-level and are not a leftover score. Complete-case coverage (ADR 0168) persists to diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 235f2a216..2a10a856b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -596,10 +596,11 @@ on those same fixed parameters (Kim, 2006 FIPC). After scoring, information at the group's mean θ (Lord, 1980 max-info CAT). Rankings persist to `report_item_information`. After those IRT main effects, residual SVD leftover pairs on two Gabriel axes (Jeon et al., 2021; -ADR 0017 / 0048 / 0119 / 0162 / 0163 / 0164 / 0182) persist to +ADR 0017 / 0048 / 0119 / 0162 / 0163 / 0164 / 0182 / 0183) persist to `report_leftover_pair` with signed residual `R`, observed `Y`, expected -`E[Y|θ, item]`, full leftover-map rank, and unexplained leftover -`U = R − R̂` named on the pair row. Leftover-map axis share (Gabriel +`E[Y|θ, item]`, full leftover-map rank, unexplained leftover +`U = R − R̂`, and two-axis reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}` so +`U + R̂ = R` stays auditable. Leftover-map axis share (Gabriel inertia of residual SVD axes 1 and 2; ADR 0148) persists to `report_leftover_map_axis`. Complete-case leftover-map coverage (ADR 0168) persists to `report_leftover_map_coverage` so readers see how diff --git a/CHANGELOG.d/2.12.31-leftover-map-reconstruction.md b/CHANGELOG.d/2.12.31-leftover-map-reconstruction.md new file mode 100644 index 000000000..bf20353ba --- /dev/null +++ b/CHANGELOG.d/2.12.31-leftover-map-reconstruction.md @@ -0,0 +1,8 @@ +## 2.12.31 — Leftover-map reconstruction + +- Persist leftover-map reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}` on leftover + post–criterion pairs (ADR 0183). After `make seed`, closest and farthest + leftover pairs sit above the member list with `R̂` next to leftover-map + distance `d`; click opens that post. Omit the badge when reconstruction + is missing. Never invent a leftover score. Signed reconstruction is + stored, never clamped. Identity `U + R̂ = R` stays auditable. diff --git a/CHANGELOG.md b/CHANGELOG.md index b2040d64c..a98057097 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ All notable changes to this project are documented here. Format follows ### Added +- Period leftover pair rows now name leftover-map reconstruction + `R̂ = ξ_{1:2} · ζ_{1:2}` next to leftover-map distance `d`, then open + that post (Gabriel, 1971; Jeon et al., 2021, eq. 3; ADR 0183). A missing + reconstruction omits the badge rather than inventing a leftover score. + Signed reconstruction is stored, never clamped. Identity `U + R̂ = R` + stays auditable. - Opening a post with persisted image-region evidence now shows each region's bounding range beside its caption, OCR, and tags (ADR 0155). After `make seed`, a synthetic process-diagram region reads **Region location: diff --git a/CLAUDE.md b/CLAUDE.md index 992753de8..f8b98146a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,7 +56,8 @@ and to read its mean θ and member posts, then open a post. Those members land immediately under that next action, ahead of Other Corp and the week strip. After `make seed`, leftover closest/farthest pairs sit above the member list with leftover-map rank (rank 0 names no -leftover structure) and unexplained leftover `U` next to leftover-map +leftover structure), unexplained leftover `U`, and leftover-map +reconstruction `R̂` next to leftover-map distance `d`. Leftover-map axis share badges name Gabriel inertia of axes 1 and 2; open a leftover pair to read the post–criterion cell. The shares do not invent a leftover score. Opening Public post names the next action: read diff --git a/backend/app/report_ingestion.py b/backend/app/report_ingestion.py index a00350c81..5f4302d11 100644 --- a/backend/app/report_ingestion.py +++ b/backend/app/report_ingestion.py @@ -445,8 +445,8 @@ async def persist_period_report( grouping_kind, grouping_key, period_code, rubric_version, pair_kind, post_id, criterion_code, leftover_distance, leftover_residual, observed_response, expected_response, leftover_map_rank, - leftover_map_unexplained - ) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) + leftover_map_unexplained, leftover_map_reconstruction + ) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) """, grouping_kind, grouping_key, @@ -461,6 +461,7 @@ async def persist_period_report( pair.expected_response, pair.leftover_map_rank, pair.leftover_map_unexplained, + pair.leftover_map_reconstruction, ) for axis in report.leftover_map_axes: await conn.execute( @@ -646,7 +647,7 @@ async def fetch_period_reports( select lp.grouping_key, lp.pair_kind, lp.post_id, lp.criterion_code, lp.leftover_distance, lp.leftover_residual, lp.observed_response, lp.expected_response, lp.leftover_map_rank, - lp.leftover_map_unexplained, p.post_title, + lp.leftover_map_unexplained, lp.leftover_map_reconstruction, p.post_title, p.visibility_code, p.corporate_entity_id, ({_SOURCE_CONTEXT_PRESENT_SQL}) as has_real_source_context from report_leftover_pair lp @@ -790,6 +791,11 @@ async def fetch_period_reports( if row["leftover_map_unexplained"] is None else float(row["leftover_map_unexplained"]) ), + "leftover_map_reconstruction": ( + None + if row["leftover_map_reconstruction"] is None + else float(row["leftover_map_reconstruction"]) + ), "visibility_code": row["visibility_code"], "corporate_entity_id": str(row["corporate_entity_id"]), "has_real_source_context": bool(row["has_real_source_context"]), diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 4cdf0729b..81ba13a1d 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -152,6 +152,11 @@ / "migrations" / "0182_report_leftover_map_unexplained.sql" ) +_LEFTOVER_MAP_RECONSTRUCTION_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0183_report_leftover_map_reconstruction.sql" +) def _postgres_available() -> bool: @@ -273,6 +278,7 @@ def seeded_db(demo_analyst_token): cur.execute(_GLOBAL_ASK_JOB_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_AXIS_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_UNEXPLAINED_MIGRATION.read_text()) + cur.execute(_LEFTOVER_MAP_RECONSTRUCTION_MIGRATION.read_text()) cur.execute( "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values " "('corporate_entity_level', 'group', 'Group'), " @@ -5014,7 +5020,10 @@ def test_seed_period_report_surfaces_on_get_reports(client, demo_analyst_token, assert pair["leftover_map_rank"] >= 0 unexplained = pair.get("leftover_map_unexplained") assert unexplained is None or isinstance(unexplained, (int, float)) - assert "leftover_map_reconstruction" not in pair + reconstruction = pair.get("leftover_map_reconstruction") + assert reconstruction is None or isinstance(reconstruction, (int, float)) + if unexplained is not None and reconstruction is not None: + assert abs(unexplained + reconstruction - pair["leftover_residual"]) < 1e-6 observed = pair.get("observed_response") expected = pair.get("expected_response") if observed is None or expected is None: diff --git a/docs/adr/0003-fast-mlsirm-report-integration.md b/docs/adr/0003-fast-mlsirm-report-integration.md index ed4ca22c8..d809fb2d4 100644 --- a/docs/adr/0003-fast-mlsirm-report-integration.md +++ b/docs/adr/0003-fast-mlsirm-report-integration.md @@ -100,13 +100,12 @@ than one large PR: `information_polytomous` (Lord, 1980 max-info). Persist the ranking (`report_item_information`) and show the rank-1 item on the Period reports panel. Do not reimplement an information function here. -7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018 / 0048 / 0049 / 0182): after +7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018 / 0048 / 0049 / 0182 / 0183): after IRT main effects, persist closest and farthest post–criterion pairs - from the residual leftover map, and name unexplained leftover - `U = R − R̂` when Gabriel coordinates exist so the leftover cell the - two-axis map does not reconstruct is not read as leftover residual - or leftover-map distance. Do not persist two-axis reconstruction - `R̂`. Do not fork LSIRM; do not invent a + from the residual leftover map, name unexplained leftover + `U = R − R̂` when Gabriel coordinates exist, and persist two-axis + reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}` so `U + R̂ = R` stays + auditable. Do not fork LSIRM; do not invent a leftover-pair API inside `fast-mlsirm` in this slice. 8. **Leftover-map axis-share slice** (ADR 0148): persist Gabriel inertia `σ_k² / Σ_j σ_j²` of leftover-map axes 1 and 2 on the same residual diff --git a/docs/adr/0048-persist-lsirm-leftover-pairs.md b/docs/adr/0048-persist-lsirm-leftover-pairs.md index 0bab2fc32..f7efb8436 100644 --- a/docs/adr/0048-persist-lsirm-leftover-pairs.md +++ b/docs/adr/0048-persist-lsirm-leftover-pairs.md @@ -5,7 +5,8 @@ **Amended by:** [ADR 0119](0119-leftover-map-two-dimensional-distance.md) (two leftover-map axes); [ADR 0163](0163-leftover-observed-expected.md) (observed Y and expected E); [ADR 0164](0164-leftover-map-rank.md) (full map rank); -[ADR 0182](0182-leftover-map-unexplained.md) (unexplained leftover U) +[ADR 0182](0182-leftover-map-unexplained.md) (unexplained leftover U); +[ADR 0183](0183-leftover-map-reconstruction.md) (two-axis reconstruction R̂) ## Context @@ -41,8 +42,9 @@ rank while distance remains on the first two axes (ADR 0164). Each leftover row also names unexplained leftover `U = R − R̂` when Gabriel coordinates exist so the leftover cell the two-axis map does not reconstruct is not read as leftover residual `R` or leftover-map -distance `d` (ADR 0182). Two-axis reconstruction `R̂` is computed -internally and is not persisted. +distance `d` (ADR 0182). Two-axis reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}` +is persisted so `U + R̂ = R` stays auditable (ADR 0183). Do not +substitute a separately centered reconstruction. Cascade the rows with `report_period_score`. A leftover post must also be a `report_member_score` row, and the leftover criterion diff --git a/docs/adr/0049-leftover-pair-report-ui.md b/docs/adr/0049-leftover-pair-report-ui.md index 729857277..81ed6bf9e 100644 --- a/docs/adr/0049-leftover-pair-report-ui.md +++ b/docs/adr/0049-leftover-pair-report-ui.md @@ -5,7 +5,8 @@ **Amended by:** [ADR 0162](0162-leftover-residual-disclosure.md) (signed residual R); [ADR 0163](0163-leftover-observed-expected.md) (observed Y and expected E); [ADR 0164](0164-leftover-map-rank.md) (full map rank); -[ADR 0182](0182-leftover-map-unexplained.md) (unexplained leftover U) +[ADR 0182](0182-leftover-map-unexplained.md) (unexplained leftover U); +[ADR 0183](0183-leftover-map-reconstruction.md) (two-axis reconstruction R̂) ## Context @@ -23,18 +24,21 @@ On each period-report group, render leftover pairs **above** the member list. Each pair is a button: closest or farthest label, post title, criterion short label, signed residual `R`, two-axis leftover-map distance, full map rank, observed `Y`, expected `E` when finite, and -unexplained leftover `U` when finite. +unexplained leftover `U` when finite, and reconstruction `R̂` when finite. The next action names every available measurement before opening the post; no amendment hides another, rank 0 explicitly names no -leftover structure, and unexplained leftover names "leftover map leaves -unexplained `U` after IRT main effects; open this post to read the -named criterion" when present. A missing unexplained leftover keeps -the existing next action. +leftover structure, reconstruction names "leftover map reconstructs +`R̂` after IRT main effects; open this post to read the named +criterion" when present, and unexplained leftover names "leftover map +leaves unexplained `U` after IRT main effects; open this post to read +the named criterion" when reconstruction is missing. A missing +reconstruction keeps the unexplained leftover next action. Clicking the button opens that post with the same handler as a member row. Residual naming is [ADR 0162](0162-leftover-residual-disclosure.md), observed/expected naming is [ADR 0163](0163-leftover-observed-expected.md), rank naming is [ADR 0164](0164-leftover-map-rank.md), unexplained -leftover naming is [ADR 0182](0182-leftover-map-unexplained.md). +leftover naming is [ADR 0182](0182-leftover-map-unexplained.md), +reconstruction naming is [ADR 0183](0183-leftover-map-reconstruction.md). After `make seed`, closest and farthest leftover pairs sit above the member list. Click a pair to open that post. diff --git a/docs/adr/0182-leftover-map-unexplained.md b/docs/adr/0182-leftover-map-unexplained.md index e7cc46fca..523353ee9 100644 --- a/docs/adr/0182-leftover-map-unexplained.md +++ b/docs/adr/0182-leftover-map-unexplained.md @@ -3,6 +3,9 @@ **Decision status:** Accepted **Date:** 2026-08-24 +**Amended by:** [ADR 0183](0183-leftover-map-reconstruction.md) +(two-axis reconstruction R̂) + Amends [ADR 0048](0048-persist-lsirm-leftover-pairs.md) and [ADR 0049](0049-leftover-pair-report-ui.md). @@ -53,8 +56,8 @@ without fabricating unexplained leftover. Fallback pairs that have no complete-case leftover map omit the value rather than inventing one. A rank-0 origin map stores `0.0` (`R = 0` and `R̂ = 0`), not a missing value. A non-finite unexplained leftover stores null rather -than inventing a leftover score. Do not persist -`leftover_map_reconstruction`. +than inventing a leftover score. Persist +`leftover_map_reconstruction` so `U + R̂ = R` stays auditable. The pair button shows `U {signed}` next to leftover-map distance `d` when the value is finite. Next action: leftover map leaves unexplained diff --git a/docs/adr/0183-leftover-map-reconstruction.md b/docs/adr/0183-leftover-map-reconstruction.md new file mode 100644 index 000000000..003171197 --- /dev/null +++ b/docs/adr/0183-leftover-map-reconstruction.md @@ -0,0 +1,93 @@ +# ADR 0183 — Name leftover-map reconstruction on period-report pair rows + +**Decision status:** Accepted +**Date:** 2026-08-25 + +Amends [ADR 0048](0048-persist-lsirm-leftover-pairs.md), +[ADR 0049](0049-leftover-pair-report-ui.md), and +[ADR 0182](0182-leftover-map-unexplained.md). Independent of landed +complete-case coverage ([ADR 0168](0168-leftover-map-complete-case-coverage.md)). + +## Context + +ADR 0182 already persists unexplained leftover `U = R − R̂` after +two-axis Gabriel reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}`. That +reconstruction is computed internally so `U` is honest, then discarded. +A buyer who reads `U` next to leftover residual `R` cannot check +`U + R̂ = R` without the reconstruction the two-axis map actually +uses. Hiding `R̂` lets leftover residual `R` or leftover-map distance +`d` be read as the leftover the map reconstructs. + +This increment persists leftover-map reconstruction `R̂`. It does not +persist leftover-map coordinates, does not name leftover-map inner +product as a separate full-rank column, does not name leftover-map +cosine, does not name leftover-map length, does not name leftover-map +explained share, unexplained share, or cross share, and does not land +Post quality on the leftover criterion. Leftover-map distance stays +two-axis Euclidean. Reconstruction is the same internal two-axis inner +product already used for `U`, so `U + R̂ = R` remains true. Do not +substitute a separately centered reconstruction `R̃` that would break +that identity. + +The unprotected-stack reconstructions for neighbouring leftover facts +use 0162–0181 and 0184–0186. This protected-main increment uses +**0183** so it does not collide with leftover-map unexplained leftover +(0182), leftover-map reconstruction on the centered stack (0186), +leftover-map unexplained share (0184 on that stack), leftover-map +explained share, leftover-map cross share, leftover residual +disclosure, leftover observed `Y` / expected `E`, leftover-map rank, +two-axis leftover-map distance, leftover coverage, leftover-map axis +share (0148), or leftover interaction-map persistence. + +## Decision + +Each leftover pair names `leftover_map_reconstruction` — two-axis +Gabriel reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}`. Unused axes pad with +zero. Hidden SVD axes after the second are dropped. Migration `0183` +is the single source of the column on every install path, fresh or +existing -- shipped migrations (`0001` / `0012`) are never edited after +the fact. The column is nullable so older leftover rows keep distance, +residual, and unexplained leftover without fabricating reconstruction. +Fallback pairs that have no complete-case leftover map omit the value +rather than inventing one. A rank-0 origin map stores `0.0` (`R = 0` +and `R̂ = 0`), not a missing value. A non-finite reconstruction stores +null rather than inventing a leftover score. A signed reconstruction +is stored, never clamped. Do not add a nonnegative CHECK. + +The pair button shows `R̂ {signed}` next to leftover-map distance `d` +when the value is finite. Next action: leftover map reconstructs `R̂` +after IRT main effects; open this post to read the named criterion. +A missing or non-finite reconstruction omits the badge and keeps the +existing unexplained-leftover next action. Do not invent a leftover +score. Do not invent a theta. + +## Consequences + +`GET /api/reports/{grouping}/{period}` returns +`leftover_map_reconstruction`. After `make seed`, closest and farthest +leftover pairs sit above the member list with named `R̂` next to `d`; +click opens that post. Hidden posts stay hidden. When both +reconstruction and unexplained leftover are finite, +`U + R̂ = R`. + +## Related + +Independent of leftover interaction-map persistence, leftover-criterion +evaluation landing, leftover residual disclosure, leftover observed +`Y` / expected `E`, leftover-map complete-case coverage, leftover-map +axis share, leftover pairs on the grouping comparison strip, two-axis +leftover-map distance, leftover-map rank, leftover-map inner product, +leftover-map cosine, leftover-map length, leftover-map unexplained +share, leftover-map explained share, leftover-map cross share, and +the centered leftover-map reconstruction stack. + +## References + +Gabriel, K. R. (1971). The biplot graphic display of matrices with +application to principal component analysis. *Biometrika, 58*(3), +453–467. https://doi.org/10.1093/biomet/58.3.453 + +Jeon, M., Jin, I. H., Schweinberger, M., & Baugh, S. (2021). Mapping +unobserved item–respondent interactions: A latent space item response +model with interaction map. *Psychometrika, 86*(2), 378–403. +https://doi.org/10.1007/s11336-021-09762-5 diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 7a2faf0ef..3b9efc4da 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -968,6 +968,7 @@ describe("App, authenticated", () => { leftover_distance: 0.12, leftover_residual: 0.4, leftover_map_unexplained: 0.05, + leftover_map_reconstruction: 0.35, observed_response: 2.4, expected_response: 2.0, leftover_map_rank: 1, @@ -980,6 +981,7 @@ describe("App, authenticated", () => { leftover_distance: 1.84, leftover_residual: -1.1, leftover_map_unexplained: -0.25, + leftover_map_reconstruction: -0.85, observed_response: 0.9, expected_response: 2.0, leftover_map_rank: 1, @@ -3715,21 +3717,23 @@ describe("App, authenticated", () => { }); expect(closestPair).toHaveTextContent("Closest leftover: Public post · sales-lead"); expect(closestPair).toHaveTextContent( - "Leftover map leaves unexplained U +0.05 after IRT main effects. Open this post to read sales-lead.", + "Leftover map reconstructs R̂ +0.35 after IRT main effects. Open this post to read sales-lead.", ); expect(closestPair).toHaveTextContent("R +0.40"); expect(closestPair).toHaveTextContent("Y 2.40 · E 2.00"); expect(closestPair).toHaveTextContent("rank 1"); expect(closestPair).toHaveTextContent("U +0.05"); + expect(closestPair).toHaveTextContent("R̂ +0.35"); expect(closestPair).toHaveTextContent("d 0.12"); expect(farthestPair).toHaveTextContent("Farthest leftover: Specification revision requested · negative"); expect(farthestPair).toHaveTextContent( - "Leftover map leaves unexplained U −0.25 after IRT main effects. Open this post to read negative.", + "Leftover map reconstructs R̂ −0.85 after IRT main effects. Open this post to read negative.", ); expect(farthestPair).toHaveTextContent("R −1.10"); expect(farthestPair).toHaveTextContent("Y 0.90 · E 2.00"); expect(farthestPair).toHaveTextContent("rank 1"); expect(farthestPair).toHaveTextContent("U −0.25"); + expect(farthestPair).toHaveTextContent("R̂ −0.85"); expect(farthestPair).toHaveTextContent("d 1.84"); const memberButton = screen.getByRole("button", { name: /open report post: public post/i }); expect(coverageCaption.compareDocumentPosition(closestPair) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); diff --git a/frontend/src/api.ts b/frontend/src/api.ts index a951207b9..6d656c523 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -878,6 +878,7 @@ export interface LeftoverPair { expected_response?: number | null; leftover_map_rank?: number | null; leftover_map_unexplained?: number | null; + leftover_map_reconstruction?: number | null; } export interface LeftoverMapAxis { diff --git a/frontend/src/components/LeftoverPairList.stories.tsx b/frontend/src/components/LeftoverPairList.stories.tsx index 680640026..637351074 100644 --- a/frontend/src/components/LeftoverPairList.stories.tsx +++ b/frontend/src/components/LeftoverPairList.stories.tsx @@ -19,6 +19,8 @@ const meta = { observed_response: 2.4, expected_response: 2.0, leftover_map_rank: 1, + leftover_map_unexplained: 0.05, + leftover_map_reconstruction: 0.35, }, { pair_kind: "farthest", @@ -30,6 +32,8 @@ const meta = { observed_response: 0.9, expected_response: 2.0, leftover_map_rank: 1, + leftover_map_unexplained: -0.25, + leftover_map_reconstruction: -0.85, }, ], }, diff --git a/frontend/src/components/LeftoverPairList.test.tsx b/frontend/src/components/LeftoverPairList.test.tsx index 19e306040..e7ef0b8d0 100644 --- a/frontend/src/components/LeftoverPairList.test.tsx +++ b/frontend/src/components/LeftoverPairList.test.tsx @@ -119,6 +119,45 @@ describe("LeftoverPairList", () => { expect(screen.getByRole("button")).toHaveTextContent(expectedAction); }); + it("names leftover-map reconstruction so the next click opens that post", () => { + render( + , + ); + + const closest = screen.getByRole("button"); + expect(closest).toHaveTextContent( + "Leftover map reconstructs R̂ +0.35 after IRT main effects. Open this post to read sales-lead.", + ); + expect(closest).toHaveTextContent("R̂ +0.35"); + expect(closest).toHaveTextContent("U +0.05"); + expect(closest).toHaveTextContent("R +0.40"); + expect(closest).toHaveTextContent("d 0.12"); + }); + + it("keeps unexplained guidance when reconstruction is missing", () => { + render( + , + ); + + expect(screen.getByRole("button")).toHaveTextContent( + "Leftover map leaves unexplained U +0.05 after IRT main effects. Open this post to read sales-lead.", + ); + }); + it("renders nothing when leftover pairs are missing", () => { const { container } = render( , diff --git a/frontend/src/components/LeftoverPairList.tsx b/frontend/src/components/LeftoverPairList.tsx index d31fbc05b..4c536699a 100644 --- a/frontend/src/components/LeftoverPairList.tsx +++ b/frontend/src/components/LeftoverPairList.tsx @@ -12,6 +12,10 @@ import { formatSignedLeftoverValue, LEFTOVER_MAP_UNEXPLAINED_ACTION, } from "../leftoverMapUnexplained"; +import { + formatLeftoverMapReconstruction, + LEFTOVER_MAP_RECONSTRUCTION_ACTION, +} from "../leftoverMapReconstruction"; export type LeftoverPairListProps = { pairs: LeftoverPair[]; @@ -25,9 +29,10 @@ export type LeftoverPairListProps = { * Distance is the two-axis leftover-map Euclidean gap. Residual is * ``R = Y − E[Y|θ, item]`` (Jeon et al., 2021, eq. 3 input). Unexplained * leftover ``U = R − R̂`` after two-axis Gabriel reconstruction (ADR 0182) - * takes priority over the residual/observed-expected/rank next action - * when finite; every badge still renders together before opening the - * named post. + * and reconstruction ``R̂ = ξ_{1:2} · ζ_{1:2}`` (ADR 0183) so + * ``U + R̂ = R`` stays auditable. Reconstruction takes priority over the + * unexplained/residual/observed-expected/rank next action when finite; + * every badge still renders together before opening the named post. */ export function LeftoverPairList({ pairs, @@ -50,8 +55,16 @@ export function LeftoverPairList({ ); const rankBadge = formatLeftoverMapRank(pair.leftover_map_rank); const unexplained = formatLeftoverMapUnexplained(pair.leftover_map_unexplained); + const reconstruction = formatLeftoverMapReconstruction(pair.leftover_map_reconstruction); let nextAction: string; - if (unexplained !== null) { + if (reconstruction !== null) { + const signedReconstruction = + formatSignedLeftoverValue(pair.leftover_map_reconstruction ?? Number.NaN) ?? "—"; + nextAction = tf(LEFTOVER_MAP_RECONSTRUCTION_ACTION, { + value: signedReconstruction, + criterion, + }); + } else if (unexplained !== null) { const signedUnexplained = formatSignedLeftoverValue(pair.leftover_map_unexplained ?? Number.NaN) ?? "—"; nextAction = tf(LEFTOVER_MAP_UNEXPLAINED_ACTION, { @@ -120,6 +133,7 @@ export function LeftoverPairList({ {observedExpected ? {observedExpected} : null} {rankBadge ? {rankBadge} : null} {unexplained ? {unexplained} : null} + {reconstruction ? {reconstruction} : null} d {pair.leftover_distance.toFixed(2)} diff --git a/frontend/src/i18n.test.ts b/frontend/src/i18n.test.ts index 713217391..f74509f77 100644 --- a/frontend/src/i18n.test.ts +++ b/frontend/src/i18n.test.ts @@ -49,6 +49,7 @@ describe("i18n", () => { "Open this post to read the criterion it sat closest to after main effects.", "Open this post to read the criterion it sat farthest from after main effects.", "Leftover map leaves unexplained U {value} after IRT main effects. Open this post to read {criterion}.", + "Leftover map reconstructs R̂ {value} after IRT main effects. Open this post to read {criterion}.", "Read observed Y {observed} and expected E {expected} after IRT main effects, then open this post.", "Leftover map has no leftover structure after IRT main effects. Open this post.", "Leftover map rank {rank} after IRT main effects. Open this post.", @@ -203,6 +204,33 @@ describe("i18n", () => { ), ).toBe(expected); }); + + it.each([ + [ + "ko", + "잔여 지도가 IRT 주효과 이후 R̂ +0.35을(를) 재구성합니다. sales-lead 기준을 읽으려면 이 글을 여세요.", + ], + [ + "zh", + "残差图在 IRT 主效应后重建 R̂ +0.35。打开这篇帖子阅读 sales-lead。", + ], + [ + "ja", + "残差マップはIRT主効果後の R̂ +0.35 を再構成します。この投稿を開いて sales-lead を読んでください。", + ], + [ + "vi", + "Bản đồ phần dư tái dựng R̂ +0.35 sau hiệu ứng chính IRT. Mở bài viết này để đọc sales-lead.", + ], + ] as const)("formats leftover-map reconstruction next action in %s", (locale, expected) => { + setLocale(locale); + expect( + tf( + "Leftover map reconstructs R̂ {value} after IRT main effects. Open this post to read {criterion}.", + { value: "+0.35", criterion: "sales-lead" }, + ), + ).toBe(expected); + }); }); describe("locale-aware source labels", () => { diff --git a/frontend/src/i18n.ts b/frontend/src/i18n.ts index 55fed1394..9dfba2ec1 100644 --- a/frontend/src/i18n.ts +++ b/frontend/src/i18n.ts @@ -456,6 +456,8 @@ const TRANSLATIONS: Partial>> = { "주효과 이후 가장 멀리 앉은 기준을 읽으려면 이 글을 여세요.", "Leftover map leaves unexplained U {value} after IRT main effects. Open this post to read {criterion}.": "잔여 지도가 IRT 주효과 이후 설명되지 않은 U {value}을(를) 남깁니다. {criterion} 기준을 읽으려면 이 글을 여세요.", + "Leftover map reconstructs R̂ {value} after IRT main effects. Open this post to read {criterion}.": + "잔여 지도가 IRT 주효과 이후 R̂ {value}을(를) 재구성합니다. {criterion} 기준을 읽으려면 이 글을 여세요.", "Read observed Y {observed} and expected E {expected} after IRT main effects, then open this post.": "IRT 주효과 이후 관측 Y {observed}와 기대 E {expected}를 읽은 다음, 이 글을 여세요.", "Leftover map has no leftover structure after IRT main effects. Open this post.": @@ -903,6 +905,8 @@ const TRANSLATIONS: Partial>> = { "打开这篇帖子，阅读主效应后距离最远的准则。", "Leftover map leaves unexplained U {value} after IRT main effects. Open this post to read {criterion}.": "残差图在 IRT 主效应后留下未解释的 U {value}。打开这篇帖子阅读 {criterion}。", + "Leftover map reconstructs R̂ {value} after IRT main effects. Open this post to read {criterion}.": + "残差图在 IRT 主效应后重建 R̂ {value}。打开这篇帖子阅读 {criterion}。", "Read observed Y {observed} and expected E {expected} after IRT main effects, then open this post.": "阅读 IRT 主效应后的观测 Y {observed} 与期望 E {expected}，然后打开这篇帖子。", "Leftover map has no leftover structure after IRT main effects. Open this post.": @@ -1351,6 +1355,8 @@ const TRANSLATIONS: Partial>> = { "主効果後に最も遠くなった基準を読むには、この投稿を開いてください。", "Leftover map leaves unexplained U {value} after IRT main effects. Open this post to read {criterion}.": "残差マップはIRT主効果後の未説明 U {value} を残します。この投稿を開いて {criterion} を読んでください。", + "Leftover map reconstructs R̂ {value} after IRT main effects. Open this post to read {criterion}.": + "残差マップはIRT主効果後の R̂ {value} を再構成します。この投稿を開いて {criterion} を読んでください。", "Read observed Y {observed} and expected E {expected} after IRT main effects, then open this post.": "IRT主効果後の観測 Y {observed} と期待 E {expected} を読んでから、この投稿を開いてください。", "Leftover map has no leftover structure after IRT main effects. Open this post.": @@ -1799,6 +1805,8 @@ const TRANSLATIONS: Partial>> = { "Mở bài viết này để đọc tiêu chí nằm xa nhất sau hiệu ứng chính.", "Leftover map leaves unexplained U {value} after IRT main effects. Open this post to read {criterion}.": "Bản đồ phần dư để lại U {value} chưa giải thích sau hiệu ứng chính IRT. Mở bài viết này để đọc {criterion}.", + "Leftover map reconstructs R̂ {value} after IRT main effects. Open this post to read {criterion}.": + "Bản đồ phần dư tái dựng R̂ {value} sau hiệu ứng chính IRT. Mở bài viết này để đọc {criterion}.", "Read observed Y {observed} and expected E {expected} after IRT main effects, then open this post.": "Đọc Y quan sát {observed} và E kỳ vọng {expected} sau hiệu ứng chính IRT, rồi mở bài viết này.", "Leftover map has no leftover structure after IRT main effects. Open this post.": diff --git a/frontend/src/leftoverMapReconstruction.test.ts b/frontend/src/leftoverMapReconstruction.test.ts new file mode 100644 index 000000000..5a35a5168 --- /dev/null +++ b/frontend/src/leftoverMapReconstruction.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vitest"; +import { formatLeftoverMapReconstruction } from "./leftoverMapReconstruction"; + +describe("formatLeftoverMapReconstruction", () => { + it("names leftover-map reconstruction without inventing a leftover score", () => { + expect(formatLeftoverMapReconstruction(0.35)).toBe("R\u0302 +0.35"); + expect(formatLeftoverMapReconstruction(-0.85)).toBe("R\u0302 \u22120.85"); + expect(formatLeftoverMapReconstruction(0)).toBe("R\u0302 0.00"); + }); + + it("omits the badge when reconstruction is missing or non-finite", () => { + expect(formatLeftoverMapReconstruction(null)).toBeNull(); + expect(formatLeftoverMapReconstruction(undefined)).toBeNull(); + expect(formatLeftoverMapReconstruction(Number.NaN)).toBeNull(); + expect(formatLeftoverMapReconstruction(Number.POSITIVE_INFINITY)).toBeNull(); + }); +}); diff --git a/frontend/src/leftoverMapReconstruction.ts b/frontend/src/leftoverMapReconstruction.ts new file mode 100644 index 000000000..895192933 --- /dev/null +++ b/frontend/src/leftoverMapReconstruction.ts @@ -0,0 +1,16 @@ +/** Two-axis leftover-map reconstruction ``R̂ = ξ_{1:2} · ζ_{1:2}``. */ + +import { formatSignedLeftoverValue } from "./leftoverMapUnexplained"; + +export const LEFTOVER_MAP_RECONSTRUCTION_ACTION = + "Leftover map reconstructs R̂ {value} after IRT main effects. Open this post to read {criterion}."; + +const RECONSTRUCTION_BADGE = "R\u0302"; + +export function formatLeftoverMapReconstruction(value: number | null | undefined): string | null { + if (value == null) { + return null; + } + const signed = formatSignedLeftoverValue(value); + return signed === null ? null : `${RECONSTRUCTION_BADGE} ${signed}`; +} diff --git a/lineageweave/leftover_pairs.py b/lineageweave/leftover_pairs.py index cb7ae265d..71283d542 100644 --- a/lineageweave/leftover_pairs.py +++ b/lineageweave/leftover_pairs.py @@ -1,6 +1,7 @@ """Jeon leftover post–criterion pairs after a main-effect IRT. -Implements ADR 0048 as amended by ADR 0119, ADR 0163, ADR 0164, and ADR 0182. +Implements ADR 0048 as amended by ADR 0119, ADR 0163, ADR 0164, ADR 0168, +ADR 0182, and ADR 0183. Does not import ``fast_mlsirm`` or ``period_report``. A Gabriel biplot of the residual ``R = Y − E[Y|θ, item]`` supplies person and item @@ -14,11 +15,12 @@ rank so a rank-0 collapse is not read as leftover structure. Axis share is the Gabriel inertia of the first two leftover-map axes (ADR 0148). Complete-case coverage (ADR 0168) names how many scored posts entered -that rectangle. Each pair also names unexplained leftover ``U = R − R̂`` -after two-axis Gabriel reconstruction ``R̂ = ξ_{1:2} · ζ_{1:2}`` so the -leftover cell the map does not reconstruct is not confused with -leftover residual ``R`` or leftover-map distance ``d``. Reconstruction -is computed internally and is not persisted. +that rectangle. Each pair names two-axis Gabriel reconstruction +``R̂ = ξ_{1:2} · ζ_{1:2}`` and unexplained leftover ``U = R − R̂`` so +the leftover cell the map reconstructs is not confused with leftover +residual ``R`` or leftover-map distance ``d``. Identity ``U + R̂ = R`` +stays auditable. Do not persist leftover-map coordinates or leftover-map +shares on the pair row. """ from __future__ import annotations @@ -47,6 +49,7 @@ class LeftoverPair: expected_response: float leftover_map_rank: int leftover_map_unexplained: float | None = None + leftover_map_reconstruction: float | None = None @dataclass(frozen=True) @@ -86,10 +89,11 @@ def leftover_pairs_from_residual( invent a leftover score. Stored residual equals observed ``Y`` minus expected ``E[Y|θ, item]``. Stored leftover-map rank is the number of Gabriel singular values above the floor. When Gabriel coordinates - exist, unexplained leftover ``U = R − R̂`` names the leftover cell - the two-axis map does not reconstruct; ``R̂`` stays internal and is - never persisted. Fallback pairs (no complete-case map) omit - unexplained leftover rather than fabricating one. + exist, reconstruction ``R̂ = ξ_{1:2} · ζ_{1:2}`` and unexplained + leftover ``U = R − R̂`` name the leftover cell the two-axis map + reconstructs and the leftover it leaves. Identity ``U + R̂ = R`` + stays auditable. Without a complete-case map there is no pair + (ADR 0168); reconstruction is omitted rather than fabricated. """ pairs, _axes = leftover_map_from_residual(post_ids, item_codes, matrix, expected) return pairs @@ -137,7 +141,7 @@ def leftover_map_from_residual( ) axes = leftover_map_axes_from_singular(singular) leftover_map_rank = int(singular.size) - candidates: list[tuple[float, str, str, float, float, float, float | None]] = [] + candidates: list[tuple[float, str, str, float, float, float, float | None, float | None]] = [] if person_pos is not None and item_pos is not None: person_index = np.flatnonzero(keep_person) item_index = np.flatnonzero(keep_item) @@ -153,10 +157,13 @@ def leftover_map_from_residual( ) if not np.isfinite(distance): continue - reconstruction = float( - np.dot(person_xy[local_person[person]], item_xy[local_item[item]]) + reconstruction = _finite_reconstruction( + float(np.dot(person_xy[local_person[person]], item_xy[local_item[item]])) + ) + unexplained = _unexplained_leftover( + float(residual[person, item]), + reconstruction if reconstruction is not None else float("nan"), ) - unexplained = _unexplained_leftover(float(residual[person, item]), reconstruction) candidates.append( _candidate_row( post_ids, @@ -168,6 +175,7 @@ def leftover_map_from_residual( item, distance, unexplained, + reconstruction, ) ) if not candidates: @@ -184,6 +192,13 @@ def leftover_map_from_residual( return pairs, axes +def _finite_reconstruction(reconstruction: float) -> float | None: + """Return two-axis reconstruction ``R̂`` when finite; otherwise omit.""" + if not np.isfinite(reconstruction): + return None + return float(reconstruction) + + def _unexplained_leftover(residual: float, reconstruction: float) -> float | None: """Return ``U = R − R̂`` when both terms are finite; otherwise omit.""" if not np.isfinite(reconstruction): @@ -204,8 +219,9 @@ def _candidate_row( item: int, distance: float, leftover_map_unexplained: float | None, -) -> tuple[float, str, str, float, float, float, float | None]: - """One observed leftover cell: distance, ids, residual, Y, E, unexplained U.""" + leftover_map_reconstruction: float | None, +) -> tuple[float, str, str, float, float, float, float | None, float | None]: + """One observed leftover cell: distance, ids, residual, Y, E, U, R̂.""" leftover_residual = float(residual[person, item]) observed_response = float(matrix[person, item]) expected_response = float(expected[person, item]) @@ -219,12 +235,13 @@ def _candidate_row( observed_response, expected_response, leftover_map_unexplained, + leftover_map_reconstruction, ) def _pair_from_candidate( pair_kind: str, - row: tuple[float, str, str, float, float, float, float | None], + row: tuple[float, str, str, float, float, float, float | None, float | None], leftover_map_rank: int, ) -> LeftoverPair: """Build a leftover pair from a candidate row.""" @@ -240,6 +257,7 @@ def _pair_from_candidate( expected_response=row[5], leftover_map_rank=leftover_map_rank, leftover_map_unexplained=row[6], + leftover_map_reconstruction=row[7], ) @@ -376,7 +394,7 @@ def _pad_map_axes(positions: np.ndarray) -> np.ndarray: Unused axes pad with zero rather than inventing a second component. Hidden SVD axes after the second are dropped so reconstruction is ``ξ_{1:2} · ζ_{1:2}``, not the full-rank inner product. That - reconstruction stays internal; only unexplained leftover is named. + reconstruction is persisted so ``U + R̂ = R`` stays auditable. """ padded = np.zeros((positions.shape[0], _LEFTOVER_MAP_AXES), dtype=np.float64) width = min(_LEFTOVER_MAP_AXES, positions.shape[1]) diff --git a/migrations/0183_report_leftover_map_reconstruction.sql b/migrations/0183_report_leftover_map_reconstruction.sql new file mode 100644 index 000000000..8662777f3 --- /dev/null +++ b/migrations/0183_report_leftover_map_reconstruction.sql @@ -0,0 +1,11 @@ +-- ADR 0183: persist leftover-map reconstruction R̂ = ξ_{1:2} · ζ_{1:2} +-- so unexplained leftover U = R − R̂ stays auditable as U + R̂ = R. +-- Distance stays Euclidean leftover-map d. No nonnegative CHECK: a +-- signed reconstruction is stored, never clamped. Upgrade column is +-- nullable so older leftover rows keep distance, residual, and +-- unexplained leftover without fabricating reconstruction. This +-- migration is the single source of the column on fresh and existing +-- installations. + +alter table report_leftover_pair + add column if not exists leftover_map_reconstruction numeric; diff --git a/migrations/rollback/0183_report_leftover_map_reconstruction.sql b/migrations/rollback/0183_report_leftover_map_reconstruction.sql new file mode 100644 index 000000000..379f6f30e --- /dev/null +++ b/migrations/rollback/0183_report_leftover_map_reconstruction.sql @@ -0,0 +1,4 @@ +-- Reverse 0183. Leftover distance, residual, and unexplained leftover stay. + +alter table report_leftover_pair + drop column if exists leftover_map_reconstruction; diff --git a/scripts/seed_demo_data.py b/scripts/seed_demo_data.py index 0034b35a6..6c6450e90 100644 --- a/scripts/seed_demo_data.py +++ b/scripts/seed_demo_data.py @@ -123,6 +123,7 @@ def seed( cur.execute((migrations / "0168_report_leftover_map_coverage.sql").read_text()) cur.execute((migrations / "0169_report_leftover_map_axis.sql").read_text()) cur.execute((migrations / "0182_report_leftover_map_unexplained.sql").read_text()) + cur.execute((migrations / "0183_report_leftover_map_reconstruction.sql").read_text()) cur.execute((migrations / "0060_role_responsibility_agent_type.sql").read_text()) cur.execute((migrations / "0013_person_job_title.sql").read_text()) cur.execute((migrations / "0014_role_responsibility_team_actor_type.sql").read_text()) @@ -1203,8 +1204,8 @@ def _persist_seed_period_report( "grouping_kind, grouping_key, period_code, rubric_version, " "pair_kind, post_id, criterion_code, leftover_distance, leftover_residual, " "observed_response, expected_response, leftover_map_rank, " - "leftover_map_unexplained" - ") values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + "leftover_map_unexplained, leftover_map_reconstruction" + ") values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", ( grouping_kind, grouping_key, @@ -1219,6 +1220,7 @@ def _persist_seed_period_report( pair.expected_response, pair.leftover_map_rank, pair.leftover_map_unexplained, + pair.leftover_map_reconstruction, ), ) for axis in report.leftover_map_axes: diff --git a/tests/test_leftover_pairs.py b/tests/test_leftover_pairs.py index a411e1e13..e3d98f825 100644 --- a/tests/test_leftover_pairs.py +++ b/tests/test_leftover_pairs.py @@ -1,7 +1,7 @@ """Leftover post–criterion pairs after the main-effect IRT. Covers ADR 0048 as amended by ADR 0119, ADR 0148, ADR 0163, ADR 0164, -and ADR 0182. +ADR 0168, ADR 0182, and ADR 0183. Uses a constructed residual matrix so the closest and farthest pair are known without calling ``fit_polytomous``. Loads @@ -103,10 +103,14 @@ def test_leftover_residual_biplot_separates_aligned_and_opposed_cells() -> None: assert farthest.leftover_distance == pytest.approx(2.0 * np.sqrt(2.0), rel=1e-6) assert closest.leftover_map_unexplained == pytest.approx(0.0, abs=1e-6) assert farthest.leftover_map_unexplained == pytest.approx(0.0, abs=1e-6) - assert not hasattr(closest, "leftover_map_reconstruction") + assert closest.leftover_map_reconstruction == pytest.approx(closest.leftover_residual, abs=1e-6) + assert farthest.leftover_map_reconstruction == pytest.approx(-2.0, abs=1e-6) for pair in pairs: _assert_residual_reconciles(pair) assert pair.leftover_map_rank == 1 + assert pair.leftover_map_unexplained + pair.leftover_map_reconstruction == pytest.approx( + pair.leftover_residual, abs=1e-6 + ) coverage = leftover_map_coverage_from_residual(post_ids, item_codes, matrix, expected) assert coverage.map_post_count == 3 assert coverage.scored_post_count == 3 @@ -133,6 +137,8 @@ def test_zero_residual_still_emits_stable_leftover_pairs() -> None: assert pairs[1].criterion_code == "item_two" assert pairs[0].leftover_map_unexplained == pytest.approx(0.0) assert pairs[1].leftover_map_unexplained == pytest.approx(0.0) + assert pairs[0].leftover_map_reconstruction == pytest.approx(0.0) + assert pairs[1].leftover_map_reconstruction == pytest.approx(0.0) for pair in pairs: _assert_residual_reconciles(pair) assert pair.leftover_map_rank == 0 @@ -170,6 +176,7 @@ def test_partial_observation_does_not_treat_missing_as_zero_residual() -> None: _assert_residual_reconciles(pair) assert pair.leftover_map_rank == 1 assert pair.leftover_map_unexplained == pytest.approx(0.0, abs=1e-6) + assert pair.leftover_map_reconstruction == pytest.approx(pair.leftover_residual, abs=1e-6) coverage = leftover_map_coverage_from_residual(post_ids, item_codes, matrix, expected) assert coverage.map_post_count == 2 assert coverage.scored_post_count == 3 @@ -241,6 +248,7 @@ def test_leftover_residual_rejects_database_tolerance_boundary() -> None: 0, 0.0, None, + None, ) @@ -379,9 +387,15 @@ def test_unexplained_equals_residual_minus_two_axis_reconstruction() -> None: item = item_index[pair.criterion_code] expected_unexplained = float(pair.leftover_residual) - float(reconstruction[person, item]) assert pair.leftover_map_unexplained == pytest.approx(expected_unexplained) + assert pair.leftover_map_reconstruction == pytest.approx(float(reconstruction[person, item])) + assert pair.leftover_map_unexplained + pair.leftover_map_reconstruction == pytest.approx( + pair.leftover_residual, abs=1e-6 + ) + assert pair.leftover_map_reconstruction != pytest.approx(pair.leftover_residual) + assert pair.leftover_map_reconstruction != pytest.approx(pair.leftover_distance) + assert pair.leftover_map_reconstruction != pytest.approx(float(full_inner[person, item])) assert pair.leftover_map_unexplained != pytest.approx(pair.leftover_residual) assert pair.leftover_map_unexplained != pytest.approx(pair.leftover_distance) - assert not hasattr(pair, "leftover_map_reconstruction") def test_pad_map_axes_truncates_hidden_svd_components() -> None: @@ -453,7 +467,7 @@ def test_leftover_map_rank_rejects_negative_rank() -> None: with pytest.raises(ValueError, match="non-negative integer"): leftover._pair_from_candidate( PAIR_KIND_CLOSEST, - (0.0, "public-post", "sales_lead_specificity", 0.0, 1.0, 1.0, None), + (0.0, "public-post", "sales_lead_specificity", 0.0, 1.0, 1.0, None, None), -1, ) diff --git a/tests/test_period_report.py b/tests/test_period_report.py index 95021ac4b..3f4e10f4a 100644 --- a/tests/test_period_report.py +++ b/tests/test_period_report.py @@ -272,7 +272,15 @@ def test_calibrated_report_attaches_leftover_pairs() -> None: assert pair.leftover_map_rank >= 0 if pair.leftover_map_unexplained is not None: assert np.isfinite(pair.leftover_map_unexplained) - assert not hasattr(pair, "leftover_map_reconstruction") + if pair.leftover_map_reconstruction is not None: + assert np.isfinite(pair.leftover_map_reconstruction) + if ( + pair.leftover_map_unexplained is not None + and pair.leftover_map_reconstruction is not None + ): + assert pair.leftover_map_unexplained + pair.leftover_map_reconstruction == pytest.approx( + pair.leftover_residual, abs=1e-6 + ) assert [axis.axis_index for axis in report.leftover_map_axes] == [1, 2] for axis in report.leftover_map_axes: assert axis.leftover_singular_value >= 0.0 diff --git a/tests/test_schema.py b/tests/test_schema.py index 0f9fd18a3..7999841f5 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -68,6 +68,11 @@ / "migrations" / "0182_report_leftover_map_unexplained.sql" ) +_LEFTOVER_MAP_RECONSTRUCTION_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0183_report_leftover_map_reconstruction.sql" +) def _postgres_available() -> bool: @@ -109,6 +114,7 @@ def schema_db(): cur.execute(_LEFTOVER_MAP_COVERAGE_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_AXIS_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_UNEXPLAINED_MIGRATION.read_text()) + cur.execute(_LEFTOVER_MAP_RECONSTRUCTION_MIGRATION.read_text()) conn.commit() yield conn finally: @@ -254,7 +260,21 @@ def test_leftover_pair_names_nullable_unexplained_column(schema_db) -> None: assert columns["leftover_map_unexplained"] == "YES" assert columns["leftover_residual"] == "NO" assert columns["leftover_distance"] == "NO" - assert "leftover_map_reconstruction" not in columns + + +def test_leftover_pair_names_nullable_reconstruction_column(schema_db) -> None: + """Every install path preserves legacy pairs while naming reconstruction R̂.""" + with schema_db.cursor() as cur: + cur.execute( + """ + select column_name, is_nullable + from information_schema.columns + where table_name = 'report_leftover_pair' + """ + ) + columns = dict(cur.fetchall()) + assert columns["leftover_map_reconstruction"] == "YES" + assert columns["leftover_map_unexplained"] == "YES" def test_leftover_map_axis_references_period_score(schema_db) -> None: From 2f5e197e04ecc6111f96d8a48c7ddb0bb0156207 Mon Sep 17 00:00:00 2001 From: seonghobae Date: Tue, 25 Aug 2026 09:31:36 +0900 Subject: [PATCH 2/6] fix(adr): allocate reconstruction decision 0201 --- CHANGELOG.md | 2 +- docs/adr/0048-persist-lsirm-leftover-pairs.md | 4 ++-- docs/adr/0049-leftover-pair-report-ui.md | 4 ++-- docs/adr/0182-leftover-map-unexplained.md | 2 +- ...-reconstruction.md => 0201-leftover-map-reconstruction.md} | 2 +- frontend/src/components/LeftoverPairList.tsx | 2 +- lineageweave/leftover_pairs.py | 2 +- tests/test_leftover_pairs.py | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) rename docs/adr/{0183-leftover-map-reconstruction.md => 0201-leftover-map-reconstruction.md} (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77705be96..cc1f3f4d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this project are documented here. Format follows - Period leftover pair rows now name leftover-map reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}` next to leftover-map distance `d`, then open - that post (Gabriel, 1971; Jeon et al., 2021, eq. 3; ADR 0183). A missing + that post (Gabriel, 1971; Jeon et al., 2021, eq. 3; ADR 0201). A missing reconstruction omits the badge rather than inventing a leftover score. Signed reconstruction is stored, never clamped. Identity `U + R̂ = R` stays auditable. diff --git a/docs/adr/0048-persist-lsirm-leftover-pairs.md b/docs/adr/0048-persist-lsirm-leftover-pairs.md index f7efb8436..85022ba14 100644 --- a/docs/adr/0048-persist-lsirm-leftover-pairs.md +++ b/docs/adr/0048-persist-lsirm-leftover-pairs.md @@ -6,7 +6,7 @@ [ADR 0163](0163-leftover-observed-expected.md) (observed Y and expected E); [ADR 0164](0164-leftover-map-rank.md) (full map rank); [ADR 0182](0182-leftover-map-unexplained.md) (unexplained leftover U); -[ADR 0183](0183-leftover-map-reconstruction.md) (two-axis reconstruction R̂) +[ADR 0201](0201-leftover-map-reconstruction.md) (two-axis reconstruction R̂) ## Context @@ -43,7 +43,7 @@ leftover row also names unexplained leftover `U = R − R̂` when Gabriel coordinates exist so the leftover cell the two-axis map does not reconstruct is not read as leftover residual `R` or leftover-map distance `d` (ADR 0182). Two-axis reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}` -is persisted so `U + R̂ = R` stays auditable (ADR 0183). Do not +is persisted so `U + R̂ = R` stays auditable (ADR 0201). Do not substitute a separately centered reconstruction. Cascade the rows with `report_period_score`. A leftover post must diff --git a/docs/adr/0049-leftover-pair-report-ui.md b/docs/adr/0049-leftover-pair-report-ui.md index 81ed6bf9e..33daa9a98 100644 --- a/docs/adr/0049-leftover-pair-report-ui.md +++ b/docs/adr/0049-leftover-pair-report-ui.md @@ -6,7 +6,7 @@ [ADR 0163](0163-leftover-observed-expected.md) (observed Y and expected E); [ADR 0164](0164-leftover-map-rank.md) (full map rank); [ADR 0182](0182-leftover-map-unexplained.md) (unexplained leftover U); -[ADR 0183](0183-leftover-map-reconstruction.md) (two-axis reconstruction R̂) +[ADR 0201](0201-leftover-map-reconstruction.md) (two-axis reconstruction R̂) ## Context @@ -38,7 +38,7 @@ row. Residual naming is [ADR 0162](0162-leftover-residual-disclosure.md), observed/expected naming is [ADR 0163](0163-leftover-observed-expected.md), rank naming is [ADR 0164](0164-leftover-map-rank.md), unexplained leftover naming is [ADR 0182](0182-leftover-map-unexplained.md), -reconstruction naming is [ADR 0183](0183-leftover-map-reconstruction.md). +reconstruction naming is [ADR 0201](0201-leftover-map-reconstruction.md). After `make seed`, closest and farthest leftover pairs sit above the member list. Click a pair to open that post. diff --git a/docs/adr/0182-leftover-map-unexplained.md b/docs/adr/0182-leftover-map-unexplained.md index 523353ee9..fecd979b9 100644 --- a/docs/adr/0182-leftover-map-unexplained.md +++ b/docs/adr/0182-leftover-map-unexplained.md @@ -3,7 +3,7 @@ **Decision status:** Accepted **Date:** 2026-08-24 -**Amended by:** [ADR 0183](0183-leftover-map-reconstruction.md) +**Amended by:** [ADR 0201](0201-leftover-map-reconstruction.md) (two-axis reconstruction R̂) Amends [ADR 0048](0048-persist-lsirm-leftover-pairs.md) and diff --git a/docs/adr/0183-leftover-map-reconstruction.md b/docs/adr/0201-leftover-map-reconstruction.md similarity index 98% rename from docs/adr/0183-leftover-map-reconstruction.md rename to docs/adr/0201-leftover-map-reconstruction.md index 003171197..2c6d777e5 100644 --- a/docs/adr/0183-leftover-map-reconstruction.md +++ b/docs/adr/0201-leftover-map-reconstruction.md @@ -1,4 +1,4 @@ -# ADR 0183 — Name leftover-map reconstruction on period-report pair rows +# ADR 0201 — Name leftover-map reconstruction on period-report pair rows **Decision status:** Accepted **Date:** 2026-08-25 diff --git a/frontend/src/components/LeftoverPairList.tsx b/frontend/src/components/LeftoverPairList.tsx index 4c536699a..febfd157b 100644 --- a/frontend/src/components/LeftoverPairList.tsx +++ b/frontend/src/components/LeftoverPairList.tsx @@ -29,7 +29,7 @@ export type LeftoverPairListProps = { * Distance is the two-axis leftover-map Euclidean gap. Residual is * ``R = Y − E[Y|θ, item]`` (Jeon et al., 2021, eq. 3 input). Unexplained * leftover ``U = R − R̂`` after two-axis Gabriel reconstruction (ADR 0182) - * and reconstruction ``R̂ = ξ_{1:2} · ζ_{1:2}`` (ADR 0183) so + * and reconstruction ``R̂ = ξ_{1:2} · ζ_{1:2}`` (ADR 0201) so * ``U + R̂ = R`` stays auditable. Reconstruction takes priority over the * unexplained/residual/observed-expected/rank next action when finite; * every badge still renders together before opening the named post. diff --git a/lineageweave/leftover_pairs.py b/lineageweave/leftover_pairs.py index 71283d542..133ef0807 100644 --- a/lineageweave/leftover_pairs.py +++ b/lineageweave/leftover_pairs.py @@ -1,7 +1,7 @@ """Jeon leftover post–criterion pairs after a main-effect IRT. Implements ADR 0048 as amended by ADR 0119, ADR 0163, ADR 0164, ADR 0168, -ADR 0182, and ADR 0183. +ADR 0182, and ADR 0201. Does not import ``fast_mlsirm`` or ``period_report``. A Gabriel biplot of the residual ``R = Y − E[Y|θ, item]`` supplies person and item diff --git a/tests/test_leftover_pairs.py b/tests/test_leftover_pairs.py index e3d98f825..607a163a1 100644 --- a/tests/test_leftover_pairs.py +++ b/tests/test_leftover_pairs.py @@ -1,7 +1,7 @@ """Leftover post–criterion pairs after the main-effect IRT. Covers ADR 0048 as amended by ADR 0119, ADR 0148, ADR 0163, ADR 0164, -ADR 0168, ADR 0182, and ADR 0183. +ADR 0168, ADR 0182, and ADR 0201. Uses a constructed residual matrix so the closest and farthest pair are known without calling ``fit_polytomous``. Loads From 3a67a802cd5f1af0755c0e74de6398d60557b36a Mon Sep 17 00:00:00 2001 From: seonghobae Date: Tue, 25 Aug 2026 11:33:09 +0900 Subject: [PATCH 3/6] =?UTF-8?q?fix:=20restore=20R=CC=82-persistence=20docs?= =?UTF-8?q?=20and=20finish=20ADR=200183=E2=86=920201=20renumbering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prior merge of origin/main into this branch (91d8e4f5) resolved the AGENTS.md/ARCHITECTURE.md/CLAUDE.md conflicts by dropping this PR's own leftover-map reconstruction documentation: - AGENTS.md reverted to "Two-axis reconstruction R̂ is not persisted", contradicting the shipped migration/ADR 0201 that persists it, and dropped ADR 0201 from the governing-ADR list. - ARCHITECTURE.md kept a redundant "0183 / 0201" pair (0183 is this repo's real, unrelated ADR 0183 "GNB four Korean chrome"; the stale 0183 leftover-map reference should have been renumbered to 0201, not kept alongside it). - CLAUDE.md's "Where the rest lives" pointer never got the ADR 0201 cross-reference added. Also finishes the ADR-number renumbering the PR's own history had started (docs/adr/0201-leftover-map-reconstruction.md is the actual ADR; ADR 0183 already belongs to a different, already-landed decision on main): fixes remaining stale "ADR 0183" citations in docs/adr/0003, the 2.12.31 CHANGELOG.d fragment, and the migration 0183 SQL header comment, all of which should cite ADR 0201. docs/adr/0049's own conflict resolution (0158 + 0201 amendments) was already correct and is unchanged. --- AGENTS.md | 12 ++++++------ ARCHITECTURE.md | 2 +- CHANGELOG.d/2.12.31-leftover-map-reconstruction.md | 2 +- CLAUDE.md | 2 +- docs/adr/0003-fast-mlsirm-report-integration.md | 2 +- .../0183_report_leftover_map_reconstruction.sql | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6544da08d..8fa30fec3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -259,18 +259,18 @@ stops startup instead of leaving a healthy-looking partial schema, and application code must not compensate for a missing table. Period leftover pairs (ADR 0017 / 0018 / 0048 / 0049 / 0119 / 0158 / 0162 / -0163 / 0164 / 0182) are computed in `lineageweave/leftover_pairs.py` from the +0163 / 0164 / 0182 / 0201) are computed in `lineageweave/leftover_pairs.py` from the residual after a real GRM/GPCM score, never invented. Distances are Euclidean on the two-dimensional Gabriel leftover map; missing cells stay out of the factorization. Closest and farthest post–criterion pairs persist to `report_leftover_pair` with signed residual `R`, observed `Y`, and expected `E[Y|θ, item]` so `R = Y − E` remains auditable, -plus leftover-map rank so rank 0 is not read as structure, and -unexplained leftover `U = R − R̂` next to leftover-map distance `d` -after two-axis Gabriel reconstruction. They sit above the member +plus leftover-map rank so rank 0 is not read as structure, +unexplained leftover `U = R − R̂`, and leftover-map reconstruction +`R̂ = ξ_{1:2} · ζ_{1:2}` next to leftover-map distance `d` so +`U + R̂ = R` stays auditable. They sit above the member list so a click opens that post with the leftover criterion current -in Post quality (ADR 0158). Two-axis reconstruction `R̂` is not -persisted. Leftover-map axis share (ADR 0148) is Gabriel inertia of +in Post quality (ADR 0158). Leftover-map axis share (ADR 0148) is Gabriel inertia of residual SVD axes 1 and 2 and persists to `report_leftover_map_axis`. Rank-0 residuals emit two zero-share axes; the shares are report-level and are not a leftover score. Complete-case coverage (ADR 0168) persists to diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 187a7336c..98435ab1e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -597,7 +597,7 @@ information at the group's mean θ (Lord, 1980 max-info CAT). Rankings persist to `report_item_information`. After those IRT main effects, residual SVD leftover pairs on two Gabriel axes (Jeon et al., 2021; ADR 0017 / 0048 / 0049 / 0119 / 0158 / 0162 / 0163 / 0164 / 0182 / -0183 / 0201) persist to +0201) persist to `report_leftover_pair` with signed residual `R`, observed `Y`, expected `E[Y|θ, item]`, full leftover-map rank, unexplained leftover `U = R − R̂`, and two-axis reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}` so diff --git a/CHANGELOG.d/2.12.31-leftover-map-reconstruction.md b/CHANGELOG.d/2.12.31-leftover-map-reconstruction.md index bf20353ba..af7898531 100644 --- a/CHANGELOG.d/2.12.31-leftover-map-reconstruction.md +++ b/CHANGELOG.d/2.12.31-leftover-map-reconstruction.md @@ -1,7 +1,7 @@ ## 2.12.31 — Leftover-map reconstruction - Persist leftover-map reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}` on leftover - post–criterion pairs (ADR 0183). After `make seed`, closest and farthest + post–criterion pairs (ADR 0201). After `make seed`, closest and farthest leftover pairs sit above the member list with `R̂` next to leftover-map distance `d`; click opens that post. Omit the badge when reconstruction is missing. Never invent a leftover score. Signed reconstruction is diff --git a/CLAUDE.md b/CLAUDE.md index f32325abc..eb9e85eab 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,7 +48,7 @@ cutoff. Create/start endpoint rules (ADR 0017 / 0021), tie-vs-miss similarity (ADR 0026), R&R catalog ids (ADR 0019 / 0027), leftover pairs -(ADR 0048–0164 / 0182), the text-channel embedding swap and cosine +(ADR 0048–0164 / 0182 / 0201), the text-channel embedding swap and cosine clamp (ADR 0190), per-edge channel-score persistence (ADR 0195), migration replay (ADR 0166), docstring coverage, and the measurement boundary are all stated in [AGENTS.md](AGENTS.md) -- read it before diff --git a/docs/adr/0003-fast-mlsirm-report-integration.md b/docs/adr/0003-fast-mlsirm-report-integration.md index d809fb2d4..fb9e79c36 100644 --- a/docs/adr/0003-fast-mlsirm-report-integration.md +++ b/docs/adr/0003-fast-mlsirm-report-integration.md @@ -100,7 +100,7 @@ than one large PR: `information_polytomous` (Lord, 1980 max-info). Persist the ranking (`report_item_information`) and show the rank-1 item on the Period reports panel. Do not reimplement an information function here. -7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018 / 0048 / 0049 / 0182 / 0183): after +7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018 / 0048 / 0049 / 0182 / 0201): after IRT main effects, persist closest and farthest post–criterion pairs from the residual leftover map, name unexplained leftover `U = R − R̂` when Gabriel coordinates exist, and persist two-axis diff --git a/migrations/0183_report_leftover_map_reconstruction.sql b/migrations/0183_report_leftover_map_reconstruction.sql index 8662777f3..c44df5311 100644 --- a/migrations/0183_report_leftover_map_reconstruction.sql +++ b/migrations/0183_report_leftover_map_reconstruction.sql @@ -1,4 +1,4 @@ --- ADR 0183: persist leftover-map reconstruction R̂ = ξ_{1:2} · ζ_{1:2} +-- ADR 0201: persist leftover-map reconstruction R̂ = ξ_{1:2} · ζ_{1:2} -- so unexplained leftover U = R − R̂ stays auditable as U + R̂ = R. -- Distance stays Euclidean leftover-map d. No nonnegative CHECK: a -- signed reconstruction is stored, never clamped. Upgrade column is From e4234bceffac09b5aeee01090dcc1b3825f45676 Mon Sep 17 00:00:00 2001 From: seonghobae Date: Tue, 25 Aug 2026 13:42:09 +0900 Subject: [PATCH 4/6] docs: reconcile ADR 0182 reconstruction context --- docs/adr/0182-leftover-map-unexplained.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/adr/0182-leftover-map-unexplained.md b/docs/adr/0182-leftover-map-unexplained.md index fecd979b9..13ab5a573 100644 --- a/docs/adr/0182-leftover-map-unexplained.md +++ b/docs/adr/0182-leftover-map-unexplained.md @@ -24,15 +24,17 @@ shows. Hiding unexplained leftover `U = R − R̂` lets a buyer read leftover residual `R` or leftover-map distance `d` as the leftover the two-axis map does not reconstruct. -This increment does not persist leftover-map reconstruction `R̂`, does +At ADR 0182's initial acceptance, this increment did not persist +leftover-map reconstruction `R̂`; ADR 0201 now persists that value so +`U + R̂ = R` remains directly auditable. It still does not persist leftover-map coordinates, does not name leftover-map inner product as a separate full-rank column, does not name leftover-map cosine, does not name leftover-map length, does not name observed `Y` / expected `E`, does not name leftover-map rank, does not split leftover-map distance onto two axes, and does not land Post quality on the leftover criterion. Leftover-map distance stays full-rank Euclidean. -Reconstruction `R̂` is computed internally so `U` is honest, then -discarded. +Reconstruction `R̂` is computed internally so `U` is honest and is now +retained under ADR 0201. The unprotected-stack reconstructions for neighbouring leftover facts use 0162–0181. This protected-main increment uses **0182** so it does From a390d2398c51fa5dfc82196ad235aa7553145447 Mon Sep 17 00:00:00 2001 From: seonghobae Date: Tue, 25 Aug 2026 13:54:58 +0900 Subject: [PATCH 5/6] test(leftover): cover reconstruction candidate contract --- tests/test_leftover_pairs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_leftover_pairs.py b/tests/test_leftover_pairs.py index 14b3e8d7d..f607ebea3 100644 --- a/tests/test_leftover_pairs.py +++ b/tests/test_leftover_pairs.py @@ -259,6 +259,7 @@ def test_leftover_residual_rejects_database_tolerance_boundary() -> None: 0.0, None, None, + None, ) From 6185f2ae58d8fe831c628266bb7a63f5514c73ec Mon Sep 17 00:00:00 2001 From: seonghobae Date: Tue, 25 Aug 2026 13:59:57 +0900 Subject: [PATCH 6/6] fix(leftover): prove rank-zero reconstruction identity --- AGENTS.md | 6 ++--- ARCHITECTURE.md | 4 ++-- .../2.12.31-leftover-map-reconstruction.md | 9 +++----- backend/tests/test_api.py | 10 ++++++++- .../0003-fast-mlsirm-report-integration.md | 22 ++++++++----------- docs/adr/0201-leftover-map-reconstruction.md | 5 +++-- tests/test_leftover_pairs.py | 21 ++++++++++++++++++ 7 files changed, 50 insertions(+), 27 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0f46a82a6..f334b18e0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -278,9 +278,9 @@ out of the factorization. Closest and farthest post–criterion pairs persist to `report_leftover_pair` with signed residual `R`, observed `Y`, and expected `E[Y|θ, item]` so `R = Y − E` remains auditable, plus leftover-map rank so rank 0 is not read as structure, -unexplained leftover `U = R − R̂`, and leftover-map reconstruction -`R̂ = ξ_{1:2} · ζ_{1:2}` next to leftover-map distance `d` so -`U + R̂ = R` stays auditable. They sit above the member +unexplained leftover, and the ADR 0201 reconstruction evidence. ADR 0201 +is the sole normative reconstruction formula, storage, and audit contract; +do not duplicate or reinterpret it here. The pairs sit above the member list so a click opens that post with the leftover criterion current in Post quality (ADR 0158). Leftover-map axis share (ADR 0148) is Gabriel inertia of residual SVD axes 1 and 2 and persists to `report_leftover_map_axis`. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2ab7ff9ba..0c554bc8c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -605,8 +605,8 @@ residual SVD leftover pairs on two Gabriel axes (Jeon et al., 2021; ADR 0017 / 0048 / 0049 / 0119 / 0148 / 0158 / 0162 / 0163 / 0164 / 0168 / 0182 / 0185 / 0201) persist to `report_leftover_pair` with signed residual `R`, observed `Y`, expected `E[Y|θ, item]`, full leftover-map rank, unexplained -leftover `U = R − R̂`, signed two-axis reconstruction `R̂`, and leftover-map -cross share `x = 2 R̂ U / R²` of raw residual. Leftover-map axis share +leftover, ADR 0201 reconstruction evidence, and ADR 0185 cross-share evidence. +Those ADRs are the normative mathematical and storage contracts. Leftover-map axis share (Gabriel inertia of residual SVD axes 1 and 2; ADR 0148) persists to `report_leftover_map_axis`. Complete-case leftover-map coverage (ADR 0168) persists to `report_leftover_map_coverage` so readers see how diff --git a/CHANGELOG.d/2.12.31-leftover-map-reconstruction.md b/CHANGELOG.d/2.12.31-leftover-map-reconstruction.md index af7898531..1bae12df2 100644 --- a/CHANGELOG.d/2.12.31-leftover-map-reconstruction.md +++ b/CHANGELOG.d/2.12.31-leftover-map-reconstruction.md @@ -1,8 +1,5 @@ ## 2.12.31 — Leftover-map reconstruction -- Persist leftover-map reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}` on leftover - post–criterion pairs (ADR 0201). After `make seed`, closest and farthest - leftover pairs sit above the member list with `R̂` next to leftover-map - distance `d`; click opens that post. Omit the badge when reconstruction - is missing. Never invent a leftover score. Signed reconstruction is - stored, never clamped. Identity `U + R̂ = R` stays auditable. +- Period leftover pairs now expose reconstruction evidence governed by ADR + 0201. After `make seed`, open a closest or farthest pair to inspect it on + the named post. Missing reconstruction omits the badge. diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index aebbd23ae..dee21f626 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -5343,11 +5343,19 @@ def test_seed_period_report_surfaces_on_get_reports(client, demo_analyst_token, assert leftover_kinds <= {"closest", "farthest"} assert all(pair["post_title"] for pair in high_report.get("leftover_pairs", [])) assert all(pair["leftover_distance"] >= 0 for pair in high_report.get("leftover_pairs", [])) + assert all( + "leftover_map_reconstruction" in pair + for pair in high_report.get("leftover_pairs", []) + ) + assert any( + pair["leftover_map_reconstruction"] is not None + for pair in high_report.get("leftover_pairs", []) + ) for pair in high_report.get("leftover_pairs", []): assert pair["leftover_map_rank"] >= 0 unexplained = pair.get("leftover_map_unexplained") assert unexplained is None or isinstance(unexplained, (int, float)) - reconstruction = pair.get("leftover_map_reconstruction") + reconstruction = pair["leftover_map_reconstruction"] assert reconstruction is None or isinstance(reconstruction, (int, float)) observed = pair.get("observed_response") expected = pair.get("expected_response") diff --git a/docs/adr/0003-fast-mlsirm-report-integration.md b/docs/adr/0003-fast-mlsirm-report-integration.md index 43929f204..27f9007fe 100644 --- a/docs/adr/0003-fast-mlsirm-report-integration.md +++ b/docs/adr/0003-fast-mlsirm-report-integration.md @@ -100,19 +100,15 @@ than one large PR: `information_polytomous` (Lord, 1980 max-info). Persist the ranking (`report_item_information`) and show the rank-1 item on the Period reports panel. Do not reimplement an information function here. -7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018 / 0048 / 0049 / - 0182 / 0185 / 0201): after IRT main effects, persist closest and farthest - post–criterion pairs from the residual leftover map, and name - unexplained leftover `U = R − R̂`, signed two-axis reconstruction `R̂`, - and leftover-map cross share - `x = 2 R̂ U / R²` of raw residual when Gabriel coordinates - exist so the identity remainder after two-axis reconstruction is not - read as leftover residual, leftover-map distance, explained leftover - share, or unexplained leftover share. Do not persist leftover-map - explained leftover share `e`, unexplained leftover share `s`, or any other - unsupported share alias in this slice. Do not fork LSIRM; do not invent a - leftover-pair API inside `fast-mlsirm` in this slice. -8. **Leftover-map axis-share slice** (ADR 0148): persist Gabriel inertia +7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018 / 0048 / + 0049): after IRT main effects, persist closest and farthest + post–criterion pairs from the residual leftover map. Do not fork LSIRM or + invent a leftover-pair API inside `fast-mlsirm` in this slice. +8. **Leftover evidence extensions:** unexplained leftover shipped in 2.12.26 + (ADR 0182), cross-share evidence shipped in 2.12.29 (ADR 0185), and + reconstruction evidence is Unreleased for 2.12.31 (ADR 0201). Do not + persist explained share, unexplained share, or another unsupported alias. +9. **Leftover-map axis-share slice** (ADR 0148): persist Gabriel inertia `σ_k² / Σ_j σ_j²` of leftover-map axes 1 and 2 on the same residual SVD. Rank-0 residuals emit two zero-share axes. Do not invent a leftover score. diff --git a/docs/adr/0201-leftover-map-reconstruction.md b/docs/adr/0201-leftover-map-reconstruction.md index 912cd29a9..049208105 100644 --- a/docs/adr/0201-leftover-map-reconstruction.md +++ b/docs/adr/0201-leftover-map-reconstruction.md @@ -51,8 +51,9 @@ existing -- shipped migrations (`0001` / `0012`) are never edited after the fact. The column is nullable so older leftover rows keep distance, residual, and unexplained leftover without fabricating reconstruction. Fallback pairs that have no complete-case leftover map omit the value -rather than inventing one. A rank-0 origin map stores `0.0` (`R = 0` -and `R̂ = 0`), not a missing value. A non-finite reconstruction stores +rather than inventing one. A rank-0 map stores `0.0` for `R̂`; raw residual +`R` may be a nonzero constant after centering, in which case `U = R` and +`U + R̂ = R` still holds. A non-finite reconstruction stores null rather than inventing a leftover score. A signed reconstruction is stored, never clamped. Do not add a nonnegative CHECK. diff --git a/tests/test_leftover_pairs.py b/tests/test_leftover_pairs.py index f607ebea3..a1080e172 100644 --- a/tests/test_leftover_pairs.py +++ b/tests/test_leftover_pairs.py @@ -158,6 +158,27 @@ def test_zero_residual_still_emits_stable_leftover_pairs() -> None: assert coverage.incomplete_post_count == 0 +def test_rank_zero_nonzero_constant_residual_keeps_raw_identity() -> None: + """Centering a constant nonzero residual gives R̂=0 while U remains R.""" + matrix = np.ones((2, 2), dtype=np.float64) + pairs = leftover_pairs_from_residual( + ["post-a", "post-b"], + ("item-a", "item-b"), + matrix, + np.zeros_like(matrix), + ) + + assert pairs + for pair in pairs: + assert pair.leftover_map_rank == 0 + assert pair.leftover_residual == pytest.approx(1.0) + assert pair.leftover_map_reconstruction == pytest.approx(0.0) + assert pair.leftover_map_unexplained == pytest.approx(1.0) + assert pair.leftover_map_unexplained + pair.leftover_map_reconstruction == pytest.approx( + pair.leftover_residual + ) + + def test_partial_observation_does_not_treat_missing_as_zero_residual() -> None: """A missing cell must not enter the Gabriel factorization as 0.""" post_ids = ["aligned-post", "opposed-post", "sparse-post"]