From 68eadf7e21366d9eae9e6f31ddef35104e869fd9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 10:18:27 +0000 Subject: [PATCH 01/11] feat: persist leftover interaction-map coordinates (v2.12.7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep Gabriel leftover-map ξ / ζ after IRT main effects, persist complete-case axes, and render a 2D biplot above leftover pairs. Click a post node to open that post. Rank-0/1 maps pad unused axes with zero; missing cells stay out of the factorization (ADR 0121). --- ARCHITECTURE.md | 9 +- .../2.12.7-leftover-interaction-map.md | 12 ++ CHANGELOG.md | 10 + backend/app/main.py | 20 +- backend/app/report_ingestion.py | 85 +++++++- backend/tests/test_api.py | 12 ++ docker/postgres-init/migrate.sh | 2 +- .../0121-persist-leftover-interaction-map.md | 73 +++++++ frontend/package.json | 2 +- frontend/src/App.css | 71 +++++++ frontend/src/App.test.tsx | 26 +++ frontend/src/App.tsx | 11 + frontend/src/LeftoverInteractionMap.test.tsx | 68 +++++++ frontend/src/LeftoverInteractionMap.tsx | 192 ++++++++++++++++++ frontend/src/api.ts | 15 ++ lineageweave/leftover_pairs.py | 111 ++++++++-- lineageweave/period_report.py | 38 +++- migrations/0001_initial_schema.sql | 40 ++++ migrations/0103_tenant_settings.sql | 5 +- .../0104_report_leftover_interaction_map.sql | 65 ++++++ .../0104_report_leftover_interaction_map.sql | 2 + pyproject.toml | 2 +- scripts/seed_demo_data.py | 33 +++ tests/test_leftover_pairs.py | 71 +++++++ tests/test_migration_replay.py | 2 + tests/test_period_report.py | 10 +- tests/test_schema.py | 24 +++ uv.lock | 2 +- 28 files changed, 975 insertions(+), 38 deletions(-) create mode 100644 CHANGELOG.d/2.12.7-leftover-interaction-map.md create mode 100644 docs/adr/0121-persist-leftover-interaction-map.md create mode 100644 frontend/src/LeftoverInteractionMap.test.tsx create mode 100644 frontend/src/LeftoverInteractionMap.tsx create mode 100644 migrations/0104_report_leftover_interaction_map.sql create mode 100644 migrations/rollback/0104_report_leftover_interaction_map.sql diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d0280ff97..41b84d1e9 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -588,8 +588,10 @@ on those same fixed parameters (Kim, 2006 FIPC). After scoring, `information_polytomous` ranks the shared-bank items by Fisher information at the group's mean θ (Lord, 1980 max-info CAT). Rankings persist to `report_item_information`. After those IRT main effects, -residual SVD leftover pairs (Jeon et al., 2021; ADR 0017) persist to -`report_leftover_pair`. Results persist to +residual SVD leftover pairs (Jeon et al., 2021; ADR 0048) persist to +`report_leftover_pair`. Complete-case leftover-map coordinates ξ / ζ +(ADR 0121) persist to `report_leftover_map_person` and +`report_leftover_map_item`. Results persist to `report_period_score` / `report_member_score`. `GET /api/reports/{grouping}` lists the trend; `GET /api/reports/{grouping}/{period}` is ABAC-filtered; @@ -601,7 +603,8 @@ bank as the dummy high/low band rows, so comparison-strip click through opens those DAG posts. Report members include the earliest open ticket title, status lookup label, and due date when one exists. The home page renders the actual mean θ, the FIPC delta, the CAT-selected item, leftover -closest/farthest pairs above the member list, and the +closest/farthest pairs and the leftover interaction map above the +member list, and the PU / corp / thread comparison -- never a placeholder. TEPP is unchanged. ## Phase 6b: Knowledge Graph as a real Ontology + Semantic Layer diff --git a/CHANGELOG.d/2.12.7-leftover-interaction-map.md b/CHANGELOG.d/2.12.7-leftover-interaction-map.md new file mode 100644 index 000000000..9ecda081f --- /dev/null +++ b/CHANGELOG.d/2.12.7-leftover-interaction-map.md @@ -0,0 +1,12 @@ +## 2.12.7 — Leftover interaction map + +- Persist complete-case leftover-map coordinates ξ / ζ after IRT main + effects (Jeon et al., 2021; Gabriel, 1971) in + `report_leftover_map_person` and `report_leftover_map_item` (ADR 0121). +- Render a 2D leftover interaction map above the leftover pair list. + Click a post node to open that post. Closest and farthest pairs stay + highlighted. Hidden posts stay hidden. +- Rank-0 and rank-1 maps pad the unused axis with zero. Missing residual + cells never enter the factorization as zero. +- Replay `0103_tenant_settings.sql` and + `0104_report_leftover_interaction_map.sql` on existing volumes. diff --git a/CHANGELOG.md b/CHANGELOG.md index c8ed1a099..98977ee27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,16 @@ All notable changes to this project are documented here. Format follows environment, so local OIDC and synthetic-data workflows resolve the same pinned dependencies as CI. +## [2.12.7] - 2026-08-23 + +### Added + +- Period reports now persist leftover interaction-map coordinates + (Jeon et al. 2021 ξ / ζ after IRT main effects) and render a 2D + biplot above the leftover pair list. Click a post node to open that + post. Rank-0 and rank-1 maps pad the unused axis with zero; missing + cells stay out of the factorization (ADR 0121). + ## [2.12.6] - 2026-08-20 ### Added diff --git a/backend/app/main.py b/backend/app/main.py index fb943315f..e2a8c2905 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -2361,6 +2361,12 @@ async def read_period_reports( if _can_see_post(account, pair) and not _is_synthetic_demo_member(pair, demo_entity_ids) ] + leftover_map_persons = [ + person + for person in report.get("leftover_map_persons", []) + if _can_see_post(account, person) + and not _is_synthetic_demo_member(person, demo_entity_ids) + ] members = [ {key: value for key, value in member.items() if key != "has_real_source_context"} for member in members @@ -2369,8 +2375,20 @@ async def read_period_reports( {key: value for key, value in pair.items() if key != "has_real_source_context"} for pair in leftover_pairs ] + leftover_map_persons = [ + {key: value for key, value in person.items() if key != "has_real_source_context"} + for person in leftover_map_persons + ] + leftover_map_items = list(report.get("leftover_map_items", [])) visible.append( - {**report, "members": members, "leftover_pairs": leftover_pairs, "post_count": len(members)} + { + **report, + "members": members, + "leftover_pairs": leftover_pairs, + "leftover_map_persons": leftover_map_persons, + "leftover_map_items": leftover_map_items, + "post_count": len(members), + } ) return {"grouping_kind": grouping_kind, "period_code": period_code, "reports": visible} diff --git a/backend/app/report_ingestion.py b/backend/app/report_ingestion.py index 50614b0ad..dc577adbe 100644 --- a/backend/app/report_ingestion.py +++ b/backend/app/report_ingestion.py @@ -351,7 +351,7 @@ async def persist_period_report( period_code: str, report: PeriodReport, ) -> None: - """Replace the stored report, member scores, leftover pairs, and item bank.""" + """Replace the stored report, member scores, leftover pairs, leftover map, and item bank.""" await conn.execute( """ delete from report_period_score @@ -456,6 +456,38 @@ async def persist_period_report( pair.leftover_distance, pair.leftover_residual, ) + for person in report.leftover_map_persons: + await conn.execute( + """ + insert into report_leftover_map_person ( + grouping_kind, grouping_key, period_code, rubric_version, + post_id, axis_one, axis_two + ) values ($1,$2,$3,$4,$5,$6,$7) + """, + grouping_kind, + grouping_key, + period_code, + RUBRIC_VERSION, + person.post_id, + person.axis_one, + person.axis_two, + ) + for item in report.leftover_map_items: + await conn.execute( + """ + insert into report_leftover_map_item ( + grouping_kind, grouping_key, period_code, rubric_version, + criterion_code, axis_one, axis_two + ) values ($1,$2,$3,$4,$5,$6,$7) + """, + grouping_kind, + grouping_key, + period_code, + RUBRIC_VERSION, + item.criterion_code, + item.axis_one, + item.axis_two, + ) def _groups_from_rows( @@ -615,6 +647,31 @@ async def fetch_period_reports( period_code, RUBRIC_VERSION, ) + leftover_map_persons = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" + select lp.grouping_key, lp.post_id, lp.axis_one, lp.axis_two, p.post_title, + p.visibility_code, p.corporate_entity_id, + ({_SOURCE_CONTEXT_PRESENT_SQL}) as has_real_source_context + from report_leftover_map_person lp + join source_post p on p.post_id = lp.post_id + where lp.grouping_kind = $1 and lp.period_code = $2 and lp.rubric_version = $3 + order by lp.grouping_key, p.post_title + """, + grouping_kind, + period_code, + RUBRIC_VERSION, + ) + leftover_map_items = await conn.fetch( + """ + select grouping_key, criterion_code, axis_one, axis_two + from report_leftover_map_item + where grouping_kind = $1 and period_code = $2 and rubric_version = $3 + order by grouping_key, criterion_code + """, + grouping_kind, + period_code, + RUBRIC_VERSION, + ) status_labels = await labels_for_codes( conn, [row["ticket_status_code"] for row in members if row["ticket_status_code"]], @@ -628,6 +685,12 @@ async def fetch_period_reports( leftover_by_group: dict[str, list[asyncpg.Record]] = defaultdict(list) for row in leftover: leftover_by_group[row["grouping_key"]].append(row) + leftover_persons_by_group: dict[str, list[asyncpg.Record]] = defaultdict(list) + for row in leftover_map_persons: + leftover_persons_by_group[row["grouping_key"]].append(row) + leftover_items_by_group: dict[str, list[asyncpg.Record]] = defaultdict(list) + for row in leftover_map_items: + leftover_items_by_group[row["grouping_key"]].append(row) payload: list[dict[str, Any]] = [] for header in headers: grouping_key = header["grouping_key"] @@ -704,6 +767,26 @@ async def fetch_period_reports( } for row in leftover_by_group.get(header["grouping_key"], []) ], + "leftover_map_persons": [ + { + "post_id": str(row["post_id"]), + "post_title": row["post_title"], + "axis_one": float(row["axis_one"]), + "axis_two": float(row["axis_two"]), + "visibility_code": row["visibility_code"], + "corporate_entity_id": str(row["corporate_entity_id"]), + "has_real_source_context": bool(row["has_real_source_context"]), + } + for row in leftover_persons_by_group.get(header["grouping_key"], []) + ], + "leftover_map_items": [ + { + "criterion_code": str(row["criterion_code"]), + "axis_one": float(row["axis_one"]), + "axis_two": float(row["axis_two"]), + } + for row in leftover_items_by_group.get(header["grouping_key"], []) + ], } ) return payload diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 438b4786a..b86694bc0 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -4518,6 +4518,18 @@ def test_seed_period_report_surfaces_on_get_reports(client, demo_analyst_token, assert leftover_kinds <= {"closest", "farthest"} assert all(pair["post_title"] for pair in high_report.get("leftover_pairs", [])) assert all(pair["leftover_distance"] >= 0 for pair in high_report.get("leftover_pairs", [])) + leftover_map_persons = high_report.get("leftover_map_persons", []) + leftover_map_items = high_report.get("leftover_map_items", []) + member_ids = {member["post_id"] for member in high_report["members"]} + item_codes = {item["item_code"] for item in high_report.get("selected_items", [])} + assert leftover_map_persons + assert leftover_map_items + assert all(person["post_title"] for person in leftover_map_persons) + assert {person["post_id"] for person in leftover_map_persons} <= member_ids + assert {item["criterion_code"] for item in leftover_map_items} <= item_codes + for point in leftover_map_persons + leftover_map_items: + assert isinstance(point["axis_one"], (int, float)) + assert isinstance(point["axis_two"], (int, float)) week3 = client.get( "/api/reports/process_unit/2026-W03", diff --git a/docker/postgres-init/migrate.sh b/docker/postgres-init/migrate.sh index f329117d6..3a16458d4 100644 --- a/docker/postgres-init/migrate.sh +++ b/docker/postgres-init/migrate.sh @@ -18,7 +18,7 @@ for migration in /opt/lineageweave/migrations/*.sql; do migration_name=${migration##*/} case "$migration_name" in 0012_*|0013_*|0014_*|0015_*|0016_*|0017_*|0018_*|0019_*|0020_*|0021_*|0022_*|0023_*|0024_*|0025_*|0026_*|0027_*|0028_*|0029_*|0030_*|0031_*|0032_*|0033_*|0034_*|0035_*|0036_*|0037_*|0038_*|0039_*|0040_*|0041_*|0042_*|0043_*|0044_*|0045_*|0046_*|0047_*|0048_*|0049_*|0050_*) ;; - 0060_*|0100_*|0101_*|0102_*) ;; + 0060_*|0100_*|0101_*|0102_*|0103_*|0104_*) ;; *) continue ;; esac printf 'Applying %s\n' "$migration_name" diff --git a/docs/adr/0121-persist-leftover-interaction-map.md b/docs/adr/0121-persist-leftover-interaction-map.md new file mode 100644 index 000000000..678285721 --- /dev/null +++ b/docs/adr/0121-persist-leftover-interaction-map.md @@ -0,0 +1,73 @@ +# ADR 0121 — Persist leftover interaction-map coordinates + +**Decision status:** Accepted +**Date:** 2026-08-23 + +## Context + +ADR 0048 persists the closest and farthest leftover post–criterion +pairs after IRT main effects. Those pairs are two cells on the Jeon +et al. (2021, eq. 3) leftover interaction map `−γ‖ξ_p − ζ_i‖`. The +Gabriel (1971) biplot that produces the pairs already computes person +positions `ξ` and item positions `ζ`, then discards them. A buyer +who sees only two named pairs cannot see *why* those cells sat +closest or farthest, or where the other complete-case posts and +criteria sit on the same leftover map. + +`fast-mlsirm` still exposes no leftover-map API. LineageWeave must +not fork LSIRM, invent a second IRT fit, or treat a missing residual +cell as a zero residual. + +## Decision + +After a real GRM/GPCM score, keep the complete-case Gabriel +coordinates that leftover pairs already use. Persist every complete- +case post as `report_leftover_map_person` (`axis_one`, `axis_two`) +and every complete-case criterion as `report_leftover_map_item`. +Pad unused axes with zero when residual rank is below two. Do not +invent a second component. Incomplete rows and columns stay out of +the factorization. + +Cascade the rows with `report_period_score`. A leftover-map post +must also be a `report_member_score` row. A leftover-map criterion +must be a `report_item_information` item on that same report. Do not +store a second theta. A rank-0 residual still emits origin +coordinates so `make seed` is not empty; those zeros are not a +fabricated interaction. + +Closest and farthest pairs remain ADR 0048 / ADR 0049. The map sits +**above** that pair list on the period-report group. Clicking a +person node opens that post with the same handler as a leftover +pair. Hidden posts stay hidden: leftover-map persons join +`source_post` and use the same ABAC gate as members and leftover +pairs. Missing map rows render nothing. + +The biplot remains in `lineageweave/leftover_pairs.py` so leftover +tests do not import `period_report` or `fast_mlsirm`. + +## Consequences + +Rebuild and seed write leftover-map coordinates in the same +transaction as leftover pairs. `GET /api/reports/{grouping}/{period}` +returns `leftover_map_persons` (with post title) and +`leftover_map_items`. Migration +`0104_report_leftover_interaction_map.sql` upgrades volumes that +already applied `0001` / `0012`. `migrate.sh` also replays `0103` +(`tenant_settings`) so existing volumes pick up that table. + +## Related + +Depends on [ADR 0048](0048-persist-lsirm-leftover-pairs.md), +[ADR 0049](0049-leftover-pair-report-ui.md), and +[ADR 0003](0003-fast-mlsirm-report-integration.md). + +## References + +Gabriel, K. R. (1971). The biplot graphic display of matrices with +application to principal component analysis. *Biometrika, 58*(3), +453–467. https://doi.org/10.1093/biomet/58.3.453 + +Jeon, M., Jin, I. H., Schweinberger, M., & Baugh, S. (2021). Mapping +unobserved item–respondent interactions: A latent space item response +model with interaction map. *Psychometrika, 86*(2), 378–403. +https://doi.org/10.1007/s11336-021-09762-5 diff --git a/frontend/package.json b/frontend/package.json index e2e996bbe..cea549188 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "2.12.6", + "version": "2.12.7", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/App.css b/frontend/src/App.css index c72aab078..a3816814a 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -529,6 +529,77 @@ fill: var(--text-h); } +.leftover-interaction-map { + margin: 0.5rem 0 0.75rem; +} + +.leftover-interaction-map figcaption { + font-size: 0.85rem; + color: var(--text-muted); + margin-bottom: 0.4rem; +} + +.leftover-interaction-map svg { + border: 1px solid var(--border); + border-radius: 8px; + background: var(--surface); +} + +.leftover-map-pair { + stroke-width: 1.5; +} + +.leftover-map-pair.leftover-map-closest { + stroke: var(--badge-status-success-text); +} + +.leftover-map-pair.leftover-map-farthest { + stroke: var(--badge-status-danger-text); +} + +.leftover-map-person { + cursor: pointer; +} + +.leftover-map-person circle { + fill: var(--badge-actor-person-bg); + stroke: var(--badge-actor-person-text); + stroke-width: 1.5; +} + +.leftover-map-item rect { + fill: var(--badge-actor-organization-bg); + stroke: var(--badge-actor-organization-text); + stroke-width: 1.5; +} + +.leftover-map-person.leftover-map-closest circle, +.leftover-map-item.leftover-map-closest rect { + stroke: var(--badge-status-success-text); + stroke-width: 2; +} + +.leftover-map-person.leftover-map-farthest circle, +.leftover-map-item.leftover-map-farthest rect { + stroke: var(--badge-status-danger-text); + stroke-width: 2; +} + +.leftover-map-person text, +.leftover-map-item text { + font-size: 0.7rem; + fill: var(--color-text-heading); +} + +.leftover-map-person:focus { + outline: none; +} + +.leftover-map-person:focus circle, +.leftover-map-person:hover circle { + stroke-width: 2.5; +} + .lineage-dag-node:focus { outline: none; } diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 7462abd2c..07d5748e3 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -942,6 +942,19 @@ describe("App, authenticated", () => { leftover_residual: -1.1, }, ], + leftover_map_persons: [ + { post_id: "post-1", post_title: "Public post", axis_one: -0.5, axis_two: 0.1 }, + { + post_id: "post-2", + post_title: "Specification revision requested", + axis_one: 0.8, + axis_two: -0.4, + }, + ], + leftover_map_items: [ + { criterion_code: "sales_lead_specificity", axis_one: -0.4, axis_two: 0.05 }, + { criterion_code: "general_sentiment_negative", axis_one: 1.2, axis_two: -0.9 }, + ], members: [ { post_id: "post-1", @@ -3351,6 +3364,8 @@ describe("App, authenticated", () => { expect(screen.getByRole("button", { name: /open report post: public post/i })).toHaveTextContent("Open"); expect(screen.getByRole("button", { name: /open report post: public post/i })).toHaveTextContent("due 2026-01-12"); expect(screen.getByLabelText("Leftover pairs")).toBeInTheDocument(); + expect(screen.getByLabelText("Leftover interaction map")).toBeInTheDocument(); + const mapPost = screen.getByRole("button", { name: "Open leftover map post: Public post" }); const closestPair = screen.getByRole("button", { name: /open leftover closest pair: public post/i }); const farthestPair = screen.getByRole("button", { name: /open leftover farthest pair: specification revision requested/i, @@ -3366,6 +3381,7 @@ describe("App, authenticated", () => { ); expect(farthestPair).toHaveTextContent("d 1.84"); const memberButton = screen.getByRole("button", { name: /open report post: public post/i }); + expect(mapPost.compareDocumentPosition(closestPair) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); expect(closestPair.compareDocumentPosition(memberButton) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); expect( screen.getByRole("button", { name: /open report post: specification revision requested/i }), @@ -3425,6 +3441,16 @@ describe("App, authenticated", () => { await waitFor(() => expect(screen.getByText("The full body text.")).toBeInTheDocument()); }); + it("opens a leftover-map post from the report panel", async () => { + stubBackend(); + render(); + + await userEvent.click( + await screen.findByRole("button", { name: "Open leftover map post: Public post" }), + ); + await waitFor(() => expect(screen.getByText("The full body text.")).toBeInTheDocument()); + }); + it("opens Event Lineage, Keyman, and evaluation from a report member click", async () => { stubBackend(); render(); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6fba0dd41..0cff77997 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -88,6 +88,7 @@ import { LineageEntityPicker } from "./components/LineageEntityPicker"; import { PopupCloseButton } from "./components/PopupCloseButton"; import { BuyerNav, type BuyerDestination } from "./components/BuyerNav"; import { LineageDag } from "./LineageDag"; +import { LeftoverInteractionMap } from "./LeftoverInteractionMap"; import { PostBody } from "./PostBody"; import { decodeHtmlEntities } from "./postBodyDisplay"; import { FiveW1H } from "./components/FiveW1H"; @@ -3389,6 +3390,16 @@ function ReportsPanel({ {report.selected_items[0].information.toFixed(2)} )} + {((report.leftover_map_persons && report.leftover_map_persons.length > 0) || + (report.leftover_map_items && report.leftover_map_items.length > 0)) && ( + + )} {report.leftover_pairs && report.leftover_pairs.length > 0 && (
    {report.leftover_pairs.map((pair) => { diff --git a/frontend/src/LeftoverInteractionMap.test.tsx b/frontend/src/LeftoverInteractionMap.test.tsx new file mode 100644 index 000000000..e34d72271 --- /dev/null +++ b/frontend/src/LeftoverInteractionMap.test.tsx @@ -0,0 +1,68 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import { LeftoverInteractionMap, projectLeftoverMap } from "./LeftoverInteractionMap"; + +const itemLabel = (code: string) => (code === "sales_lead_specificity" ? "sales-lead" : code); + +describe("LeftoverInteractionMap", () => { + it("projects coincident origin points to the map center", () => { + const projected = projectLeftoverMap( + [{ post_id: "post-1", post_title: "Public post", axis_one: 0, axis_two: 0 }], + [{ criterion_code: "sales_lead_specificity", axis_one: 0, axis_two: 0 }], + itemLabel, + [], + 360, + 240, + 36, + ); + expect(projected.persons[0]).toMatchObject({ x: 180, y: 120 }); + expect(projected.items[0]).toMatchObject({ x: 180, y: 120 }); + }); + + it("renders closest and farthest leftover-map nodes and opens a post", async () => { + const onSelectPost = vi.fn(); + render( + , + ); + + expect(screen.getByLabelText("Leftover interaction map")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Open leftover map post: Public post" })).toBeInTheDocument(); + await userEvent.click(screen.getByRole("button", { name: "Open leftover map post: Public post" })); + expect(onSelectPost).toHaveBeenCalledWith("post-1"); + }); +}); diff --git a/frontend/src/LeftoverInteractionMap.tsx b/frontend/src/LeftoverInteractionMap.tsx new file mode 100644 index 000000000..0d3ee6fe3 --- /dev/null +++ b/frontend/src/LeftoverInteractionMap.tsx @@ -0,0 +1,192 @@ +import type { LeftoverMapItem, LeftoverMapPerson, LeftoverPair } from "./api"; + +const MAP_WIDTH = 360; +const MAP_HEIGHT = 240; +const MAP_PAD = 36; + +export type ProjectedLeftoverPoint = { + id: string; + label: string; + x: number; + y: number; + kind: "person" | "item"; + pairKind?: "closest" | "farthest"; +}; + +function pairKindFor( + id: string, + kind: "person" | "item", + pairs: LeftoverPair[], +): "closest" | "farthest" | undefined { + for (const pair of pairs) { + const match = kind === "person" ? pair.post_id === id : pair.criterion_code === id; + if (!match) continue; + if (pair.pair_kind === "closest" || pair.pair_kind === "farthest") { + return pair.pair_kind; + } + } + return undefined; +} + +export function projectLeftoverMap( + persons: LeftoverMapPerson[], + items: LeftoverMapItem[], + itemLabel: (criterionCode: string) => string, + pairs: LeftoverPair[] = [], + width = MAP_WIDTH, + height = MAP_HEIGHT, + pad = MAP_PAD, +): { persons: ProjectedLeftoverPoint[]; items: ProjectedLeftoverPoint[] } { + const raw = [ + ...persons.map((person) => ({ + id: person.post_id, + label: person.post_title, + axisOne: person.axis_one, + axisTwo: person.axis_two, + kind: "person" as const, + })), + ...items.map((item) => ({ + id: item.criterion_code, + label: itemLabel(item.criterion_code), + axisOne: item.axis_one, + axisTwo: item.axis_two, + kind: "item" as const, + })), + ]; + const xs = raw.map((point) => point.axisOne); + const ys = raw.map((point) => point.axisTwo); + const minX = Math.min(...xs); + const maxX = Math.max(...xs); + const minY = Math.min(...ys); + const maxY = Math.max(...ys); + const spanX = maxX - minX; + const spanY = maxY - minY; + const toSvg = (axisOne: number, axisTwo: number) => ({ + x: spanX === 0 ? width / 2 : pad + ((axisOne - minX) / spanX) * (width - 2 * pad), + y: spanY === 0 ? height / 2 : pad + ((maxY - axisTwo) / spanY) * (height - 2 * pad), + }); + return { + persons: raw + .filter((point) => point.kind === "person") + .map((point) => { + const { x, y } = toSvg(point.axisOne, point.axisTwo); + return { + id: point.id, + label: point.label, + x, + y, + kind: "person" as const, + pairKind: pairKindFor(point.id, "person", pairs), + }; + }), + items: raw + .filter((point) => point.kind === "item") + .map((point) => { + const { x, y } = toSvg(point.axisOne, point.axisTwo); + return { + id: point.id, + label: point.label, + x, + y, + kind: "item" as const, + pairKind: pairKindFor(point.id, "item", pairs), + }; + }), + }; +} + +function truncateLabel(label: string): string { + return label.length > 22 ? `${label.slice(0, 21)}…` : label; +} + +export function LeftoverInteractionMap({ + persons, + items, + pairs, + itemLabel, + onSelectPost, +}: { + persons: LeftoverMapPerson[]; + items: LeftoverMapItem[]; + pairs: LeftoverPair[]; + itemLabel: (criterionCode: string) => string; + onSelectPost: (postId: string) => void; +}) { + if (persons.length === 0 && items.length === 0) { + return null; + } + const projected = projectLeftoverMap(persons, items, itemLabel, pairs); + const personById = Object.fromEntries(projected.persons.map((point) => [point.id, point])); + const itemById = Object.fromEntries(projected.items.map((point) => [point.id, point])); + return ( +
    +
    Leftover interaction map after main effects
    + + {pairs.map((pair) => { + const person = personById[pair.post_id]; + const item = itemById[pair.criterion_code]; + if (!person || !item) return null; + const kindClass = + pair.pair_kind === "farthest" ? "leftover-map-farthest" : "leftover-map-closest"; + return ( + + + {pair.pair_kind === "farthest" ? "Farthest leftover" : "Closest leftover"}:{" "} + {person.label} · {item.label} + + + ); + })} + {projected.items.map((point) => ( + + + + {truncateLabel(point.label)} + + {`Criterion: ${point.label}`} + + ))} + {projected.persons.map((point) => ( + onSelectPost(point.id)} + onKeyDown={(event) => { + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + onSelectPost(point.id); + } + }} + > + + + {truncateLabel(point.label)} + + {`Open this post on the leftover map: ${point.label}`} + + ))} + +
    + ); +} diff --git a/frontend/src/api.ts b/frontend/src/api.ts index cd0141a32..486a004f2 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -761,6 +761,19 @@ export interface LeftoverPair { leftover_residual: number; } +export interface LeftoverMapPerson { + post_id: string; + post_title: string; + axis_one: number; + axis_two: number; +} + +export interface LeftoverMapItem { + criterion_code: string; + axis_one: number; + axis_two: number; +} + export interface PeriodGroupReport { grouping_key: string; grouping_label?: string; @@ -776,6 +789,8 @@ export interface PeriodGroupReport { members: ReportMember[]; selected_items: SelectedReportItem[]; leftover_pairs: LeftoverPair[]; + leftover_map_persons?: LeftoverMapPerson[]; + leftover_map_items?: LeftoverMapItem[]; } export interface PeriodReports { diff --git a/lineageweave/leftover_pairs.py b/lineageweave/leftover_pairs.py index 353b1a6ef..8f1d22631 100644 --- a/lineageweave/leftover_pairs.py +++ b/lineageweave/leftover_pairs.py @@ -3,7 +3,9 @@ Does not import ``fast_mlsirm`` or ``period_report``. A Gabriel biplot of the residual ``R = Y − E[Y|θ, item]`` supplies person and item positions. Missing response cells are excluded from the factorization; -they are never treated as zero residuals. +they are never treated as zero residuals. Rank-0 and rank-1 maps pad +the unused leftover-map axis with zero rather than inventing a second +component. """ from __future__ import annotations @@ -15,6 +17,7 @@ PAIR_KIND_CLOSEST = "closest" PAIR_KIND_FARTHEST = "farthest" _LEFTOVER_SINGULAR_FLOOR = 1e-12 +_LEFTOVER_MAP_AXES = 2 @dataclass(frozen=True) @@ -28,19 +31,57 @@ class LeftoverPair: leftover_residual: float +@dataclass(frozen=True) +class LeftoverMapPerson: + """One post's leftover-map coordinates ξ after IRT main effects.""" + + post_id: str + axis_one: float + axis_two: float + + +@dataclass(frozen=True) +class LeftoverMapItem: + """One criterion's leftover-map coordinates ζ after IRT main effects.""" + + criterion_code: str + axis_one: float + axis_two: float + + +@dataclass(frozen=True) +class LeftoverInteractionMap: + """Gabriel leftover-map points plus the closest and farthest observed pairs.""" + + pairs: tuple[LeftoverPair, ...] + persons: tuple[LeftoverMapPerson, ...] + items: tuple[LeftoverMapItem, ...] + + def leftover_pairs_from_residual( post_ids: list[str], item_codes: tuple[str, ...], matrix: np.ndarray, expected: np.ndarray, ) -> tuple[LeftoverPair, ...]: - """Closest and farthest leftover-map pairs from residual SVD biplot. + """Closest and farthest leftover-map pairs from residual SVD biplot.""" + return leftover_map_from_residual(post_ids, item_codes, matrix, expected).pairs + + +def leftover_map_from_residual( + post_ids: list[str], + item_codes: tuple[str, ...], + matrix: np.ndarray, + expected: np.ndarray, +) -> LeftoverInteractionMap: + """Person and item leftover-map coordinates plus closest/farthest pairs. Jeon et al. (2021) leftover interaction is ``−γ‖ξ_p − ζ_i‖``. This estimator places persons and items from the residual after IRT main effects (Gabriel, 1971). Only observed cells become pairs. A rank-0 residual still emits a stable closest/farthest pair so seed is not - empty; it does not invent a leftover score. + empty; it does not invent a leftover score. Unused map axes stay + zero when the residual rank is below two. """ if matrix.shape != (len(post_ids), len(item_codes)): raise ValueError( @@ -58,7 +99,7 @@ def leftover_pairs_from_residual( if observed_mask[person, item] ] if not observed: - return () + return LeftoverInteractionMap(pairs=(), persons=(), items=()) keep_person, keep_item = _complete_case_masks(observed_mask) person_index = np.flatnonzero(keep_person) @@ -68,10 +109,30 @@ def leftover_pairs_from_residual( else: center = float(np.mean([residual[person, item] for person, item in observed])) person_pos, item_pos = _complete_case_positions(residual, center, keep_person, keep_item) + persons: tuple[LeftoverMapPerson, ...] = () + items: tuple[LeftoverMapItem, ...] = () candidates: list[tuple[float, str, str, float]] = [] if person_pos is not None and item_pos is not None: person_index = np.flatnonzero(keep_person) item_index = np.flatnonzero(keep_item) + person_xy = _pad_map_axes(person_pos) + item_xy = _pad_map_axes(item_pos) + persons = tuple( + LeftoverMapPerson( + post_id=post_ids[int(person)], + axis_one=float(person_xy[local, 0]), + axis_two=float(person_xy[local, 1]), + ) + for local, person in enumerate(person_index) + ) + items = tuple( + LeftoverMapItem( + criterion_code=item_codes[int(item)], + axis_one=float(item_xy[local, 0]), + axis_two=float(item_xy[local, 1]), + ) + for local, item in enumerate(item_index) + ) local_person = {int(person): local for local, person in enumerate(person_index)} local_item = {int(item): local for local, item in enumerate(item_index)} for person, item in observed: @@ -103,21 +164,25 @@ def leftover_pairs_from_residual( ) closest = min(candidates, key=lambda row: (row[0], row[1], row[2])) farthest = max(candidates, key=lambda row: (row[0], row[1], row[2])) - return ( - LeftoverPair( - pair_kind=PAIR_KIND_CLOSEST, - post_id=closest[1], - criterion_code=closest[2], - leftover_distance=closest[0], - leftover_residual=closest[3], - ), - LeftoverPair( - pair_kind=PAIR_KIND_FARTHEST, - post_id=farthest[1], - criterion_code=farthest[2], - leftover_distance=farthest[0], - leftover_residual=farthest[3], + return LeftoverInteractionMap( + pairs=( + LeftoverPair( + pair_kind=PAIR_KIND_CLOSEST, + post_id=closest[1], + criterion_code=closest[2], + leftover_distance=closest[0], + leftover_residual=closest[3], + ), + LeftoverPair( + pair_kind=PAIR_KIND_FARTHEST, + post_id=farthest[1], + criterion_code=farthest[2], + leftover_distance=farthest[0], + leftover_residual=farthest[3], + ), ), + persons=persons, + items=items, ) @@ -166,3 +231,13 @@ def _leftover_map_positions(filled: np.ndarray) -> tuple[np.ndarray, np.ndarray] person_pos = left[:, keep] * scale item_pos = right[keep, :].T * scale return person_pos, item_pos + + +def _pad_map_axes(positions: np.ndarray) -> np.ndarray: + """Pad Gabriel coordinates to two leftover-map axes; unused axes stay 0.""" + padded = np.zeros((positions.shape[0], _LEFTOVER_MAP_AXES), dtype=np.float64) + if positions.size == 0: + return padded + width = min(_LEFTOVER_MAP_AXES, positions.shape[1]) + padded[:, :width] = positions[:, :width] + return padded diff --git a/lineageweave/period_report.py b/lineageweave/period_report.py index 0b7fabc79..d874408eb 100644 --- a/lineageweave/period_report.py +++ b/lineageweave/period_report.py @@ -16,8 +16,9 @@ ``fast_mlsirm.information_polytomous`` -- Samejima (1969) GRM / Muraki (1993) GPCM, computed in Rust. A missing bank is not invented. -Leftover post–criterion pairs (ADR 0017) come from the residual -interaction after those IRT main effects: ``R = Y − E[Y|θ, item]``. +Leftover post–criterion pairs (ADR 0048) and leftover-map coordinates +(ADR 0121) come from the residual interaction after those IRT main +effects: ``R = Y − E[Y|θ, item]``. A Gabriel biplot of ``R`` supplies person and item leftover-map positions. Closest / farthest pairs are the min / max Euclidean distances on that map (Jeon et al., 2021, eq. 3). ``fast-mlsirm`` @@ -43,7 +44,8 @@ validate_irt_response_matrix, ) -from .leftover_pairs import LeftoverPair, leftover_pairs_from_residual +from .leftover_pairs import LeftoverInteractionMap, LeftoverMapItem, LeftoverMapPerson, LeftoverPair +from .leftover_pairs import leftover_map_from_residual from .leftover_pairs import PAIR_KIND_CLOSEST as PAIR_KIND_CLOSEST from .leftover_pairs import PAIR_KIND_FARTHEST as PAIR_KIND_FARTHEST from .post_evaluation import CRITERION_CODES, IRT_CATEGORY_COUNT @@ -111,6 +113,8 @@ class PeriodReport: delta_mean_theta: float | None = None selected_items: tuple[SelectedItem, ...] = () leftover_pairs: tuple[LeftoverPair, ...] = () + leftover_map_persons: tuple[LeftoverMapPerson, ...] = () + leftover_map_items: tuple[LeftoverMapItem, ...] = () def _sigmoid(value: np.ndarray) -> np.ndarray: @@ -252,9 +256,21 @@ def leftover_pairs_for_fit( fit: PolytomousFit, ) -> tuple[LeftoverPair, ...]: """Leftover pairs from the already-fitted GRM/GPCM main effects.""" + return leftover_map_for_fit(post_ids, item_codes, matrix, model, theta, fit).pairs + + +def leftover_map_for_fit( + post_ids: list[str], + item_codes: tuple[str, ...], + matrix: np.ndarray, + model: str, + theta: np.ndarray, + fit: PolytomousFit, +) -> LeftoverInteractionMap: + """Leftover-map coordinates and pairs from already-fitted GRM/GPCM main effects.""" probs = _category_probabilities(model, theta, fit) expected = expected_category_matrix(matrix, probs) - return leftover_pairs_from_residual(post_ids, item_codes, matrix, expected) + return leftover_map_from_residual(post_ids, item_codes, matrix, expected) def _member_scores(post_ids: list[str], scores: dict[str, np.ndarray]) -> tuple[MemberScore, ...]: @@ -306,6 +322,7 @@ def calibrate_period_report( theta = np.asarray(scores["theta_eap"], dtype=np.float64) mean_theta = float(theta.mean()) item_bank = item_bank_from_fit(fit, item_codes, source_period_code) + leftover_map = leftover_map_for_fit(post_ids, item_codes, matrix, selected, theta, fit) return PeriodReport( selected_model=selected, mean_theta=mean_theta, @@ -319,7 +336,9 @@ def calibrate_period_report( item_bank=item_bank, link_method=LINK_METHOD_FREE, selected_items=rank_items_by_information(item_bank, mean_theta), - leftover_pairs=leftover_pairs_for_fit(post_ids, item_codes, matrix, selected, theta, fit), + leftover_pairs=leftover_map.pairs, + leftover_map_persons=leftover_map.persons, + leftover_map_items=leftover_map.items, ) @@ -348,6 +367,9 @@ def score_period_on_bank( item_type="polytomous", response_process="cumulative", ) + leftover_map = leftover_map_for_fit( + post_ids, item_bank.item_codes, matrix, item_bank.model, theta, fit + ) return PeriodReport( selected_model=item_bank.model, mean_theta=mean_theta, @@ -367,9 +389,9 @@ def score_period_on_bank( else mean_theta - float(previous_mean_theta) ), selected_items=rank_items_by_information(item_bank, mean_theta), - leftover_pairs=leftover_pairs_for_fit( - post_ids, item_bank.item_codes, matrix, item_bank.model, theta, fit - ), + leftover_pairs=leftover_map.pairs, + leftover_map_persons=leftover_map.persons, + leftover_map_items=leftover_map.items, ) diff --git a/migrations/0001_initial_schema.sql b/migrations/0001_initial_schema.sql index 483893a1d..f44b3f507 100644 --- a/migrations/0001_initial_schema.sql +++ b/migrations/0001_initial_schema.sql @@ -356,6 +356,46 @@ create table report_leftover_pair ( create index report_leftover_pair_post_idx on report_leftover_pair (post_id); +-- Leftover interaction-map coordinates after IRT main effects (ADR 0121). +-- Complete-case Gabriel ξ / ζ on two leftover-map axes. Cascade with +-- the period score. A leftover-map post must be a member of this +-- report; a leftover-map criterion must be a CAT item on this report. +create table report_leftover_map_person ( + grouping_kind text not null, + grouping_key text not null, + period_code text not null, + rubric_version text not null, + post_id uuid not null references source_post (post_id), + axis_one numeric not null, + axis_two numeric not null, + primary key (grouping_kind, grouping_key, period_code, rubric_version, post_id), + foreign key (grouping_kind, grouping_key, period_code, rubric_version) + references report_period_score (grouping_kind, grouping_key, period_code, rubric_version) + on delete cascade, + foreign key (grouping_kind, grouping_key, period_code, rubric_version, post_id) + references report_member_score (grouping_kind, grouping_key, period_code, rubric_version, post_id) + on delete cascade +); + +create index report_leftover_map_person_post_idx on report_leftover_map_person (post_id); + +create table report_leftover_map_item ( + grouping_kind text not null, + grouping_key text not null, + period_code text not null, + rubric_version text not null, + criterion_code text not null references common_lookup_value (lookup_code), + axis_one numeric not null, + axis_two numeric not null, + primary key (grouping_kind, grouping_key, period_code, rubric_version, criterion_code), + foreign key (grouping_kind, grouping_key, period_code, rubric_version) + references report_period_score (grouping_kind, grouping_key, period_code, rubric_version) + on delete cascade, + foreign key (grouping_kind, grouping_key, period_code, rubric_version, criterion_code) + references report_item_information (grouping_kind, grouping_key, period_code, rubric_version, item_code) + on delete cascade +); + -- --------------------------------------------------------------------- -- Keyman: real (or, in this repo's default synthetic configuration, -- fabricated -- see ADR 0001) people mentioned in posts. A person may diff --git a/migrations/0103_tenant_settings.sql b/migrations/0103_tenant_settings.sql index 9470ebe9a..2b78b6b0e 100644 --- a/migrations/0103_tenant_settings.sql +++ b/migrations/0103_tenant_settings.sql @@ -1,6 +1,7 @@ -CREATE TABLE tenant_settings ( +CREATE TABLE IF NOT EXISTS tenant_settings ( id int PRIMARY KEY CHECK (id = 1), brand_name text NOT NULL DEFAULT 'LineageWeave', updated_at timestamptz NOT NULL DEFAULT now() ); -INSERT INTO tenant_settings (id, brand_name) VALUES (1, 'LineageWeave'); +INSERT INTO tenant_settings (id, brand_name) VALUES (1, 'LineageWeave') +ON CONFLICT (id) DO NOTHING; diff --git a/migrations/0104_report_leftover_interaction_map.sql b/migrations/0104_report_leftover_interaction_map.sql new file mode 100644 index 000000000..1d8c93359 --- /dev/null +++ b/migrations/0104_report_leftover_interaction_map.sql @@ -0,0 +1,65 @@ +-- ADR 0121: persist leftover interaction-map coordinates after IRT main +-- effects. CREATE IF NOT EXISTS so a volume that already ran 0001 still +-- upgrades. Composite FKs are added below so an existing table from an +-- earlier 0104 still gains member/item integrity. + +create table if not exists report_leftover_map_person ( + grouping_kind text not null, + grouping_key text not null, + period_code text not null, + rubric_version text not null, + post_id uuid not null references source_post (post_id), + axis_one numeric not null, + axis_two numeric not null, + primary key (grouping_kind, grouping_key, period_code, rubric_version, post_id), + foreign key (grouping_kind, grouping_key, period_code, rubric_version) + references report_period_score (grouping_kind, grouping_key, period_code, rubric_version) + on delete cascade +); + +create index if not exists report_leftover_map_person_post_idx + on report_leftover_map_person (post_id); + +create table if not exists report_leftover_map_item ( + grouping_kind text not null, + grouping_key text not null, + period_code text not null, + rubric_version text not null, + criterion_code text not null references common_lookup_value (lookup_code), + axis_one numeric not null, + axis_two numeric not null, + primary key (grouping_kind, grouping_key, period_code, rubric_version, criterion_code), + foreign key (grouping_kind, grouping_key, period_code, rubric_version) + references report_period_score (grouping_kind, grouping_key, period_code, rubric_version) + on delete cascade +); + +do $$ +begin + if not exists ( + select 1 + from pg_constraint + where conname = 'leftover_map_person_member_score_fk' + ) then + alter table report_leftover_map_person + add constraint leftover_map_person_member_score_fk + foreign key (grouping_kind, grouping_key, period_code, rubric_version, post_id) + references report_member_score ( + grouping_kind, grouping_key, period_code, rubric_version, post_id + ) + on delete cascade; + end if; + if not exists ( + select 1 + from pg_constraint + where conname = 'leftover_map_item_information_fk' + ) then + alter table report_leftover_map_item + add constraint leftover_map_item_information_fk + foreign key (grouping_kind, grouping_key, period_code, rubric_version, criterion_code) + references report_item_information ( + grouping_kind, grouping_key, period_code, rubric_version, item_code + ) + on delete cascade; + end if; +end $$; diff --git a/migrations/rollback/0104_report_leftover_interaction_map.sql b/migrations/rollback/0104_report_leftover_interaction_map.sql new file mode 100644 index 000000000..c6f41988b --- /dev/null +++ b/migrations/rollback/0104_report_leftover_interaction_map.sql @@ -0,0 +1,2 @@ +drop table if exists report_leftover_map_item; +drop table if exists report_leftover_map_person; diff --git a/pyproject.toml b/pyproject.toml index cb4be2916..7e230986f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lineageweave" -version = "2.12.6" +version = "2.12.7" description = "Reconstructs git-branch-style lineage DAGs from scattered short records using multi-channel score fusion and LLM adjudication." readme = "README.md" license = { text = "MIT" } diff --git a/scripts/seed_demo_data.py b/scripts/seed_demo_data.py index 8ec15a065..9359b47c7 100644 --- a/scripts/seed_demo_data.py +++ b/scripts/seed_demo_data.py @@ -117,6 +117,7 @@ def seed( cur.execute((migrations / "0010_report_item_information.sql").read_text()) cur.execute((migrations / "0011_post_chat_result.sql").read_text()) cur.execute((migrations / "0012_report_leftover_pair.sql").read_text()) + cur.execute((migrations / "0104_report_leftover_interaction_map.sql").read_text()) cur.execute((migrations / "0060_role_responsibility_agent_type.sql").read_text()) cur.execute((migrations / "0013_person_job_title.sql").read_text()) cur.execute((migrations / "0014_role_responsibility_team_actor_type.sql").read_text()) @@ -1209,6 +1210,38 @@ def _persist_seed_period_report( pair.leftover_residual, ), ) + for person in report.leftover_map_persons: + cur.execute( + "insert into report_leftover_map_person (" + "grouping_kind, grouping_key, period_code, rubric_version, " + "post_id, axis_one, axis_two" + ") values (%s,%s,%s,%s,%s,%s,%s)", + ( + grouping_kind, + grouping_key, + period_code, + RUBRIC_VERSION, + person.post_id, + person.axis_one, + person.axis_two, + ), + ) + for item in report.leftover_map_items: + cur.execute( + "insert into report_leftover_map_item (" + "grouping_kind, grouping_key, period_code, rubric_version, " + "criterion_code, axis_one, axis_two" + ") values (%s,%s,%s,%s,%s,%s,%s)", + ( + grouping_kind, + grouping_key, + period_code, + RUBRIC_VERSION, + item.criterion_code, + item.axis_one, + item.axis_two, + ), + ) def _seed_demo_period_report(cur, author_account_id, corporate_entity_id, process_unit_id) -> None: diff --git a/tests/test_leftover_pairs.py b/tests/test_leftover_pairs.py index 72dea89b6..8a5f02c06 100644 --- a/tests/test_leftover_pairs.py +++ b/tests/test_leftover_pairs.py @@ -41,6 +41,7 @@ def _load_leftover(): PAIR_KIND_CLOSEST = leftover.PAIR_KIND_CLOSEST PAIR_KIND_FARTHEST = leftover.PAIR_KIND_FARTHEST leftover_pairs_from_residual = leftover.leftover_pairs_from_residual +leftover_map_from_residual = leftover.leftover_map_from_residual def test_leftover_residual_biplot_separates_aligned_and_opposed_cells() -> None: @@ -116,3 +117,73 @@ def test_leftover_is_empty_without_observed_cells() -> None: matrix = np.array([[np.nan]], dtype=np.float64) expected = np.array([[0.0]], dtype=np.float64) assert leftover_pairs_from_residual(post_ids, item_codes, matrix, expected) == () + empty_map = leftover_map_from_residual(post_ids, item_codes, matrix, expected) + assert empty_map.pairs == () + assert empty_map.persons == () + assert empty_map.items == () + + +def test_leftover_map_pads_rank_one_axis_two_to_zero() -> None: + """A rank-1 residual still emits two map axes; the unused axis is zero.""" + post_ids = ["post-a", "post-b", "post-c"] + item_codes = ("item_near", "item_mid", "item_far") + matrix = np.array( + [ + [2.0, 0.0, -2.0], + [0.0, 0.0, 0.0], + [-2.0, 0.0, 2.0], + ], + dtype=np.float64, + ) + leftover_map = leftover_map_from_residual(post_ids, item_codes, matrix, np.zeros_like(matrix)) + assert leftover_map.pairs == leftover_pairs_from_residual( + post_ids, item_codes, matrix, np.zeros_like(matrix) + ) + assert {person.post_id for person in leftover_map.persons} == set(post_ids) + assert {item.criterion_code for item in leftover_map.items} == set(item_codes) + for person in leftover_map.persons: + assert person.axis_two == pytest.approx(0.0) + assert np.isfinite(person.axis_one) + for item in leftover_map.items: + assert item.axis_two == pytest.approx(0.0) + assert np.isfinite(item.axis_one) + by_post = {person.post_id: person for person in leftover_map.persons} + by_item = {item.criterion_code: item for item in leftover_map.items} + closest = leftover_map.pairs[0] + person = by_post[closest.post_id] + item = by_item[closest.criterion_code] + distance = np.hypot(person.axis_one - item.axis_one, person.axis_two - item.axis_two) + assert distance == pytest.approx(closest.leftover_distance, abs=1e-9) + + +def test_zero_residual_map_places_every_complete_case_at_the_origin() -> None: + post_ids = ["alpha-post", "beta-post"] + item_codes = ("item_one", "item_two") + matrix = np.ones((2, 2), dtype=np.float64) + leftover_map = leftover_map_from_residual(post_ids, item_codes, matrix, matrix) + assert leftover_map.pairs[0].leftover_distance == pytest.approx(0.0) + assert {person.post_id for person in leftover_map.persons} == set(post_ids) + assert {item.criterion_code for item in leftover_map.items} == set(item_codes) + for person in leftover_map.persons: + assert person.axis_one == pytest.approx(0.0) + assert person.axis_two == pytest.approx(0.0) + for item in leftover_map.items: + assert item.axis_one == pytest.approx(0.0) + assert item.axis_two == pytest.approx(0.0) + + +def test_sparse_person_does_not_receive_invented_map_coordinates() -> None: + """Incomplete rows are dropped from the Gabriel rectangle, not filled with zero.""" + post_ids = ["aligned-post", "opposed-post", "sparse-post"] + item_codes = ("item_near", "item_far") + matrix = np.array( + [ + [2.0, -2.0], + [-2.0, 2.0], + [2.0, np.nan], + ], + dtype=np.float64, + ) + leftover_map = leftover_map_from_residual(post_ids, item_codes, matrix, np.zeros_like(matrix)) + assert {person.post_id for person in leftover_map.persons} == {"aligned-post", "opposed-post"} + assert "sparse-post" not in {person.post_id for person in leftover_map.persons} diff --git a/tests/test_migration_replay.py b/tests/test_migration_replay.py index 29fe1c176..9beb407ec 100644 --- a/tests/test_migration_replay.py +++ b/tests/test_migration_replay.py @@ -33,3 +33,5 @@ def test_migrate_sh_replays_leftover_pair_migration_on_existing_volumes() -> Non ).read_text(encoding="utf-8") assert "0012_*" in script + assert "0103_*" in script + assert "0104_*" in script diff --git a/tests/test_period_report.py b/tests/test_period_report.py index 04c9c3921..0dd1d7baf 100644 --- a/tests/test_period_report.py +++ b/tests/test_period_report.py @@ -10,6 +10,7 @@ import numpy as np import pytest +from lineageweave.leftover_pairs import leftover_pairs_from_residual from lineageweave.period_report import ( LINK_METHOD_FIPC, LINK_METHOD_FREE, @@ -18,7 +19,6 @@ ItemBank, assemble_response_matrix, calibrate_period_report, - leftover_pairs_from_residual, link_or_calibrate_period_report, rank_items_by_information, score_groups_on_shared_metric, @@ -266,6 +266,14 @@ def test_calibrated_report_attaches_leftover_pairs() -> None: assert pair.criterion_code in items assert pair.leftover_distance >= 0.0 assert np.isfinite(pair.leftover_residual) + assert {person.post_id for person in report.leftover_map_persons} <= member_ids + assert {item.criterion_code for item in report.leftover_map_items} <= set(items) + for person in report.leftover_map_persons: + assert np.isfinite(person.axis_one) + assert np.isfinite(person.axis_two) + for item in report.leftover_map_items: + assert np.isfinite(item.axis_one) + assert np.isfinite(item.axis_two) diff --git a/tests/test_schema.py b/tests/test_schema.py index 1e2c708a3..da47a2445 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -122,6 +122,8 @@ def test_migration_applies_cleanly(schema_db) -> None: "report_item_parameter", "report_item_information", "report_leftover_pair", + "report_leftover_map_person", + "report_leftover_map_item", "post_summary_result", "post_summary_event", "post_summary_role", @@ -172,6 +174,28 @@ def test_leftover_pair_references_member_and_item_rows(schema_db) -> None: assert "report_period_score" in targets +def test_leftover_map_references_member_and_item_rows(schema_db) -> None: + """A leftover-map point cannot name a post or item from another report.""" + with schema_db.cursor() as cur: + cur.execute( + """ + select conrelid::regclass::text, confrelid::regclass::text + from pg_constraint + where conrelid in ( + 'report_leftover_map_person'::regclass, + 'report_leftover_map_item'::regclass + ) and contype = 'f' + """ + ) + by_table: dict[str, set[str]] = {} + for table_name, target in cur.fetchall(): + by_table.setdefault(table_name, set()).add(target) + assert "report_member_score" in by_table["report_leftover_map_person"] + assert "report_period_score" in by_table["report_leftover_map_person"] + assert "report_item_information" in by_table["report_leftover_map_item"] + assert "report_period_score" in by_table["report_leftover_map_item"] + + def test_corporate_hierarchy_recursive_query_returns_correct_shape(schema_db) -> None: """The real product requirement: 'Acme Group -> Acme Electronics Korea diff --git a/uv.lock b/uv.lock index 10bcf9ff1..b4288a2c6 100644 --- a/uv.lock +++ b/uv.lock @@ -454,7 +454,7 @@ wheels = [ [[package]] name = "lineageweave" -version = "2.12.6" +version = "2.12.7" source = { editable = "." } dependencies = [ { name = "certifi" }, From b20d70c99ff8f91c6213d21acb4b0e3fe7ec88b9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 19:47:00 +0900 Subject: [PATCH 02/11] fix(migrations): stop duplicating leftover-map schema into 0001 migrations/0001_initial_schema.sql is baked into the Postgres image and only replays on fresh installs; the leftover-interaction-map tables were added there directly as well as in the new, correctly idempotent 0104_report_leftover_interaction_map.sql, leaving existing volumes and fresh installs on different schemas. Removes the duplicate from 0001 and keeps 0104 as the single source of truth. Updates tests/test_schema.py's schema_db fixture to apply 0104 explicitly (it lists each schema migration by hand rather than globbing), and fixes a second, unrelated bug the schema test then surfaced: report_ingestion.py's new leftover_map_persons query had the required nosemgrep suppression but was missing its "Safe SQL:" reason comment, and the suppression count constant wasn't bumped for the new site. --- backend/app/report_ingestion.py | 1 + migrations/0001_initial_schema.sql | 40 ----------------------- tests/test_schema.py | 6 ++++ tests/test_static_sql_review_contracts.py | 2 +- 4 files changed, 8 insertions(+), 41 deletions(-) diff --git a/backend/app/report_ingestion.py b/backend/app/report_ingestion.py index dc577adbe..9ba7d0de8 100644 --- a/backend/app/report_ingestion.py +++ b/backend/app/report_ingestion.py @@ -647,6 +647,7 @@ async def fetch_period_reports( period_code, RUBRIC_VERSION, ) + # Safe SQL: the source-context expression is an immutable schema fragment; report keys are bound. leftover_map_persons = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli f""" select lp.grouping_key, lp.post_id, lp.axis_one, lp.axis_two, p.post_title, diff --git a/migrations/0001_initial_schema.sql b/migrations/0001_initial_schema.sql index f44b3f507..483893a1d 100644 --- a/migrations/0001_initial_schema.sql +++ b/migrations/0001_initial_schema.sql @@ -356,46 +356,6 @@ create table report_leftover_pair ( create index report_leftover_pair_post_idx on report_leftover_pair (post_id); --- Leftover interaction-map coordinates after IRT main effects (ADR 0121). --- Complete-case Gabriel ξ / ζ on two leftover-map axes. Cascade with --- the period score. A leftover-map post must be a member of this --- report; a leftover-map criterion must be a CAT item on this report. -create table report_leftover_map_person ( - grouping_kind text not null, - grouping_key text not null, - period_code text not null, - rubric_version text not null, - post_id uuid not null references source_post (post_id), - axis_one numeric not null, - axis_two numeric not null, - primary key (grouping_kind, grouping_key, period_code, rubric_version, post_id), - foreign key (grouping_kind, grouping_key, period_code, rubric_version) - references report_period_score (grouping_kind, grouping_key, period_code, rubric_version) - on delete cascade, - foreign key (grouping_kind, grouping_key, period_code, rubric_version, post_id) - references report_member_score (grouping_kind, grouping_key, period_code, rubric_version, post_id) - on delete cascade -); - -create index report_leftover_map_person_post_idx on report_leftover_map_person (post_id); - -create table report_leftover_map_item ( - grouping_kind text not null, - grouping_key text not null, - period_code text not null, - rubric_version text not null, - criterion_code text not null references common_lookup_value (lookup_code), - axis_one numeric not null, - axis_two numeric not null, - primary key (grouping_kind, grouping_key, period_code, rubric_version, criterion_code), - foreign key (grouping_kind, grouping_key, period_code, rubric_version) - references report_period_score (grouping_kind, grouping_key, period_code, rubric_version) - on delete cascade, - foreign key (grouping_kind, grouping_key, period_code, rubric_version, criterion_code) - references report_item_information (grouping_kind, grouping_key, period_code, rubric_version, item_code) - on delete cascade -); - -- --------------------------------------------------------------------- -- Keyman: real (or, in this repo's default synthetic configuration, -- fabricated -- see ADR 0001) people mentioned in posts. A person may diff --git a/tests/test_schema.py b/tests/test_schema.py index da47a2445..d676d7ef9 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -43,6 +43,11 @@ / "migrations" / "0102_project_bound_summary_event.sql" ) +_LEFTOVER_INTERACTION_MAP_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0104_report_leftover_interaction_map.sql" +) def _postgres_available() -> bool: @@ -79,6 +84,7 @@ def schema_db(): cur.execute(_MAJOR_EVENT_ACTION_MIGRATION.read_text()) cur.execute(_PROJECT_BOUND_ACTION_MIGRATION.read_text()) cur.execute(_PROJECT_BOUND_EVENT_MIGRATION.read_text()) + cur.execute(_LEFTOVER_INTERACTION_MAP_MIGRATION.read_text()) conn.commit() yield conn finally: diff --git a/tests/test_static_sql_review_contracts.py b/tests/test_static_sql_review_contracts.py index 7269e6acf..31c7896bc 100644 --- a/tests/test_static_sql_review_contracts.py +++ b/tests/test_static_sql_review_contracts.py @@ -28,7 +28,7 @@ ) ASYNC_STATEMENT_METHODS = {"execute", "fetch", "fetchrow", "fetchval"} SQL_REVIEW_RULE = "python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli" -EXPECTED_SQL_SUPPRESSION_COUNT = 35 +EXPECTED_SQL_SUPPRESSION_COUNT = 36 @pytest.mark.parametrize("relative_path", SQL_REVIEW_PATHS) From 732f2b25f8ce26001e621a4d0528ae0722a6beea Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 19:46:57 +0900 Subject: [PATCH 03/11] fix: restore PR 481 CI contracts --- frontend/src/App.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 0cff77997..89f361e23 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -4621,7 +4621,8 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean
    - {destination === "admin" ? : null}