From 03c194653b7ed359ebc4b2c231f55acc4bc04d70 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 16:47:52 +0000 Subject: [PATCH 01/43] feat: persist and explain Event Lineage channel evidence Reconstruct already computed per-channel scores, but live Event Lineage collapsed each edge to a fused score. Persist the active signals beside each edge, return them on GET /api/lineage, and disclose exact inferred values in the Buyer DAG. --- ARCHITECTURE.md | 19 +- .../2.14.0-event-lineage-channel-evidence.md | 17 ++ CHANGELOG.md | 8 + backend/app/lineage_ingestion.py | 103 +++++++- docker/postgres-init/migrate.sh | 2 +- .../0124-event-lineage-channel-evidence.md | 95 +++++++ docs/adr/README.md | 2 +- docs/storybook-inventory.md | 2 +- frontend/package.json | 2 +- frontend/src/App.css | 93 +++++++ frontend/src/LineageDag.stories.tsx | 73 ++++++ frontend/src/LineageDag.test.tsx | 134 ++++++++++ frontend/src/LineageDag.tsx | 138 +++++++++- frontend/src/api.ts | 19 ++ frontend/src/i18n.test.ts | 4 + frontend/src/i18n.ts | 78 ++++++ frontend/src/lineageLayout.ts | 2 + lineageweave/__init__.py | 12 +- lineageweave/lineage_persistence.py | 243 +++++++++++++++++- migrations/0105_post_lineage_edge_signal.sql | 57 ++++ .../0105_post_lineage_edge_signal.sql | 10 + pyproject.toml | 2 +- scripts/seed_demo_data.py | 33 ++- tests/test_lineage_channel_evidence.py | 143 +++++++++++ tests/test_lineage_ingestion.py | 221 +++++++++++++++- tests/test_schema.py | 59 ++++- uv.lock | 2 +- 27 files changed, 1531 insertions(+), 42 deletions(-) create mode 100644 CHANGELOG.d/2.14.0-event-lineage-channel-evidence.md create mode 100644 docs/adr/0124-event-lineage-channel-evidence.md create mode 100644 frontend/src/LineageDag.stories.tsx create mode 100644 frontend/src/LineageDag.test.tsx create mode 100644 migrations/0105_post_lineage_edge_signal.sql create mode 100644 migrations/rollback/0105_post_lineage_edge_signal.sql create mode 100644 tests/test_lineage_channel_evidence.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d0280ff97..64f29e646 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -68,7 +68,7 @@ flowchart LR | `tepp_client.py` | TEPP's published `AnalysisRunRequest` wire contract, pluggable transport | | `rankweave_client.py` | Fail-closed RankWeave ranking port (`weighted_reciprocal_rank_fuse` in-process; never invent a fused score or a theta) | | `reconstruct.py` | The pipeline: group → candidate window → score → fuse → thread | -| `lineage_persistence.py` | Flattens reconstruct trees into `post_lineage_edge` row specs (parent, child, fused_score) | +| `lineage_persistence.py` | Flattens reconstruct trees into `post_lineage_edge` rows plus `post_lineage_edge_signal` channel evidence | | `knowledge_graph.py` | Random-walk-with-restart relevance + per-node adaptive related-node cutoff (Tong et al., 2006) -- pure graph math, no Postgres | | `keyman_extraction.py` | Pluggable LLM extraction of two-sided (our-side/counterparty) person mentions + N:N org affiliations from a post | | `entity_relationship_classification.py` | Pluggable LLM classification of a named organization's relationship to the post author (`rel_voc`/`rel_vom`/`rel_vop`/`rel_vocc`/`rel_voco`/`rel_vos`) | @@ -213,13 +213,16 @@ lives in `lineageweave/keyman_extraction.py` and talks to contextual-orchestrator; persist is `backend/app/keyman_ingestion.py`. `GET /api/lineage` returns the ABAC-filtered reconstruct graph -(`{nodes, edges}`) from persisted `post_lineage_edge` rows. Each node -includes `group` from the same `reconstruct_group_key()` rebuild uses -(persisted `thread_group_key`, else process unit, else corp). -`POST /api/lineage/rebuild` (`post_admin`) re-runs `reconstruct()` over -every `source_post` and rewrites those edges. Reconstruct grouping is -stored on the post as `thread_group_key` / `secondary_grouping_key` -(not derived from process unit or voc type). +(`{nodes, edges, reconstruction}`) from persisted `post_lineage_edge` +rows. Each edge includes additive `channel_evidence` from +`post_lineage_edge_signal`. Evidence for an endpoint the account cannot +see is omitted. Each node includes `group` from the same +`reconstruct_group_key()` rebuild uses (persisted `thread_group_key`, +else process unit, else corp). `POST /api/lineage/rebuild` (`post_admin`) +re-runs `reconstruct()` over every `source_post` and rewrites those +edges and signals atomically. Reconstruct grouping is stored on the post +as `thread_group_key` / `secondary_grouping_key` (not derived from +process unit or voc type). Phase 3 adds `GET /api/posts/{post_id}/counterparties` (same RBAC+ABAC gate) and extends `POST /api/posts/{post_id}/extract-keymen` to also diff --git a/CHANGELOG.d/2.14.0-event-lineage-channel-evidence.md b/CHANGELOG.d/2.14.0-event-lineage-channel-evidence.md new file mode 100644 index 000000000..84a4e3486 --- /dev/null +++ b/CHANGELOG.d/2.14.0-event-lineage-channel-evidence.md @@ -0,0 +1,17 @@ +# 2.14.0 — Event Lineage channel evidence + +Live Event Lineage now persists and explains the independent signals that +produced each reconstructed connection. + +- `post_lineage_edge_signal` stores per-channel score, the normalized + active weight actually used, and `weight * score` contribution beside + each `post_lineage_edge` row. The optional LLM channel is omitted when + it did not participate; it is never fabricated. +- `event_lineage_rebuild` records reconstruction version, generated-at + time, and the active weight profile so a later rebuild cannot silently + rewrite historic evidence. +- `GET /api/lineage` returns additive `channel_evidence` on each visible + edge. ABAC never reveals evidence for an invisible endpoint. +- The Buyer DAG discloses exact values with keyboard and screen-reader + access, labels the relation as inferred rather than causal, and keeps + the same values in print. diff --git a/CHANGELOG.md b/CHANGELOG.md index c8ed1a099..2e2aeec84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ All notable changes to this project are documented here. Format follows ## [Unreleased] +### Added + +- Event Lineage now persists each reconstructed connection's independent + channel scores, the normalized weights actually used, and their + contributions. The Buyer DAG discloses those exact values as inferred + evidence, not a causal claim, and omits the LLM channel when it did + not participate. + ### Fixed - `make smoke` and `make seed` now run through the locked project `uv` diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index f1e76d495..b2ee707a9 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -10,13 +10,19 @@ from __future__ import annotations +from collections import defaultdict from datetime import datetime from typing import Any, Mapping import asyncpg from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL -from lineageweave.lineage_persistence import lineage_edge_specs +from lineageweave.lineage_persistence import ( + LOOKUP_CODE_TO_SIGNAL, + lineage_edge_specs, + lineage_rebuild_spec, + rank_channel_evidence, +) from lineageweave.models import Edge, Record @@ -60,8 +66,32 @@ def records_from_source_posts(rows: list[Mapping[str, Any]]) -> list[Record]: async def persist_lineage_edges(conn: asyncpg.Connection, edges: list[Edge]) -> None: - """Replace ``post_lineage_edge`` with ``edges`` (reconstruct is source of truth).""" + """Replace live Event Lineage with ``edges`` and their channel evidence. + + Reconstruct is the source of truth. The delete is cascaded onto + ``post_lineage_edge_signal`` so a rebuild cannot leave orphan or + stale signal rows. Rebuild metadata is replaced in the same + connection so version, weights, and generated-at stay aligned with + the new graph. + """ + spec = lineage_rebuild_spec(edges) await conn.execute("delete from post_lineage_edge") + await conn.execute("delete from event_lineage_rebuild") + await conn.execute( + "insert into event_lineage_rebuild " + "(rebuild_lock, reconstruction_version, generated_at, min_fused_score, candidate_window) " + "values (true, $1, now(), $2, $3)", + spec.reconstruction_version, + spec.min_fused_score, + spec.candidate_window, + ) + for signal_code, signal_weight in spec.channel_weights: + await conn.execute( + "insert into event_lineage_rebuild_channel " + "(rebuild_lock, signal_code, signal_weight) values (true, $1, $2)", + signal_code, + signal_weight, + ) for edge in edges: await conn.execute( "insert into post_lineage_edge (parent_post_id, child_post_id, fused_score) " @@ -70,6 +100,18 @@ async def persist_lineage_edges(conn: asyncpg.Connection, edges: list[Edge]) -> edge.child_id, edge.fused_score, ) + for row in spec.signal_rows: + await conn.execute( + "insert into post_lineage_edge_signal " + "(parent_post_id, child_post_id, signal_code, signal_score, signal_weight, signal_contribution) " + "values ($1::uuid, $2::uuid, $3, $4, $5, $6)", + row["parent_post_id"], + row["child_post_id"], + row["signal_code"], + row["signal_score"], + row["signal_weight"], + row["signal_contribution"], + ) async def rebuild_lineage(conn: asyncpg.Connection) -> list[Edge]: @@ -84,6 +126,12 @@ async def rebuild_lineage(conn: asyncpg.Connection) -> list[Edge]: return edges +def _isoformat(value: object) -> str: + if hasattr(value, "isoformat"): + return value.isoformat() # type: ignore[no-any-return] + return str(value) + + async def visible_lineage_graph( conn: asyncpg.Connection, can_see_post, @@ -94,7 +142,8 @@ async def visible_lineage_graph( The persisted graph can contain tens of thousands of posts. The UI opens individual posts for complete lineage, while this landing projection keeps - only the newest ``limit`` visible nodes and edges between them. + only the newest ``limit`` visible nodes and edges between them. Channel + evidence is attached only after both endpoints are visible. """ posts = await conn.fetch( "select post_id, post_title, voc_type_code, visibility_code, " @@ -105,6 +154,17 @@ async def visible_lineage_graph( edge_rows = await conn.fetch( "select parent_post_id, child_post_id, fused_score from post_lineage_edge" ) + signal_rows = await conn.fetch( + "select parent_post_id, child_post_id, signal_code, signal_score, " + "signal_weight, signal_contribution from post_lineage_edge_signal" + ) + rebuild_rows = await conn.fetch( + "select reconstruction_version, generated_at, min_fused_score, candidate_window " + "from event_lineage_rebuild" + ) + weight_rows = await conn.fetch( + "select signal_code, signal_weight from event_lineage_rebuild_channel" + ) if focus_post_id is None: visible = sorted( @@ -165,12 +225,47 @@ async def visible_lineage_graph( "is_branch_point": len(children_of.get(post_id, [])) >= 2, } ) + + signals_by_edge: dict[tuple[str, str], list[dict[str, object]]] = defaultdict(list) + for row in signal_rows: + parent_id = str(row["parent_post_id"]) + child_id = str(row["child_post_id"]) + if parent_id not in visible_ids or child_id not in visible_ids: + continue + payload = dict(row) + payload["channel_name"] = LOOKUP_CODE_TO_SIGNAL.get(str(row["signal_code"]), str(row["signal_code"])) + signals_by_edge[(parent_id, child_id)].append(payload) + edges = [ { "source": str(row["parent_post_id"]), "target": str(row["child_post_id"]), "fused_score": float(row["fused_score"]), + "channel_evidence": rank_channel_evidence( + signals_by_edge[(str(row["parent_post_id"]), str(row["child_post_id"]))] + ), } for row in visible_edges ] - return {"nodes": nodes, "edges": edges, "truncated": truncated} + reconstruction = None + if rebuild_rows: + rebuild = rebuild_rows[0] + reconstruction = { + "reconstruction_version": rebuild["reconstruction_version"], + "generated_at": _isoformat(rebuild["generated_at"]), + "min_fused_score": float(rebuild["min_fused_score"]), + "candidate_window": int(rebuild["candidate_window"]), + "active_weights": [ + { + "signal_code": LOOKUP_CODE_TO_SIGNAL.get(str(row["signal_code"]), str(row["signal_code"])), + "signal_weight": float(row["signal_weight"]), + } + for row in weight_rows + ], + } + return { + "nodes": nodes, + "edges": edges, + "truncated": truncated, + "reconstruction": reconstruction, + } diff --git a/docker/postgres-init/migrate.sh b/docker/postgres-init/migrate.sh index f329117d6..88f168a43 100644 --- a/docker/postgres-init/migrate.sh +++ b/docker/postgres-init/migrate.sh @@ -18,7 +18,7 @@ for migration in /opt/lineageweave/migrations/*.sql; do migration_name=${migration##*/} case "$migration_name" in 0012_*|0013_*|0014_*|0015_*|0016_*|0017_*|0018_*|0019_*|0020_*|0021_*|0022_*|0023_*|0024_*|0025_*|0026_*|0027_*|0028_*|0029_*|0030_*|0031_*|0032_*|0033_*|0034_*|0035_*|0036_*|0037_*|0038_*|0039_*|0040_*|0041_*|0042_*|0043_*|0044_*|0045_*|0046_*|0047_*|0048_*|0049_*|0050_*) ;; - 0060_*|0100_*|0101_*|0102_*) ;; + 0060_*|0100_*|0101_*|0102_*|0103_*|0104_*|0105_*) ;; *) continue ;; esac printf 'Applying %s\n' "$migration_name" diff --git a/docs/adr/0124-event-lineage-channel-evidence.md b/docs/adr/0124-event-lineage-channel-evidence.md new file mode 100644 index 000000000..ea964c3ea --- /dev/null +++ b/docs/adr/0124-event-lineage-channel-evidence.md @@ -0,0 +1,95 @@ +# ADR 0124: Persist and explain Event Lineage channel evidence + +**Status:** Accepted +**Date:** 2026-08-21 +**Issue:** [#274](https://github.com/ContextualWisdomLab/LineageWeave/issues/274) + +## Context + +`reconstruct()` already computes the winning edge's per-channel scores +(`temporal`, `secondary_key`, `text`, optional `llm`) and RankWeave +fuses them with a weighted convex combination. Production persistence +collapsed each edge to `(parent_post_id, child_post_id, fused_score)`, +so `/api/lineage` and the Buyer DAG exposed only the fused score. + +A buyer could see that two posts were linked but could not answer which +independent signals supported the edge, whether the optional LLM channel +participated, which signal dominated, or how to audit a later +reconstruction after model or weight changes. ADR 0064 already treats +every accepted edge as uncertainty-bearing evidence, not a proven +business fact; that contract was not visible in PostgreSQL or the UI. + +This is distinct from typed Knowledge Graph path repair. Event Lineage +explains reconstructed post-to-post links. PostgreSQL remains +authoritative; PROV-O/RDF export is a projection. + +## Decision + +1. Persist a child table `post_lineage_edge_signal` with a composite + foreign key to `post_lineage_edge` (`ON DELETE CASCADE`), one row per + edge and active signal, and exact `numeric(8,6)` score, weight, and + contribution. No JSONB for billable or auditable numeric facts. +2. Controlled lookup values are globally unique: + `lineage_signal_temporal`, `lineage_signal_secondary_key`, + `lineage_signal_text`, `lineage_signal_llm`. The LLM row is omitted + when the adjudication client is unavailable; it is never fabricated. +3. Weights are the normalized active weights actually used + (`reconstruct.active_weights`). Contribution is `weight * score` and + must reconcile with `fused_score` within + `CHANNEL_EVIDENCE_TOLERANCE` (`1e-6`). +4. Live Event Lineage is replaced atomically. A singleton + `event_lineage_rebuild` stores reconstruction version, generated-at + time, minimum fused score, and candidate window; + `event_lineage_rebuild_channel` stores the active weight profile. + Analysis-run reconstruction (`analysis_run_lineage_edge`) stays a + separate immutable run-scoped table. +5. `GET /api/lineage` returns an additive `channel_evidence` collection + on each visible edge (`signal_code`, `signal_label`, `score`, + `weight`, `contribution`, `rank`) ordered by contribution, then + controlled signal order. ABAC never reveals evidence for an invisible + endpoint. +6. The Buyer DAG provides an accessible edge-detail disclosure (not + hover-only), labels the relation as inferred rather than causal, and + states when no LLM channel participated only when at least one + recorded channel exists. Print/export uses the same values. + +## Consequences + +- Buyers can inspect why a connection was selected and distinguish + inference from source evidence. +- A later rebuild rewrites live Event Lineage as a whole; historic + meaning is not silently mutated in place. +- Completeness is lower when the LLM channel is unavailable, matching + ADR 0064: missing channels are dropped and weights renormalize. + +## References + +Allen, J. F. (1983). Maintaining knowledge about temporal intervals. +*Communications of the ACM, 26*(11), 832–843. +https://doi.org/10.1145/182.358434 + +Cormack, G. V., Clarke, C. L. A., & Buettcher, S. (2009). Reciprocal +rank fusion outperforms Condorcet and individual rank learning methods. +In *Proceedings of the 32nd International ACM SIGIR Conference on +Research and Development in Information Retrieval* (pp. 758–759). ACM. +https://doi.org/10.1145/1571941.1572114 + +Hearst, M. A. (1997). TextTiling: Segmenting text into multi-paragraph +subtopic passages. *Computational Linguistics, 23*(1), 33–64. + +Jeon, J.-J., Kim, I., Vanli, N. D., & Choi, T. (2021). Logistic +structured interaction model for binary item response (arXiv:2007.08719). +https://arxiv.org/abs/2007.08719 + +Lebo, T., Sahoo, S., McGuinness, D., Belhajjame, K., Cheney, J., +Corsar, D., Garijo, D., Soiland-Reyes, S., Zednik, S., & Zhao, J. +(2013). *PROV-O: The PROV ontology* (W3C Recommendation). World Wide +Web Consortium. https://www.w3.org/TR/2013/REC-prov-o-20130430/ + +Tong, H., Faloutsos, C., & Pan, J.-Y. (2006). Fast random walk with +restart and its applications. In *Proceedings of the Sixth International +Conference on Data Mining* (pp. 613–622). IEEE. +https://doi.org/10.1109/ICDM.2006.70 + +ADR 0064 (uncertainty-bearing lineage evidence) +ADR 0024 (RankWeave fusion fail-closed) diff --git a/docs/adr/README.md b/docs/adr/README.md index 762f1c051..3b2c63ac3 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -10,7 +10,7 @@ decision from them. | Supporting document | Normative ADR | |---|---| | [`product-technical-gap-baseline.md`](../product-technical-gap-baseline.md) | Product/technical traceability projection across the ADR set; ADRs remain normative | -| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md) | +| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0124](0124-event-lineage-channel-evidence.md) | | [`PROV_O_IMPLEMENTATION.md`](../PROV_O_IMPLEMENTATION.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`PROV_O_IMPLEMENTATION_MATRIX.md`](../PROV_O_IMPLEMENTATION_MATRIX.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`image-content-schema.md`](../image-content-schema.md) | [0066](0066-position-preserving-image-content.md) | diff --git a/docs/storybook-inventory.md b/docs/storybook-inventory.md index 28c59bd48..03f0e9cd8 100644 --- a/docs/storybook-inventory.md +++ b/docs/storybook-inventory.md @@ -8,7 +8,7 @@ buyer-facing control you can click before changing product CSS. | `Evidence/CitationChip` | Click a cited title to open that source post. | `--color-chip-border`, `--radius-chip`, `CitationChip` | | `AnalysisRun/CutoffKnownBody` | Read the cutoff-known sentence, then compare it with the live body below. | `--color-accent-border`, `--space-panel-block`, `--radius-panel`, `CutoffKnownBody` | | `Analysis/LineageEntityPicker` | Choose which corp to reconstruct, then click Request a lineage reconstruction. | `--space-control-gap`, `--size-control-min`, `--radius-control`, `LineageEntityPicker` | -| `Chrome/PopupCloseButton` | Close the evidence panel or post popup. | `--space-close-inset`, `--font-size-close`, `PopupCloseButton` | +| `Evidence/LineageDag` | Open a reconstructed connection and read the inferred channel scores. | `--color-border`, `--radius-control`, `LineageDag` | Repeated web objects must use `frontend/src/styles/tokens.css` and a module under `frontend/src/components/`. Do not add a second Node package manager; diff --git a/frontend/package.json b/frontend/package.json index e2e996bbe..c6a4389de 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "2.12.6", + "version": "2.14.0", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/App.css b/frontend/src/App.css index c72aab078..cb477861a 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -503,6 +503,99 @@ stroke: var(--border); stroke-width: 1.5; fill: none; + cursor: pointer; +} + +.lineage-dag-edge:focus, +.lineage-dag-edge:focus-visible { + outline: 2px solid var(--color-primary, var(--text-h)); + outline-offset: 2px; +} + +.lineage-dag-edge-selected { + stroke: var(--text-h); + stroke-width: 2.5; +} + +.lineage-edge-evidence { + margin-top: 1rem; +} + +.lineage-edge-evidence h3 { + font-size: 1rem; + margin: 0 0 0.4rem; +} + +.lineage-edge-evidence p { + margin: 0 0 0.75rem; +} + +.lineage-rebuild-profile { + display: grid; + gap: 0.35rem; + margin: 0 0 0.75rem; +} + +.lineage-rebuild-profile div { + display: grid; + grid-template-columns: minmax(8rem, 14rem) 1fr; + gap: 0.5rem; +} + +.lineage-rebuild-profile dt { + font-weight: 600; +} + +.lineage-rebuild-profile dd { + margin: 0; +} + +.lineage-edge-evidence-item { + margin: 0 0 0.5rem; + padding: 0.4rem 0.6rem; + border: 1px solid var(--color-border, var(--border)); + border-radius: var(--radius-control, 8px); + background: var(--surface); +} + +.lineage-edge-evidence table { + width: 100%; + border-collapse: collapse; + margin-top: 0.5rem; +} + +.lineage-edge-evidence caption { + text-align: left; + font-weight: 600; + margin-bottom: 0.25rem; +} + +.lineage-edge-evidence th, +.lineage-edge-evidence td { + padding: 0.25rem 0.4rem; + border-bottom: 1px solid var(--color-border, var(--border)); +} + +.lineage-edge-evidence th:nth-child(1), +.lineage-edge-evidence td:nth-child(1), +.lineage-edge-evidence th:nth-child(n + 3), +.lineage-edge-evidence td:nth-child(n + 3) { + text-align: right; +} + +.lineage-edge-evidence th:nth-child(2), +.lineage-edge-evidence td:nth-child(2) { + text-align: left; +} + +@media print { + .lineage-edge-evidence details { + display: block; + } + + .lineage-edge-evidence details > * { + display: block !important; + } } .lineage-dag-node { diff --git a/frontend/src/LineageDag.stories.tsx b/frontend/src/LineageDag.stories.tsx new file mode 100644 index 000000000..b5da3c9db --- /dev/null +++ b/frontend/src/LineageDag.stories.tsx @@ -0,0 +1,73 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { LineageDag } from "./LineageDag"; + +const meta = { + title: "Evidence/LineageDag", + component: LineageDag, + args: { + onSelectPost: () => undefined, + graph: { + nodes: [ + { + id: "rec-002", + group: "A-100", + label: "Kickoff recap", + occurred_at: "2026-01-02T00:00:00", + is_root: true, + is_branch_point: true, + }, + { + id: "rec-003", + group: "A-100", + label: "Pricing follow-up", + occurred_at: "2026-01-03T00:00:00", + is_root: false, + is_branch_point: false, + }, + ], + edges: [ + { + source: "rec-002", + target: "rec-003", + fused_score: 0.7, + channel_evidence: [ + { + signal_code: "text", + signal_label: "Text similarity", + score: 0.5, + weight: 0.5, + contribution: 0.25, + rank: 1, + }, + { + signal_code: "temporal", + signal_label: "Temporal proximity", + score: 0.8, + weight: 0.25, + contribution: 0.2, + rank: 2, + }, + ], + }, + ], + reconstruction: { + reconstruction_version: "lineageweave.reconstruct/2.14.0", + generated_at: "2026-08-21T12:00:00+00:00", + min_fused_score: 0.3, + candidate_window: 50, + active_weights: [ + { signal_code: "temporal", signal_weight: 0.25 }, + { signal_code: "text", signal_weight: 0.5 }, + ], + }, + }, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const ConnectionEvidence: Story = {}; + +export const NoLlmChannel: Story = {}; diff --git a/frontend/src/LineageDag.test.tsx b/frontend/src/LineageDag.test.tsx new file mode 100644 index 000000000..559125f99 --- /dev/null +++ b/frontend/src/LineageDag.test.tsx @@ -0,0 +1,134 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import type { LineageGraph } from "./api"; +import { LineageDag } from "./LineageDag"; + +const graph: LineageGraph = { + nodes: [ + { + id: "rec-002", + group: "A-100", + label: "Kickoff recap", + occurred_at: "2026-01-02T00:00:00", + is_root: true, + is_branch_point: true, + }, + { + id: "rec-003", + group: "A-100", + label: "Pricing follow-up", + occurred_at: "2026-01-03T00:00:00", + is_root: false, + is_branch_point: false, + }, + ], + edges: [ + { + source: "rec-002", + target: "rec-003", + fused_score: 0.7, + channel_evidence: [ + { + signal_code: "text", + signal_label: "Text similarity", + score: 0.5, + weight: 0.5, + contribution: 0.25, + rank: 1, + }, + { + signal_code: "secondary_key", + signal_label: "Secondary key match", + score: 1.0, + weight: 0.25, + contribution: 0.25, + rank: 2, + }, + { + signal_code: "temporal", + signal_label: "Temporal proximity", + score: 0.8, + weight: 0.25, + contribution: 0.2, + rank: 3, + }, + ], + }, + ], + reconstruction: { + reconstruction_version: "lineageweave.reconstruct/2.14.0", + generated_at: "2026-08-21T12:00:00+00:00", + min_fused_score: 0.3, + candidate_window: 50, + active_weights: [ + { signal_code: "temporal", signal_weight: 0.25 }, + { signal_code: "secondary_key", signal_weight: 0.25 }, + { signal_code: "text", signal_weight: 0.5 }, + ], + }, +}; + +describe("LineageDag channel evidence", () => { + it("discloses exact inferred values without hover-only interaction", async () => { + render(); + expect( + screen.getByText("Each connection is inferred from independent signals. It is not a causal claim."), + ).toBeInTheDocument(); + expect(screen.getByText("No LLM adjudication participated in this connection.")).toBeInTheDocument(); + expect(screen.getByText("lineageweave.reconstruct/2.14.0")).toBeInTheDocument(); + expect(screen.getAllByText("0.250000").length).toBeGreaterThan(0); + expect(screen.getByText("0.200000")).toBeInTheDocument(); + expect(screen.getByText(/fused score 0.700000/)).toBeInTheDocument(); + expect(screen.queryByText(/causal relationship/i)).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Open connection evidence: Kickoff recap to Pricing follow-up" })).toHaveAttribute( + "tabindex", + "0", + ); + }); + + it("does not claim a missing LLM channel when no evidence was recorded", () => { + render( + , + ); + expect(screen.queryByText("No LLM adjudication participated in this connection.")).not.toBeInTheDocument(); + }); + + it("keeps the LLM channel visible when it participated", async () => { + render( + , + ); + await userEvent.click(screen.getByText(/fused score 0.780000/)); + expect(screen.getByText("LLM adjudication")).toBeInTheDocument(); + expect(screen.queryByText("No LLM adjudication participated in this connection.")).not.toBeInTheDocument(); + }); +}); diff --git a/frontend/src/LineageDag.tsx b/frontend/src/LineageDag.tsx index f082296f8..babb61794 100644 --- a/frontend/src/LineageDag.tsx +++ b/frontend/src/LineageDag.tsx @@ -1,4 +1,5 @@ -import type { LineageGraph } from "./api"; +import { useMemo, useState } from "react"; +import type { LineageChannelEvidence, LineageGraph, LineageGraphEdge } from "./api"; import { t, tf } from "./i18n"; import { layoutLineageDag } from "./lineageLayout"; @@ -6,6 +7,18 @@ function truncateLabel(label: string): string { return label.length > 34 ? `${label.slice(0, 33)}…` : label; } +function edgeKey(edge: LineageGraphEdge): string { + return `${edge.source}:${edge.target}`; +} + +function formatExact(value: number): string { + return value.toFixed(6); +} + +function llmParticipated(evidence: LineageChannelEvidence[]): boolean { + return evidence.some((item) => item.signal_code === "llm"); +} + export function LineageDag({ graph, onSelectPost, @@ -16,6 +29,12 @@ export function LineageDag({ currentPostId?: string; }) { const groups = layoutLineageDag(graph); + const labelById = useMemo( + () => Object.fromEntries(graph.nodes.map((node) => [node.id, node.label])), + [graph.nodes], + ); + const [selectedEdge, setSelectedEdge] = useState(null); + if (graph.nodes.length === 0) { return

{t("No reconstructed lineage yet. Rebuild after seeding posts.")}

; } @@ -45,11 +64,26 @@ export function LineageDag({ const to = byId[edge.target]; if (!from || !to) return null; const midX = (from.x + to.x) / 2; + const key = edgeKey(edge); + const selected = selectedEdge === key; return ( setSelectedEdge(key)} + onKeyDown={(event) => { + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + setSelectedEdge(key); + } + }} > {tf("{from} follows {to} ({score})", { @@ -98,6 +132,104 @@ export function LineageDag({ </figure> ); })} + <section className="lineage-edge-evidence" aria-label={t("Connection evidence")}> + <h3>{t("Connection evidence")}</h3> + <p> + {t("Each connection is inferred from independent signals. It is not a causal claim.")} + </p> + {graph.reconstruction ? ( + <dl className="lineage-rebuild-profile"> + <div> + <dt>{t("Reconstruction version")}</dt> + <dd>{graph.reconstruction.reconstruction_version}</dd> + </div> + <div> + <dt>{t("Generated at")}</dt> + <dd>{graph.reconstruction.generated_at}</dd> + </div> + <div> + <dt>{t("Active weight profile")}</dt> + <dd> + {graph.reconstruction.active_weights + .map((item) => `${t(signalLabel(item.signal_code))}: ${formatExact(item.signal_weight)}`) + .join(", ")} + </dd> + </div> + </dl> + ) : null} + {graph.edges.map((edge) => { + const key = edgeKey(edge); + const evidence = edge.channel_evidence ?? []; + const fromLabel = labelById[edge.source] ?? edge.source; + const toLabel = labelById[edge.target] ?? edge.target; + return ( + <details + key={key} + className="lineage-edge-evidence-item" + open + onToggle={(event) => { + const details = event.currentTarget; + if (details.open) { + setSelectedEdge(key); + } else if (selectedEdge === key) { + setSelectedEdge(null); + } + }} + > + <summary> + {tf("{from} follows {to}, fused score {score}", { + from: fromLabel, + to: toLabel, + score: formatExact(edge.fused_score), + })} + </summary> + {evidence.length > 0 && !llmParticipated(evidence) ? ( + <p>{t("No LLM adjudication participated in this connection.")}</p> + ) : null} + {evidence.length > 0 ? ( + <table> + <caption>{t("Connection evidence")}</caption> + <thead> + <tr> + <th scope="col">{t("Rank")}</th> + <th scope="col">{t("Signal")}</th> + <th scope="col">{t("Score")}</th> + <th scope="col">{t("Weight")}</th> + <th scope="col">{t("Contribution")}</th> + </tr> + </thead> + <tbody> + {evidence.map((item) => ( + <tr key={item.signal_code}> + <td>{item.rank}</td> + <td>{t(item.signal_label)}</td> + <td>{formatExact(item.score)}</td> + <td>{formatExact(item.weight)}</td> + <td>{formatExact(item.contribution)}</td> + </tr> + ))} + </tbody> + </table> + ) : null} + </details> + ); + })} + </section> </div> ); } + +function signalLabel(signalCode: string): string { + switch (signalCode) { + case "temporal": + return "Temporal proximity"; + case "secondary_key": + return "Secondary key match"; + case "text": + return "Text similarity"; + case "llm": + return "LLM adjudication"; + default: + return signalCode; + } +} diff --git a/frontend/src/api.ts b/frontend/src/api.ts index cd0141a32..68fe128f1 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -402,16 +402,35 @@ export interface LineageGraphNode { is_branch_point: boolean; } +export interface LineageChannelEvidence { + signal_code: string; + signal_label: string; + score: number; + weight: number; + contribution: number; + rank: number; +} + +export interface LineageRebuildProfile { + reconstruction_version: string; + generated_at: string; + min_fused_score: number; + candidate_window: number; + active_weights: { signal_code: string; signal_weight: number }[]; +} + export interface LineageGraphEdge { source: string; target: string; fused_score: number; + channel_evidence?: LineageChannelEvidence[]; } export interface LineageGraph { nodes: LineageGraphNode[]; edges: LineageGraphEdge[]; truncated?: boolean; + reconstruction?: LineageRebuildProfile | null; } export function fetchLineageGraph(accessToken: string, postId?: string): Promise<LineageGraph> { diff --git a/frontend/src/i18n.test.ts b/frontend/src/i18n.test.ts index 5a3afbfc3..6858bc5a2 100644 --- a/frontend/src/i18n.test.ts +++ b/frontend/src/i18n.test.ts @@ -36,6 +36,10 @@ describe("i18n", () => { "Page", "Answer", "Showing the first {shown} of {total} posts known at this cutoff.", + "Connection evidence", + "Each connection is inferred from independent signals. It is not a causal claim.", + "No LLM adjudication participated in this connection.", + "Temporal proximity", ] as const; it("supports the five product locales", () => { diff --git a/frontend/src/i18n.ts b/frontend/src/i18n.ts index 650acfca8..887212269 100644 --- a/frontend/src/i18n.ts +++ b/frontend/src/i18n.ts @@ -303,6 +303,26 @@ const TRANSLATIONS: Partial<Record<Locale, Record<string, string>>> = { "Open evidence: {title}": "근거 열기: {title}", "Open post: {label}": "글 열기: {label}", "{from} follows {to} ({score})": "{from}이(가) {to}을(를) 따름 ({score})", + "Connection evidence": "연결 근거", + "Each connection is inferred from independent signals. It is not a causal claim.": + "각 연결은 독립된 신호로부터 추론된 것이며, 인과 관계가 아닙니다.", + "No LLM adjudication participated in this connection.": + "이 연결에는 LLM 판정이 참여하지 않았습니다.", + "Open connection evidence: {from} to {to}": "연결 근거 열기: {from} → {to}", + "{from} follows {to}, fused score {score}": + "{from}이(가) {to}을(를) 따름, 융합 점수 {score}", + Signal: "신호", + Score: "점수", + Weight: "가중치", + Contribution: "기여", + Rank: "순위", + "Reconstruction version": "재구성 버전", + "Generated at": "생성 시각", + "Active weight profile": "사용한 가중치 프로필", + "Temporal proximity": "시간 근접성", + "Secondary key match": "보조 키 일치", + "Text similarity": "텍스트 유사도", + "LLM adjudication": "LLM 판정", "{label} — {date}": "{label} — {date}", "Corporate entity to reconstruct": "재구성할 법인", "Next action": "다음 작업", @@ -641,6 +661,24 @@ const TRANSLATIONS: Partial<Record<Locale, Record<string, string>>> = { "Open evidence: {title}": "打开证据：{title}", "Open post: {label}": "打开文章：{label}", "{from} follows {to} ({score})": "{from} 接续 {to}（{score}）", + "Connection evidence": "连接证据", + "Each connection is inferred from independent signals. It is not a causal claim.": + "每条连接均由独立信号推断得出，并非因果关系。", + "No LLM adjudication participated in this connection.": "此连接未使用 LLM 裁定。", + "Open connection evidence: {from} to {to}": "打开连接证据：{from} 至 {to}", + "{from} follows {to}, fused score {score}": "{from} 接续 {to}，融合分数 {score}", + Signal: "信号", + Score: "分数", + Weight: "权重", + Contribution: "贡献", + Rank: "排名", + "Reconstruction version": "重建版本", + "Generated at": "生成时间", + "Active weight profile": "所用权重配置", + "Temporal proximity": "时间接近", + "Secondary key match": "次级键匹配", + "Text similarity": "文本相似度", + "LLM adjudication": "LLM 裁定", "{label} — {date}": "{label} — {date}", "Corporate entity to reconstruct": "要重建的法人实体", "Next action": "下一步操作", @@ -979,6 +1017,26 @@ const TRANSLATIONS: Partial<Record<Locale, Record<string, string>>> = { "{group} lineage": "{group}の系譜", "Open post: {label}": "投稿を開く: {label}", "{from} follows {to} ({score})": "{from}は{to}に続く（{score}）", + "Connection evidence": "接続の根拠", + "Each connection is inferred from independent signals. It is not a causal claim.": + "各接続は独立した信号から推論されたものであり、因果関係ではありません。", + "No LLM adjudication participated in this connection.": + "この接続に LLM 判定は関与していません。", + "Open connection evidence: {from} to {to}": "接続の根拠を開く: {from} → {to}", + "{from} follows {to}, fused score {score}": + "{from}は{to}に続く、融合スコア {score}", + Signal: "信号", + Score: "スコア", + Weight: "重み", + Contribution: "寄与", + Rank: "順位", + "Reconstruction version": "再構成バージョン", + "Generated at": "生成日時", + "Active weight profile": "使用した重みプロファイル", + "Temporal proximity": "時間的近接", + "Secondary key match": "副次キー一致", + "Text similarity": "テキスト類似度", + "LLM adjudication": "LLM 判定", "{label} — {date}": "{label} — {date}", "Corporate entity to reconstruct": "再構成する法人", "Next action": "次の操作", @@ -1317,6 +1375,26 @@ const TRANSLATIONS: Partial<Record<Locale, Record<string, string>>> = { "{group} lineage": "Dòng sự kiện {group}", "Open post: {label}": "Mở bài viết: {label}", "{from} follows {to} ({score})": "{from} tiếp nối {to} ({score})", + "Connection evidence": "Bằng chứng liên kết", + "Each connection is inferred from independent signals. It is not a causal claim.": + "Mỗi liên kết được suy ra từ các tín hiệu độc lập. Đây không phải là quan hệ nhân quả.", + "No LLM adjudication participated in this connection.": + "Kết nối này không có sự tham gia của phán định LLM.", + "Open connection evidence: {from} to {to}": "Mở bằng chứng liên kết: {from} đến {to}", + "{from} follows {to}, fused score {score}": + "{from} tiếp nối {to}, điểm hợp nhất {score}", + Signal: "Tín hiệu", + Score: "Điểm", + Weight: "Trọng số", + Contribution: "Đóng góp", + Rank: "Hạng", + "Reconstruction version": "Phiên bản tái dựng", + "Generated at": "Thời điểm tạo", + "Active weight profile": "Hồ sơ trọng số đã dùng", + "Temporal proximity": "Gần về thời gian", + "Secondary key match": "Khớp khóa phụ", + "Text similarity": "Độ tương đồng văn bản", + "LLM adjudication": "Phán định LLM", "{label} — {date}": "{label} — {date}", "Corporate entity to reconstruct": "Pháp nhân cần tái dựng", "Next action": "Thao tác tiếp theo", diff --git a/frontend/src/lineageLayout.ts b/frontend/src/lineageLayout.ts index ceb1a54aa..9b32c92d2 100644 --- a/frontend/src/lineageLayout.ts +++ b/frontend/src/lineageLayout.ts @@ -92,6 +92,8 @@ export function subgraphForPost(graph: LineageGraph, postId: string): LineageGra return { nodes, edges: graph.edges.filter((edge) => ids.has(edge.source) && ids.has(edge.target)), + truncated: graph.truncated, + reconstruction: graph.reconstruction, }; } diff --git a/lineageweave/__init__.py b/lineageweave/__init__.py index 95330cb50..821377c54 100644 --- a/lineageweave/__init__.py +++ b/lineageweave/__init__.py @@ -11,7 +11,12 @@ from .corporate_hierarchy_resolution import resolve_corporate_entity from .entity_relationship_classification import OrganizationRelationship from .knowledge_graph import random_walk_with_restart, select_related_nodes -from .lineage_persistence import lineage_edge_specs +from .lineage_persistence import ( + CHANNEL_EVIDENCE_TOLERANCE, + lineage_edge_specs, + rank_channel_evidence, + reconstruction_version, +) from .models import Edge, Record, Tree from .post_chat import ChatAnswer, cited_post_summaries from .post_summary import PostSummary @@ -47,12 +52,15 @@ "Tree", "build_affiliate_forest", "cited_post_summaries", + "CHANNEL_EVIDENCE_TOLERANCE", "lineage_edge_specs", + "rank_channel_evidence", "random_walk_with_restart", "reconstruct", + "reconstruction_version", "resolve_corporate_entity", "select_related_nodes", "sentence_excerpts", ] -__version__ = "2.12.6" +__version__ = "2.14.0" diff --git a/lineageweave/lineage_persistence.py b/lineageweave/lineage_persistence.py index 8b9f4f16d..bcf51c11f 100644 --- a/lineageweave/lineage_persistence.py +++ b/lineageweave/lineage_persistence.py @@ -1,28 +1,62 @@ -"""Flatten ``reconstruct()`` trees into the rows ``post_lineage_edge`` stores. +"""Flatten ``reconstruct()`` trees into the rows Event Lineage persists. The reconstruction algorithm stays in ``reconstruct.py``. This module is -only the persistence contract: one ``Edge`` becomes one -``(parent_post_id, child_post_id, fused_score)`` row. Seed scripts and a -future rebuild endpoint share this so they cannot drift from what the -Event Lineage panel reads. +the persistence contract shared by seed scripts and the live rebuild +writer so they cannot drift from what the Event Lineage panel reads. + +Each parent→child edge is still one ``post_lineage_edge`` row. The +winning edge's active channel scores are persisted beside it as +``post_lineage_edge_signal`` rows (ADR 0124). A missing LLM channel is +dropped, never fabricated. Contribution is ``weight * score`` and must +reconcile with ``fused_score`` within :data:`CHANNEL_EVIDENCE_TOLERANCE`. """ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from decimal import ROUND_HALF_EVEN, Decimal from .adjudication_client import AdjudicationClient from .models import Edge, Record -from .reconstruct import reconstruct +from .reconstruct import ( + DEFAULT_CANDIDATE_WINDOW, + DEFAULT_CHANNEL_WEIGHTS, + DEFAULT_MIN_FUSED_SCORE, + reconstruct, +) + +RECONSTRUCTION_VERSION_PREFIX = "lineageweave.reconstruct" +CHANNEL_EVIDENCE_TOLERANCE = 1e-6 +SIGNAL_QUANTUM = Decimal("0.000001") + +LINEAGE_SIGNAL_ORDER = ("temporal", "secondary_key", "text", "llm") + +LINEAGE_SIGNAL_LOOKUP_CODES = { + "temporal": "lineage_signal_temporal", + "secondary_key": "lineage_signal_secondary_key", + "text": "lineage_signal_text", + "llm": "lineage_signal_llm", +} + +LINEAGE_SIGNAL_LABELS = { + "temporal": "Temporal proximity", + "secondary_key": "Secondary key match", + "text": "Text similarity", + "llm": "LLM adjudication", +} + +LOOKUP_CODE_TO_SIGNAL = {code: name for name, code in LINEAGE_SIGNAL_LOOKUP_CODES.items()} def lineage_edge_specs(records: Sequence[Record], *, llm: AdjudicationClient | None = None) -> list[Edge]: """Run reconstruct and return every resulting parent→child edge. - Callers persist these as ``post_lineage_edge`` rows. Record ids must - already be the ids the database will store (UUIDs for product posts, - fixture ids for the library-only demo) -- this function does not - invent or rewrite identifiers. + Callers persist these as ``post_lineage_edge`` rows plus matching + ``post_lineage_edge_signal`` rows. Record ids must already be the ids + the database will store (UUIDs for product posts, fixture ids for the + library-only demo) -- this function does not invent or rewrite + identifiers. ``llm`` defaults to ``None``, which ``reconstruct()`` treats as the unavailable :class:`~lineageweave.adjudication_client.NullAdjudicationClient` @@ -32,3 +66,190 @@ def lineage_edge_specs(records: Sequence[Record], *, llm: AdjudicationClient | N """ trees = reconstruct(list(records), llm=llm) return [edge for tree in trees for edge in tree.edges] + + +def reconstruction_version(package_version: str | None = None) -> str: + """Return the reconstruction identity stored on ``event_lineage_rebuild``. + + PostgreSQL remains the authority for live Event Lineage. This string + names the reconstruct implementation that produced the current graph + so a later rebuild cannot silently rewrite historic evidence. + """ + if package_version is None: + from lineageweave import __version__ as package_version + return f"{RECONSTRUCTION_VERSION_PREFIX}/{package_version}" + + +def quantize_signal_value(value: float) -> float: + """Quantize a score, weight, or contribution onto the persisted numeric scale.""" + quantized = Decimal(str(value)).quantize(SIGNAL_QUANTUM, rounding=ROUND_HALF_EVEN) + return float(quantized) + + +def weights_for_channel_scores(channel_scores: Mapping[str, float]) -> dict[str, float]: + """Return the normalized active weights implied by recorded channel scores. + + Channels absent from ``channel_scores`` (including ``llm`` when the + adjudication client was unavailable) are dropped and the remainder is + renormalized. This is the same rule ``reconstruct.active_weights`` + applies at fusion time. + """ + active = { + name: DEFAULT_CHANNEL_WEIGHTS[name] + for name in channel_scores + if name in DEFAULT_CHANNEL_WEIGHTS + } + total = sum(active.values()) + if total <= 0: + return {} + return {name: weight / total for name, weight in active.items()} + + +def default_no_llm_weights() -> dict[str, float]: + """Normalized default weights when the LLM channel did not participate.""" + return weights_for_channel_scores( + {name: 0.0 for name in LINEAGE_SIGNAL_ORDER if name != "llm"} + ) + + +@dataclass(frozen=True) +class LineageRebuildSpec: + """Rows one atomic Event Lineage rebuild writes besides the edge list.""" + + reconstruction_version: str + min_fused_score: float + candidate_window: int + channel_weights: tuple[tuple[str, float], ...] + signal_rows: tuple[dict[str, object], ...] + + +def channel_signal_rows( + edge: Edge, + weights: Mapping[str, float] | None = None, +) -> list[dict[str, object]]: + """Build persistable signal rows for one reconstructed edge. + + One row per active channel. The LLM channel is omitted when it did + not participate. ``signal_weight`` is the normalized active weight + actually used. ``signal_contribution`` is ``weight * score``. + + Raises: + ValueError: if recorded contributions do not reconcile with + ``edge.fused_score`` within :data:`CHANNEL_EVIDENCE_TOLERANCE`. + """ + active_weights = dict(weights) if weights is not None else weights_for_channel_scores(edge.channel_scores) + rows: list[dict[str, object]] = [] + contribution_sum = 0.0 + for channel in LINEAGE_SIGNAL_ORDER: + if channel not in edge.channel_scores or channel not in active_weights: + continue + score = quantize_signal_value(float(edge.channel_scores[channel])) + weight = quantize_signal_value(float(active_weights[channel])) + contribution = quantize_signal_value(float(active_weights[channel]) * float(edge.channel_scores[channel])) + contribution_sum += contribution + rows.append( + { + "parent_post_id": edge.parent_id, + "child_post_id": edge.child_id, + "signal_code": LINEAGE_SIGNAL_LOOKUP_CODES[channel], + "channel_name": channel, + "signal_score": score, + "signal_weight": weight, + "signal_contribution": contribution, + } + ) + residual = abs(contribution_sum - float(edge.fused_score)) + if rows and residual > CHANNEL_EVIDENCE_TOLERANCE: + raise ValueError( + f"channel contributions {contribution_sum} do not reconcile with " + f"fused_score {edge.fused_score} (tolerance {CHANNEL_EVIDENCE_TOLERANCE})" + ) + return rows + + +def rank_channel_evidence(rows: Sequence[Mapping[str, object]]) -> list[dict[str, object]]: + """Project persisted signal rows onto the additive API collection. + + Ordering is contribution descending, then the controlled signal order + ``temporal``, ``secondary_key``, ``text``, ``llm``. Rank is 1-based. + The payload never includes prompts, responses, credentials, or source + text. + """ + + def sort_key(row: Mapping[str, object]) -> tuple[float, int]: + channel = _channel_name(row) + order = LINEAGE_SIGNAL_ORDER.index(channel) if channel in LINEAGE_SIGNAL_ORDER else len(LINEAGE_SIGNAL_ORDER) + return (-float(row["signal_contribution"]), order) + + evidence: list[dict[str, object]] = [] + for rank, row in enumerate(sorted(rows, key=sort_key), start=1): + channel = _channel_name(row) + label = str(row["signal_label"]) if row.get("signal_label") else LINEAGE_SIGNAL_LABELS.get(channel, channel) + evidence.append( + { + "signal_code": channel, + "signal_label": label, + "score": float(row["signal_score"]), + "weight": float(row["signal_weight"]), + "contribution": float(row["signal_contribution"]), + "rank": rank, + } + ) + return evidence + + +def llm_participated(evidence: Sequence[Mapping[str, object]]) -> bool: + """Return whether the optional LLM channel is present in ``evidence``.""" + return any(item.get("signal_code") == "llm" for item in evidence) + + +def lineage_rebuild_spec( + edges: Sequence[Edge], + *, + weights: Mapping[str, float] | None = None, + min_fused_score: float = DEFAULT_MIN_FUSED_SCORE, + candidate_window: int = DEFAULT_CANDIDATE_WINDOW, + package_version: str | None = None, +) -> LineageRebuildSpec: + """Assemble the rebuild metadata and signal rows for ``edges``. + + ``weights`` defaults to the normalized active weights implied by the + first edge that recorded channel scores, or the no-LLM default when + the rebuild produced no edges. Every signal row uses that same + profile so a later audit can see the weights that actually fused the + graph. + """ + active_weights = dict(weights) if weights is not None else {} + if not active_weights: + for edge in edges: + inferred = weights_for_channel_scores(edge.channel_scores) + if inferred: + active_weights = inferred + break + if not active_weights: + active_weights = default_no_llm_weights() + + signal_rows: list[dict[str, object]] = [] + for edge in edges: + signal_rows.extend(channel_signal_rows(edge, active_weights)) + + ordered_weights = tuple( + (LINEAGE_SIGNAL_LOOKUP_CODES[name], quantize_signal_value(active_weights[name])) + for name in LINEAGE_SIGNAL_ORDER + if name in active_weights + ) + return LineageRebuildSpec( + reconstruction_version=reconstruction_version(package_version), + min_fused_score=min_fused_score, + candidate_window=candidate_window, + channel_weights=ordered_weights, + signal_rows=tuple(signal_rows), + ) + + +def _channel_name(row: Mapping[str, object]) -> str: + stored = row.get("channel_name") + if isinstance(stored, str) and stored: + return stored + lookup = str(row.get("signal_code") or "") + return LOOKUP_CODE_TO_SIGNAL.get(lookup, lookup) diff --git a/migrations/0105_post_lineage_edge_signal.sql b/migrations/0105_post_lineage_edge_signal.sql new file mode 100644 index 000000000..826cd6368 --- /dev/null +++ b/migrations/0105_post_lineage_edge_signal.sql @@ -0,0 +1,57 @@ +-- ADR 0124: persist Event Lineage channel evidence beside each fused edge. +-- lookup_code is globally unique, so signal codes are prefixed. +-- CREATE IF NOT EXISTS / ON CONFLICT so migrate.sh replay is idempotent. + +insert into common_lookup_value (lookup_category, lookup_code, lookup_label, display_order) values + ('lineage_signal', 'lineage_signal_temporal', 'Temporal proximity', 0), + ('lineage_signal', 'lineage_signal_secondary_key', 'Secondary key match', 1), + ('lineage_signal', 'lineage_signal_text', 'Text similarity', 2), + ('lineage_signal', 'lineage_signal_llm', 'LLM adjudication', 3) +on conflict (lookup_code) do nothing; + +create table if not exists event_lineage_rebuild ( + rebuild_lock boolean primary key default true check (rebuild_lock), + reconstruction_version text not null, + generated_at timestamptz not null, + min_fused_score numeric(8,6) not null, + candidate_window integer not null, + check (min_fused_score >= 0 and min_fused_score <= 1), + check (candidate_window >= 1) +); + +comment on table event_lineage_rebuild is + 'Singleton identity of the live Event Lineage rebuild; replaced atomically.'; + +create table if not exists event_lineage_rebuild_channel ( + rebuild_lock boolean not null default true + references event_lineage_rebuild (rebuild_lock) on delete cascade, + signal_code text not null references common_lookup_value (lookup_code), + signal_weight numeric(8,6) not null, + primary key (rebuild_lock, signal_code), + check (signal_weight > 0 and signal_weight <= 1) +); + +comment on table event_lineage_rebuild_channel is + 'Normalized active channel weights used by the live Event Lineage rebuild.'; + +create table if not exists post_lineage_edge_signal ( + parent_post_id uuid not null, + child_post_id uuid not null, + signal_code text not null references common_lookup_value (lookup_code), + signal_score numeric(8,6) not null, + signal_weight numeric(8,6) not null, + signal_contribution numeric(8,6) not null, + primary key (parent_post_id, child_post_id, signal_code), + foreign key (parent_post_id, child_post_id) + references post_lineage_edge (parent_post_id, child_post_id) + on delete cascade, + check (signal_score >= 0 and signal_score <= 1), + check (signal_weight > 0 and signal_weight <= 1), + check (signal_contribution >= 0 and signal_contribution <= 1) +); + +comment on table post_lineage_edge_signal is + 'Per-channel score, active weight, and contribution for one reconstructed lineage edge.'; + +create index if not exists post_lineage_edge_signal_child_idx + on post_lineage_edge_signal (child_post_id, parent_post_id); diff --git a/migrations/rollback/0105_post_lineage_edge_signal.sql b/migrations/rollback/0105_post_lineage_edge_signal.sql new file mode 100644 index 000000000..6e484775b --- /dev/null +++ b/migrations/rollback/0105_post_lineage_edge_signal.sql @@ -0,0 +1,10 @@ +drop table if exists post_lineage_edge_signal; +drop table if exists event_lineage_rebuild_channel; +drop table if exists event_lineage_rebuild; +delete from common_lookup_value + where lookup_code in ( + 'lineage_signal_temporal', + 'lineage_signal_secondary_key', + 'lineage_signal_text', + 'lineage_signal_llm' + ); diff --git a/pyproject.toml b/pyproject.toml index cb4be2916..e7d731270 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lineageweave" -version = "2.12.6" +version = "2.14.0" description = "Reconstructs git-branch-style lineage DAGs from scattered short records using multi-channel score fusion and LLM adjudication." readme = "README.md" license = { text = "MIT" } diff --git a/scripts/seed_demo_data.py b/scripts/seed_demo_data.py index 8ec15a065..070561324 100644 --- a/scripts/seed_demo_data.py +++ b/scripts/seed_demo_data.py @@ -503,7 +503,7 @@ def _seed_reconstructed_lineage(cur, author_account_id, corporate_entity_id, pro reconstruct() already knows the A-100 fork. """ from lineageweave.fixtures import sample_records - from lineageweave.lineage_persistence import lineage_edge_specs + from lineageweave.lineage_persistence import lineage_edge_specs, lineage_rebuild_spec records = sample_records() cur.execute("select 1 from source_post where post_title = %s", (records[0].label,)) @@ -513,12 +513,41 @@ def _seed_reconstructed_lineage(cur, author_account_id, corporate_entity_id, pro persisted = insert_fixture_source_posts( cur, author_account_id, corporate_entity_id, process_unit_id ) - for edge in lineage_edge_specs(persisted): + edges = lineage_edge_specs(persisted) + spec = lineage_rebuild_spec(edges) + cur.execute("delete from event_lineage_rebuild") + cur.execute( + "insert into event_lineage_rebuild " + "(rebuild_lock, reconstruction_version, generated_at, min_fused_score, candidate_window) " + "values (true, %s, now(), %s, %s)", + (spec.reconstruction_version, spec.min_fused_score, spec.candidate_window), + ) + for signal_code, signal_weight in spec.channel_weights: + cur.execute( + "insert into event_lineage_rebuild_channel " + "(rebuild_lock, signal_code, signal_weight) values (true, %s, %s)", + (signal_code, signal_weight), + ) + for edge in edges: cur.execute( "insert into post_lineage_edge (parent_post_id, child_post_id, fused_score) " "values (%s, %s, %s) on conflict do nothing", (edge.parent_id, edge.child_id, edge.fused_score), ) + for row in spec.signal_rows: + cur.execute( + "insert into post_lineage_edge_signal " + "(parent_post_id, child_post_id, signal_code, signal_score, signal_weight, signal_contribution) " + "values (%s, %s, %s, %s, %s, %s) on conflict do nothing", + ( + row["parent_post_id"], + row["child_post_id"], + row["signal_code"], + row["signal_score"], + row["signal_weight"], + row["signal_contribution"], + ), + ) def _write_post_summary(cur, post_id, summary) -> None: diff --git a/tests/test_lineage_channel_evidence.py b/tests/test_lineage_channel_evidence.py new file mode 100644 index 000000000..b69a7836f --- /dev/null +++ b/tests/test_lineage_channel_evidence.py @@ -0,0 +1,143 @@ +"""Event Lineage channel evidence must round-trip without fabricating LLM scores.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from lineageweave.fixtures import sample_records +from lineageweave.lineage_persistence import ( + CHANNEL_EVIDENCE_TOLERANCE, + LINEAGE_SIGNAL_LOOKUP_CODES, + channel_signal_rows, + default_no_llm_weights, + lineage_edge_specs, + lineage_rebuild_spec, + llm_participated, + rank_channel_evidence, + reconstruction_version, + weights_for_channel_scores, +) +from lineageweave.models import Edge + +_ROOT = Path(__file__).resolve().parents[1] + + +def _four_channel_edge() -> Edge: + scores = {"temporal": 0.8, "secondary_key": 1.0, "text": 0.5, "llm": 0.9} + weights = {"temporal": 0.15, "secondary_key": 0.15, "text": 0.30, "llm": 0.40} + fused = sum(weights[name] * scores[name] for name in scores) + return Edge("parent-a", "child-b", fused, scores) + + +def _no_llm_edge() -> Edge: + scores = {"temporal": 0.8, "secondary_key": 1.0, "text": 0.5} + weights = default_no_llm_weights() + fused = sum(weights[name] * scores[name] for name in scores) + return Edge("parent-a", "child-b", fused, scores) + + +def test_four_channel_edge_round_trips_scores_and_normalized_weights() -> None: + edge = _four_channel_edge() + rows = channel_signal_rows(edge) + codes = [row["channel_name"] for row in rows] + assert codes == ["temporal", "secondary_key", "text", "llm"] + by_name = {row["channel_name"]: row for row in rows} + assert by_name["llm"]["signal_code"] == "lineage_signal_llm" + assert by_name["temporal"]["signal_weight"] == pytest.approx(0.15) + assert by_name["text"]["signal_weight"] == pytest.approx(0.30) + evidence = rank_channel_evidence(rows) + assert [item["rank"] for item in evidence] == [1, 2, 3, 4] + assert evidence[0]["signal_code"] == "llm" + assert sum(item["contribution"] for item in evidence) == pytest.approx(edge.fused_score) + + +def test_no_llm_reconstruction_persists_exactly_three_channels() -> None: + edge = _no_llm_edge() + rows = channel_signal_rows(edge) + assert [row["channel_name"] for row in rows] == ["temporal", "secondary_key", "text"] + assert "llm" not in {row["channel_name"] for row in rows} + assert "lineage_signal_llm" not in {row["signal_code"] for row in rows} + evidence = rank_channel_evidence(rows) + assert llm_participated(evidence) is False + weights = weights_for_channel_scores(edge.channel_scores) + assert weights == pytest.approx({"temporal": 0.25, "secondary_key": 0.25, "text": 0.5}) + + +def test_contributions_reconcile_to_fused_score_within_tolerance() -> None: + edge = _four_channel_edge() + rows = channel_signal_rows(edge) + residual = abs(sum(float(row["signal_contribution"]) for row in rows) - edge.fused_score) + assert residual <= CHANNEL_EVIDENCE_TOLERANCE + + +def test_mismatched_fused_score_is_rejected() -> None: + edge = Edge("parent-a", "child-b", 0.99, {"temporal": 0.1, "secondary_key": 0.1, "text": 0.1}) + with pytest.raises(ValueError, match="do not reconcile"): + channel_signal_rows(edge) + + +def test_fixture_reconstruction_never_fabricates_llm() -> None: + edges = lineage_edge_specs(sample_records()) + assert edges + spec = lineage_rebuild_spec(edges) + assert all(row["channel_name"] != "llm" for row in spec.signal_rows) + assert "lineage_signal_llm" not in {code for code, _weight in spec.channel_weights} + for edge in edges: + rows = channel_signal_rows(edge) + residual = abs(sum(float(row["signal_contribution"]) for row in rows) - edge.fused_score) + assert residual <= CHANNEL_EVIDENCE_TOLERANCE + + +def test_rebuild_spec_is_idempotent_for_the_same_edges() -> None: + edges = lineage_edge_specs(sample_records()) + first = lineage_rebuild_spec(edges, package_version="2.14.0") + second = lineage_rebuild_spec(edges, package_version="2.14.0") + assert first == second + assert first.reconstruction_version == reconstruction_version("2.14.0") + assert first.reconstruction_version == "lineageweave.reconstruct/2.14.0" + + +def test_rank_is_contribution_then_controlled_signal_order() -> None: + rows = [ + { + "channel_name": "text", + "signal_code": LINEAGE_SIGNAL_LOOKUP_CODES["text"], + "signal_score": 1.0, + "signal_weight": 0.5, + "signal_contribution": 0.2, + }, + { + "channel_name": "temporal", + "signal_code": LINEAGE_SIGNAL_LOOKUP_CODES["temporal"], + "signal_score": 1.0, + "signal_weight": 0.25, + "signal_contribution": 0.2, + }, + { + "channel_name": "secondary_key", + "signal_code": LINEAGE_SIGNAL_LOOKUP_CODES["secondary_key"], + "signal_score": 0.4, + "signal_weight": 0.25, + "signal_contribution": 0.1, + }, + ] + evidence = rank_channel_evidence(rows) + assert [item["signal_code"] for item in evidence] == ["temporal", "text", "secondary_key"] + assert [item["rank"] for item in evidence] == [1, 2, 3] + + +def test_migrate_sh_replays_channel_evidence_and_tenant_settings() -> None: + migrate = (_ROOT / "docker/postgres-init/migrate.sh").read_text() + assert "0103_*" in migrate + assert "0104_*" in migrate + assert "0105_*" in migrate + + +def test_channel_evidence_migration_has_no_jsonb() -> None: + migration = (_ROOT / "migrations" / "0105_post_lineage_edge_signal.sql").read_text() + assert "jsonb" not in migration.casefold() + assert "post_lineage_edge_signal" in migration + assert "event_lineage_rebuild" in migration + assert "on delete cascade" in migration.casefold() diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index de8f289c6..4f131d0a4 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from backend.app.lineage_ingestion import ( + persist_lineage_edges, reconstruct_group_key, records_from_source_posts, visible_lineage_graph, @@ -133,7 +134,13 @@ class FakeConnection: {"parent_post_id": "post-a", "child_post_id": "post-b", "fused_score": 0.8} ] - async def fetch(self, query: str): + async def fetch(self, query: str, *_args): + if "post_lineage_edge_signal" in query: + return getattr(self, "signals", []) + if "event_lineage_rebuild_channel" in query: + return getattr(self, "rebuild_channels", []) + if "event_lineage_rebuild" in query: + return getattr(self, "rebuilds", []) return self.edges if "post_lineage_edge" in query else self.posts connection = FakeConnection() @@ -149,4 +156,214 @@ async def fetch(self, query: str): assert {node["id"] for node in focused["nodes"]} == {"post-a", "post-b"} assert len(focused["edges"]) == 1 assert focused["truncated"] is False - assert isolated == {"nodes": [], "edges": [], "truncated": False} + assert isolated["nodes"] == [] + assert isolated["edges"] == [] + assert isolated["truncated"] is False + assert isolated["reconstruction"] is None + assert focused["edges"][0]["channel_evidence"] == [] + + +class _RecordingConnection: + def __init__(self) -> None: + self.statements: list[tuple[str, tuple]] = [] + + async def execute(self, query: str, *args): + self.statements.append((query, args)) + + async def fetch(self, query: str, *_args): + return [] + + +def test_visible_graph_attaches_ranked_channel_evidence() -> None: + class FakeConnection: + posts = [ + { + "post_id": "post-a", + "post_title": "A", + "voc_type_code": "voc", + "visibility_code": "public", + "corporate_entity_id": "corp", + "process_unit_id": "pu", + "thread_group_key": "thread-a", + "created_at": datetime(2026, 1, 1), + }, + { + "post_id": "post-b", + "post_title": "B", + "voc_type_code": "voc", + "visibility_code": "public", + "corporate_entity_id": "corp", + "process_unit_id": "pu", + "thread_group_key": "thread-a", + "created_at": datetime(2026, 1, 2), + }, + ] + edges = [{"parent_post_id": "post-a", "child_post_id": "post-b", "fused_score": 0.7}] + signals = [ + { + "parent_post_id": "post-a", + "child_post_id": "post-b", + "signal_code": "lineage_signal_text", + "signal_score": 0.5, + "signal_weight": 0.5, + "signal_contribution": 0.25, + }, + { + "parent_post_id": "post-a", + "child_post_id": "post-b", + "signal_code": "lineage_signal_temporal", + "signal_score": 0.8, + "signal_weight": 0.25, + "signal_contribution": 0.2, + }, + { + "parent_post_id": "post-a", + "child_post_id": "post-b", + "signal_code": "lineage_signal_secondary_key", + "signal_score": 1.0, + "signal_weight": 0.25, + "signal_contribution": 0.25, + }, + ] + rebuilds = [ + { + "reconstruction_version": "lineageweave.reconstruct/2.14.0", + "generated_at": datetime(2026, 8, 21, 12, 0, 0), + "min_fused_score": 0.3, + "candidate_window": 50, + } + ] + rebuild_channels = [ + {"signal_code": "lineage_signal_temporal", "signal_weight": 0.25}, + {"signal_code": "lineage_signal_text", "signal_weight": 0.5}, + ] + + async def fetch(self, query: str, *_args): + if "post_lineage_edge_signal" in query: + return self.signals + if "event_lineage_rebuild_channel" in query: + return self.rebuild_channels + if "event_lineage_rebuild" in query: + return self.rebuilds + return self.edges if "post_lineage_edge" in query else self.posts + + graph = asyncio.run(visible_lineage_graph(FakeConnection(), lambda row: True)) + evidence = graph["edges"][0]["channel_evidence"] + assert [item["signal_code"] for item in evidence] == ["secondary_key", "text", "temporal"] + assert [item["rank"] for item in evidence] == [1, 2, 3] + assert "llm" not in {item["signal_code"] for item in evidence} + assert graph["reconstruction"]["reconstruction_version"] == "lineageweave.reconstruct/2.14.0" + assert graph["reconstruction"]["active_weights"][0]["signal_code"] == "temporal" + + +def test_abac_never_reveals_channel_evidence_for_an_invisible_endpoint() -> None: + class FakeConnection: + posts = [ + { + "post_id": "post-public", + "post_title": "Public", + "voc_type_code": "voc", + "visibility_code": "public", + "corporate_entity_id": "corp", + "process_unit_id": "pu", + "thread_group_key": "thread-a", + "created_at": datetime(2026, 1, 1), + }, + { + "post_id": "post-secret", + "post_title": "Secret", + "voc_type_code": "voc", + "visibility_code": "restricted", + "corporate_entity_id": "corp", + "process_unit_id": "pu", + "thread_group_key": "thread-a", + "created_at": datetime(2026, 1, 2), + }, + ] + edges = [ + {"parent_post_id": "post-public", "child_post_id": "post-secret", "fused_score": 0.8} + ] + signals = [ + { + "parent_post_id": "post-public", + "child_post_id": "post-secret", + "signal_code": "lineage_signal_text", + "signal_score": 0.9, + "signal_weight": 0.5, + "signal_contribution": 0.45, + } + ] + rebuilds = [] + rebuild_channels = [] + + async def fetch(self, query: str, *_args): + if "post_lineage_edge_signal" in query: + return self.signals + if "event_lineage_rebuild_channel" in query: + return self.rebuild_channels + if "event_lineage_rebuild" in query: + return self.rebuilds + return self.edges if "post_lineage_edge" in query else self.posts + + graph = asyncio.run( + visible_lineage_graph(FakeConnection(), lambda row: row["post_id"] == "post-public") + ) + assert [node["id"] for node in graph["nodes"]] == ["post-public"] + assert graph["edges"] == [] + serialized = str(graph) + assert "post-secret" not in serialized + assert "0.45" not in serialized + assert "lineage_signal_text" not in serialized + + +def test_persist_lineage_edges_replaces_signals_atomically_without_llm() -> None: + from lineageweave.lineage_persistence import lineage_rebuild_spec + from lineageweave.models import Edge + + scores = {"temporal": 0.8, "secondary_key": 1.0, "text": 0.5} + weights = {"temporal": 0.25, "secondary_key": 0.25, "text": 0.5} + fused = sum(weights[name] * scores[name] for name in scores) + edge = Edge( + "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", + fused, + scores, + ) + connection = _RecordingConnection() + asyncio.run(persist_lineage_edges(connection, [edge])) + statements = [sql.casefold() for sql, _args in connection.statements] + assert statements[0].startswith("delete from post_lineage_edge") + assert any("delete from event_lineage_rebuild" in sql for sql in statements) + assert any("insert into post_lineage_edge_signal" in sql for sql in statements) + inserted_codes = [ + args[2] + for sql, args in connection.statements + if "insert into post_lineage_edge_signal" in sql.casefold() + ] + assert inserted_codes == [ + "lineage_signal_temporal", + "lineage_signal_secondary_key", + "lineage_signal_text", + ] + spec = lineage_rebuild_spec([edge], package_version="2.14.0") + assert spec.reconstruction_version == "lineageweave.reconstruct/2.14.0" + + +def test_duplicate_rebuild_replays_the_same_delete_insert_sequence() -> None: + from lineageweave.models import Edge + + scores = {"temporal": 0.8, "secondary_key": 1.0, "text": 0.5} + weights = {"temporal": 0.25, "secondary_key": 0.25, "text": 0.5} + fused = sum(weights[name] * scores[name] for name in scores) + edge = Edge( + "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", + fused, + scores, + ) + first = _RecordingConnection() + second = _RecordingConnection() + asyncio.run(persist_lineage_edges(first, [edge])) + asyncio.run(persist_lineage_edges(second, [edge])) + assert [sql for sql, _args in first.statements] == [sql for sql, _args in second.statements] + assert [args for _sql, args in first.statements] == [args for _sql, args in second.statements] diff --git a/tests/test_schema.py b/tests/test_schema.py index 1e2c708a3..b58d04c6a 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -38,10 +38,8 @@ / "migrations" / "0101_project_bound_major_event_action.sql" ) -_PROJECT_BOUND_EVENT_MIGRATION = ( - Path(__file__).resolve().parents[1] - / "migrations" - / "0102_project_bound_summary_event.sql" +_CHANNEL_EVIDENCE_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0105_post_lineage_edge_signal.sql" ) @@ -79,6 +77,7 @@ def schema_db(): cur.execute(_MAJOR_EVENT_ACTION_MIGRATION.read_text()) cur.execute(_PROJECT_BOUND_ACTION_MIGRATION.read_text()) cur.execute(_PROJECT_BOUND_EVENT_MIGRATION.read_text()) + cur.execute(_CHANNEL_EVIDENCE_MIGRATION.read_text()) conn.commit() yield conn finally: @@ -116,6 +115,9 @@ def test_migration_applies_cleanly(schema_db) -> None: "knowledge_graph_edge_evidence", "issue_ticket", "post_lineage_edge", + "post_lineage_edge_signal", + "event_lineage_rebuild", + "event_lineage_rebuild_channel", "post_evaluation_response", "report_period_score", "report_member_score", @@ -172,6 +174,55 @@ def test_leftover_pair_references_member_and_item_rows(schema_db) -> None: assert "report_period_score" in targets +def test_lineage_channel_evidence_is_cascaded_and_lookup_controlled(schema_db) -> None: + """Signal rows cannot outlive their edge or name an unknown channel.""" + with schema_db.cursor() as cur: + cur.execute( + """ + select pg_get_constraintdef(oid) + from pg_constraint + where conrelid = 'post_lineage_edge_signal'::regclass + order by conname + """ + ) + definitions = " ".join(row[0].casefold() for row in cur.fetchall()) + assert "references post_lineage_edge" in definitions + assert "on delete cascade" in definitions + assert "references common_lookup_value" in definitions + cur.execute( + "select lookup_code from common_lookup_value " + "where lookup_category = 'lineage_signal' order by display_order" + ) + assert [row[0] for row in cur.fetchall()] == [ + "lineage_signal_temporal", + "lineage_signal_secondary_key", + "lineage_signal_text", + "lineage_signal_llm", + ] + cur.execute( + """ + select data_type, numeric_precision, numeric_scale + from information_schema.columns + where table_name = 'post_lineage_edge_signal' + and column_name in ('signal_score', 'signal_weight', 'signal_contribution') + """ + ) + for data_type, precision, scale in cur.fetchall(): + assert data_type == "numeric" + assert precision == 8 + assert scale == 6 + cur.execute( + """ + select column_name from information_schema.columns + where table_name in ( + 'post_lineage_edge_signal', + 'event_lineage_rebuild', + 'event_lineage_rebuild_channel' + ) and data_type = 'jsonb' + """ + ) + assert cur.fetchall() == [] + def test_corporate_hierarchy_recursive_query_returns_correct_shape(schema_db) -> None: """The real product requirement: 'Acme Group -> Acme Electronics Korea diff --git a/uv.lock b/uv.lock index 10bcf9ff1..eed84dc7a 100644 --- a/uv.lock +++ b/uv.lock @@ -454,7 +454,7 @@ wheels = [ [[package]] name = "lineageweave" -version = "2.12.6" +version = "2.14.0" source = { editable = "." } dependencies = [ { name = "certifi" }, From 780fd57af15b9e433c80cfeaeb4c6defa76413bb Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 01:53:20 +0900 Subject: [PATCH 02/43] test(schema): retain project event migration fixture --- tests/test_schema.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_schema.py b/tests/test_schema.py index b58d04c6a..c220e7982 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -38,6 +38,11 @@ / "migrations" / "0101_project_bound_major_event_action.sql" ) +_PROJECT_BOUND_EVENT_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0102_project_bound_summary_event.sql" +) _CHANNEL_EVIDENCE_MIGRATION = ( Path(__file__).resolve().parents[1] / "migrations" / "0105_post_lineage_edge_signal.sql" ) From cd7044a3f7e077a16c58cf76b327e3a6c7867be4 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 01:57:03 +0900 Subject: [PATCH 03/43] fix(frontend): keep admin panel behind authentication --- frontend/src/App.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6fba0dd41..666888a4d 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -101,7 +101,6 @@ import { tf, useLocale, } from "./i18n"; -import { rememberOidcReturnUrl, returnUrlFromLocation } from "./oidcReturnUrl"; import "./App.css"; function orchestratorUnavailableMessage(err: unknown, action: string): string { @@ -4620,7 +4619,6 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean <small>Enterprise SSO Authentication</small> </div> </div> - {destination === "admin" ? <AdminPanel currentBrandName={brandName} onBrandNameChange={setBrandName} accessToken={accessToken} /> : null} </main> <footer className="app-footer" role="contentinfo"> <div className="app-footer-title"> From c40d3a8e36834dabd96213aa9a7d3bad22a3ca05 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 01:56:50 +0900 Subject: [PATCH 04/43] test: apply lineage evidence migration to API fixtures --- backend/tests/test_api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 438b4786a..8d5c5533a 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -113,6 +113,11 @@ / "migrations" / "0102_project_bound_summary_event.sql" ) +_CHANNEL_EVIDENCE_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0105_post_lineage_edge_signal.sql" +) def _postgres_available() -> bool: @@ -226,6 +231,7 @@ def seeded_db(demo_analyst_token): cur.execute(_MAJOR_EVENT_ACTION_MIGRATION.read_text()) cur.execute(_PROJECT_BOUND_ACTION_MIGRATION.read_text()) cur.execute(_PROJECT_BOUND_EVENT_MIGRATION.read_text()) + cur.execute(_CHANNEL_EVIDENCE_MIGRATION.read_text()) cur.execute( "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values " "('corporate_entity_level', 'group', 'Group'), " From 10cf59eef692d16a8e828b799f7e859251c09c00 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 02:02:22 +0900 Subject: [PATCH 05/43] feat: route rebuild adjudication through orchestrator --- backend/app/lineage_ingestion.py | 19 +++++++++++++++---- backend/app/main.py | 2 +- tests/test_lineage_ingestion.py | 25 +++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index b2ee707a9..4e7878324 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -17,6 +17,7 @@ import asyncpg from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL +from lineageweave.adjudication_client import AdjudicationClient from lineageweave.lineage_persistence import ( LOOKUP_CODE_TO_SIGNAL, lineage_edge_specs, @@ -114,14 +115,23 @@ async def persist_lineage_edges(conn: asyncpg.Connection, edges: list[Edge]) -> ) -async def rebuild_lineage(conn: asyncpg.Connection) -> list[Edge]: - """Reconstruct lineage for every ``source_post`` and persist the edges.""" +async def rebuild_lineage( + conn: asyncpg.Connection, + *, + llm: AdjudicationClient | None = None, +) -> list[Edge]: + """Reconstruct lineage for every ``source_post`` and persist the edges. + + A configured contextual-orchestrator client is passed through so the + optional LLM channel is recorded when available; ``None`` preserves the + fail-closed three-channel rebuild. + """ rows = await conn.fetch( "select post_id, post_title, voc_type_code, created_at, corporate_entity_id, " "process_unit_id, thread_group_key, secondary_grouping_key " f"from source_post where {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}" ) - edges = lineage_edge_specs(records_from_source_posts(rows)) + edges = lineage_edge_specs(records_from_source_posts(rows), llm=llm) await persist_lineage_edges(conn, edges) return edges @@ -163,7 +173,8 @@ async def visible_lineage_graph( "from event_lineage_rebuild" ) weight_rows = await conn.fetch( - "select signal_code, signal_weight from event_lineage_rebuild_channel" + "select signal_code, signal_weight from event_lineage_rebuild_channel " + "order by signal_code" ) if focus_post_id is None: diff --git a/backend/app/main.py b/backend/app/main.py index fb943315f..413b4f024 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1112,7 +1112,7 @@ async def rebuild_lineage_graph( _require_post_admin(account) async with pool.acquire() as conn: async with conn.transaction(): - edges = await rebuild_lineage(conn) + edges = await rebuild_lineage(conn, llm=_adjudication_client()) return {"edge_count": len(edges)} diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index 4f131d0a4..104b1d466 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from backend.app.lineage_ingestion import ( + rebuild_lineage, persist_lineage_edges, reconstruct_group_key, records_from_source_posts, @@ -34,6 +35,30 @@ def test_records_use_persisted_thread_keys_not_process_unit_or_voc_type() -> Non assert records[0].occurred_at.tzinfo is None +def test_rebuild_passes_the_configured_adjudication_client(monkeypatch) -> None: + class FakeConnection: + async def fetch(self, _query: str, *_args): + return [] + + client = object() + captured: dict[str, object] = {} + + def fake_lineage_edge_specs(_records, *, llm=None): + captured["llm"] = llm + return [] + + async def fake_persist_lineage_edges(_conn, _edges): + return None + + import backend.app.lineage_ingestion as ingestion + + monkeypatch.setattr(ingestion, "lineage_edge_specs", fake_lineage_edge_specs) + monkeypatch.setattr(ingestion, "persist_lineage_edges", fake_persist_lineage_edges) + asyncio.run(rebuild_lineage(FakeConnection(), llm=client)) + + assert captured["llm"] is client + + def test_records_fall_back_to_corporate_entity_when_thread_keys_are_empty() -> None: rows = [ { From 2c47b353bd71dba78b720edf14bf5708c314f4a0 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 02:05:31 +0900 Subject: [PATCH 06/43] fix: preserve adjudication during PostgreSQL imports --- backend/app/main.py | 8 ++++---- scripts/import_postgresql_posts.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/backend/app/main.py b/backend/app/main.py index 413b4f024..cb0186e7d 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -332,10 +332,10 @@ def _adjudication_client(): reconstruct.py's DEFAULT_CHANNEL_WEIGHTS gives this channel the most weight (0.40) of the four -- it is the only one that reasons about - content instead of approximating it (ADR 0064) -- but nothing ever - passed a real client through lineage_edge_specs() to reconstruct(), - so every lineage reconstruction had silently run on the weaker - 3-channel fallback since the feature was built. + content instead of approximating it (ADR 0064). The same client is + shared by analysis-run reconstruction and the administrator-triggered + live graph rebuild; an unconfigured gateway remains an explicit + three-channel fallback. """ settings = load_settings() if not (settings.orchestrator_base_url and settings.orchestrator_api_key): diff --git a/scripts/import_postgresql_posts.py b/scripts/import_postgresql_posts.py index e6ccf5449..5e62b2608 100644 --- a/scripts/import_postgresql_posts.py +++ b/scripts/import_postgresql_posts.py @@ -27,6 +27,10 @@ sys.path.insert(0, str(REPOSITORY_ROOT)) from backend.app.lineage_ingestion import rebuild_lineage +from lineageweave.adjudication_client import ( + ContextualOrchestratorAdjudicationClient, + NullAdjudicationClient, +) from lineageweave.synthetic_seed_cleanup import cleanup_synthetic_seed from lineageweave.embedding_client import orchestrator_embedding_client from lineageweave.image_content import orchestrator_vision_client @@ -403,6 +407,14 @@ async def import_rows(args: argparse.Namespace) -> dict[str, int]: if orchestrator_base_url and orchestrator_api_key else NullPostStructureClient() ) + adjudication_client = ( + ContextualOrchestratorAdjudicationClient( + orchestrator_base_url, + orchestrator_api_key, + ) + if orchestrator_base_url and orchestrator_api_key + else NullAdjudicationClient() + ) for row in rows: if _source_code_matches(row, mapping.draft, args.exclude_draft_value) or _source_code_matches( row, mapping.deleted, args.exclude_deleted_value @@ -535,7 +547,7 @@ async def import_rows(args: argparse.Namespace) -> dict[str, int]: ) imported += 1 cleanup = await cleanup_synthetic_seed(target, apply=True) - edges = await rebuild_lineage(target) + edges = await rebuild_lineage(target, llm=adjudication_client) return { "source_rows": len(rows), "imported_rows": imported, From 516cc558b30eacc0058528d8953d1fcd1c658507 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 02:04:13 +0900 Subject: [PATCH 07/43] fix: order persisted lineage weights deterministically --- backend/app/lineage_ingestion.py | 8 ++++++-- tests/test_lineage_ingestion.py | 10 +++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index 4e7878324..6edec204a 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -173,8 +173,12 @@ async def visible_lineage_graph( "from event_lineage_rebuild" ) weight_rows = await conn.fetch( - "select signal_code, signal_weight from event_lineage_rebuild_channel " - "order by signal_code" + "select channel.signal_code, channel.signal_weight " + "from event_lineage_rebuild_channel as channel " + "join common_lookup_value as lookup " + "on lookup.lookup_code = channel.signal_code " + "where channel.rebuild_lock = true " + "order by lookup.display_order, channel.signal_code" ) if focus_post_id is None: diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index 104b1d466..33e902e69 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -201,6 +201,9 @@ async def fetch(self, query: str, *_args): def test_visible_graph_attaches_ranked_channel_evidence() -> None: class FakeConnection: + def __init__(self) -> None: + self.queries: list[str] = [] + posts = [ { "post_id": "post-a", @@ -264,6 +267,7 @@ class FakeConnection: ] async def fetch(self, query: str, *_args): + self.queries.append(query) if "post_lineage_edge_signal" in query: return self.signals if "event_lineage_rebuild_channel" in query: @@ -272,13 +276,17 @@ async def fetch(self, query: str, *_args): return self.rebuilds return self.edges if "post_lineage_edge" in query else self.posts - graph = asyncio.run(visible_lineage_graph(FakeConnection(), lambda row: True)) + connection = FakeConnection() + graph = asyncio.run(visible_lineage_graph(connection, lambda row: True)) evidence = graph["edges"][0]["channel_evidence"] assert [item["signal_code"] for item in evidence] == ["secondary_key", "text", "temporal"] assert [item["rank"] for item in evidence] == [1, 2, 3] assert "llm" not in {item["signal_code"] for item in evidence} assert graph["reconstruction"]["reconstruction_version"] == "lineageweave.reconstruct/2.14.0" assert graph["reconstruction"]["active_weights"][0]["signal_code"] == "temporal" + weight_query = next(query for query in connection.queries if "event_lineage_rebuild_channel" in query) + assert "join common_lookup_value as lookup" in weight_query + assert "order by lookup.display_order, channel.signal_code" in weight_query def test_abac_never_reveals_channel_evidence_for_an_invisible_endpoint() -> None: From 4faf9a31371195c5ec63fca42a5afbb93a95369b Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 02:10:13 +0900 Subject: [PATCH 08/43] docs: record orchestrated lineage rebuild policy --- docs/adr/0124-event-lineage-channel-evidence.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/adr/0124-event-lineage-channel-evidence.md b/docs/adr/0124-event-lineage-channel-evidence.md index ea964c3ea..c5bf9e4df 100644 --- a/docs/adr/0124-event-lineage-channel-evidence.md +++ b/docs/adr/0124-event-lineage-channel-evidence.md @@ -43,11 +43,16 @@ authoritative; PROV-O/RDF export is a projection. `event_lineage_rebuild_channel` stores the active weight profile. Analysis-run reconstruction (`analysis_run_lineage_edge`) stays a separate immutable run-scoped table. + The administrator-triggered live rebuild and PostgreSQL import pass the + configured contextual-orchestrator adjudication client through the same + reconstruction boundary; when it is unavailable, the LLM channel is + dropped and the remaining weights are renormalized. 5. `GET /api/lineage` returns an additive `channel_evidence` collection on each visible edge (`signal_code`, `signal_label`, `score`, `weight`, `contribution`, `rank`) ordered by contribution, then - controlled signal order. ABAC never reveals evidence for an invisible - endpoint. + controlled signal order. The rebuild profile is returned in the same + controlled order using `common_lookup_value.display_order`. ABAC never + reveals evidence for an invisible endpoint. 6. The Buyer DAG provides an accessible edge-detail disclosure (not hover-only), labels the relation as inferred rather than causal, and states when no LLM channel participated only when at least one From f145b83f68731d6f31c3e1b1de59fd7be8f62001 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 02:18:46 +0900 Subject: [PATCH 09/43] fix(frontend): preserve structured footnote roles --- frontend/src/postBodyDisplay.test.ts | 16 +++++++++ frontend/src/postBodyDisplay.ts | 51 +++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 07ca9514d..52c8c65eb 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -61,6 +61,22 @@ describe("splitPostBody", () => { ]); }); + it("labels HTML, Word, and OOXML footnotes in the fallback renderer", () => { + expect( + splitPostBody( + '<p>Body text</p>' + + '<ol class="footnotes"><li id="fn1"><p>HTML footnote body</p></li></ol>' + + '<p class="MsoFootnoteText"><a href="#_ftnref1"><sup>1</sup></a> Word footnote body</p>' + + "<w:footnote w:id='1'><w:p>OOXML footnote body</w:p></w:footnote>", + ), + ).toEqual([ + { kind: "text", text: "Body text" }, + { kind: "text", text: "HTML footnote body", role: "footnote" }, + { kind: "text", text: "^1 Word footnote body", role: "footnote" }, + { kind: "text", text: "OOXML footnote body", role: "footnote" }, + ]); + }); + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { expect(splitPostBody("The full body text.")).toEqual([ { kind: "text", text: "The full body text." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 919e8c0ca..7e7c4b39c 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -24,10 +24,47 @@ const FOOTNOTE_START = /^\s*[*†‡](?=\S)/; const INDENT_MARKER = "\u0001lw-indent:"; const INDENT_MARKER_END = "\u0002"; const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g; +const FOOTNOTE_MARKER = "\u0001lw-footnote\u0002"; +const FOOTNOTE_MARKER_PATTERN = new RegExp(FOOTNOTE_MARKER, "g"); + +function markFootnoteTags(markup: string): string { + let footnoteDepth = 0; + return markup.replace(HTML_TAG, (tag) => { + const match = tag.match(/^<\s*(\/?)\s*([a-z][a-z0-9:-]*)\b/i); + if (!match) return tag; + const closing = Boolean(match[1]); + const name = match[2].toLowerCase(); + const hasFootnoteLabel = /\b(?:footnotes?|endnotes?|msofootnotetext|msoendnotetext)\b/i.test(tag); + const isContainer = + (name === "ol" || name === "ul") && hasFootnoteLabel; + const isWordParagraph = + name === "p" && hasFootnoteLabel; + const isOoxmlContainer = name === "w:footnote" || name === "w:endnote"; + + if (closing) { + if (isOoxmlContainer || isContainer) { + footnoteDepth = Math.max(0, footnoteDepth - 1); + } + return tag; + } + if (isOoxmlContainer || isContainer) { + if (!/\/\s*>$/.test(tag)) footnoteDepth += 1; + return `${tag}${FOOTNOTE_MARKER}`; + } + if ( + isWordParagraph || + (footnoteDepth > 0 && (name === "li" || name === "p" || name === "w:p")) + ) { + return `${tag}${FOOTNOTE_MARKER}`; + } + return tag; + }); +} function stripIndentMarkers(value: string): string { return value .replace(INDENT_MARKER_PATTERN, "") + .replace(FOOTNOTE_MARKER_PATTERN, "") .split(String.fromCharCode(1)) .join("") .split(String.fromCharCode(2)) @@ -90,7 +127,7 @@ function indentMarker(width: number): string { } function stripHtmlTags(text: string): string { - text = text.replace(/<sup[^>]*>(.*?)<\/sup>/gi, "^$1"); + text = markFootnoteTags(text).replace(/<sup[^>]*>(.*?)<\/sup>/gi, "^$1"); const withBoundaries = text .replace(BREAK_TAG, "\n") .replace(BLOCK_TAG, (tag) => { @@ -98,9 +135,10 @@ function stripHtmlTags(text: string): string { return `\n\n${indentMarker(declaredIndentWidth(tag))}`; }) .replace(WORD_INDENT_TAG, (tag) => indentMarker(declaredIndentWidth(tag))); - const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => - /^<\/?w:/i.test(tag) ? "" : " ", - ); + const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => { + if (/^<\/?(?:a|w:)/i.test(tag)) return ""; + return " "; + }); const decoded = decodeHtmlEntities(withoutTags); return decoded .split("\n") @@ -200,6 +238,7 @@ function isDecodableBase64(raw: string): boolean { function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void { const text = stripHtmlTags(raw); for (const paragraph of splitSemanticParagraphs(text)) { + const isMarkedFootnote = paragraph.includes(FOOTNOTE_MARKER); const indentLevel = indentationLevel(paragraph, indentUnit); const normalized = stripIndentMarkers(paragraph) .replace(/^[ \t]+/, "") @@ -209,7 +248,9 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): kind: "text", text: normalized, ...(indentLevel > 0 ? { indentLevel } : {}), - ...(FOOTNOTE_START.test(normalized) ? { role: "footnote" as const } : {}), + ...(isMarkedFootnote || FOOTNOTE_START.test(normalized) + ? { role: "footnote" as const } + : {}), }); } } From df2519c09b224f05ac84e6997abd3b4f1bb94cb6 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 02:25:09 +0900 Subject: [PATCH 10/43] fix(lineage): budget signal rounding tolerance --- lineageweave/lineage_persistence.py | 15 +++++++++++---- tests/test_lineage_channel_evidence.py | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/lineageweave/lineage_persistence.py b/lineageweave/lineage_persistence.py index bcf51c11f..a21104b18 100644 --- a/lineageweave/lineage_persistence.py +++ b/lineageweave/lineage_persistence.py @@ -8,7 +8,8 @@ winning edge's active channel scores are persisted beside it as ``post_lineage_edge_signal`` rows (ADR 0124). A missing LLM channel is dropped, never fabricated. Contribution is ``weight * score`` and must -reconcile with ``fused_score`` within :data:`CHANNEL_EVIDENCE_TOLERANCE`. +reconcile with ``fused_score`` within the base tolerance plus the bounded +persistence quantization budget. """ from __future__ import annotations @@ -135,7 +136,8 @@ def channel_signal_rows( Raises: ValueError: if recorded contributions do not reconcile with - ``edge.fused_score`` within :data:`CHANNEL_EVIDENCE_TOLERANCE`. + ``edge.fused_score`` after accounting for one half quantum per + persisted channel and a small floating-point guard. """ active_weights = dict(weights) if weights is not None else weights_for_channel_scores(edge.channel_scores) rows: list[dict[str, object]] = [] @@ -158,11 +160,16 @@ def channel_signal_rows( "signal_contribution": contribution, } ) + # Each numeric(8,6) contribution can differ from its exact product by + # half a quantum. A fixed tolerance fails valid 3/4-channel edges when + # those independent rounding errors accumulate. + rounding_budget = len(rows) * float(SIGNAL_QUANTUM) / 2 + float(SIGNAL_QUANTUM) + reconciliation_tolerance = max(CHANNEL_EVIDENCE_TOLERANCE, rounding_budget) residual = abs(contribution_sum - float(edge.fused_score)) - if rows and residual > CHANNEL_EVIDENCE_TOLERANCE: + if rows and residual > reconciliation_tolerance: raise ValueError( f"channel contributions {contribution_sum} do not reconcile with " - f"fused_score {edge.fused_score} (tolerance {CHANNEL_EVIDENCE_TOLERANCE})" + f"fused_score {edge.fused_score} (tolerance {reconciliation_tolerance})" ) return rows diff --git a/tests/test_lineage_channel_evidence.py b/tests/test_lineage_channel_evidence.py index b69a7836f..74f6f8e3d 100644 --- a/tests/test_lineage_channel_evidence.py +++ b/tests/test_lineage_channel_evidence.py @@ -72,6 +72,26 @@ def test_contributions_reconcile_to_fused_score_within_tolerance() -> None: assert residual <= CHANNEL_EVIDENCE_TOLERANCE +def test_rebuild_accepts_expected_multi_channel_quantization_error() -> None: + scores = { + "temporal": 0.1234567, + "secondary_key": 0.2345678, + "text": 0.3456789, + "llm": 0.4567891, + } + weights = {"temporal": 0.15, "secondary_key": 0.15, "text": 0.30, "llm": 0.40} + edge = Edge( + "parent-a", + "child-b", + sum(weights[name] * scores[name] for name in scores), + scores, + ) + + rows = channel_signal_rows(edge) + assert len(rows) == 4 + assert lineage_rebuild_spec([edge]).signal_rows + + def test_mismatched_fused_score_is_rejected() -> None: edge = Edge("parent-a", "child-b", 0.99, {"temporal": 0.1, "secondary_key": 0.1, "text": 0.1}) with pytest.raises(ValueError, match="do not reconcile"): From 81cd12fcfe24fd8073ef86b96a29035a6d1cec23 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 02:29:58 +0900 Subject: [PATCH 11/43] fix(frontend): close HTML footnote containers reliably --- frontend/src/postBodyDisplay.test.ts | 11 +++++++++++ frontend/src/postBodyDisplay.ts | 20 ++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 52c8c65eb..71b056e75 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -77,6 +77,17 @@ describe("splitPostBody", () => { ]); }); + it("stops labeling ordinary content after an HTML footnote list", () => { + expect( + splitPostBody( + '<ol class="footnotes"><li>HTML footnote body</li></ol><p>Ordinary body after footnotes</p>', + ), + ).toEqual([ + { kind: "text", text: "HTML footnote body", role: "footnote" }, + { kind: "text", text: "Ordinary body after footnotes" }, + ]); + }); + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { expect(splitPostBody("The full body text.")).toEqual([ { kind: "text", text: "The full body text." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 7e7c4b39c..a22de0bef 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -29,6 +29,8 @@ const FOOTNOTE_MARKER_PATTERN = new RegExp(FOOTNOTE_MARKER, "g"); function markFootnoteTags(markup: string): string { let footnoteDepth = 0; + const openTags: Array<{ name: string; isFootnote: boolean }> = []; + const voidTags = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "w:br"]); return markup.replace(HTML_TAG, (tag) => { const match = tag.match(/^<\s*(\/?)\s*([a-z][a-z0-9:-]*)\b/i); if (!match) return tag; @@ -42,13 +44,23 @@ function markFootnoteTags(markup: string): string { const isOoxmlContainer = name === "w:footnote" || name === "w:endnote"; if (closing) { - if (isOoxmlContainer || isContainer) { - footnoteDepth = Math.max(0, footnoteDepth - 1); + const matchingIndex = openTags.map((entry) => entry.name).lastIndexOf(name); + if (matchingIndex >= 0) { + const closedTags = openTags.splice(matchingIndex); + footnoteDepth = Math.max( + 0, + footnoteDepth - closedTags.filter((entry) => entry.isFootnote).length, + ); } return tag; } - if (isOoxmlContainer || isContainer) { - if (!/\/\s*>$/.test(tag)) footnoteDepth += 1; + const selfClosing = /\/\s*>$/.test(tag) || voidTags.has(name); + const opensFootnote = isOoxmlContainer || isContainer; + if (!selfClosing) { + openTags.push({ name, isFootnote: opensFootnote }); + } + if (opensFootnote) { + if (!selfClosing) footnoteDepth += 1; return `${tag}${FOOTNOTE_MARKER}`; } if ( From 146cc56e07db9479ab8dba93aedb0ffd06d2e795 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 02:31:05 +0900 Subject: [PATCH 12/43] fix: hide empty footnote markers --- frontend/src/postBodyDisplay.test.ts | 4 ++++ frontend/src/postBodyDisplay.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 71b056e75..bd28646d6 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -88,6 +88,10 @@ describe("splitPostBody", () => { ]); }); + it("does not expose control markers for an empty footnote container", () => { + expect(splitPostBody('<ol class="footnotes"></ol>')).toEqual([{ kind: "text", text: "" }]); + }); + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { expect(splitPostBody("The full body text.")).toEqual([ { kind: "text", text: "The full body text." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index a22de0bef..16cfb1004 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -296,7 +296,7 @@ export function splitPostBody(body: string): PostBodySegment[] { } pushText(segments, body.slice(lastIndex), indentUnit); if (segments.length === 0) { - return [{ kind: "text", text: stripHtmlTags(body) }]; + return [{ kind: "text", text: stripIndentMarkers(stripHtmlTags(body)) }]; } return segments; } From 383a01324ef9b7b9ee2b1fb508b4f6dac20e2847 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 02:42:53 +0900 Subject: [PATCH 13/43] fix(lineage): offload synchronous reconstruction --- backend/app/lineage_ingestion.py | 4 +++- tests/test_lineage_ingestion.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index 6edec204a..486beaa90 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -10,6 +10,7 @@ from __future__ import annotations +import asyncio from collections import defaultdict from datetime import datetime from typing import Any, Mapping @@ -131,7 +132,8 @@ async def rebuild_lineage( "process_unit_id, thread_group_key, secondary_grouping_key " f"from source_post where {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}" ) - edges = lineage_edge_specs(records_from_source_posts(rows), llm=llm) + records = records_from_source_posts(rows) + edges = await asyncio.to_thread(lineage_edge_specs, records, llm=llm) await persist_lineage_edges(conn, edges) return edges diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index 33e902e69..bc84017e7 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -50,13 +50,19 @@ def fake_lineage_edge_specs(_records, *, llm=None): async def fake_persist_lineage_edges(_conn, _edges): return None + async def fake_to_thread(function, *args, **kwargs): + captured["offloaded_function"] = function + return function(*args, **kwargs) + import backend.app.lineage_ingestion as ingestion monkeypatch.setattr(ingestion, "lineage_edge_specs", fake_lineage_edge_specs) monkeypatch.setattr(ingestion, "persist_lineage_edges", fake_persist_lineage_edges) + monkeypatch.setattr(asyncio, "to_thread", fake_to_thread) asyncio.run(rebuild_lineage(FakeConnection(), llm=client)) assert captured["llm"] is client + assert captured["offloaded_function"] is fake_lineage_edge_specs def test_records_fall_back_to_corporate_entity_when_thread_keys_are_empty() -> None: From 4bf061314516a6d824dcc41b24a021ca69661aa4 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 02:44:16 +0900 Subject: [PATCH 14/43] perf(lineage): bound channel evidence reads --- backend/app/lineage_ingestion.py | 13 +++++++++---- tests/test_lineage_ingestion.py | 3 +++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index 486beaa90..c5642f5ba 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -166,10 +166,6 @@ async def visible_lineage_graph( edge_rows = await conn.fetch( "select parent_post_id, child_post_id, fused_score from post_lineage_edge" ) - signal_rows = await conn.fetch( - "select parent_post_id, child_post_id, signal_code, signal_score, " - "signal_weight, signal_contribution from post_lineage_edge_signal" - ) rebuild_rows = await conn.fetch( "select reconstruction_version, generated_at, min_fused_score, candidate_window " "from event_lineage_rebuild" @@ -225,6 +221,15 @@ async def visible_lineage_graph( for row in edge_rows if str(row["parent_post_id"]) in visible_ids and str(row["child_post_id"]) in visible_ids ] + visible_id_list = sorted(visible_ids) + signal_rows = await conn.fetch( + "select parent_post_id, child_post_id, signal_code, signal_score, " + "signal_weight, signal_contribution from post_lineage_edge_signal " + "where parent_post_id = any($1::uuid[]) " + "and child_post_id = any($2::uuid[])", + visible_id_list, + visible_id_list, + ) children_of: dict[str, list[str]] = {} for row in visible_edges: children_of.setdefault(str(row["parent_post_id"]), []).append(str(row["child_post_id"])) diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index bc84017e7..f7ca8e8a1 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -290,6 +290,9 @@ async def fetch(self, query: str, *_args): assert "llm" not in {item["signal_code"] for item in evidence} assert graph["reconstruction"]["reconstruction_version"] == "lineageweave.reconstruct/2.14.0" assert graph["reconstruction"]["active_weights"][0]["signal_code"] == "temporal" + signal_query = next(query for query in connection.queries if "post_lineage_edge_signal" in query) + assert "parent_post_id = any($1::uuid[])" in signal_query + assert "child_post_id = any($2::uuid[])" in signal_query weight_query = next(query for query in connection.queries if "event_lineage_rebuild_channel" in query) assert "join common_lookup_value as lookup" in weight_query assert "order by lookup.display_order, channel.signal_code" in weight_query From 1b680a27e6eaca544f1d99512e31220278c43110 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 03:00:46 +0900 Subject: [PATCH 15/43] fix(frontend): bound footnote and anchor tag detection --- frontend/src/postBodyDisplay.test.ts | 18 ++++++++++++++++++ frontend/src/postBodyDisplay.ts | 7 +++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index bd28646d6..29bad7fa3 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -92,6 +92,24 @@ describe("splitPostBody", () => { expect(splitPostBody('<ol class="footnotes"></ol>')).toEqual([{ kind: "text", text: "" }]); }); + it("does not infer footnotes from unrelated attribute values", () => { + expect( + splitPostBody( + '<ol data-purpose="footnotes"><li>Ordinary list</li></ol>' + + '<p data-purpose="footnote">Ordinary paragraph</p>', + ), + ).toEqual([ + { kind: "text", text: "Ordinary list" }, + { kind: "text", text: "Ordinary paragraph" }, + ]); + }); + + it("keeps text boundaries for tags whose names start with a", () => { + expect(splitPostBody('<p>Alpha<abbr title="expanded">Beta</abbr>Gamma</p>')).toEqual([ + { kind: "text", text: "Alpha Beta Gamma" }, + ]); + }); + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { expect(splitPostBody("The full body text.")).toEqual([ { kind: "text", text: "The full body text." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 16cfb1004..a3eeeb9fd 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -36,7 +36,10 @@ function markFootnoteTags(markup: string): string { if (!match) return tag; const closing = Boolean(match[1]); const name = match[2].toLowerCase(); - const hasFootnoteLabel = /\b(?:footnotes?|endnotes?|msofootnotetext|msoendnotetext)\b/i.test(tag); + const hasFootnoteLabel = [...tag.matchAll(/\b(?:class|role)\s*=\s*(["'])(.*?)\1/gi)].some( + (attribute) => + /\b(?:footnotes?|endnotes?|msofootnotetext|msoendnotetext)\b/i.test(attribute[2]), + ); const isContainer = (name === "ol" || name === "ul") && hasFootnoteLabel; const isWordParagraph = @@ -148,7 +151,7 @@ function stripHtmlTags(text: string): string { }) .replace(WORD_INDENT_TAG, (tag) => indentMarker(declaredIndentWidth(tag))); const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => { - if (/^<\/?(?:a|w:)/i.test(tag)) return ""; + if (/^<\/?(?:a\b|w:)/i.test(tag)) return ""; return " "; }); const decoded = decodeHtmlEntities(withoutTags); From cb79302831c0889007699ba7dca2aafa02c01e50 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 03:12:30 +0900 Subject: [PATCH 16/43] fix: close rebuild transaction before reconstruction --- backend/app/lineage_ingestion.py | 7 ++++++- backend/app/main.py | 3 +-- tests/test_lineage_ingestion.py | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index c5642f5ba..3fd1b0628 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -127,6 +127,10 @@ async def rebuild_lineage( optional LLM channel is recorded when available; ``None`` preserves the fail-closed three-channel rebuild. """ + # Keep the pooled connection out of an open transaction while the + # optional orchestrator evaluates the whole corpus. Only the destructive + # replacement is transactional, so a slow model call cannot hold an idle + # database transaction open. rows = await conn.fetch( "select post_id, post_title, voc_type_code, created_at, corporate_entity_id, " "process_unit_id, thread_group_key, secondary_grouping_key " @@ -134,7 +138,8 @@ async def rebuild_lineage( ) records = records_from_source_posts(rows) edges = await asyncio.to_thread(lineage_edge_specs, records, llm=llm) - await persist_lineage_edges(conn, edges) + async with conn.transaction(): + await persist_lineage_edges(conn, edges) return edges diff --git a/backend/app/main.py b/backend/app/main.py index cb0186e7d..983db4f8b 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1111,8 +1111,7 @@ async def rebuild_lineage_graph( """ _require_post_admin(account) async with pool.acquire() as conn: - async with conn.transaction(): - edges = await rebuild_lineage(conn, llm=_adjudication_client()) + edges = await rebuild_lineage(conn, llm=_adjudication_client()) return {"edge_count": len(edges)} diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index f7ca8e8a1..131885478 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -36,10 +36,23 @@ def test_records_use_persisted_thread_keys_not_process_unit_or_voc_type() -> Non def test_rebuild_passes_the_configured_adjudication_client(monkeypatch) -> None: + events: list[str] = [] + + class FakeTransaction: + async def __aenter__(self): + events.append("transaction_enter") + + async def __aexit__(self, *_args): + events.append("transaction_exit") + class FakeConnection: async def fetch(self, _query: str, *_args): + events.append("fetch") return [] + def transaction(self): + return FakeTransaction() + client = object() captured: dict[str, object] = {} @@ -48,9 +61,11 @@ def fake_lineage_edge_specs(_records, *, llm=None): return [] async def fake_persist_lineage_edges(_conn, _edges): + events.append("persist") return None async def fake_to_thread(function, *args, **kwargs): + events.append("reconstruct") captured["offloaded_function"] = function return function(*args, **kwargs) @@ -63,6 +78,7 @@ async def fake_to_thread(function, *args, **kwargs): assert captured["llm"] is client assert captured["offloaded_function"] is fake_lineage_edge_specs + assert events == ["fetch", "reconstruct", "transaction_enter", "persist", "transaction_exit"] def test_records_fall_back_to_corporate_entity_when_thread_keys_are_empty() -> None: From caea21be9e3485086ae3967fe5e6d23199b5459e Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 03:15:58 +0900 Subject: [PATCH 17/43] fix(frontend): recognize wrapped footnote lists --- frontend/src/postBodyDisplay.test.ts | 14 ++++++++++++++ frontend/src/postBodyDisplay.ts | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 29bad7fa3..bde6bd936 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -88,6 +88,20 @@ describe("splitPostBody", () => { ]); }); + it("labels footnotes inside a labeled wrapper around an HTML list", () => { + expect( + splitPostBody( + '<p>Body text</p>' + + '<div class="footnotes"><ol><li><p>Wrapped footnote body</p></li></ol></div>' + + "<p>Ordinary body after footnotes</p>", + ), + ).toEqual([ + { kind: "text", text: "Body text" }, + { kind: "text", text: "Wrapped footnote body", role: "footnote" }, + { kind: "text", text: "Ordinary body after footnotes" }, + ]); + }); + it("does not expose control markers for an empty footnote container", () => { expect(splitPostBody('<ol class="footnotes"></ol>')).toEqual([{ kind: "text", text: "" }]); }); diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index a3eeeb9fd..51c040f17 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -41,7 +41,7 @@ function markFootnoteTags(markup: string): string { /\b(?:footnotes?|endnotes?|msofootnotetext|msoendnotetext)\b/i.test(attribute[2]), ); const isContainer = - (name === "ol" || name === "ul") && hasFootnoteLabel; + hasFootnoteLabel && (name === "div" || name === "ol" || name === "ul"); const isWordParagraph = name === "p" && hasFootnoteLabel; const isOoxmlContainer = name === "w:footnote" || name === "w:endnote"; From 6e32ecac3677b220adf42ea7e7d5157d5dd80b8a Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 03:14:37 +0900 Subject: [PATCH 18/43] fix: release lineage pool during reconstruction --- .../2.14.0-event-lineage-channel-evidence.md | 3 + backend/app/lineage_ingestion.py | 63 +++++++++----- backend/app/main.py | 9 +- docker/postgres-init/migrate.sh | 2 +- tests/test_lineage_channel_evidence.py | 2 +- tests/test_lineage_ingestion.py | 85 +++++++++++++++++-- 6 files changed, 132 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.d/2.14.0-event-lineage-channel-evidence.md b/CHANGELOG.d/2.14.0-event-lineage-channel-evidence.md index 84a4e3486..ec2ecaea4 100644 --- a/CHANGELOG.d/2.14.0-event-lineage-channel-evidence.md +++ b/CHANGELOG.d/2.14.0-event-lineage-channel-evidence.md @@ -10,6 +10,9 @@ produced each reconstructed connection. - `event_lineage_rebuild` records reconstruction version, generated-at time, and the active weight profile so a later rebuild cannot silently rewrite historic evidence. +- Administrator rebuilds release their read connection before CPU and + orchestrator work, then acquire one short transaction for the atomic + live-graph replacement. - `GET /api/lineage` returns additive `channel_evidence` on each visible edge. ABAC never reveals evidence for an invisible endpoint. - The Buyer DAG discloses exact values with keyboard and screen-reader diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index 3fd1b0628..b4f339695 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -12,8 +12,9 @@ import asyncio from collections import defaultdict +from collections.abc import Mapping from datetime import datetime -from typing import Any, Mapping +from typing import Any import asyncpg @@ -116,6 +117,24 @@ async def persist_lineage_edges(conn: asyncpg.Connection, edges: list[Edge]) -> ) +async def _load_lineage_records(conn: asyncpg.Connection) -> list[Record]: + """Load the eligible source snapshot used by one reconstruction.""" + rows = await conn.fetch( + "select post_id, post_title, voc_type_code, created_at, corporate_entity_id, " + "process_unit_id, thread_group_key, secondary_grouping_key " + f"from source_post where {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}" + ) + return records_from_source_posts(rows) + + +async def _reconstruct_lineage_records( + records: list[Record], + llm: AdjudicationClient | None, +) -> list[Edge]: + """Run the CPU/provider reconstruction without blocking the event loop.""" + return await asyncio.to_thread(lineage_edge_specs, records, llm=llm) + + async def rebuild_lineage( conn: asyncpg.Connection, *, @@ -127,26 +146,30 @@ async def rebuild_lineage( optional LLM channel is recorded when available; ``None`` preserves the fail-closed three-channel rebuild. """ - # Keep the pooled connection out of an open transaction while the - # optional orchestrator evaluates the whole corpus. Only the destructive - # replacement is transactional, so a slow model call cannot hold an idle - # database transaction open. - rows = await conn.fetch( - "select post_id, post_title, voc_type_code, created_at, corporate_entity_id, " - "process_unit_id, thread_group_key, secondary_grouping_key " - f"from source_post where {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}" - ) - records = records_from_source_posts(rows) - edges = await asyncio.to_thread(lineage_edge_specs, records, llm=llm) + records = await _load_lineage_records(conn) + edges = await _reconstruct_lineage_records(records, llm) async with conn.transaction(): await persist_lineage_edges(conn, edges) return edges -def _isoformat(value: object) -> str: - if hasattr(value, "isoformat"): - return value.isoformat() # type: ignore[no-any-return] - return str(value) +async def rebuild_lineage_from_pool( + pool: asyncpg.Pool, + *, + llm: AdjudicationClient | None = None, +) -> list[Edge]: + """Reconstruct without holding a pooled connection during provider work. + + The source snapshot is read and released first. Only the replacement + writes run in a transaction, preserving ADR 0124 atomicity without an + idle-in-transaction connection during CPU or orchestrator calls. + """ + async with pool.acquire() as conn: + records = await _load_lineage_records(conn) + edges = await _reconstruct_lineage_records(records, llm) + async with pool.acquire() as conn, conn.transaction(): + await persist_lineage_edges(conn, edges) + return edges async def visible_lineage_graph( @@ -202,13 +225,11 @@ async def visible_lineage_graph( neighbors.setdefault(child_id, set()).add(parent_id) component_ids: set[str] = set() - frontier = [focus_id] if focus_visible else [] + frontier = {focus_id} if focus_visible else set() while frontier: current_id = frontier.pop() - if current_id in component_ids: - continue component_ids.add(current_id) - frontier.extend(neighbors.get(current_id, set()) - component_ids) + frontier.update(neighbors.get(current_id, set()) - component_ids) # An isolated post has no DAG to render; the post-lineage endpoint # still reports its empty direct/indirect lists. @@ -279,7 +300,7 @@ async def visible_lineage_graph( rebuild = rebuild_rows[0] reconstruction = { "reconstruction_version": rebuild["reconstruction_version"], - "generated_at": _isoformat(rebuild["generated_at"]), + "generated_at": rebuild["generated_at"].isoformat(), "min_fused_score": float(rebuild["min_fused_score"]), "candidate_window": int(rebuild["candidate_window"]), "active_weights": [ diff --git a/backend/app/main.py b/backend/app/main.py index 983db4f8b..2d63bade9 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -165,7 +165,10 @@ visible_mention_post_ids, visible_team_mention_post_ids, ) -from backend.app.lineage_ingestion import rebuild_lineage, visible_lineage_graph +from backend.app.lineage_ingestion import ( + rebuild_lineage_from_pool, + visible_lineage_graph, +) from backend.app.post_chat_ingestion import ( fetch_persisted_chat, fetch_persisted_chats, @@ -183,7 +186,6 @@ from backend.app.demo_scope import ( fetch_demo_corporate_entity_ids, has_real_source_context, - is_demo_scope, ) from lineageweave.http_client import HttpClientError @@ -1110,8 +1112,7 @@ async def rebuild_lineage_graph( post_admin only: this is a corpus-wide write. Reads stay ABAC-gated. """ _require_post_admin(account) - async with pool.acquire() as conn: - edges = await rebuild_lineage(conn, llm=_adjudication_client()) + edges = await rebuild_lineage_from_pool(pool, llm=_adjudication_client()) return {"edge_count": len(edges)} diff --git a/docker/postgres-init/migrate.sh b/docker/postgres-init/migrate.sh index 88f168a43..300ab6b8c 100644 --- a/docker/postgres-init/migrate.sh +++ b/docker/postgres-init/migrate.sh @@ -18,7 +18,7 @@ for migration in /opt/lineageweave/migrations/*.sql; do migration_name=${migration##*/} case "$migration_name" in 0012_*|0013_*|0014_*|0015_*|0016_*|0017_*|0018_*|0019_*|0020_*|0021_*|0022_*|0023_*|0024_*|0025_*|0026_*|0027_*|0028_*|0029_*|0030_*|0031_*|0032_*|0033_*|0034_*|0035_*|0036_*|0037_*|0038_*|0039_*|0040_*|0041_*|0042_*|0043_*|0044_*|0045_*|0046_*|0047_*|0048_*|0049_*|0050_*) ;; - 0060_*|0100_*|0101_*|0102_*|0103_*|0104_*|0105_*) ;; + 0060_*|0100_*|0101_*|0102_*|0103_*|0105_*) ;; *) continue ;; esac printf 'Applying %s\n' "$migration_name" diff --git a/tests/test_lineage_channel_evidence.py b/tests/test_lineage_channel_evidence.py index 74f6f8e3d..a23a3e559 100644 --- a/tests/test_lineage_channel_evidence.py +++ b/tests/test_lineage_channel_evidence.py @@ -151,7 +151,7 @@ def test_rank_is_contribution_then_controlled_signal_order() -> None: def test_migrate_sh_replays_channel_evidence_and_tenant_settings() -> None: migrate = (_ROOT / "docker/postgres-init/migrate.sh").read_text() assert "0103_*" in migrate - assert "0104_*" in migrate + assert "0104_*" not in migrate assert "0105_*" in migrate diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index 131885478..886c56e56 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -6,8 +6,9 @@ from datetime import datetime, timezone from backend.app.lineage_ingestion import ( - rebuild_lineage, persist_lineage_edges, + rebuild_lineage, + rebuild_lineage_from_pool, reconstruct_group_key, records_from_source_posts, visible_lineage_graph, @@ -69,10 +70,12 @@ async def fake_to_thread(function, *args, **kwargs): captured["offloaded_function"] = function return function(*args, **kwargs) - import backend.app.lineage_ingestion as ingestion - - monkeypatch.setattr(ingestion, "lineage_edge_specs", fake_lineage_edge_specs) - monkeypatch.setattr(ingestion, "persist_lineage_edges", fake_persist_lineage_edges) + monkeypatch.setattr( + "backend.app.lineage_ingestion.lineage_edge_specs", fake_lineage_edge_specs + ) + monkeypatch.setattr( + "backend.app.lineage_ingestion.persist_lineage_edges", fake_persist_lineage_edges + ) monkeypatch.setattr(asyncio, "to_thread", fake_to_thread) asyncio.run(rebuild_lineage(FakeConnection(), llm=client)) @@ -81,6 +84,78 @@ async def fake_to_thread(function, *args, **kwargs): assert events == ["fetch", "reconstruct", "transaction_enter", "persist", "transaction_exit"] +def test_pooled_rebuild_releases_the_connection_during_reconstruction(monkeypatch) -> None: + """Keep provider work outside the pool and transaction, then replace atomically.""" + events: list[str] = [] + + class FakeTransaction: + async def __aenter__(self): + events.append("transaction-enter") + + async def __aexit__(self, *_args): + events.append("transaction-exit") + + class FakeConnection: + async def fetch(self, _query: str, *_args): + events.append("fetch") + return [] + + def transaction(self): + return FakeTransaction() + + class FakeAcquire: + def __init__(self, pool): + self.pool = pool + + async def __aenter__(self): + self.pool.active += 1 + events.append("acquire") + return self.pool.connection + + async def __aexit__(self, *_args): + self.pool.active -= 1 + events.append("release") + + class FakePool: + def __init__(self): + self.active = 0 + self.connection = FakeConnection() + + def acquire(self): + return FakeAcquire(self) + + pool = FakePool() + + async def fake_to_thread(function, *args, **kwargs): + assert pool.active == 0 + events.append("reconstruct") + return function(*args, **kwargs) + + async def fake_persist(_conn, _edges): + assert pool.active == 1 + assert events[-1] == "transaction-enter" + events.append("persist") + + monkeypatch.setattr(asyncio, "to_thread", fake_to_thread) + monkeypatch.setattr( + "backend.app.lineage_ingestion.persist_lineage_edges", fake_persist + ) + + asyncio.run(rebuild_lineage_from_pool(pool)) + + assert events == [ + "acquire", + "fetch", + "release", + "reconstruct", + "acquire", + "transaction-enter", + "persist", + "transaction-exit", + "release", + ] + + def test_records_fall_back_to_corporate_entity_when_thread_keys_are_empty() -> None: rows = [ { From 068ed6a44a7235e2f996450f0d6a7948bdd8732a Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Fri, 21 Aug 2026 11:19:19 -0700 Subject: [PATCH 19/43] fix(frontend): preserve structured footnote roles (#388) * fix(frontend): preserve structured footnote roles * fix(frontend): close HTML footnote containers reliably * fix: hide empty footnote markers * fix(frontend): bound footnote and anchor tag detection * fix(frontend): recognize wrapped footnote lists --- frontend/src/postBodyDisplay.test.ts | 63 ++++++++++++++++++++++++++ frontend/src/postBodyDisplay.ts | 68 +++++++++++++++++++++++++--- 2 files changed, 125 insertions(+), 6 deletions(-) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 07ca9514d..bde6bd936 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -61,6 +61,69 @@ describe("splitPostBody", () => { ]); }); + it("labels HTML, Word, and OOXML footnotes in the fallback renderer", () => { + expect( + splitPostBody( + '<p>Body text</p>' + + '<ol class="footnotes"><li id="fn1"><p>HTML footnote body</p></li></ol>' + + '<p class="MsoFootnoteText"><a href="#_ftnref1"><sup>1</sup></a> Word footnote body</p>' + + "<w:footnote w:id='1'><w:p>OOXML footnote body</w:p></w:footnote>", + ), + ).toEqual([ + { kind: "text", text: "Body text" }, + { kind: "text", text: "HTML footnote body", role: "footnote" }, + { kind: "text", text: "^1 Word footnote body", role: "footnote" }, + { kind: "text", text: "OOXML footnote body", role: "footnote" }, + ]); + }); + + it("stops labeling ordinary content after an HTML footnote list", () => { + expect( + splitPostBody( + '<ol class="footnotes"><li>HTML footnote body</li></ol><p>Ordinary body after footnotes</p>', + ), + ).toEqual([ + { kind: "text", text: "HTML footnote body", role: "footnote" }, + { kind: "text", text: "Ordinary body after footnotes" }, + ]); + }); + + it("labels footnotes inside a labeled wrapper around an HTML list", () => { + expect( + splitPostBody( + '<p>Body text</p>' + + '<div class="footnotes"><ol><li><p>Wrapped footnote body</p></li></ol></div>' + + "<p>Ordinary body after footnotes</p>", + ), + ).toEqual([ + { kind: "text", text: "Body text" }, + { kind: "text", text: "Wrapped footnote body", role: "footnote" }, + { kind: "text", text: "Ordinary body after footnotes" }, + ]); + }); + + it("does not expose control markers for an empty footnote container", () => { + expect(splitPostBody('<ol class="footnotes"></ol>')).toEqual([{ kind: "text", text: "" }]); + }); + + it("does not infer footnotes from unrelated attribute values", () => { + expect( + splitPostBody( + '<ol data-purpose="footnotes"><li>Ordinary list</li></ol>' + + '<p data-purpose="footnote">Ordinary paragraph</p>', + ), + ).toEqual([ + { kind: "text", text: "Ordinary list" }, + { kind: "text", text: "Ordinary paragraph" }, + ]); + }); + + it("keeps text boundaries for tags whose names start with a", () => { + expect(splitPostBody('<p>Alpha<abbr title="expanded">Beta</abbr>Gamma</p>')).toEqual([ + { kind: "text", text: "Alpha Beta Gamma" }, + ]); + }); + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { expect(splitPostBody("The full body text.")).toEqual([ { kind: "text", text: "The full body text." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 919e8c0ca..51c040f17 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -24,10 +24,62 @@ const FOOTNOTE_START = /^\s*[*†‡](?=\S)/; const INDENT_MARKER = "\u0001lw-indent:"; const INDENT_MARKER_END = "\u0002"; const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g; +const FOOTNOTE_MARKER = "\u0001lw-footnote\u0002"; +const FOOTNOTE_MARKER_PATTERN = new RegExp(FOOTNOTE_MARKER, "g"); + +function markFootnoteTags(markup: string): string { + let footnoteDepth = 0; + const openTags: Array<{ name: string; isFootnote: boolean }> = []; + const voidTags = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "w:br"]); + return markup.replace(HTML_TAG, (tag) => { + const match = tag.match(/^<\s*(\/?)\s*([a-z][a-z0-9:-]*)\b/i); + if (!match) return tag; + const closing = Boolean(match[1]); + const name = match[2].toLowerCase(); + const hasFootnoteLabel = [...tag.matchAll(/\b(?:class|role)\s*=\s*(["'])(.*?)\1/gi)].some( + (attribute) => + /\b(?:footnotes?|endnotes?|msofootnotetext|msoendnotetext)\b/i.test(attribute[2]), + ); + const isContainer = + hasFootnoteLabel && (name === "div" || name === "ol" || name === "ul"); + const isWordParagraph = + name === "p" && hasFootnoteLabel; + const isOoxmlContainer = name === "w:footnote" || name === "w:endnote"; + + if (closing) { + const matchingIndex = openTags.map((entry) => entry.name).lastIndexOf(name); + if (matchingIndex >= 0) { + const closedTags = openTags.splice(matchingIndex); + footnoteDepth = Math.max( + 0, + footnoteDepth - closedTags.filter((entry) => entry.isFootnote).length, + ); + } + return tag; + } + const selfClosing = /\/\s*>$/.test(tag) || voidTags.has(name); + const opensFootnote = isOoxmlContainer || isContainer; + if (!selfClosing) { + openTags.push({ name, isFootnote: opensFootnote }); + } + if (opensFootnote) { + if (!selfClosing) footnoteDepth += 1; + return `${tag}${FOOTNOTE_MARKER}`; + } + if ( + isWordParagraph || + (footnoteDepth > 0 && (name === "li" || name === "p" || name === "w:p")) + ) { + return `${tag}${FOOTNOTE_MARKER}`; + } + return tag; + }); +} function stripIndentMarkers(value: string): string { return value .replace(INDENT_MARKER_PATTERN, "") + .replace(FOOTNOTE_MARKER_PATTERN, "") .split(String.fromCharCode(1)) .join("") .split(String.fromCharCode(2)) @@ -90,7 +142,7 @@ function indentMarker(width: number): string { } function stripHtmlTags(text: string): string { - text = text.replace(/<sup[^>]*>(.*?)<\/sup>/gi, "^$1"); + text = markFootnoteTags(text).replace(/<sup[^>]*>(.*?)<\/sup>/gi, "^$1"); const withBoundaries = text .replace(BREAK_TAG, "\n") .replace(BLOCK_TAG, (tag) => { @@ -98,9 +150,10 @@ function stripHtmlTags(text: string): string { return `\n\n${indentMarker(declaredIndentWidth(tag))}`; }) .replace(WORD_INDENT_TAG, (tag) => indentMarker(declaredIndentWidth(tag))); - const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => - /^<\/?w:/i.test(tag) ? "" : " ", - ); + const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => { + if (/^<\/?(?:a\b|w:)/i.test(tag)) return ""; + return " "; + }); const decoded = decodeHtmlEntities(withoutTags); return decoded .split("\n") @@ -200,6 +253,7 @@ function isDecodableBase64(raw: string): boolean { function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void { const text = stripHtmlTags(raw); for (const paragraph of splitSemanticParagraphs(text)) { + const isMarkedFootnote = paragraph.includes(FOOTNOTE_MARKER); const indentLevel = indentationLevel(paragraph, indentUnit); const normalized = stripIndentMarkers(paragraph) .replace(/^[ \t]+/, "") @@ -209,7 +263,9 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): kind: "text", text: normalized, ...(indentLevel > 0 ? { indentLevel } : {}), - ...(FOOTNOTE_START.test(normalized) ? { role: "footnote" as const } : {}), + ...(isMarkedFootnote || FOOTNOTE_START.test(normalized) + ? { role: "footnote" as const } + : {}), }); } } @@ -243,7 +299,7 @@ export function splitPostBody(body: string): PostBodySegment[] { } pushText(segments, body.slice(lastIndex), indentUnit); if (segments.length === 0) { - return [{ kind: "text", text: stripHtmlTags(body) }]; + return [{ kind: "text", text: stripIndentMarkers(stripHtmlTags(body)) }]; } return segments; } From 778c5df1223ed60a6494e8896079b3ece97669f4 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Fri, 21 Aug 2026 11:20:13 -0700 Subject: [PATCH 20/43] fix(frontend): render markdown tables in post bodies (#389) * fix(frontend): render markdown tables in post bodies * fix(frontend): harden markdown table rendering * fix(frontend): preserve separator-free OCR tables * fix(frontend): confirm markdown tables before splitting prose --- frontend/src/PostBody.test.tsx | 65 ++++++++++++++++++++++++++++ frontend/src/PostBody.tsx | 53 ++++++++++++++++++----- frontend/src/postBodyDisplay.test.ts | 12 +++++ frontend/src/postBodyDisplay.ts | 24 ++++++++++ 4 files changed, 142 insertions(+), 12 deletions(-) diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index 1a9b00c4a..6a0a029a1 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -295,6 +295,71 @@ describe("PostBody", () => { expect(screen.getByText("Panel")).toBeInTheDocument(); }); + it("keeps separator-free OCR rows in the existing image table path", () => { + render( + <PostBody + body={'<img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" />'} + imageContent={[ + { + unit_index: 0, + mime_type: "image/png", + status_code: "described", + extracted_text: "No. | Item\n1 | Panel", + caption: "A table image", + tags: [], + }, + ]} + />, + ); + + expect(screen.getByRole("table")).toHaveClass("post-image-text-table"); + expect(screen.getAllByRole("row")).toHaveLength(2); + }); + + it("renders a Markdown table in the source body and keeps empty cells", () => { + render( + <PostBody + body={"Before\n\n| Field | Value | Note |\n| --- | --- | --- |\n| Owner | Buyer | |\n\nAfter"} + />, + ); + + expect(screen.getByRole("table")).toHaveClass("post-markdown-table"); + expect(screen.getAllByRole("row")).toHaveLength(2); + expect(screen.getByText("Owner")).toBeInTheDocument(); + expect(screen.getByText("Before")).toBeInTheDocument(); + expect(screen.getByText("After")).toBeInTheDocument(); + }); + + it("does not turn pipe-delimited prose into a table", () => { + render(<PostBody body={"Alice | manager\nBob | engineer"} />); + + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + expect(screen.getByText((text) => text.includes("Alice | manager"))).toBeInTheDocument(); + }); + + it("renders Markdown tables when persisted text units are present", () => { + const table = "| Field | Value |\n| --- | --- |\n| Owner | Buyer |"; + render( + <PostBody + body={table} + structureUnits={[ + { + unit_index: 0, + unit_kind_code: "dom", + unit_text: table, + indent_level: 0, + indent_source_code: "unresolved", + indent_confidence: 0, + indent_evidence: "", + }, + ]} + />, + ); + + expect(screen.getByRole("table")).toHaveClass("post-markdown-table"); + expect(screen.getByText("Owner")).toBeInTheDocument(); + }); + it("keeps source-image placement while showing persisted OCR and caption evidence", () => { render( <PostBody diff --git a/frontend/src/PostBody.tsx b/frontend/src/PostBody.tsx index 6769d0192..30113026e 100644 --- a/frontend/src/PostBody.tsx +++ b/frontend/src/PostBody.tsx @@ -3,32 +3,46 @@ import { t } from "./i18n"; import type { PostContentUnit, PostImageContent } from "./api"; import type { ReactNode } from "react"; -function parsePipeDelimitedTable(text: string): string[][] | null { - const rows = text +function parsePipeDelimitedTable(text: string, requireSeparator = true): string[][] | null { + const rawRows = text .split(/\r?\n/) .map((row) => { const cells = row.split("|").map((cell) => cell.trim()); if (cells[0] === "") cells.shift(); if (cells[cells.length - 1] === "") cells.pop(); return cells; - }) - .filter((row) => !row.every((cell) => /^:?-{3,}:?$/.test(cell))) + }); + const separatorIndex = rawRows.findIndex( + (row) => row.length > 1 && row.every((cell) => /^:?-{3,}:?$/.test(cell)), + ); + if (requireSeparator && separatorIndex !== 1) return null; + const rows = rawRows + .filter((_row, rowIndex) => rowIndex !== separatorIndex) .filter((row) => row.length > 1 && row.some(Boolean)); if (rows.length < 2 || rows.some((row) => row.length !== rows[0].length)) return null; if (rows[0].length < 2) return null; return rows; } -function renderImageText(text: string) { - const rows = parsePipeDelimitedTable(text); - if (!rows) return <p>{text}</p>; +function renderPipeTable( + text: string, + className: string, + keyPrefix: string, + requireSeparator = true, +): ReactNode | null { + const rows = parsePipeDelimitedTable(text, requireSeparator); + if (!rows) return null; return ( - <table className="post-body-table post-image-text-table"> + <table + key={`${keyPrefix}-table`} + className={className} + data-content-kind="table" + > <tbody> {rows.map((row, rowIndex) => ( - <tr key={`post-image-text-row-${rowIndex}`}> + <tr key={`${keyPrefix}-row-${rowIndex}`}> {row.map((cell, cellIndex) => ( - <td key={`post-image-text-cell-${rowIndex}-${cellIndex}`}>{cell}</td> + <td key={`${keyPrefix}-cell-${rowIndex}-${cellIndex}`}>{cell}</td> ))} </tr> ))} @@ -37,6 +51,14 @@ function renderImageText(text: string) { ); } +function renderImageText(text: string) { + return ( + renderPipeTable(text, "post-body-table post-image-text-table", "post-image-text", false) ?? ( + <p>{text}</p> + ) + ); +} + const SAFE_EMBEDDED_IMAGE_SOURCE = /^data:image\/(?:png|jpe?g|gif|webp|avif|bmp|x-icon|vnd\.microsoft\.icon);base64,[A-Za-z0-9+/]+={0,2}$/i; @@ -114,6 +136,13 @@ function renderSegment(segment: PostBodySegment, index: number, imageContent?: P } } +function renderTextSegment(segment: Extract<PostBodySegment, { kind: "text" }>, index: number) { + return ( + renderPipeTable(segment.text, "post-body-table post-markdown-table", `post-markdown-${index}`) ?? + renderSegment(segment, index) + ); +} + function isStructuredTableRow(unit: PostContentUnit): boolean { return ( unit.unit_label === "tr" || @@ -236,7 +265,7 @@ function renderStructuredUnits( ? unit.indent_level : undefined; rendered.push( - renderSegment( + renderTextSegment( { kind: "text", text: unit.unit_text, @@ -274,7 +303,7 @@ export function PostBody({ {splitPostBody(body).map((segment, index) => { const content = segment.kind === "image" ? imageContent[imageOrdinal++] : undefined; if (segment.kind !== "text") return renderSegment(segment, index, content); - return renderSegment(segment, index, content); + return renderTextSegment(segment, index); })} </div> ); diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index bde6bd936..743739697 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -124,6 +124,18 @@ describe("splitPostBody", () => { ]); }); + it("keeps a stray pipe line inside its surrounding paragraph", () => { + expect(splitPostBody("<p>Before<br>ratio A | B<br>After</p>")).toEqual([ + { kind: "text", text: "Before ratio A | B After" }, + ]); + }); + + it("space-joins consecutive pipe prose when no Markdown separator exists", () => { + expect(splitPostBody("Alice | manager\nBob | engineer")).toEqual([ + { kind: "text", text: "Alice | manager Bob | engineer" }, + ]); + }); + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { expect(splitPostBody("The full body text.")).toEqual([ { kind: "text", text: "The full body text." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 51c040f17..1f69962e9 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -172,13 +172,36 @@ function stripHtmlTags(text: string): string { function splitSemanticParagraphs(text: string): string[] { const paragraphs: string[] = []; let lines: string[] = []; + let pipeTableRows: string[] = []; const flush = () => { const paragraph = lines.join(" ").trimEnd(); if (paragraph.trim()) paragraphs.push(paragraph); lines = []; }; + const flushPipeTableRows = () => { + const hasSeparator = pipeTableRows.some((row) => { + const cells = row.trim().replace(/^\|/, "").replace(/\|$/, "").split("|"); + return cells.length >= 2 && cells.every((cell) => /^\s*:?-{3,}:?\s*$/.test(cell)); + }); + if (pipeTableRows.length >= 2 && hasSeparator) { + flush(); + paragraphs.push(pipeTableRows.map((row) => row.trim()).join("\n")); + } else { + lines.push(...pipeTableRows); + } + pipeTableRows = []; + }; for (const line of text.split("\n")) { + const trimmed = line.trim(); + if (trimmed.includes("|")) { + const cells = trimmed.replace(/^\|/, "").replace(/\|$/, "").split("|"); + if (cells.length >= 2 && cells.some((cell) => cell.trim())) { + pipeTableRows.push(line); + continue; + } + } + if (pipeTableRows.length > 0) flushPipeTableRows(); if (!line.trim()) { flush(); continue; @@ -186,6 +209,7 @@ function splitSemanticParagraphs(text: string): string[] { if (lines.length > 0 && LIST_ITEM_START.test(line)) flush(); lines.push(lines.length === 0 ? line.replace(/[ \t]+$/g, "") : line.trim()); } + if (pipeTableRows.length > 0) flushPipeTableRows(); flush(); return paragraphs; } From 7a0a5f649c766d967d73265ae7833aa7c070f542 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 03:47:05 +0900 Subject: [PATCH 21/43] fix: make lineage evidence disclosure interactive --- backend/tests/test_api.py | 6 ++++++ frontend/src/LineageDag.test.tsx | 16 ++++++++++++---- frontend/src/LineageDag.tsx | 3 ++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 8d5c5533a..8322ce9eb 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -113,6 +113,11 @@ / "migrations" / "0102_project_bound_summary_event.sql" ) +_TENANT_SETTINGS_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0103_tenant_settings.sql" +) _CHANNEL_EVIDENCE_MIGRATION = ( Path(__file__).resolve().parents[2] / "migrations" @@ -231,6 +236,7 @@ def seeded_db(demo_analyst_token): cur.execute(_MAJOR_EVENT_ACTION_MIGRATION.read_text()) cur.execute(_PROJECT_BOUND_ACTION_MIGRATION.read_text()) cur.execute(_PROJECT_BOUND_EVENT_MIGRATION.read_text()) + cur.execute(_TENANT_SETTINGS_MIGRATION.read_text()) cur.execute(_CHANNEL_EVIDENCE_MIGRATION.read_text()) cur.execute( "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values " diff --git a/frontend/src/LineageDag.test.tsx b/frontend/src/LineageDag.test.tsx index 559125f99..9dd007d0a 100644 --- a/frontend/src/LineageDag.test.tsx +++ b/frontend/src/LineageDag.test.tsx @@ -72,6 +72,13 @@ const graph: LineageGraph = { describe("LineageDag channel evidence", () => { it("discloses exact inferred values without hover-only interaction", async () => { render(<LineageDag graph={graph} onSelectPost={vi.fn()} />); + const disclosure = screen.getByText(/fused score 0.700000/).closest("details"); + const edgeButton = screen.getByRole("button", { + name: "Open connection evidence: Kickoff recap to Pricing follow-up", + }); + expect(disclosure).not.toHaveAttribute("open"); + await userEvent.click(edgeButton); + expect(disclosure).toHaveAttribute("open"); expect( screen.getByText("Each connection is inferred from independent signals. It is not a causal claim."), ).toBeInTheDocument(); @@ -81,10 +88,11 @@ describe("LineageDag channel evidence", () => { expect(screen.getByText("0.200000")).toBeInTheDocument(); expect(screen.getByText(/fused score 0.700000/)).toBeInTheDocument(); expect(screen.queryByText(/causal relationship/i)).not.toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Open connection evidence: Kickoff recap to Pricing follow-up" })).toHaveAttribute( - "tabindex", - "0", - ); + expect(edgeButton).toHaveAttribute("tabindex", "0"); + expect(edgeButton).toHaveAttribute("aria-pressed", "true"); + await userEvent.click(screen.getByText(/fused score 0.700000/)); + expect(disclosure).not.toHaveAttribute("open"); + expect(edgeButton).toHaveAttribute("aria-pressed", "false"); }); it("does not claim a missing LLM channel when no evidence was recorded", () => { diff --git a/frontend/src/LineageDag.tsx b/frontend/src/LineageDag.tsx index babb61794..cf51bf6e4 100644 --- a/frontend/src/LineageDag.tsx +++ b/frontend/src/LineageDag.tsx @@ -73,6 +73,7 @@ export function LineageDag({ d={`M ${from.x} ${from.y} C ${midX} ${from.y}, ${midX} ${to.y}, ${to.x} ${to.y}`} role="button" tabIndex={0} + aria-pressed={selected} aria-label={tf("Open connection evidence: {from} to {to}", { from: from.label, to: to.label, @@ -166,7 +167,7 @@ export function LineageDag({ <details key={key} className="lineage-edge-evidence-item" - open + open={selectedEdge === key} onToggle={(event) => { const details = event.currentTarget; if (details.open) { From 16f2b13caad10f4d999293d623405aefadeda52e Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Fri, 21 Aug 2026 11:54:34 -0700 Subject: [PATCH 22/43] fix(frontend): preserve nested list indentation (#391) * fix(frontend): preserve nested list indentation * fix(frontend): indent block children in nested lists --- frontend/src/postBodyDisplay.test.ts | 22 ++++++++++++++++++++++ frontend/src/postBodyDisplay.ts | 13 +++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 743739697..d757a77b0 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -61,6 +61,28 @@ describe("splitPostBody", () => { ]); }); + it("preserves nested HTML list depth as semantic indentation", () => { + expect( + splitPostBody("<ol><li>Parent<ol><li>Child</li></ol></li><li>Sibling</li></ol>"), + ).toEqual([ + { kind: "text", text: "Parent" }, + { kind: "text", text: "Child", indentLevel: 1 }, + { kind: "text", text: "Sibling" }, + ]); + }); + + it("preserves nested list depth when item text is wrapped in a block child", () => { + expect( + splitPostBody( + "<ol><li><p>Parent</p><ol><li><p>Child</p></li></ol></li><li><p>Sibling</p></li></ol>", + ), + ).toEqual([ + { kind: "text", text: "Parent" }, + { kind: "text", text: "Child", indentLevel: 1 }, + { kind: "text", text: "Sibling" }, + ]); + }); + it("labels HTML, Word, and OOXML footnotes in the fallback renderer", () => { expect( splitPostBody( diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 1f69962e9..5c59470f1 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -143,11 +143,20 @@ function indentMarker(width: number): string { function stripHtmlTags(text: string): string { text = markFootnoteTags(text).replace(/<sup[^>]*>(.*?)<\/sup>/gi, "^$1"); + let listDepth = 0; const withBoundaries = text .replace(BREAK_TAG, "\n") .replace(BLOCK_TAG, (tag) => { - if (/^<\//.test(tag)) return "\n\n"; - return `\n\n${indentMarker(declaredIndentWidth(tag))}`; + const name = tag.match(/^<\/?\s*([a-z0-9:]+)/i)?.[1]?.toLowerCase() ?? ""; + const closing = /^<\//.test(tag); + if (name === "ul" || name === "ol") { + if (closing) listDepth = Math.max(0, listDepth - 1); + else listDepth += 1; + return "\n\n"; + } + if (closing) return "\n\n"; + const nestedListIndent = !closing && listDepth > 0 ? Math.max(0, listDepth - 1) * 4 : 0; + return `\n\n${indentMarker(declaredIndentWidth(tag) + nestedListIndent)}`; }) .replace(WORD_INDENT_TAG, (tag) => indentMarker(declaredIndentWidth(tag))); const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => { From eaea56d3b2f07f89a5dfcc7d81b032148048982d Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 04:06:36 +0900 Subject: [PATCH 23/43] fix(db): make tenant settings migration replay-safe --- backend/tests/test_api.py | 2 ++ migrations/0103_tenant_settings.sql | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 8322ce9eb..750955c16 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -237,6 +237,8 @@ def seeded_db(demo_analyst_token): cur.execute(_PROJECT_BOUND_ACTION_MIGRATION.read_text()) cur.execute(_PROJECT_BOUND_EVENT_MIGRATION.read_text()) cur.execute(_TENANT_SETTINGS_MIGRATION.read_text()) + # Compose replays this gate on restart; the second apply is the contract. + cur.execute(_TENANT_SETTINGS_MIGRATION.read_text()) cur.execute(_CHANNEL_EVIDENCE_MIGRATION.read_text()) cur.execute( "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values " diff --git a/migrations/0103_tenant_settings.sql b/migrations/0103_tenant_settings.sql index 9470ebe9a..46d52ced2 100644 --- a/migrations/0103_tenant_settings.sql +++ b/migrations/0103_tenant_settings.sql @@ -1,6 +1,8 @@ -CREATE TABLE tenant_settings ( +-- migrate.sh replays gated migrations on every Compose start. +CREATE TABLE IF NOT EXISTS tenant_settings ( id int PRIMARY KEY CHECK (id = 1), brand_name text NOT NULL DEFAULT 'LineageWeave', updated_at timestamptz NOT NULL DEFAULT now() ); -INSERT INTO tenant_settings (id, brand_name) VALUES (1, 'LineageWeave'); +INSERT INTO tenant_settings (id, brand_name) VALUES (1, 'LineageWeave') +ON CONFLICT (id) DO NOTHING; From 16f6341a0feec262904c1ad9275ed73449444cf5 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sat, 22 Aug 2026 04:36:47 +0900 Subject: [PATCH 24/43] fix: bound live lineage adjudication work --- CHANGELOG.md | 4 +++ backend/app/lineage_ingestion.py | 18 ++++++++++--- .../0124-event-lineage-channel-evidence.md | 7 +++-- tests/test_lineage_ingestion.py | 27 +++++++++++++++++++ 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e2aeec84..e8a5a13a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ All notable changes to this project are documented here. Format follows ### Fixed +- Full-corpus Event Lineage rebuilds now count candidate pairs before provider + work and omit the optional LLM channel above the 5,000-pair ADR budget, + preventing millions of synchronous orchestrator calls while retaining one + auditable weight profile for the entire rebuild. - `make smoke` and `make seed` now run through the locked project `uv` environment, so local OIDC and synthetic-data workflows resolve the same pinned dependencies as CI. diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index b4f339695..4134034d9 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -27,6 +27,9 @@ rank_channel_evidence, ) from lineageweave.models import Edge, Record +from lineageweave.reconstruct import DEFAULT_CANDIDATE_WINDOW + +MAXIMUM_LIVE_LLM_PAIR_EVALUATIONS = 5_000 def _occurred_at(value: datetime) -> datetime: @@ -132,6 +135,14 @@ async def _reconstruct_lineage_records( llm: AdjudicationClient | None, ) -> list[Edge]: """Run the CPU/provider reconstruction without blocking the event loop.""" + pair_count = 0 + records_per_group: defaultdict[str, int] = defaultdict(int) + for record in records: + pair_count += min(records_per_group[record.group_key], DEFAULT_CANDIDATE_WINDOW) + if pair_count > MAXIMUM_LIVE_LLM_PAIR_EVALUATIONS: + llm = None + break + records_per_group[record.group_key] += 1 return await asyncio.to_thread(lineage_edge_specs, records, llm=llm) @@ -142,9 +153,10 @@ async def rebuild_lineage( ) -> list[Edge]: """Reconstruct lineage for every ``source_post`` and persist the edges. - A configured contextual-orchestrator client is passed through so the - optional LLM channel is recorded when available; ``None`` preserves the - fail-closed three-channel rebuild. + A configured contextual-orchestrator client is passed through only when + the exact candidate-pair work fits the ADR 0124 budget. Larger snapshots + drop the optional channel before any provider call and preserve one + fail-closed three-channel profile across the rebuild. """ records = await _load_lineage_records(conn) edges = await _reconstruct_lineage_records(records, llm) diff --git a/docs/adr/0124-event-lineage-channel-evidence.md b/docs/adr/0124-event-lineage-channel-evidence.md index c5bf9e4df..82d0baf9d 100644 --- a/docs/adr/0124-event-lineage-channel-evidence.md +++ b/docs/adr/0124-event-lineage-channel-evidence.md @@ -45,8 +45,11 @@ authoritative; PROV-O/RDF export is a projection. separate immutable run-scoped table. The administrator-triggered live rebuild and PostgreSQL import pass the configured contextual-orchestrator adjudication client through the same - reconstruction boundary; when it is unavailable, the LLM channel is - dropped and the remaining weights are renormalized. + reconstruction boundary only when the exact candidate-pair count is at + most 5,000. Larger snapshots drop the LLM channel before any provider call + and renormalize the remaining weights. This is an operational work bound, + not a model-quality or provider-ranking heuristic. One rebuild never mixes + LLM and non-LLM weight profiles across edges. 5. `GET /api/lineage` returns an additive `channel_evidence` collection on each visible edge (`signal_code`, `signal_label`, `score`, `weight`, `contribution`, `rank`) ordered by contribution, then diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index 886c56e56..b03a81fb4 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from backend.app.lineage_ingestion import ( + _reconstruct_lineage_records, persist_lineage_edges, rebuild_lineage, rebuild_lineage_from_pool, @@ -15,6 +16,7 @@ ) from lineageweave.fixtures import sample_records from lineageweave.lineage_persistence import lineage_edge_specs +from lineageweave.models import Record def test_records_use_persisted_thread_keys_not_process_unit_or_voc_type() -> None: @@ -84,6 +86,31 @@ async def fake_to_thread(function, *args, **kwargs): assert events == ["fetch", "reconstruct", "transaction_enter", "persist", "transaction_exit"] +def test_rebuild_drops_llm_before_candidate_pair_budget_is_exceeded(monkeypatch) -> None: + """Keep a large live rebuild from issuing unbounded provider calls.""" + + records = [ + Record(f"record-{index}", "shared-group", f"Record {index}", datetime(2026, 1, index + 1)) + for index in range(3) + ] + captured: dict[str, object] = {} + + def fake_lineage_edge_specs(_records, *, llm=None): + captured["llm"] = llm + return [] + + monkeypatch.setattr( + "backend.app.lineage_ingestion.MAXIMUM_LIVE_LLM_PAIR_EVALUATIONS", 1 + ) + monkeypatch.setattr( + "backend.app.lineage_ingestion.lineage_edge_specs", fake_lineage_edge_specs + ) + + asyncio.run(_reconstruct_lineage_records(records, object())) + + assert captured["llm"] is None + + def test_pooled_rebuild_releases_the_connection_during_reconstruction(monkeypatch) -> None: """Keep provider work outside the pool and transaction, then replace atomically.""" events: list[str] = [] From 55a13f3473789a9481061ca2cd1f9ea042fc5902 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sun, 23 Aug 2026 15:23:36 +0900 Subject: [PATCH 25/43] chore: nudge CI re-review (opencode-agent's prior REQUEST_CHANGES was against a transient coverage-evidence flake on this same head; later reruns of the same head passed, but opencode-review only posts once per head SHA) From c34681fdc692a25e688fe4a5eb06ad3fe50f2281 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Sun, 23 Aug 2026 17:22:52 +0900 Subject: [PATCH 26/43] fix: guard corpus lineage rebuild against orchestrator failures; drop Buyer wording - Wrap rebuild_lineage_from_pool() in main.py's /api/lineage/rebuild in the same except (HttpClientError, OSError) -> HTTPException(503) pattern used at this file's other orchestrator call sites, so a transient hiccup during a corpus-wide (up to 5,000-call) rebuild degrades cleanly instead of discarding the whole reconstruction as a raw 500. Adds a focused endpoint test covering the new 503 path. - Reword the new ADR 0124 prose and CHANGELOG/CHANGELOG.d entries added by this PR to say "reader" / "Event Lineage DAG" instead of "buyer"/"Buyer", so this PR doesn't reintroduce naming PR #474 (ADR 0119) is retiring. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B5M79L945DMyMs3sg5yJ14 --- .../2.14.0-event-lineage-channel-evidence.md | 2 +- CHANGELOG.md | 2 +- backend/app/main.py | 13 ++++++- backend/tests/test_api.py | 34 +++++++++++++++++++ .../0124-event-lineage-channel-evidence.md | 8 ++--- 5 files changed, 52 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.d/2.14.0-event-lineage-channel-evidence.md b/CHANGELOG.d/2.14.0-event-lineage-channel-evidence.md index ec2ecaea4..dc8e9217c 100644 --- a/CHANGELOG.d/2.14.0-event-lineage-channel-evidence.md +++ b/CHANGELOG.d/2.14.0-event-lineage-channel-evidence.md @@ -15,6 +15,6 @@ produced each reconstructed connection. live-graph replacement. - `GET /api/lineage` returns additive `channel_evidence` on each visible edge. ABAC never reveals evidence for an invisible endpoint. -- The Buyer DAG discloses exact values with keyboard and screen-reader +- The Event Lineage DAG discloses exact values with keyboard and screen-reader access, labels the relation as inferred rather than causal, and keeps the same values in print. diff --git a/CHANGELOG.md b/CHANGELOG.md index e8a5a13a8..c958422bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this project are documented here. Format follows - Event Lineage now persists each reconstructed connection's independent channel scores, the normalized weights actually used, and their - contributions. The Buyer DAG discloses those exact values as inferred + contributions. The Event Lineage DAG discloses those exact values as inferred evidence, not a causal claim, and omits the LLM channel when it did not participate. diff --git a/backend/app/main.py b/backend/app/main.py index 2d63bade9..ca696382d 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1112,7 +1112,18 @@ async def rebuild_lineage_graph( post_admin only: this is a corpus-wide write. Reads stay ABAC-gated. """ _require_post_admin(account) - edges = await rebuild_lineage_from_pool(pool, llm=_adjudication_client()) + try: + edges = await rebuild_lineage_from_pool(pool, llm=_adjudication_client()) + except (HttpClientError, OSError) as exc: + # This can issue up to MAXIMUM_LIVE_LLM_PAIR_EVALUATIONS sequential + # adjudication calls across the whole corpus (lineage_ingestion.py); + # a transient orchestrator hiccup on any one of them must not + # discard the rest of the reconstruction as a raw 500 -- same + # discipline as this file's other orchestrator call sites. + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Lineage rebuild is unavailable: the orchestrator did not respond", + ) from exc return {"edge_count": len(edges)} diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 750955c16..d86501b12 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -3669,6 +3669,40 @@ def test_rebuild_lineage_requires_post_admin(client, demo_analyst_token) -> None assert response.status_code == 403 +def test_rebuild_lineage_reports_503_on_orchestrator_failure( + monkeypatch, client, demo_analyst_token, seeded_db +) -> None: + """A transient orchestrator failure mid-rebuild must degrade to a clean + 503, not discard the whole corpus reconstruction as a raw 500 (same + discipline as this file's other orchestrator call sites). + """ + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) " + "values ('permission', 'post_admin', 'Administer posts') on conflict (lookup_code) do nothing" + ) + cur.execute("select access_role_id from account_role_assignment limit 1") + role_id = cur.fetchone()[0] + cur.execute( + "insert into role_permission (access_role_id, permission_code) values (%s, 'post_admin') " + "on conflict do nothing", + (role_id,), + ) + finally: + admin_conn.close() + + async def _raise(pool, *, llm): + raise HttpClientError("orchestrator hiccup") + + monkeypatch.setattr("backend.app.main.rebuild_lineage_from_pool", _raise) + + response = client.post("/api/lineage/rebuild", headers={"Authorization": f"Bearer {demo_analyst_token}"}) + assert response.status_code == 503 + + def test_rebuild_lineage_recovers_the_a100_fork(client, demo_analyst_token, seeded_db) -> None: """Rebuild on the same A-100+B-200 rows seed writes (grouping keys + occurred_at), not a hand-picked A-100-only insert that hides mapping bugs. diff --git a/docs/adr/0124-event-lineage-channel-evidence.md b/docs/adr/0124-event-lineage-channel-evidence.md index 82d0baf9d..81a0aa9d9 100644 --- a/docs/adr/0124-event-lineage-channel-evidence.md +++ b/docs/adr/0124-event-lineage-channel-evidence.md @@ -10,9 +10,9 @@ (`temporal`, `secondary_key`, `text`, optional `llm`) and RankWeave fuses them with a weighted convex combination. Production persistence collapsed each edge to `(parent_post_id, child_post_id, fused_score)`, -so `/api/lineage` and the Buyer DAG exposed only the fused score. +so `/api/lineage` and the Event Lineage DAG exposed only the fused score. -A buyer could see that two posts were linked but could not answer which +A reader could see that two posts were linked but could not answer which independent signals supported the edge, whether the optional LLM channel participated, which signal dominated, or how to audit a later reconstruction after model or weight changes. ADR 0064 already treats @@ -56,14 +56,14 @@ authoritative; PROV-O/RDF export is a projection. controlled signal order. The rebuild profile is returned in the same controlled order using `common_lookup_value.display_order`. ABAC never reveals evidence for an invisible endpoint. -6. The Buyer DAG provides an accessible edge-detail disclosure (not +6. The Event Lineage DAG provides an accessible edge-detail disclosure (not hover-only), labels the relation as inferred rather than causal, and states when no LLM channel participated only when at least one recorded channel exists. Print/export uses the same values. ## Consequences -- Buyers can inspect why a connection was selected and distinguish +- Readers can inspect why a connection was selected and distinguish inference from source evidence. - A later rebuild rewrites live Event Lineage as a whole; historic meaning is not silently mutated in place. From 115bd3e2d5ff18b63bea8e0452947ad1f7c9a591 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Mon, 24 Aug 2026 04:40:48 +0900 Subject: [PATCH 27/43] fix: allocate migration 0174 for lineage signals --- backend/tests/test_api.py | 2 +- docker/postgres-init/migrate.sh | 2 +- ...eage_edge_signal.sql => 0174_post_lineage_edge_signal.sql} | 2 +- ...eage_edge_signal.sql => 0174_post_lineage_edge_signal.sql} | 0 tests/test_lineage_channel_evidence.py | 4 ++-- tests/test_schema.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) rename migrations/{0105_post_lineage_edge_signal.sql => 0174_post_lineage_edge_signal.sql} (96%) rename migrations/rollback/{0105_post_lineage_edge_signal.sql => 0174_post_lineage_edge_signal.sql} (100%) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index d86501b12..52bff1bdb 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -121,7 +121,7 @@ _CHANNEL_EVIDENCE_MIGRATION = ( Path(__file__).resolve().parents[2] / "migrations" - / "0105_post_lineage_edge_signal.sql" + / "0174_post_lineage_edge_signal.sql" ) diff --git a/docker/postgres-init/migrate.sh b/docker/postgres-init/migrate.sh index 300ab6b8c..de0fa613c 100644 --- a/docker/postgres-init/migrate.sh +++ b/docker/postgres-init/migrate.sh @@ -18,7 +18,7 @@ for migration in /opt/lineageweave/migrations/*.sql; do migration_name=${migration##*/} case "$migration_name" in 0012_*|0013_*|0014_*|0015_*|0016_*|0017_*|0018_*|0019_*|0020_*|0021_*|0022_*|0023_*|0024_*|0025_*|0026_*|0027_*|0028_*|0029_*|0030_*|0031_*|0032_*|0033_*|0034_*|0035_*|0036_*|0037_*|0038_*|0039_*|0040_*|0041_*|0042_*|0043_*|0044_*|0045_*|0046_*|0047_*|0048_*|0049_*|0050_*) ;; - 0060_*|0100_*|0101_*|0102_*|0103_*|0105_*) ;; + 0060_*|0100_*|0101_*|0102_*|0103_*|0174_*) ;; *) continue ;; esac printf 'Applying %s\n' "$migration_name" diff --git a/migrations/0105_post_lineage_edge_signal.sql b/migrations/0174_post_lineage_edge_signal.sql similarity index 96% rename from migrations/0105_post_lineage_edge_signal.sql rename to migrations/0174_post_lineage_edge_signal.sql index 826cd6368..b476717b2 100644 --- a/migrations/0105_post_lineage_edge_signal.sql +++ b/migrations/0174_post_lineage_edge_signal.sql @@ -1,6 +1,6 @@ -- ADR 0124: persist Event Lineage channel evidence beside each fused edge. -- lookup_code is globally unique, so signal codes are prefixed. --- CREATE IF NOT EXISTS / ON CONFLICT so migrate.sh replay is idempotent. +-- Migration 0174 uses CREATE IF NOT EXISTS / ON CONFLICT for idempotent replay. insert into common_lookup_value (lookup_category, lookup_code, lookup_label, display_order) values ('lineage_signal', 'lineage_signal_temporal', 'Temporal proximity', 0), diff --git a/migrations/rollback/0105_post_lineage_edge_signal.sql b/migrations/rollback/0174_post_lineage_edge_signal.sql similarity index 100% rename from migrations/rollback/0105_post_lineage_edge_signal.sql rename to migrations/rollback/0174_post_lineage_edge_signal.sql diff --git a/tests/test_lineage_channel_evidence.py b/tests/test_lineage_channel_evidence.py index a23a3e559..f855d1387 100644 --- a/tests/test_lineage_channel_evidence.py +++ b/tests/test_lineage_channel_evidence.py @@ -152,11 +152,11 @@ def test_migrate_sh_replays_channel_evidence_and_tenant_settings() -> None: migrate = (_ROOT / "docker/postgres-init/migrate.sh").read_text() assert "0103_*" in migrate assert "0104_*" not in migrate - assert "0105_*" in migrate + assert "0174_*" in migrate def test_channel_evidence_migration_has_no_jsonb() -> None: - migration = (_ROOT / "migrations" / "0105_post_lineage_edge_signal.sql").read_text() + migration = (_ROOT / "migrations" / "0174_post_lineage_edge_signal.sql").read_text() assert "jsonb" not in migration.casefold() assert "post_lineage_edge_signal" in migration assert "event_lineage_rebuild" in migration diff --git a/tests/test_schema.py b/tests/test_schema.py index c220e7982..da31a8199 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -44,7 +44,7 @@ / "0102_project_bound_summary_event.sql" ) _CHANNEL_EVIDENCE_MIGRATION = ( - Path(__file__).resolve().parents[1] / "migrations" / "0105_post_lineage_edge_signal.sql" + Path(__file__).resolve().parents[1] / "migrations" / "0174_post_lineage_edge_signal.sql" ) From 3c3bcb51e68d7a353490b3f2d6f099df48d79145 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Mon, 24 Aug 2026 05:29:00 +0900 Subject: [PATCH 28/43] fix: close lineage evidence review gaps --- backend/app/lineage_ingestion.py | 7 ++- ...=> 0172-event-lineage-channel-evidence.md} | 2 +- docs/adr/README.md | 2 +- frontend/src/LineageDag.test.tsx | 1 + frontend/src/LineageDag.tsx | 2 +- lineageweave/__init__.py | 14 ++--- lineageweave/lineage_persistence.py | 2 +- migrations/0174_post_lineage_edge_signal.sql | 35 ++++++++++++- tests/test_lineage_ingestion.py | 45 ++++++++++++++++ tests/test_schema.py | 52 +++++++++++++++++++ 10 files changed, 148 insertions(+), 14 deletions(-) rename docs/adr/{0124-event-lineage-channel-evidence.md => 0172-event-lineage-channel-evidence.md} (98%) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index 4134034d9..6e70065b6 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -154,7 +154,7 @@ async def rebuild_lineage( """Reconstruct lineage for every ``source_post`` and persist the edges. A configured contextual-orchestrator client is passed through only when - the exact candidate-pair work fits the ADR 0124 budget. Larger snapshots + the exact candidate-pair work fits the ADR 0172 budget. Larger snapshots drop the optional channel before any provider call and preserve one fail-closed three-channel profile across the rebuild. """ @@ -173,7 +173,7 @@ async def rebuild_lineage_from_pool( """Reconstruct without holding a pooled connection during provider work. The source snapshot is read and released first. Only the replacement - writes run in a transaction, preserving ADR 0124 atomicity without an + writes run in a transaction, preserving ADR 0172 atomicity without an idle-in-transaction connection during CPU or orchestrator calls. """ async with pool.acquire() as conn: @@ -203,6 +203,7 @@ async def visible_lineage_graph( f"from source_post where {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}" ) visible_all = [row for row in posts if can_see_post(row)] + visible_all_ids = {str(row["post_id"]) for row in visible_all} edge_rows = await conn.fetch( "select parent_post_id, child_post_id, fused_score from post_lineage_edge" ) @@ -233,6 +234,8 @@ async def visible_lineage_graph( for edge in edge_rows: parent_id = str(edge["parent_post_id"]) child_id = str(edge["child_post_id"]) + if parent_id not in visible_all_ids or child_id not in visible_all_ids: + continue neighbors.setdefault(parent_id, set()).add(child_id) neighbors.setdefault(child_id, set()).add(parent_id) diff --git a/docs/adr/0124-event-lineage-channel-evidence.md b/docs/adr/0172-event-lineage-channel-evidence.md similarity index 98% rename from docs/adr/0124-event-lineage-channel-evidence.md rename to docs/adr/0172-event-lineage-channel-evidence.md index 81a0aa9d9..717fe61ec 100644 --- a/docs/adr/0124-event-lineage-channel-evidence.md +++ b/docs/adr/0172-event-lineage-channel-evidence.md @@ -1,4 +1,4 @@ -# ADR 0124: Persist and explain Event Lineage channel evidence +# ADR 0172: Persist and explain Event Lineage channel evidence **Status:** Accepted **Date:** 2026-08-21 diff --git a/docs/adr/README.md b/docs/adr/README.md index 3b2c63ac3..99a8b76e8 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -10,7 +10,7 @@ decision from them. | Supporting document | Normative ADR | |---|---| | [`product-technical-gap-baseline.md`](../product-technical-gap-baseline.md) | Product/technical traceability projection across the ADR set; ADRs remain normative | -| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0124](0124-event-lineage-channel-evidence.md) | +| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0172](0172-event-lineage-channel-evidence.md) | | [`PROV_O_IMPLEMENTATION.md`](../PROV_O_IMPLEMENTATION.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`PROV_O_IMPLEMENTATION_MATRIX.md`](../PROV_O_IMPLEMENTATION_MATRIX.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`image-content-schema.md`](../image-content-schema.md) | [0066](0066-position-preserving-image-content.md) | diff --git a/frontend/src/LineageDag.test.tsx b/frontend/src/LineageDag.test.tsx index 9dd007d0a..1cf92387f 100644 --- a/frontend/src/LineageDag.test.tsx +++ b/frontend/src/LineageDag.test.tsx @@ -76,6 +76,7 @@ describe("LineageDag channel evidence", () => { const edgeButton = screen.getByRole("button", { name: "Open connection evidence: Kickoff recap to Pricing follow-up", }); + expect(screen.getByRole("group", { name: "A-100 lineage" })).toBeInTheDocument(); expect(disclosure).not.toHaveAttribute("open"); await userEvent.click(edgeButton); expect(disclosure).toHaveAttribute("open"); diff --git a/frontend/src/LineageDag.tsx b/frontend/src/LineageDag.tsx index cf51bf6e4..ace18ad5e 100644 --- a/frontend/src/LineageDag.tsx +++ b/frontend/src/LineageDag.tsx @@ -56,7 +56,7 @@ export function LineageDag({ viewBox={`0 0 ${group.width} ${group.height}`} width="100%" height={Math.max(120, group.height)} - role="img" + role="group" aria-label={tf("{group} lineage", { group: group.heading })} > {group.edges.map((edge) => { diff --git a/lineageweave/__init__.py b/lineageweave/__init__.py index 821377c54..113c6b3f1 100644 --- a/lineageweave/__init__.py +++ b/lineageweave/__init__.py @@ -24,8 +24,8 @@ PROV, PROV_CLASSES, PROV_QUALIFICATIONS, - PROV_RELATIONS, PROV_RECOMMENDED_INVERSES, + PROV_RELATIONS, ProvAssertion, ProvGraph, ProvLiteral, @@ -35,14 +35,15 @@ from .voc_evidence import sentence_excerpts __all__ = [ - "ChatAnswer", - "Edge", - "OrganizationRelationship", + "CHANNEL_EVIDENCE_TOLERANCE", "PROV", "PROV_CLASSES", "PROV_QUALIFICATIONS", - "PROV_RELATIONS", "PROV_RECOMMENDED_INVERSES", + "PROV_RELATIONS", + "ChatAnswer", + "Edge", + "OrganizationRelationship", "PostSummary", "ProvAssertion", "ProvGraph", @@ -52,10 +53,9 @@ "Tree", "build_affiliate_forest", "cited_post_summaries", - "CHANNEL_EVIDENCE_TOLERANCE", "lineage_edge_specs", - "rank_channel_evidence", "random_walk_with_restart", + "rank_channel_evidence", "reconstruct", "reconstruction_version", "resolve_corporate_entity", diff --git a/lineageweave/lineage_persistence.py b/lineageweave/lineage_persistence.py index a21104b18..5032fd670 100644 --- a/lineageweave/lineage_persistence.py +++ b/lineageweave/lineage_persistence.py @@ -6,7 +6,7 @@ Each parent→child edge is still one ``post_lineage_edge`` row. The winning edge's active channel scores are persisted beside it as -``post_lineage_edge_signal`` rows (ADR 0124). A missing LLM channel is +``post_lineage_edge_signal`` rows (ADR 0172). A missing LLM channel is dropped, never fabricated. Contribution is ``weight * score`` and must reconcile with ``fused_score`` within the base tolerance plus the bounded persistence quantization budget. diff --git a/migrations/0174_post_lineage_edge_signal.sql b/migrations/0174_post_lineage_edge_signal.sql index b476717b2..fe237ab07 100644 --- a/migrations/0174_post_lineage_edge_signal.sql +++ b/migrations/0174_post_lineage_edge_signal.sql @@ -1,4 +1,4 @@ --- ADR 0124: persist Event Lineage channel evidence beside each fused edge. +-- ADR 0172: persist Event Lineage channel evidence beside each fused edge. -- lookup_code is globally unique, so signal codes are prefixed. -- Migration 0174 uses CREATE IF NOT EXISTS / ON CONFLICT for idempotent replay. @@ -53,5 +53,38 @@ create table if not exists post_lineage_edge_signal ( comment on table post_lineage_edge_signal is 'Per-channel score, active weight, and contribution for one reconstructed lineage edge.'; +do $migration$ +begin + if not exists ( + select 1 from pg_constraint + where conname = 'event_lineage_rebuild_channel_signal_code_check' + and conrelid = 'event_lineage_rebuild_channel'::regclass + ) then + alter table event_lineage_rebuild_channel + add constraint event_lineage_rebuild_channel_signal_code_check + check (signal_code in ( + 'lineage_signal_temporal', + 'lineage_signal_secondary_key', + 'lineage_signal_text', + 'lineage_signal_llm' + )); + end if; + if not exists ( + select 1 from pg_constraint + where conname = 'post_lineage_edge_signal_code_check' + and conrelid = 'post_lineage_edge_signal'::regclass + ) then + alter table post_lineage_edge_signal + add constraint post_lineage_edge_signal_code_check + check (signal_code in ( + 'lineage_signal_temporal', + 'lineage_signal_secondary_key', + 'lineage_signal_text', + 'lineage_signal_llm' + )); + end if; +end; +$migration$; + create index if not exists post_lineage_edge_signal_child_idx on post_lineage_edge_signal (child_post_id, parent_post_id); diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index b03a81fb4..838cfc2cf 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -476,6 +476,51 @@ async def fetch(self, query: str, *_args): assert "lineage_signal_text" not in serialized +def test_focused_graph_cannot_bridge_through_an_invisible_post() -> None: + class FakeConnection: + posts = tuple( + { + "post_id": post_id, + "post_title": post_id, + "voc_type_code": "voc", + "visibility_code": visibility, + "corporate_entity_id": "corp", + "process_unit_id": "pu", + "thread_group_key": "thread-a", + "created_at": datetime(2026, 1, day, tzinfo=timezone.utc), + } + for day, post_id, visibility in ( + (1, "post-a", "public"), + (2, "post-hidden", "restricted"), + (3, "post-b", "public"), + ) + ) + edges = ( + {"parent_post_id": "post-a", "child_post_id": "post-hidden", "fused_score": 0.8}, + {"parent_post_id": "post-hidden", "child_post_id": "post-b", "fused_score": 0.8}, + ) + + async def fetch(self, query: str, *_args): + if "post_lineage_edge_signal" in query: + return [] + if "event_lineage_rebuild_channel" in query: + return [] + if "event_lineage_rebuild" in query: + return [] + return self.edges if "post_lineage_edge" in query else self.posts + + graph = asyncio.run( + visible_lineage_graph( + FakeConnection(), + lambda row: row["visibility_code"] == "public", + focus_post_id="post-a", + ) + ) + + assert graph["nodes"] == [] + assert graph["edges"] == [] + + def test_persist_lineage_edges_replaces_signals_atomically_without_llm() -> None: from lineageweave.lineage_persistence import lineage_rebuild_spec from lineageweave.models import Edge diff --git a/tests/test_schema.py b/tests/test_schema.py index da31a8199..274eff859 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -229,6 +229,58 @@ def test_lineage_channel_evidence_is_cascaded_and_lookup_controlled(schema_db) - assert cur.fetchall() == [] +def test_lineage_signal_tables_reject_other_lookup_categories(schema_db) -> None: + with schema_db.cursor() as cur: + cur.execute( + "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) " + "values ('test_category', 'not_lineage_signal', 'Not lineage')" + ) + cur.execute( + "insert into event_lineage_rebuild " + "(rebuild_lock, reconstruction_version, generated_at, min_fused_score, candidate_window) " + "values (true, 'test', now(), 0.3, 50)" + ) + statements = ( + "insert into event_lineage_rebuild_channel " + "(rebuild_lock, signal_code, signal_weight) " + "values (true, 'not_lineage_signal', 0.5)", + "insert into post_lineage_edge_signal " + "(parent_post_id, child_post_id, signal_code, signal_score, signal_weight, signal_contribution) " + "values ('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', " + "'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', 'not_lineage_signal', 0.5, 0.5, 0.25)", + ) + for index, statement in enumerate(statements): + savepoint = f"wrong_signal_{index}" + cur.execute(f"savepoint {savepoint}") + with pytest.raises(psycopg2.errors.CheckViolation): + cur.execute(statement) + cur.execute(f"rollback to savepoint {savepoint}") + schema_db.rollback() + + +def test_lineage_channel_evidence_migration_upgrades_existing_tables(schema_db) -> None: + with schema_db.cursor() as cur: + cur.execute( + "alter table event_lineage_rebuild_channel " + "drop constraint event_lineage_rebuild_channel_signal_code_check" + ) + cur.execute( + "alter table post_lineage_edge_signal " + "drop constraint post_lineage_edge_signal_code_check" + ) + cur.execute(_CHANNEL_EVIDENCE_MIGRATION.read_text()) + cur.execute( + "select conname from pg_constraint where conname in (" + "'event_lineage_rebuild_channel_signal_code_check', " + "'post_lineage_edge_signal_code_check') order by conname" + ) + assert [row[0] for row in cur.fetchall()] == [ + "event_lineage_rebuild_channel_signal_code_check", + "post_lineage_edge_signal_code_check", + ] + schema_db.commit() + + def test_corporate_hierarchy_recursive_query_returns_correct_shape(schema_db) -> None: """The real product requirement: 'Acme Group -> Acme Electronics Korea -> Acme Electronics Gwangju Plant' must be walkable with one query, From 6bcd52f1d8b1efb76f160506e15b5fb2cea1dbf4 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Mon, 24 Aug 2026 05:44:04 +0900 Subject: [PATCH 29/43] Batch Event Lineage persistence writes --- backend/app/lineage_ingestion.py | 53 ++++++++++++++++---------------- tests/test_lineage_ingestion.py | 7 +++++ 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index 6e70065b6..b2a324062 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -91,33 +91,32 @@ async def persist_lineage_edges(conn: asyncpg.Connection, edges: list[Edge]) -> spec.min_fused_score, spec.candidate_window, ) - for signal_code, signal_weight in spec.channel_weights: - await conn.execute( - "insert into event_lineage_rebuild_channel " - "(rebuild_lock, signal_code, signal_weight) values (true, $1, $2)", - signal_code, - signal_weight, - ) - for edge in edges: - await conn.execute( - "insert into post_lineage_edge (parent_post_id, child_post_id, fused_score) " - "values ($1::uuid, $2::uuid, $3)", - edge.parent_id, - edge.child_id, - edge.fused_score, - ) - for row in spec.signal_rows: - await conn.execute( - "insert into post_lineage_edge_signal " - "(parent_post_id, child_post_id, signal_code, signal_score, signal_weight, signal_contribution) " - "values ($1::uuid, $2::uuid, $3, $4, $5, $6)", - row["parent_post_id"], - row["child_post_id"], - row["signal_code"], - row["signal_score"], - row["signal_weight"], - row["signal_contribution"], - ) + await conn.executemany( + "insert into event_lineage_rebuild_channel " + "(rebuild_lock, signal_code, signal_weight) values (true, $1, $2)", + spec.channel_weights, + ) + await conn.executemany( + "insert into post_lineage_edge (parent_post_id, child_post_id, fused_score) " + "values ($1::uuid, $2::uuid, $3)", + [(edge.parent_id, edge.child_id, edge.fused_score) for edge in edges], + ) + await conn.executemany( + "insert into post_lineage_edge_signal " + "(parent_post_id, child_post_id, signal_code, signal_score, signal_weight, signal_contribution) " + "values ($1::uuid, $2::uuid, $3, $4, $5, $6)", + [ + ( + row["parent_post_id"], + row["child_post_id"], + row["signal_code"], + row["signal_score"], + row["signal_weight"], + row["signal_contribution"], + ) + for row in spec.signal_rows + ], + ) async def _load_lineage_records(conn: asyncpg.Connection) -> list[Record]: diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index 838cfc2cf..5468de42f 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -315,10 +315,16 @@ async def fetch(self, query: str, *_args): class _RecordingConnection: def __init__(self) -> None: self.statements: list[tuple[str, tuple]] = [] + self.batches: list[tuple[str, list[tuple]]] = [] async def execute(self, query: str, *args): self.statements.append((query, args)) + async def executemany(self, query: str, args): + rows = list(args) + self.batches.append((query, rows)) + self.statements.extend((query, row) for row in rows) + async def fetch(self, query: str, *_args): return [] @@ -550,6 +556,7 @@ def test_persist_lineage_edges_replaces_signals_atomically_without_llm() -> None "lineage_signal_secondary_key", "lineage_signal_text", ] + assert [len(rows) for _query, rows in connection.batches] == [3, 1, 3] spec = lineage_rebuild_spec([edge], package_version="2.14.0") assert spec.reconstruction_version == "lineageweave.reconstruct/2.14.0" From 5ef0f2e6f2256008698c6144285133cd249cf639 Mon Sep 17 00:00:00 2001 From: Seongho Bae <me@seonghobae.me> Date: Mon, 24 Aug 2026 11:06:44 +0900 Subject: [PATCH 30/43] fix(frontend): restore shared OIDC return-URL helpers on login The login button had regressed to a raw window.location.pathname + window.location.search concat for the OIDC state.returnUrl, dropping the hash fragment and the isSafeReturnUrl validation that returnUrlFromLocation() already provides, and never called rememberOidcReturnUrl() to persist a storage fallback for restoreOidcReturnUrl() in main.tsx's onSigninCallback. Use the shared helpers again, matching the existing recurring-bug pattern already fixed elsewhere in this stack (the admin-panel-on-unauthenticated-branch half of the same class of regression). --- frontend/src/App.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 666888a4d..1b5b351ab 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -101,6 +101,7 @@ import { tf, useLocale, } from "./i18n"; +import { rememberOidcReturnUrl, returnUrlFromLocation } from "./oidcReturnUrl"; import "./App.css"; function orchestratorUnavailableMessage(err: unknown, action: string): string { @@ -4609,7 +4610,8 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean </div> <div className="login-controls"> <button className="btn-primary" onClick={() => { - const returnUrl = window.location.pathname + window.location.search; + const returnUrl = returnUrlFromLocation(); + rememberOidcReturnUrl(returnUrl); void auth.signinRedirect({ state: { returnUrl } }); }}> {t("Log in")} From 5f02940d18fe8c084e58c24468b03d8e474e5c3e Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:48:03 +0900 Subject: [PATCH 31/43] fix: reconcile merged accessibility roles, script normalization, and migration-replay tests Post-merge fixes surfaced by full test verification after merging origin/main into this branch: - LineageDag's SVG now uses role="group" (this PR's own accessibility fix for the interactive per-edge evidence buttons), superseding main's older role="img". Updated the tests that still queried role="img" for the lineage graph, and switched the ambiguous getAllByRole("group") assertions to precise `svg[role="group"]` queries since <details> (added by this PR's channel-evidence panel) also carries an implicit ARIA group role. - postBodyDisplay.ts: adopted main's normalizeScriptText for <sup>/<sub> handling (real Unicode superscripts, later re-rendered by splitScriptRuns) instead of this PR's older ad hoc "^N" caret regex, and dropped the bare-marker FOOTNOTE_START heuristic main had already removed as a false-positive source (a bullet list starting with "*" was being misread as a footnote). Restored the FOOTNOTE_START constant only where still needed transitively, then removed it entirely once isMarkedFootnote (this PR's own container-aware footnote detection) proved sufficient on its own. Updated the one test still asserting the old "^1" caret text. - migrate.sh's replay gate: this PR's test asserted the old explicit per-file allowlist main had already replaced with ADR 0166's general four-digit filename pattern. Updated the assertion to check the new mechanism covers 0103/0174 without individual entries. - scripts/import_postgresql_posts.py's rebuild_lineage stub now accepts the llm= keyword this PR's own call site passes. - lineage_persistence.py: added the missing docstring on the nested sort_key helper the AST docstring-coverage audit flagged. Verified: backend `uv run pytest -q` 931 passed / 16 skipped / 0 failed (0:08:37); frontend `pnpm run lint`, `pnpm run build`, and `pnpm exec vitest run` (245 passed / 245). --- frontend/src/App.test.tsx | 4 ++-- frontend/src/LineageDag.test.tsx | 10 ++++++---- frontend/src/postBodyDisplay.test.ts | 2 +- frontend/src/postBodyDisplay.ts | 5 +---- lineageweave/lineage_persistence.py | 1 + tests/test_import_postgresql_posts.py | 2 +- tests/test_lineage_channel_evidence.py | 11 ++++++++--- 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 065c8ef40..446fb0d92 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -1754,8 +1754,8 @@ describe("App, authenticated", () => { expect(await screen.findByLabelText("Reconstructed lineage")).toBeInTheDocument(); // Two distinct reconstruct threads (thread-alpha, thread-beta) must // render as two independent branch-tree figures, not merged into one. - expect(screen.getByRole("img", { name: "thread-alpha lineage" })).toBeInTheDocument(); - expect(screen.getByRole("img", { name: "thread-beta lineage" })).toBeInTheDocument(); + expect(screen.getByRole("group", { name: "thread-alpha lineage" })).toBeInTheDocument(); + expect(screen.getByRole("group", { name: "thread-beta lineage" })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Open post: Follow-up post" })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Open post: Unrelated thread post" })).toBeInTheDocument(); }); diff --git a/frontend/src/LineageDag.test.tsx b/frontend/src/LineageDag.test.tsx index 26b418d0a..e7e407441 100644 --- a/frontend/src/LineageDag.test.tsx +++ b/frontend/src/LineageDag.test.tsx @@ -146,7 +146,7 @@ describe("LineageDag", () => { it("shows an empty-state message instead of an empty graph", () => { render(<LineageDag graph={{ nodes: [], edges: [] }} onSelectPost={vi.fn()} />); expect(screen.getByText("No reconstructed lineage yet. Rebuild after seeding posts.")).toBeInTheDocument(); - expect(screen.queryByRole("img")).not.toBeInTheDocument(); + expect(document.querySelector("svg")).not.toBeInTheDocument(); }); it("renders one branch figure per lineage group, git-branch style", () => { @@ -162,7 +162,7 @@ describe("LineageDag", () => { expect(screen.getByText("Project Alpha (2 records, 1 lineage edges)")).toBeInTheDocument(); expect(screen.getByText("Project Beta (1 records, 0 lineage edges)")).toBeInTheDocument(); - expect(screen.getAllByRole("img")).toHaveLength(2); + expect(container.querySelectorAll('svg[role="group"]')).toHaveLength(2); expect(container.querySelectorAll(".lineage-dag-edge")).toHaveLength(1); expect(container.querySelector(".lineage-dag-branch")).toBeInTheDocument(); expect(container.querySelector(".lineage-dag-root")).toBeInTheDocument(); @@ -177,9 +177,11 @@ describe("LineageDag", () => { ], edges: [], }; - render(<LineageDag graph={graph} onSelectPost={vi.fn()} />); + const { container } = render(<LineageDag graph={graph} onSelectPost={vi.fn()} />); - const headings = screen.getAllByRole("img").map((img) => img.getAttribute("aria-label")); + const headings = [...container.querySelectorAll('svg[role="group"]')].map((svg) => + svg.getAttribute("aria-label"), + ); expect(headings).toEqual(["Zeta Corp lineage", "Ungrouped lineage"]); }); diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 4774dd468..cc67d0b69 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -98,7 +98,7 @@ describe("splitPostBody", () => { ).toEqual([ { kind: "text", text: "Body text" }, { kind: "text", text: "HTML footnote body", role: "footnote" }, - { kind: "text", text: "^1 Word footnote body", role: "footnote" }, + { kind: "text", text: "¹ Word footnote body", role: "footnote" }, { kind: "text", text: "OOXML footnote body", role: "footnote" }, ]); }); diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 8a0d7af48..0365d6be1 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -20,7 +20,6 @@ const BLOCK_TAG = /<\/?(?:article|blockquote|div|h[1-6]|li|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi; const WORD_INDENT_TAG = /<w:ind\b[^>]*\/?\s*>/gi; const LIST_ITEM_START = /^\s*(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)/; -const FOOTNOTE_START = /^\s*[*†‡](?=\S)/; const INDENT_MARKER = "\u0001lw-indent:"; const INDENT_MARKER_END = "\u0002"; const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g; @@ -467,9 +466,7 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): kind: "text", text: normalized, ...(indentLevel > 0 ? { indentLevel } : {}), - ...(isMarkedFootnote || FOOTNOTE_START.test(normalized) - ? { role: "footnote" as const } - : {}), + ...(isMarkedFootnote ? { role: "footnote" as const } : {}), }); } } diff --git a/lineageweave/lineage_persistence.py b/lineageweave/lineage_persistence.py index e5108ee6c..bed937f10 100644 --- a/lineageweave/lineage_persistence.py +++ b/lineageweave/lineage_persistence.py @@ -196,6 +196,7 @@ def rank_channel_evidence(rows: Sequence[Mapping[str, object]]) -> list[dict[str """ def sort_key(row: Mapping[str, object]) -> tuple[float, int]: + """Contribution descending, ties broken by the canonical channel order.""" channel = _channel_name(row) order = LINEAGE_SIGNAL_ORDER.index(channel) if channel in LINEAGE_SIGNAL_ORDER else len(LINEAGE_SIGNAL_ORDER) return (-float(row["signal_contribution"]), order) diff --git a/tests/test_import_postgresql_posts.py b/tests/test_import_postgresql_posts.py index 8b18796d7..32c8e94d5 100644 --- a/tests/test_import_postgresql_posts.py +++ b/tests/test_import_postgresql_posts.py @@ -101,7 +101,7 @@ async def no_content(*_args, **_kwargs) -> None: async def no_cleanup(*_args, **_kwargs) -> dict[str, int]: return {"synthetic_rows_removed": 0} - async def no_edges(_conn) -> list[object]: + async def no_edges(_conn, *, llm=None) -> list[object]: return [] monkeypatch.setattr("scripts.import_postgresql_posts.asyncpg.connect", fake_connect) diff --git a/tests/test_lineage_channel_evidence.py b/tests/test_lineage_channel_evidence.py index f855d1387..3eef465e0 100644 --- a/tests/test_lineage_channel_evidence.py +++ b/tests/test_lineage_channel_evidence.py @@ -149,10 +149,15 @@ def test_rank_is_contribution_then_controlled_signal_order() -> None: def test_migrate_sh_replays_channel_evidence_and_tenant_settings() -> None: + """ADR 0166's portable filename boundary must still cover 0103 and 0174 + on existing volumes -- no individual allowlist entry is needed for + either, since the general four-digit pattern already replays both. + """ migrate = (_ROOT / "docker/postgres-init/migrate.sh").read_text() - assert "0103_*" in migrate - assert "0104_*" not in migrate - assert "0174_*" in migrate + assert "[0-9][0-9][0-9][0-9]_*" in migrate + assert "000[0-9]_*|001[01]_*" in migrate + assert "0103_*" not in migrate + assert "0174_*" not in migrate def test_channel_evidence_migration_has_no_jsonb() -> None: From 2a6c520a6d3c95c015d36992a08510cb5cf92cd5 Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:52:52 +0900 Subject: [PATCH 32/43] fix: resolve post-merge test breakage from combined LineageDag changes - LineageDag.test.tsx declared a module-level `graph` const twice (once for the channel-evidence tests, once for the hit-target tests), which the line-based merge could not detect since both additions landed in non-overlapping hunks. Rename the second to `nodeHitTargetGraph`. - The merged LineageDag now keeps the lineage svg's `role="group"` (needed because its edges/nodes are interactive button-role descendants), so the mobile-scroll viewport test's `getByRole("img", ...)` query no longer matches; update it to `getByRole("group", ...)`. --- frontend/src/LineageDag.responsive.test.tsx | 2 +- frontend/src/LineageDag.test.tsx | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/LineageDag.responsive.test.tsx b/frontend/src/LineageDag.responsive.test.tsx index 4ac272cb0..1b67d9b63 100644 --- a/frontend/src/LineageDag.responsive.test.tsx +++ b/frontend/src/LineageDag.responsive.test.tsx @@ -27,7 +27,7 @@ describe("LineageDag responsive viewport", () => { expect(viewport).toHaveAttribute("tabindex", "0"); expect(viewport).toHaveClass("lineage-dag-viewport"); - const svg = screen.getByRole("img", { name: "A-100 lineage" }); + const svg = screen.getByRole("group", { name: "A-100 lineage" }); expect(viewport).toContainElement(svg); expect(Number(svg.getAttribute("width"))).toBeGreaterThan(320); }); diff --git a/frontend/src/LineageDag.test.tsx b/frontend/src/LineageDag.test.tsx index 6372bd6b7..6e411b90f 100644 --- a/frontend/src/LineageDag.test.tsx +++ b/frontend/src/LineageDag.test.tsx @@ -142,7 +142,7 @@ describe("LineageDag channel evidence", () => { }); }); -const graph = { +const nodeHitTargetGraph = { nodes: [ { id: "rec-001", @@ -166,7 +166,7 @@ const graph = { describe("LineageDag", () => { it("gives every node mark a 24x24px-minimum transparent hit target ahead of the visible mark", () => { - render(<LineageDag graph={graph} onSelectPost={() => undefined} />); + render(<LineageDag graph={nodeHitTargetGraph} onSelectPost={() => undefined} />); const button = screen.getByRole("button", { name: "Open post: Initial site visit and project scope discussion" }); const circles = button.querySelectorAll("circle"); expect(circles).toHaveLength(2); @@ -183,7 +183,7 @@ describe("LineageDag", () => { it("still opens the post when the enlarged hit target is clicked", async () => { const onSelectPost = vi.fn(); - render(<LineageDag graph={graph} onSelectPost={onSelectPost} />); + render(<LineageDag graph={nodeHitTargetGraph} onSelectPost={onSelectPost} />); await userEvent.click(screen.getByRole("button", { name: "Open post: Pricing renegotiation follow-up" })); expect(onSelectPost).toHaveBeenCalledWith("rec-002"); }); From 519d0e6377c7ff973b90c9812468478e41e60209 Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:28:35 +0900 Subject: [PATCH 33/43] fix(lineage): persist estimated fusion weights --- backend/app/lineage_ingestion.py | 12 ++++++++---- tests/test_lineage_ingestion.py | 11 ++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index 9911b9ac8..763876e07 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -77,7 +77,11 @@ def records_from_source_posts(rows: list[Mapping[str, Any]]) -> list[Record]: return records -async def persist_lineage_edges(conn: asyncpg.Connection, edges: list[Edge]) -> None: +async def persist_lineage_edges( + conn: asyncpg.Connection, + edges: list[Edge], + weights: dict[str, float] | None = None, +) -> None: """Replace live Event Lineage with ``edges`` and their channel evidence. Reconstruct is the source of truth. The delete is cascaded onto @@ -86,7 +90,7 @@ async def persist_lineage_edges(conn: asyncpg.Connection, edges: list[Edge]) -> connection so version, weights, and generated-at stay aligned with the new graph. """ - spec = lineage_rebuild_spec(edges) + spec = lineage_rebuild_spec(edges, weights=weights) await conn.execute("delete from post_lineage_edge") await conn.execute("delete from event_lineage_rebuild") await conn.execute( @@ -282,7 +286,7 @@ async def rebuild_lineage( ) edges = await _reconstruct_lineage_records(records, llm, weights) async with conn.transaction(): - await persist_lineage_edges(conn, edges) + await persist_lineage_edges(conn, edges, weights) return edges @@ -304,7 +308,7 @@ async def rebuild_lineage_from_pool( ) edges = await _reconstruct_lineage_records(records, llm, weights) async with pool.acquire() as conn, conn.transaction(): - await persist_lineage_edges(conn, edges) + await persist_lineage_edges(conn, edges, weights) return edges diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index 6b6f78923..981c3d2bd 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -343,7 +343,7 @@ def fake_lineage_edge_specs(_records, *, llm=None): captured["llm"] = llm return [] - async def fake_persist_lineage_edges(_conn, _edges): + async def fake_persist_lineage_edges(_conn, _edges, _weights=None): events.append("persist") return None @@ -441,7 +441,7 @@ async def fake_to_thread(function, *args, **kwargs): events.append("reconstruct") return function(*args, **kwargs) - async def fake_persist(_conn, _edges): + async def fake_persist(_conn, _edges, _weights=None): assert pool.active == 1 assert events[-1] == "transaction-enter" events.append("persist") @@ -883,7 +883,7 @@ def test_persist_lineage_edges_replaces_signals_atomically_without_llm() -> None scores, ) connection = _RecordingConnection() - asyncio.run(persist_lineage_edges(connection, [edge])) + asyncio.run(persist_lineage_edges(connection, [edge], weights)) statements = [sql.casefold() for sql, _args in connection.statements] assert statements[0].startswith("delete from post_lineage_edge") assert any("delete from event_lineage_rebuild" in sql for sql in statements) @@ -899,6 +899,11 @@ def test_persist_lineage_edges_replaces_signals_atomically_without_llm() -> None "lineage_signal_text", ] assert [len(rows) for _query, rows in connection.batches] == [3, 1, 3] + assert connection.batches[0][1] == [ + ("lineage_signal_temporal", 0.25), + ("lineage_signal_secondary_key", 0.25), + ("lineage_signal_text", 0.5), + ] spec = lineage_rebuild_spec([edge], package_version="2.14.0") assert spec.reconstruction_version == "lineageweave.reconstruct/2.14.0" From 2b95a8be3a348e452fcdb0e4349019d68d72bdd3 Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:49:17 +0900 Subject: [PATCH 34/43] fix(lineage): load weights for the active channel set --- backend/app/lineage_ingestion.py | 16 ++++++++++------ tests/test_lineage_ingestion.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index 763876e07..7922a3ca8 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -38,6 +38,14 @@ _SUPPORTED_ANCHOR_METHOD_CODES: frozenset[str] = frozenset() +def estimated_weight_channels(llm: AdjudicationClient | None) -> set[str]: + """Return the channels that one live reconstruction can actually use.""" + channels = {"temporal", "secondary_key", "text"} + if getattr(llm, "available", False): + channels.add("llm") + return channels + + def _occurred_at(value: datetime) -> datetime: """Reconstruct expects naive datetimes; asyncpg returns timestamptz.""" return value.replace(tzinfo=None) if value.tzinfo is not None else value @@ -281,9 +289,7 @@ async def rebuild_lineage( back to the default channel weights until one is. """ records = await _load_lineage_records(conn) - weights = await load_estimated_channel_weights( - conn, {"temporal", "secondary_key", "text"} - ) + weights = await load_estimated_channel_weights(conn, estimated_weight_channels(llm)) edges = await _reconstruct_lineage_records(records, llm, weights) async with conn.transaction(): await persist_lineage_edges(conn, edges, weights) @@ -303,9 +309,7 @@ async def rebuild_lineage_from_pool( """ async with pool.acquire() as conn: records = await _load_lineage_records(conn) - weights = await load_estimated_channel_weights( - conn, {"temporal", "secondary_key", "text"} - ) + weights = await load_estimated_channel_weights(conn, estimated_weight_channels(llm)) edges = await _reconstruct_lineage_records(records, llm, weights) async with pool.acquire() as conn, conn.transaction(): await persist_lineage_edges(conn, edges, weights) diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index 981c3d2bd..b52d4be52 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -366,6 +366,21 @@ async def fake_to_thread(function, *args, **kwargs): assert events == ["fetch", "reconstruct", "transaction_enter", "persist", "transaction_exit"] +def test_estimated_weight_channels_include_only_an_available_llm() -> None: + """Weight lookup must match the exact channel set reconstruction can use.""" + assert ingestion.estimated_weight_channels(None) == { + "temporal", + "secondary_key", + "text", + } + assert ingestion.estimated_weight_channels(type("LLM", (), {"available": True})()) == { + "temporal", + "secondary_key", + "text", + "llm", + } + + def test_rebuild_drops_llm_before_candidate_pair_budget_is_exceeded(monkeypatch) -> None: """Keep a large live rebuild from issuing unbounded provider calls.""" From 47424651841157eba6f8bfbe350163a1d46063e9 Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 11:10:18 +0900 Subject: [PATCH 35/43] fix: align lineage weights with budgeted channels --- backend/app/lineage_ingestion.py | 25 +++++++++++------ frontend/src/LineageDag.test.tsx | 32 +++++++++++++++++++++ frontend/src/LineageDag.tsx | 23 +++++++++------ tests/test_lineage_ingestion.py | 48 ++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 17 deletions(-) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index 7922a3ca8..66eff119d 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -254,20 +254,27 @@ async def _load_lineage_records(conn: asyncpg.Connection) -> list[Record]: return records_from_source_posts(rows) -async def _reconstruct_lineage_records( - records: list[Record], - llm: AdjudicationClient | None, - weights: dict[str, float] | None = None, -) -> list[Edge]: - """Run the CPU/provider reconstruction without blocking the event loop.""" +def _budgeted_llm( + records: list[Record], llm: AdjudicationClient | None +) -> AdjudicationClient | None: + """Drop adjudication before weight lookup when the pair budget is exceeded.""" pair_count = 0 records_per_group: defaultdict[str, int] = defaultdict(int) for record in records: pair_count += min(records_per_group[record.group_key], DEFAULT_CANDIDATE_WINDOW) if pair_count > MAXIMUM_LIVE_LLM_PAIR_EVALUATIONS: - llm = None - break + return None records_per_group[record.group_key] += 1 + return llm + + +async def _reconstruct_lineage_records( + records: list[Record], + llm: AdjudicationClient | None, + weights: dict[str, float] | None = None, +) -> list[Edge]: + """Run the CPU/provider reconstruction without blocking the event loop.""" + llm = _budgeted_llm(records, llm) if weights is None: return await asyncio.to_thread(lineage_edge_specs, records, llm=llm) return await asyncio.to_thread(lineage_edge_specs, records, llm=llm, weights=weights) @@ -289,6 +296,7 @@ async def rebuild_lineage( back to the default channel weights until one is. """ records = await _load_lineage_records(conn) + llm = _budgeted_llm(records, llm) weights = await load_estimated_channel_weights(conn, estimated_weight_channels(llm)) edges = await _reconstruct_lineage_records(records, llm, weights) async with conn.transaction(): @@ -309,6 +317,7 @@ async def rebuild_lineage_from_pool( """ async with pool.acquire() as conn: records = await _load_lineage_records(conn) + llm = _budgeted_llm(records, llm) weights = await load_estimated_channel_weights(conn, estimated_weight_channels(llm)) edges = await _reconstruct_lineage_records(records, llm, weights) async with pool.acquire() as conn, conn.transaction(): diff --git a/frontend/src/LineageDag.test.tsx b/frontend/src/LineageDag.test.tsx index 6e411b90f..1ca0f4032 100644 --- a/frontend/src/LineageDag.test.tsx +++ b/frontend/src/LineageDag.test.tsx @@ -70,6 +70,38 @@ const graph: LineageGraph = { }; describe("LineageDag channel evidence", () => { + it("groups connection evidence by thread in a merged graph", () => { + const secondGroup: LineageGraph = { + nodes: graph.nodes.map((node) => ({ + ...node, + id: `second-${node.id}`, + group: "B-200", + label: `Second ${node.label}`, + })), + edges: graph.edges.map((edge) => ({ + ...edge, + source: `second-${edge.source}`, + target: `second-${edge.target}`, + })), + }; + + render( + <LineageDag + graph={{ + nodes: [...graph.nodes, ...secondGroup.nodes], + edges: [...graph.edges, ...secondGroup.edges], + }} + onSelectPost={vi.fn()} + />, + ); + + const firstEvidence = screen.getByRole("region", { name: "A-100" }); + const secondEvidence = screen.getByRole("region", { name: "B-200" }); + expect(firstEvidence).toHaveTextContent("Kickoff recap follows Pricing follow-up"); + expect(firstEvidence).not.toHaveTextContent("Second Kickoff recap"); + expect(secondEvidence).toHaveTextContent("Second Kickoff recap follows Second Pricing follow-up"); + }); + it("discloses exact inferred values without hover-only interaction", async () => { render(<LineageDag graph={graph} onSelectPost={vi.fn()} />); const disclosure = screen.getByText(/fused score 0.700000/).closest("details"); diff --git a/frontend/src/LineageDag.tsx b/frontend/src/LineageDag.tsx index da0a77b51..f1c842439 100644 --- a/frontend/src/LineageDag.tsx +++ b/frontend/src/LineageDag.tsx @@ -188,12 +188,15 @@ export function LineageDag({ </div> </dl> ) : null} - {graph.edges.map((edge) => { - const key = edgeKey(edge); - const evidence = edge.channel_evidence ?? []; - const fromLabel = labelById[edge.source] ?? edge.source; - const toLabel = labelById[edge.target] ?? edge.target; - return ( + {groups.map((group) => ( + <section key={group.group} aria-label={group.heading}> + <h4>{group.heading}</h4> + {group.edges.map((edge) => { + const key = edgeKey(edge); + const evidence = edge.channel_evidence ?? []; + const fromLabel = labelById[edge.source] ?? edge.source; + const toLabel = labelById[edge.target] ?? edge.target; + return ( <details key={key} className="lineage-edge-evidence-item" @@ -242,9 +245,11 @@ export function LineageDag({ </tbody> </table> ) : null} - </details> - ); - })} + </details> + ); + })} + </section> + ))} </section> </div> ); diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index b52d4be52..1398ce093 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -406,6 +406,54 @@ def fake_lineage_edge_specs(_records, *, llm=None): assert captured["llm"] is None +def test_rebuild_drops_llm_before_estimated_weight_lookup(monkeypatch) -> None: + """Never load a four-channel estimate for a three-channel reconstruction.""" + + records = [ + Record(f"record-{index}", "shared-group", f"Record {index}", datetime(2026, 1, index + 1)) + for index in range(3) + ] + captured: dict[str, object] = {} + + class FakeTransaction: + async def __aenter__(self): + return None + + async def __aexit__(self, *_args): + return None + + class FakeConnection: + def transaction(self): + return FakeTransaction() + + async def fake_load_records(_conn): + return records + + async def fake_load_weights(_conn, channels): + captured["channels"] = channels + return None + + async def fake_reconstruct(_records, llm, _weights): + captured["llm"] = llm + return [] + + async def fake_persist(_conn, _edges, _weights): + return None + + monkeypatch.setattr(ingestion, "MAXIMUM_LIVE_LLM_PAIR_EVALUATIONS", 1) + monkeypatch.setattr(ingestion, "_load_lineage_records", fake_load_records) + monkeypatch.setattr(ingestion, "load_estimated_channel_weights", fake_load_weights) + monkeypatch.setattr(ingestion, "_reconstruct_lineage_records", fake_reconstruct) + monkeypatch.setattr(ingestion, "persist_lineage_edges", fake_persist) + + asyncio.run(rebuild_lineage(FakeConnection(), llm=type("LLM", (), {"available": True})())) + + assert captured == { + "channels": {"temporal", "secondary_key", "text"}, + "llm": None, + } + + def test_pooled_rebuild_releases_the_connection_during_reconstruction(monkeypatch) -> None: """Keep provider work outside the pool and transaction, then replace atomically.""" events: list[str] = [] From b798cefcad0ded1ab9c6a5beba0e513cb543b3c7 Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:22:01 +0900 Subject: [PATCH 36/43] fix: fail closed on adjudication errors --- backend/app/analysis_run_worker.py | 2 ++ lineageweave/adjudication_client.py | 14 ++++++++--- tests/test_adjudication_client.py | 24 ++++++++++++++++++ tests/test_analysis_run_worker.py | 25 +++++++++++++++++++ ...est_estimate_llm_channel_weights_script.py | 6 ++++- 5 files changed, 66 insertions(+), 5 deletions(-) diff --git a/backend/app/analysis_run_worker.py b/backend/app/analysis_run_worker.py index 166810e13..be0d3843b 100644 --- a/backend/app/analysis_run_worker.py +++ b/backend/app/analysis_run_worker.py @@ -79,6 +79,8 @@ async def consume_analysis_run_stream_once( exc.status_code, exc.detail, ) + except Exception: + logger.exception("analysis-run %s delivery failed", analysis_run_id) last_id = str(entry_id) return last_id diff --git a/lineageweave/adjudication_client.py b/lineageweave/adjudication_client.py index ce020fc70..60cfd61ce 100644 --- a/lineageweave/adjudication_client.py +++ b/lineageweave/adjudication_client.py @@ -15,7 +15,7 @@ import re from typing import Protocol -from .http_client import chat_completion_content, post_json +from .http_client import HttpClientError, chat_completion_content, post_json class AdjudicationClient(Protocol): @@ -53,9 +53,11 @@ def judge_prompt(candidate_label: str, record_label: str) -> str: def parse_confidence(content: str) -> float: - """Clamp the judge's numeric reply into [0, 1]; no number reads as 0.""" + """Clamp a numeric reply into ``[0, 1]`` or fail without inventing zero.""" parsed = parse_confidence_or_none(content) - return 0.0 if parsed is None else parsed + if parsed is None: + raise HttpClientError("adjudication response had no confidence score") + return parsed def parse_confidence_or_none(content: str) -> float | None: @@ -102,4 +104,8 @@ def judge(self, candidate_label: str, record_label: str) -> float: headers={"authorization": f"Bearer {self._api_key}"}, timeout=self._timeout, ) - return parse_confidence(chat_completion_content(body)) + try: + content = chat_completion_content(body) + except (TypeError, ValueError) as exc: + raise HttpClientError("adjudication response did not contain text") from exc + return parse_confidence(content) diff --git a/tests/test_adjudication_client.py b/tests/test_adjudication_client.py index 576f5e9c4..0ea9f6e6c 100644 --- a/tests/test_adjudication_client.py +++ b/tests/test_adjudication_client.py @@ -1,6 +1,9 @@ from __future__ import annotations +import pytest + from lineageweave.adjudication_client import ContextualOrchestratorAdjudicationClient +from lineageweave.http_client import HttpClientError def test_adjudication_uses_supported_auto_mode_and_long_local_timeout(monkeypatch) -> None: @@ -21,3 +24,24 @@ def fake_post_json(url, payload, *, headers, timeout): assert captured["payload"]["mode"] == "auto" assert captured["payload"]["reasoning_effort"] == "auto" assert captured["timeout"] == 180.0 + + +@pytest.mark.parametrize( + "body", + [ + {"choices": [{"message": {"content": "not a score"}}]}, + {"choices": []}, + {"choices": [{"message": {"content": None}}]}, + ], +) +def test_adjudication_fails_closed_for_unscoreable_responses(monkeypatch, body) -> None: + """A provider failure must not become a genuine unrelated score of zero.""" + monkeypatch.setattr( + "lineageweave.adjudication_client.post_json", lambda *args, **kwargs: body + ) + client = ContextualOrchestratorAdjudicationClient( + base_url="http://orchestrator:8000", api_key="synthetic-token" + ) + + with pytest.raises(HttpClientError): + client.judge("workshop", "follow-up bid") diff --git a/tests/test_analysis_run_worker.py b/tests/test_analysis_run_worker.py index e492b5a2e..bb06f3ce4 100644 --- a/tests/test_analysis_run_worker.py +++ b/tests/test_analysis_run_worker.py @@ -130,3 +130,28 @@ async def fake_deliver(conn, **kwargs): assert last_id == "2-1" assert delivered == ["00000000-0000-0000-0000-000000000002"] + + +@pytest.mark.anyio +async def test_one_unexpected_delivery_failure_does_not_end_the_worker(monkeypatch): + """A malformed provider reply must not stop later durable deliveries.""" + delivered = [] + + async def fake_deliver(conn, **kwargs): + del conn + if kwargs["analysis_run_id"].endswith("1"): + raise RuntimeError("malformed provider reply") + delivered.append(kwargs["analysis_run_id"]) + + monkeypatch.setattr(analysis_run_worker, "deliver_queued_analysis_run", fake_deliver) + + last_id = await analysis_run_worker.consume_analysis_run_stream_once( + _TwoRunsValkey(), + _Pool(), + last_id="0-0", + tepp_client=TeppClient(), + adjudication_client=NullAdjudicationClient(), + ) + + assert last_id == "2-1" + assert delivered == ["00000000-0000-0000-0000-000000000002"] diff --git a/tests/test_estimate_llm_channel_weights_script.py b/tests/test_estimate_llm_channel_weights_script.py index c9ab53a05..210c95d84 100644 --- a/tests/test_estimate_llm_channel_weights_script.py +++ b/tests/test_estimate_llm_channel_weights_script.py @@ -8,7 +8,10 @@ from __future__ import annotations +import pytest + from lineageweave.adjudication_client import judge_prompt, parse_confidence +from lineageweave.http_client import HttpClientError import scripts.estimate_llm_channel_weights as script @@ -31,7 +34,8 @@ def test_shared_judge_prompt_and_confidence_parse_round_trip() -> None: assert "Record B: Follow-up record" in prompt assert parse_confidence("0.85") == 0.85 assert parse_confidence("confidence: 0.4 maybe") == 0.4 - assert parse_confidence("no number here") == 0.0 + with pytest.raises(HttpClientError): + parse_confidence("no number here") assert parse_confidence("1.7") == 1.0 From be547fdeecc649fff02398904958b67ffcbfbb96 Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:22:19 +0900 Subject: [PATCH 37/43] fix(lineage): wire the adjudication client into corpus-wide rebuild Issue #289: rebuild_lineage accepted no adjudication client, so the optional LLM channel never contributed on the corpus-wide path. Add an optional adjudication_client parameter: an available client adds llm to the active-channel set (failing closed until a four-channel estimate exists per ADR 0200) and reaches reconstruct as the reasoning channel; a missing or unavailable client keeps the three-channel path. The POST /api/lineage/rebuild endpoint now passes _adjudication_client(). Unavailable clients are normalized to None at this boundary so reconstruct receives a single canonical unavailable value. --- backend/app/lineage_ingestion.py | 34 ++++++++--- backend/app/main.py | 4 +- tests/test_lineage_ingestion.py | 99 ++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 8 deletions(-) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index 90a993ac3..e82242790 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -19,6 +19,7 @@ import asyncpg from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL +from lineageweave.adjudication_client import AdjudicationClient from lineageweave.lineage_persistence import lineage_edge_specs from lineageweave.models import Edge, Record @@ -212,26 +213,45 @@ def __init__(self, active_channels: set[str]) -> None: self.active_channels = active_channels -async def rebuild_lineage(conn: asyncpg.Connection) -> list[Edge]: +async def rebuild_lineage( + conn: asyncpg.Connection, + adjudication_client: AdjudicationClient | None = None, +) -> list[Edge]: """Reconstruct lineage for every ``source_post`` and persist the edges. - Raises :class:`ChannelWeightsNotEstimated` when no activated - estimate matches this path's active channels -- run + Pass a live :class:`~lineageweave.adjudication_client.AdjudicationClient` + to include the optional LLM channel (issue #289); omit it (the + default) to reconstruct on the three deterministic channels only. + Either way, raises :class:`ChannelWeightsNotEstimated` when no + activated estimate matches this path's active channels -- run ``scripts/estimate_channel_weights.py`` first (ADR 0200 point 1). + A missing client drops the llm channel and renormalizes; it never + fabricates an adjudication. """ rows = await conn.fetch( "select post_id, post_title, voc_type_code, created_at, corporate_entity_id, " "process_unit_id, thread_group_key, secondary_grouping_key " f"from source_post where {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}" ) - # No adjudication client is wired on this path, so the active channel - # set is the three deterministic channels (reconstruct drops llm when - # unavailable rather than faking it). + # With no adjudication client — or an unavailable one — the active + # channel set is the three deterministic channels (reconstruct drops + # llm when unavailable rather than faking it). A wired, *available* + # client adds the reasoning channel, and the weight lookup fails + # closed until a four-channel estimate exists -- never renormalizing + # a three-channel vector onto it. active_channels = {"temporal", "secondary_key", "text"} + wired_llm: AdjudicationClient | None = None + if adjudication_client is not None and getattr(adjudication_client, "available", False): + active_channels.add("llm") + wired_llm = adjudication_client weights = await load_estimated_channel_weights(conn, active_channels) if weights is None: raise ChannelWeightsNotEstimated(active_channels) - edges = lineage_edge_specs(records_from_source_posts(rows), weights=weights) + edges = lineage_edge_specs( + records_from_source_posts(rows), + llm=wired_llm, + weights=weights, + ) await persist_lineage_edges(conn, edges) return edges diff --git a/backend/app/main.py b/backend/app/main.py index 572d93ca1..196b785cb 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1170,7 +1170,9 @@ async def rebuild_lineage_graph( async with pool.acquire() as conn: async with conn.transaction(): try: - edges = await rebuild_lineage(conn) + edges = await rebuild_lineage( + conn, adjudication_client=_adjudication_client() + ) except ChannelWeightsNotEstimated: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index dc9bbd142..9f67738f8 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -5,6 +5,7 @@ import asyncio import math from datetime import UTC, datetime, timezone +from unittest import mock import pytest @@ -675,3 +676,101 @@ async def fetch(self, query: str): merged = asyncio.run(lineage_graphs_for_posts(FakeConnection(), lambda row: True, [])) assert merged == {"nodes": [], "edges": [], "truncated": False} + + +def test_rebuild_wires_available_adjudication_client_into_llm_channel() -> None: + """An available client adds the llm channel; a null one leaves it out. + + Issue #289: the corpus-wide rebuild path must pass a wired + adjudication client through to ``reconstruct`` so the optional LLM + channel actually contributes when (and only when) it is available. + The weight lookup must see the four-channel set -- failing closed, + never renormalizing a three-channel vector onto an llm run. + """ + rows = [ + { + "post_id": rec.record_id, + "process_unit_id": "shared-pu", + "corporate_entity_id": "shared-corp", + "post_title": rec.label, + "voc_type_code": "voc" if rec.secondary_key else "vom", + "thread_group_key": rec.group_key, + "secondary_grouping_key": rec.secondary_key, + "created_at": rec.occurred_at, + } + for rec in sample_records() + ] + four_channel_rows = [ + { + "channel_set_code": channel_set, + "channel_code": channel, + "weight_value": weight, + "estimation_run_id": "00000000-0000-0000-0000-000000000001", + "estimation_method_code": "mls2plm_expected_information", + "estimator_version": "1.0.0", + "anchor_method_code": "unanchored_internal_structure", + "source_snapshot_sha256": "a" * 64, + "sample_pair_count": 600, + "knowledge_cutoff": datetime(2026, 1, 1, tzinfo=UTC), + } + for channel_set, channel, weight in ( + ("channel_set_deterministic", "temporal", 0.49), + ("channel_set_deterministic", "secondary_key", 0.34), + ("channel_set_deterministic", "text", 0.17), + ("channel_set_with_llm", "temporal", 0.35), + ("channel_set_with_llm", "secondary_key", 0.24), + ("channel_set_with_llm", "text", 0.14), + ("channel_set_with_llm", "llm", 0.27), + ) + ] + + class FakeConnection: + async def fetch(self, query: str): + if "lineage_channel_weight" in query: + # The loader fetches every row and matches one persisted + # set against the active-channel set exactly. + return four_channel_rows + assert "from source_post" in query + return rows + + async def fetchval(self, _query: str): + return True + + async def execute(self, query: str, *args: object) -> None: + pass + + class AvailableClient: + available = True + + def judge(self, candidate_label: str, record_label: str) -> float: + return 0.9 + + class UnavailableClient: + available = False + + connection = FakeConnection() + captured: dict[str, object] = {} + + real_specs = ingestion.lineage_edge_specs + + def capturing_specs(records, *, llm=None, weights): + captured["llm"] = llm + return real_specs(records, llm=llm, weights=weights) + + with mock.patch.object(ingestion, "lineage_edge_specs", capturing_specs): + edges = asyncio.run( + ingestion.rebuild_lineage(connection, adjudication_client=AvailableClient()) + ) + assert edges + assert isinstance(captured["llm"], AvailableClient), ( + "an available client must reach reconstruct as the llm channel" + ) + + # An unavailable client must leave the llm channel out entirely -- + # reconstruct receives None and renormalizes onto three channels. + captured.clear() + with mock.patch.object(ingestion, "lineage_edge_specs", capturing_specs): + asyncio.run( + ingestion.rebuild_lineage(connection, adjudication_client=UnavailableClient()) + ) + assert captured["llm"] is None From 2620e0aecbf7a396b5713160a99867ed02ef6a09 Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:29:38 +0900 Subject: [PATCH 38/43] fix(ask): retain lineage reconstruction profile --- backend/app/lineage_ingestion.py | 3 +++ tests/test_lineage_ingestion.py | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index 541d97860..3531d957e 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -523,11 +523,13 @@ async def lineage_graphs_for_posts( nodes_by_id: dict[str, dict[str, Any]] = {} edges_by_key: dict[tuple[str, str], dict[str, Any]] = {} truncated = False + reconstruction = None for post_id in dict.fromkeys(post_ids): graph = await visible_lineage_graph( conn, can_see_post, focus_post_id=post_id, include_isolated=True ) truncated = truncated or graph["truncated"] + reconstruction = reconstruction or graph["reconstruction"] for node in graph["nodes"]: nodes_by_id[node["id"]] = node for edge in graph["edges"]: @@ -536,4 +538,5 @@ async def lineage_graphs_for_posts( "nodes": list(nodes_by_id.values()), "edges": list(edges_by_key.values()), "truncated": truncated, + "reconstruction": reconstruction, } diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index 66f446125..ed5e608d8 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -1185,7 +1185,14 @@ async def fetch(self, query: str, *_args): if "event_lineage_rebuild_channel" in query: return [] if "event_lineage_rebuild" in query: - return [] + return [ + { + "reconstruction_version": "lineageweave.reconstruct/2.14.0", + "generated_at": datetime(2026, 1, 5), + "min_fused_score": 0.5, + "candidate_window": 6, + } + ] return self.edges if "post_lineage_edge" in query else self.posts connection = FakeConnection() @@ -1213,6 +1220,9 @@ async def fetch(self, query: str, *_args): } assert len(merged["edges"]) == 2 assert merged["truncated"] is False + assert merged["reconstruction"]["reconstruction_version"] == ( + "lineageweave.reconstruct/2.14.0" + ) def test_lineage_graphs_for_posts_with_no_citations_is_empty() -> None: @@ -1221,4 +1231,9 @@ async def fetch(self, query: str, *_args): return [] merged = asyncio.run(lineage_graphs_for_posts(FakeConnection(), lambda row: True, [])) - assert merged == {"nodes": [], "edges": [], "truncated": False} + assert merged == { + "nodes": [], + "edges": [], + "truncated": False, + "reconstruction": None, + } From b14725f25ac11c58cc1b5efb66e8453105d6ac7b Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:18:42 +0900 Subject: [PATCH 39/43] fix(lineage): preserve worker retries and parser edge cases --- backend/app/analysis_run_worker.py | 46 +++++++++++++++------------- backend/app/lineage_ingestion.py | 2 +- frontend/src/LineageDag.test.tsx | 8 ++--- frontend/src/postBodyDisplay.test.ts | 6 ++++ frontend/src/postBodyDisplay.ts | 9 ++++-- tests/test_analysis_run_worker.py | 1 + tests/test_lineage_ingestion.py | 9 +++--- 7 files changed, 47 insertions(+), 34 deletions(-) diff --git a/backend/app/analysis_run_worker.py b/backend/app/analysis_run_worker.py index aae80fc5c..dd63fe538 100644 --- a/backend/app/analysis_run_worker.py +++ b/backend/app/analysis_run_worker.py @@ -76,17 +76,17 @@ async def consume_analysis_run_stream_once( # transaction rolls back, the durable outbox row stays # available, and an explicit HTTP start retries the run # once the operator resolves the named next action. - try: - async with pool.acquire() as conn: - owner = await conn.fetchrow( - """ - select requested_by_account_id - from analysis_run - where analysis_run_id = $1::uuid - """, - analysis_run_id, - ) - if owner is not None: + async with pool.acquire() as conn: + owner = await conn.fetchrow( + """ + select requested_by_account_id + from analysis_run + where analysis_run_id = $1::uuid + """, + analysis_run_id, + ) + if owner is not None: + try: await deliver_queued_analysis_run( pool, database_url=database_url, @@ -97,17 +97,19 @@ async def consume_analysis_run_stream_once( adjudication_client=adjudication_client, valkey_stream_entry_id=str(entry_id), ) - except AnalysisRunCreateError as exc: - _worker_logger.warning( - "analysis-run %s delivery refused (%s): %s", - analysis_run_id, - exc.status_code, - exc.detail, - ) - except Exception: - _worker_logger.exception( - "analysis-run %s delivery failed", analysis_run_id - ) + except AnalysisRunCreateError as exc: + _worker_logger.warning( + "analysis-run %s delivery refused (%s): %s", + analysis_run_id, + exc.status_code, + exc.detail, + ) + except Exception as exc: + _worker_logger.warning( + "analysis-run %s delivery failed (error_type=%s)", + analysis_run_id, + type(exc).__name__, + ) last_id = str(entry_id) return last_id diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index 8caea1b85..313c4f5c6 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -149,7 +149,7 @@ def interval_relation_code_for_edge( async def persist_lineage_edges( conn: asyncpg.Connection, edges: list[Edge], - weights: dict[str, float] | None = None, + weights: dict[str, float], points_by_post_id: Mapping[str, Mapping[str, Any]] | None = None, ) -> None: """Replace live Event Lineage with edges, channel evidence, and intervals. diff --git a/frontend/src/LineageDag.test.tsx b/frontend/src/LineageDag.test.tsx index 63213cf81..9198e5502 100644 --- a/frontend/src/LineageDag.test.tsx +++ b/frontend/src/LineageDag.test.tsx @@ -95,11 +95,11 @@ describe("LineageDag channel evidence", () => { />, ); - const firstEvidence = screen.getByRole("region", { name: "A-100" }); - const secondEvidence = screen.getByRole("region", { name: "B-200" }); - expect(firstEvidence).toHaveTextContent("Kickoff recap follows Pricing follow-up"); + const firstEvidence = screen.getByRole("region", { name: "A-100 lineage viewport" }); + const secondEvidence = screen.getByRole("region", { name: "B-200 lineage viewport" }); + expect(firstEvidence).toHaveTextContent("Pricing follow-up follows Kickoff recap"); expect(firstEvidence).not.toHaveTextContent("Second Kickoff recap"); - expect(secondEvidence).toHaveTextContent("Second Kickoff recap follows Second Pricing follow-up"); + expect(secondEvidence).toHaveTextContent("Second Pricing follow-up follows Second Kickoff recap"); }); it("discloses exact inferred values without hover-only interaction", async () => { diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index cc67d0b69..ac3afb35d 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -103,6 +103,12 @@ describe("splitPostBody", () => { ]); }); + it("labels unquoted HTML footnote attributes", () => { + expect(splitPostBody("<ol class=footnotes><li>Unquoted footnote</li></ol>")).toEqual([ + { kind: "text", text: "Unquoted footnote", role: "footnote" }, + ]); + }); + it("stops labeling ordinary content after an HTML footnote list", () => { expect( splitPostBody( diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 0365d6be1..240b63d22 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -35,9 +35,12 @@ function markFootnoteTags(markup: string): string { if (!match) return tag; const closing = Boolean(match[1]); const name = match[2].toLowerCase(); - const hasFootnoteLabel = [...tag.matchAll(/\b(?:class|role)\s*=\s*(["'])(.*?)\1/gi)].some( - (attribute) => - /\b(?:footnotes?|endnotes?|msofootnotetext|msoendnotetext)\b/i.test(attribute[2]), + const hasFootnoteLabel = [...tag.matchAll( + /\b(?:class|role)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>"']+))/gi, + )].some((attribute) => + /\b(?:footnotes?|endnotes?|msofootnotetext|msoendnotetext)\b/i.test( + attribute[1] ?? attribute[2] ?? attribute[3] ?? "", + ), ); const isContainer = hasFootnoteLabel && (name === "div" || name === "ol" || name === "ul"); diff --git a/tests/test_analysis_run_worker.py b/tests/test_analysis_run_worker.py index b1fbe0110..26466430a 100644 --- a/tests/test_analysis_run_worker.py +++ b/tests/test_analysis_run_worker.py @@ -336,6 +336,7 @@ async def fake_deliver(conn, **kwargs): _TwoRunsValkey(), _Pool(), last_id="0-0", + database_url="postgresql://synthetic", tepp_client=TeppClient(), adjudication_client=NullAdjudicationClient(), ) diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index d61d1be88..e79de17a2 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -654,10 +654,11 @@ async def execute(self, query: str, *_args): with pytest.raises(ValueError, match="child"): asyncio.run( - persist_lineage_edges( - connection, - [edge], - points_by_post_id={"parent": {"created_at": datetime(2026, 1, 1)}}, + persist_lineage_edges( + connection, + [edge], + {}, + points_by_post_id={"parent": {"created_at": datetime(2026, 1, 1)}}, ) ) assert connection.calls == [] From 8b882d8c7f4bee469506e311ce6588114b1b8d31 Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:41:45 +0900 Subject: [PATCH 40/43] fix(ui): stop WorkspaceCalendar's fail-closed placeholder announcing as role=status Its resolved empty/unavailable state carried role="status" like sibling panels' transient loading text does, so mounting it inside the Board's collapsed Advanced Review Tools details collided with every other status region on the page (4 failing App.test.tsx assertions). RankingsPanel's own resolved placeholders carry no ARIA role for the same reason -- only the "Loading..." state announces. --- frontend/src/components/WorkspaceCalendar.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/WorkspaceCalendar.tsx b/frontend/src/components/WorkspaceCalendar.tsx index 5f2631f39..0b37ae2dc 100644 --- a/frontend/src/components/WorkspaceCalendar.tsx +++ b/frontend/src/components/WorkspaceCalendar.tsx @@ -33,7 +33,7 @@ export function WorkspaceCalendar({ <section className="popup-section" aria-labelledby={`${headingId}-observed`}> <h3 id={`${headingId}-observed`}>{t("Observed calendar events")}</h3> {events.length === 0 ? ( - <p className="popup-placeholder" role="status"> + <p className="popup-placeholder"> {naruonAvailable ? t("No observed calendar events are available.") : failClosedCopy} From d6b74f53c15630b7ce9cd0f432d58c1e65226716 Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:21:08 +0900 Subject: [PATCH 41/43] fix(measurement): reject duplicated lineage channels --- lineageweave/channel_weight_estimation.py | 7 +++ tests/test_lineage_channel_evidence.py | 52 +++++++++++------------ 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/lineageweave/channel_weight_estimation.py b/lineageweave/channel_weight_estimation.py index c46b6ce5e..38c2b8191 100644 --- a/lineageweave/channel_weight_estimation.py +++ b/lineageweave/channel_weight_estimation.py @@ -183,6 +183,13 @@ def estimate_channel_weights( [dichotomize(scores[channel]) for channel in channels] for scores in pair_channel_scores ] + distinct_columns = { + tuple(row[column] for row in responses) for column in range(len(channels)) + } + if len(distinct_columns) != len(channels): + # Identical channels are one signal copied twice, not independent + # measurement evidence. Refuse instead of double-counting it. + return None for column, channel in enumerate(channels): observed = {row[column] for row in responses} if len(observed) < 2: diff --git a/tests/test_lineage_channel_evidence.py b/tests/test_lineage_channel_evidence.py index a3cfd0831..47d42f8da 100644 --- a/tests/test_lineage_channel_evidence.py +++ b/tests/test_lineage_channel_evidence.py @@ -29,28 +29,14 @@ _ROOT = Path(__file__).resolve().parents[1] -@lru_cache(maxsize=2) -def _estimated_weights(include_llm: bool = False) -> dict[str, float]: +@lru_cache(maxsize=1) +def _estimated_weights() -> dict[str, float]: """Return fast-mlsirm estimates for the declared synthetic design.""" - if not include_llm: - estimate = estimate_fixture_channel_weights() - else: - pair_scores, group_ids = simulate_fixture_pair_scores() - estimate = estimate_channel_weights( - [{**scores, "llm": scores["text"]} for scores in pair_scores], - group_ids, - ) + estimate = estimate_fixture_channel_weights() assert estimate is not None return estimate.weights -def _four_channel_edge() -> Edge: - scores = {"temporal": 0.8, "secondary_key": 1.0, "text": 0.5, "llm": 0.9} - weights = _estimated_weights(include_llm=True) - fused = sum(weights[name] * scores[name] for name in scores) - return Edge("parent-a", "child-b", fused, scores) - - def _no_llm_edge() -> Edge: scores = {"temporal": 0.8, "secondary_key": 1.0, "text": 0.5} weights = _estimated_weights() @@ -58,18 +44,17 @@ def _no_llm_edge() -> Edge: return Edge("parent-a", "child-b", fused, scores) -def test_four_channel_edge_round_trips_scores_and_normalized_weights() -> None: - edge = _four_channel_edge() - weights = _estimated_weights(include_llm=True) +def test_grounded_edge_round_trips_scores_and_normalized_weights() -> None: + edge = _no_llm_edge() + weights = _estimated_weights() rows = channel_signal_rows(edge, weights) codes = [row["channel_name"] for row in rows] - assert codes == ["temporal", "secondary_key", "text", "llm"] + assert codes == ["temporal", "secondary_key", "text"] by_name = {row["channel_name"]: row for row in rows} - assert by_name["llm"]["signal_code"] == "lineage_signal_llm" assert by_name["temporal"]["signal_weight"] == quantize_signal_value(weights["temporal"]) assert by_name["text"]["signal_weight"] == quantize_signal_value(weights["text"]) evidence = rank_channel_evidence(rows) - assert [item["rank"] for item in evidence] == [1, 2, 3, 4] + assert [item["rank"] for item in evidence] == [1, 2, 3] assert sum(item["contribution"] for item in evidence) == pytest.approx(edge.fused_score) @@ -88,8 +73,8 @@ def test_no_llm_reconstruction_persists_exactly_three_channels() -> None: def test_contributions_reconcile_to_fused_score_within_tolerance() -> None: - edge = _four_channel_edge() - rows = channel_signal_rows(edge, _estimated_weights(include_llm=True)) + edge = _no_llm_edge() + rows = channel_signal_rows(edge, _estimated_weights()) residual = abs(sum(float(row["signal_contribution"]) for row in rows) - edge.fused_score) assert residual <= CHANNEL_EVIDENCE_TOLERANCE @@ -99,9 +84,8 @@ def test_rebuild_accepts_expected_multi_channel_quantization_error() -> None: "temporal": 0.1234567, "secondary_key": 0.2345678, "text": 0.3456789, - "llm": 0.4567891, } - weights = _estimated_weights(include_llm=True) + weights = _estimated_weights() edge = Edge( "parent-a", "child-b", @@ -110,10 +94,22 @@ def test_rebuild_accepts_expected_multi_channel_quantization_error() -> None: ) rows = channel_signal_rows(edge, weights) - assert len(rows) == 4 + assert len(rows) == 3 assert lineage_rebuild_spec([edge], weights=weights).signal_rows +def test_duplicated_text_proxy_cannot_invent_an_llm_weight() -> None: + """A copied text score is not an independent LLM validity anchor.""" + pair_scores, group_ids = simulate_fixture_pair_scores() + assert ( + estimate_channel_weights( + [{**scores, "llm": scores["text"]} for scores in pair_scores], + group_ids, + ) + is None + ) + + def test_mismatched_fused_score_is_rejected() -> None: edge = Edge("parent-a", "child-b", 0.99, {"temporal": 0.1, "secondary_key": 0.1, "text": 0.1}) with pytest.raises(ValueError, match="do not reconcile"): From 34e6e6f26c3c62b322d770c2023d9285083e474d Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:08:59 +0900 Subject: [PATCH 42/43] fix(lineage): name connection direction correctly --- frontend/src/LineageDag.test.tsx | 1 + frontend/src/LineageDag.tsx | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/src/LineageDag.test.tsx b/frontend/src/LineageDag.test.tsx index 9198e5502..cb71816c0 100644 --- a/frontend/src/LineageDag.test.tsx +++ b/frontend/src/LineageDag.test.tsx @@ -105,6 +105,7 @@ describe("LineageDag channel evidence", () => { it("discloses exact inferred values without hover-only interaction", async () => { render(<LineageDag graph={graph} onSelectPost={vi.fn()} />); const disclosure = screen.getByText(/fused score 0.700000/).closest("details"); + expect(disclosure).toHaveTextContent("Pricing follow-up follows Kickoff recap"); const edgeButton = screen.getByRole("button", { name: "Open connection evidence: Kickoff recap to Pricing follow-up", }); diff --git a/frontend/src/LineageDag.tsx b/frontend/src/LineageDag.tsx index 444241e1b..3a0f3f92f 100644 --- a/frontend/src/LineageDag.tsx +++ b/frontend/src/LineageDag.tsx @@ -263,8 +263,8 @@ export function LineageDag({ > <summary> {tf("{from} follows {to}, fused score {score}", { - from: fromLabel, - to: toLabel, + from: toLabel, + to: fromLabel, score: formatExact(edge.fused_score), })} </summary> From de9fc2ebdf5e5b3f0785b6df661d102df3afd486 Mon Sep 17 00:00:00 2001 From: seonghobae <seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:09:12 +0900 Subject: [PATCH 43/43] fix(lineage): name follows direction correctly --- frontend/src/LineageDag.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/LineageDag.tsx b/frontend/src/LineageDag.tsx index 444241e1b..3a0f3f92f 100644 --- a/frontend/src/LineageDag.tsx +++ b/frontend/src/LineageDag.tsx @@ -263,8 +263,8 @@ export function LineageDag({ > <summary> {tf("{from} follows {to}, fused score {score}", { - from: fromLabel, - to: toLabel, + from: toLabel, + to: fromLabel, score: formatExact(edge.fused_score), })} </summary>