diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 0b12c6c2c..c7f17f666 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -15,17 +15,15 @@ repos in the ecosystem: trajectories, uncertainty-quantified estimates) is [TEPP](https://github.com/ContextualWisdomLab/TEPP)'s job. -This is why the org-wide rule that mathematical/psychometrics computation -layers must be Rust with GPU + CPU multithreading does not apply to this -repo: LineageWeave does no such computation. Its heaviest per-request work -is fusing a handful of `[0, 1]` channel scores over a bounded candidate -window (`reconstruct.DEFAULT_CANDIDATE_WINDOW`, default 50) -- a scheduling -and orchestration problem, not a numerical-estimation one. If a future -version added real statistical inference (e.g. estimating thread-assignment -uncertainty), that layer would move into TEPP rather than being built here, -consistent with the dependency direction the ecosystem's own architecture -docs already establish (`psychometrics-commons`'s TRD explicitly forbids a -downstream product from reimplementing a measurement engine's model). +ADR 0208 fixes the end state: LineageWeave retains wire validation, +authorization, provenance persistence, and UI projection only. The current +Python IRT/report, residual-map, similarity, graph-ranking, and fusion paths +are explicitly inventoried migration debt rather than evidence that this +repository owns their mathematics. They move by construct to TEPP, +fast-mlsirm, or RankWeave after versioned Rust CPU/GPU owner contracts pass +recovery/equivalence checks; affected product paths fail closed during each +cutover rather than substituting a local estimate. See +`docs/doctoring/python-mathematical-compute-boundary-audit.md`. ## Data flow diff --git a/backend/app/analysis_run_start.py b/backend/app/analysis_run_start.py index c7398198a..c08810078 100644 --- a/backend/app/analysis_run_start.py +++ b/backend/app/analysis_run_start.py @@ -391,6 +391,21 @@ async def _persist_tepp_result( anchor["criterion_validity_status"], anchor["validated_pair_count"], ) + await conn.execute( + """ + update lineage_channel_weight + set anchor_method_code = 'tepp_lineage_criterion_v1' + where estimation_run_id = $1 + and estimation_method_code = 'mls2plm_expected_information' + and source_snapshot_sha256 = $2 + and knowledge_cutoff = $3 + and sample_pair_count = $4 + """, + estimation_run_id, + anchor["source_snapshot_sha256"], + anchor_cutoff, + anchor["validated_pair_count"], + ) except (asyncpg.PostgresError, TypeError, ValueError): return False return True diff --git a/backend/app/global_ask_queue.py b/backend/app/global_ask_queue.py index 260712d1c..c7e570d81 100644 --- a/backend/app/global_ask_queue.py +++ b/backend/app/global_ask_queue.py @@ -29,6 +29,7 @@ from fastapi import HTTPException, status from lineageweave.ask_delivery import build_ask_delivery +from lineageweave.embedding_client import EmbeddingClient, NullEmbeddingClient from lineageweave.http_client import HttpClientError from lineageweave.observability import record_server_failure from lineageweave.post_chat import ( @@ -213,6 +214,7 @@ async def compute_global_ask_answer( process_unit_ids: set[str], process_scope_limited: bool, chat_client: PostChatClient, + embedding_client: EmbeddingClient | None = None, ) -> dict[str, Any]: """Assemble one complete Ask answer payload from authorized evidence. @@ -243,6 +245,7 @@ def can_see(row: asyncpg.Record) -> bool: process_unit_ids, question=question_text, today=today, + embedding_client=embedding_client, ) except Exception as exc: log_internal_fault("global_ask", exc) @@ -346,6 +349,7 @@ async def process_global_ask_job( *, job_id: str, chat_factory: Callable[[], PostChatClient], + embedding_factory: Callable[[], EmbeddingClient] = NullEmbeddingClient, ) -> None: """Claim, answer, and settle one Ask job. @@ -392,6 +396,7 @@ async def process_global_ask_job( process_unit_ids=process_unit_ids, process_scope_limited=process_scope_limited, chat_client=chat_client, + embedding_client=embedding_factory(), ), timeout=JOB_DEADLINE_SECONDS, ) @@ -505,6 +510,7 @@ async def consume_global_ask_stream_once( *, last_id: str, chat_factory: Callable[[], PostChatClient], + embedding_factory: Callable[[], EmbeddingClient] = NullEmbeddingClient, limiter: asyncio.Semaphore | None = None, tasks: set[asyncio.Task] | None = None, ) -> str: @@ -524,12 +530,21 @@ async def consume_global_ask_stream_once( job_id = str(fields.get("global_ask_job_id", "")).strip() if job_id: if limiter is None: - await process_global_ask_job(pool, job_id=job_id, chat_factory=chat_factory) + await process_global_ask_job( + pool, + job_id=job_id, + chat_factory=chat_factory, + embedding_factory=embedding_factory, + ) else: await limiter.acquire() task = asyncio.create_task( _process_and_release( - pool, job_id=job_id, chat_factory=chat_factory, limiter=limiter + pool, + job_id=job_id, + chat_factory=chat_factory, + embedding_factory=embedding_factory, + limiter=limiter, ) ) if tasks is not None: @@ -544,11 +559,17 @@ async def _process_and_release( *, job_id: str, chat_factory: Callable[[], PostChatClient], + embedding_factory: Callable[[], EmbeddingClient], limiter: asyncio.Semaphore, ) -> None: """Run one dispatched job and free its concurrency slot afterwards.""" try: - await process_global_ask_job(pool, job_id=job_id, chat_factory=chat_factory) + await process_global_ask_job( + pool, + job_id=job_id, + chat_factory=chat_factory, + embedding_factory=embedding_factory, + ) finally: limiter.release() @@ -568,6 +589,7 @@ async def run_global_ask_worker( pool: asyncpg.Pool, *, chat_factory: Callable[[], PostChatClient], + embedding_factory: Callable[[], EmbeddingClient] = NullEmbeddingClient, ) -> None: """Run the at-least-once Ask consumer with periodic queued-row recovery.""" last_id = await _stream_tail(client) @@ -586,6 +608,7 @@ async def run_global_ask_worker( pool, last_id=last_id, chat_factory=chat_factory, + embedding_factory=embedding_factory, limiter=limiter, tasks=tasks, ) diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index b765c4ffb..a176db247 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -221,14 +221,14 @@ async def load_estimated_channel_weights( sets: dict[str, list] = {} for row in all_rows: sets.setdefault(row["channel_set_code"], []).append(row) - rows = next( - ( - candidate - for candidate in sets.values() - if {row["channel_code"] for row in candidate} == active_channels - ), - [], - ) + matching_sets = [ + candidate + for candidate in sets.values() + if {row["channel_code"] for row in candidate} == active_channels + ] + if len(matching_sets) != 1: + return None + rows = matching_sets[0] persisted = {row["channel_code"]: float(row["weight_value"]) for row in rows} if not persisted or set(persisted) != active_channels: return None diff --git a/backend/app/main.py b/backend/app/main.py index 5e482147e..a9cd25a66 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -87,6 +87,7 @@ from lineageweave.post_summary import ContextualOrchestratorPostSummaryClient, NullPostSummaryClient from lineageweave.relation_verification import NullRelationVerificationClient, SearxngRelationVerificationClient from lineageweave.semantic_hints import customer_hint_trust, format_semantic_hints +from lineageweave.similar_voc import ContextualOrchestratorSimilarVocAnalysisClient from lineageweave.ontology import LW from lineageweave.rankweave_client import build_rankweave_client from lineageweave.naruon_calendar_workspace import ( @@ -220,6 +221,8 @@ _POST_READ = "post_read" _POST_ADMIN = "post_admin" +_SIMILAR_VOC_PAGE_SIZE = 8 +_SIMILAR_VOC_REQUEST_TIMEOUT_SECONDS = 180.0 @asynccontextmanager @@ -272,6 +275,7 @@ async def lifespan(app: FastAPI): chat_factory=lambda: _post_chat_client( timeout=load_settings().orchestrator_answer_timeout_seconds ), + embedding_factory=_embedding_client, ) ) app.state.global_ask_worker = global_ask_worker @@ -482,6 +486,18 @@ def _post_evaluation_client(): ) +def _similar_voc_client(): + """Live semantic-pair client, or ``None`` when inference is unavailable.""" + settings = load_settings() + if not (settings.orchestrator_base_url and settings.orchestrator_api_key): + return None + return ContextualOrchestratorSimilarVocAnalysisClient( + base_url=settings.orchestrator_base_url, + api_key=settings.orchestrator_api_key, + timeout=_SIMILAR_VOC_REQUEST_TIMEOUT_SECONDS, + ) + + def _rankweave_client(): """In-process RankWeave unless RANKWEAVE_DISABLED=1 (ADR 0024).""" return build_rankweave_client(disabled=load_settings().rankweave_disabled) @@ -756,7 +772,11 @@ async def operations_dashboard( async with pool.acquire() as conn: try: return await fetch_operations_dashboard( - conn, account.corporate_entity_ids, period_start, period_end + conn, + account.corporate_entity_ids, + account.process_unit_ids, + period_start, + period_end, ) except ValueError as exc: raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, str(exc)) from exc @@ -1754,7 +1774,8 @@ async def _load_visible_post( # Safe SQL: the eligibility predicate is an immutable schema fragment; post id is bound. row = await conn.fetchrow( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli """ - select source_post.post_id, source_post.post_title, source_post.voc_type_code, + select source_post.post_id, source_post.post_title, source_post.post_body, + source_post.voc_type_code, source_post.visibility_code, source_post.corporate_entity_id, source_post.process_unit_id, source_post.created_at, source_post.author_account_id, source_post.source_process_unit_code, source_post.source_author_code, @@ -1776,6 +1797,95 @@ async def _load_visible_post( return row +@app.get("/api/posts/{post_id}/similar-voc") +async def read_similar_voc( + post_id: str, + offset: int = Query(0, ge=0), + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """Return authorized, semantically adjudicated prior VOC evidence. + + Persisted ``repeat_issue`` classifications narrow the candidate corpus + without lexical matching. contextual-orchestrator then establishes each + pair; event time orders the display and is not a relevance score. + """ + focal = await _load_visible_post(post_id, account, pool) + client = _similar_voc_client() + if client is None: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "similar VOC inference is unavailable; configure contextual-orchestrator and retry", + ) + async with pool.acquire() as conn: + rows = await conn.fetch( + f""" + select post.post_id, post.post_title, post.post_body, + post.visibility_code, post.corporate_entity_id, post.process_unit_id, + coalesce(post.event_occurred_at, post.created_at) as occurred_at + from operations_case_classification classification + join source_post post on post.post_id = classification.post_id + where classification.case_kind_code = 'repeat_issue' + and post.post_id <> $1 + and post.post_body <> '' + and (post.visibility_code = 'public' + or (post.corporate_entity_id::text = any($2::text[]) + and (cardinality($3::text[]) = 0 + or post.process_unit_id::text = any($3::text[])))) + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} + order by coalesce(post.event_occurred_at, post.created_at) desc, post.post_id + offset $4 limit $5 + """, + post_id, + list(account.corporate_entity_ids), + list(account.process_unit_ids), + offset, + _SIMILAR_VOC_PAGE_SIZE + 1, + ) + candidates = [row for row in rows[:_SIMILAR_VOC_PAGE_SIZE] if _can_see_post(account, row)] + + async def _adjudicate(candidate: asyncpg.Record): + with use_llm_metadata(build_post_llm_metadata(post_id, focal)): + return await asyncio.to_thread( + client.analyze, + focal["post_title"], + focal["post_body"], + str(candidate["post_id"]), + candidate["post_title"], + candidate["post_body"], + ) + + try: + results = await asyncio.wait_for( + asyncio.gather(*(_adjudicate(candidate) for candidate in candidates), return_exceptions=True), + timeout=_SIMILAR_VOC_REQUEST_TIMEOUT_SECONDS, + ) + except TimeoutError: + results = () + items = [] + for candidate, evidence in zip(candidates, results): + if evidence is None or isinstance(evidence, BaseException): + continue + items.append( + { + "post_id": evidence.candidate_post_id, + "post_title": candidate["post_title"], + "issue_summary": evidence.issue_summary, + "focal_evidence_text": evidence.focal_evidence_text, + "candidate_evidence_text": evidence.candidate_evidence_text, + "customer_cohort_text": evidence.customer_cohort_text, + "action_history": evidence.action_history, + "occurred_at": candidate["occurred_at"].isoformat(), + } + ) + return { + "items": items, + "next_offset": offset + _SIMILAR_VOC_PAGE_SIZE + if len(rows) > _SIMILAR_VOC_PAGE_SIZE + else None, + } + + async def _load_post_semantic_hints(conn: asyncpg.Connection, post_id: str) -> str: """Render author, business-unit, sales-pool, and customer hints without treating them as proof.""" rows = await conn.fetch( diff --git a/backend/app/operations_case_ingestion.py b/backend/app/operations_case_ingestion.py index 4150b436b..a2a1fc84f 100644 --- a/backend/app/operations_case_ingestion.py +++ b/backend/app/operations_case_ingestion.py @@ -2,9 +2,9 @@ from __future__ import annotations -import hashlib from typing import Any, Protocol +from backend.app.post_content_queue import source_body_sha256 from lineageweave.operations_case_analysis import OperationsCase @@ -23,8 +23,8 @@ async def executemany(self, query: str, args: list[tuple[object, ...]]) -> Any: def source_body_digest(body: str) -> str: - """Return the digest that binds inference to an exact source body.""" - return hashlib.sha256(body.encode("utf-8")).hexdigest() + """Return the digest that binds inference to an exact focal source body.""" + return source_body_sha256(body) async def persist_operations_cases( @@ -40,22 +40,24 @@ async def persist_operations_cases( await conn.execute( "insert into operations_case_analysis (post_id, source_body_sha256, orchestrator_session_id) values ($1, $2, $3)", post_id, - source_body_digest(source_body), + source_body_sha256(source_body), orchestrator_session_id, ) for case in cases: await conn.execute( - "insert into operations_case_classification (post_id, case_kind_code, summary_text, evidence_text) values ($1, $2, $3, $4)", + "insert into operations_case_classification (post_id, case_kind_code, summary_text, evidence_text, evidence_post_id, evidence_input_sha256) values ($1, $2, $3, $4, $5, $6)", post_id, case.case_kind_code, case.summary_text, case.evidence_text, + case.evidence_post_id, + case.evidence_input_sha256, ) if case.facts: await conn.executemany( - "insert into operations_case_fact (post_id, case_kind_code, fact_ordinal, fact_type_code, value_text, evidence_text) values ($1, $2, $3, $4, $5, $6)", + "insert into operations_case_fact (post_id, case_kind_code, fact_ordinal, fact_type_code, value_text, evidence_text, evidence_post_id, evidence_input_sha256) values ($1, $2, $3, $4, $5, $6, $7, $8)", [ - (post_id, case.case_kind_code, ordinal, fact.fact_type_code, fact.value_text, fact.evidence_text) + (post_id, case.case_kind_code, ordinal, fact.fact_type_code, fact.value_text, fact.evidence_text, fact.evidence_post_id, fact.evidence_input_sha256) for ordinal, fact in enumerate(case.facts) ], ) diff --git a/backend/app/operations_dashboard.py b/backend/app/operations_dashboard.py index 7bcb70dcb..6342d8fd4 100644 --- a/backend/app/operations_dashboard.py +++ b/backend/app/operations_dashboard.py @@ -43,25 +43,28 @@ def _visible_period_sql(alias: str = "post") -> str: """Return the shared ABAC, eligibility, and event-clock predicate.""" return f""" ({alias}.visibility_code = 'public' - or {alias}.corporate_entity_id::text = any($1::text[])) + or ({alias}.corporate_entity_id::text = any($1::text[]) + and (cardinality($2::text[]) = 0 + or {alias}.process_unit_id::text = any($2::text[])))) and {SOURCE_POST_ELIGIBILITY_SQL.format(alias=alias)} - and ($2::date is null or (coalesce({alias}.event_occurred_at, {alias}.created_at) - at time zone 'Asia/Seoul')::date >= $2) and ($3::date is null or (coalesce({alias}.event_occurred_at, {alias}.created_at) - at time zone 'Asia/Seoul')::date <= $3) + at time zone 'Asia/Seoul')::date >= $3) + and ($4::date is null or (coalesce({alias}.event_occurred_at, {alias}.created_at) + at time zone 'Asia/Seoul')::date <= $4) """ async def fetch_operations_dashboard( conn: _Connection, corporate_entity_ids: tuple[str, ...] | list[str], + process_unit_ids: tuple[str, ...] | list[str] = (), period_start: date | None = None, period_end: date | None = None, ) -> dict[str, Any]: """Return quantified cases and their persisted source evidence.""" if period_start and period_end and period_start > period_end: raise ValueError("period_start must not be after period_end") - args = (list(corporate_entity_ids), period_start, period_end) + args = (list(corporate_entity_ids), list(process_unit_ids), period_start, period_end) visible = _visible_period_sql() metrics = await conn.fetchrow( f""" @@ -82,7 +85,17 @@ async def fetch_operations_dashboard( where not exists ( select 1 from operations_case_analysis analysis where analysis.post_id = visible_post.post_id - )) as pending_analysis_count + ) and not exists ( + select 1 from post_content_ingestion_job job + where job.post_id = visible_post.post_id + and job.status_code = 'post_content_ingestion_failed' + )) as pending_analysis_count, + (select count(*) from visible_post + where exists ( + select 1 from post_content_ingestion_job job + where job.post_id = visible_post.post_id + and job.status_code = 'post_content_ingestion_failed' + )) as failed_analysis_count """, *args, ) @@ -90,6 +103,7 @@ async def fetch_operations_dashboard( f""" select classification.post_id, classification.case_kind_code, classification.summary_text, classification.evidence_text, + classification.evidence_post_id, coalesce(post.event_occurred_at, post.created_at) as occurred_at, coalesce(nullif(btrim(post.source_project_name), ''), project.project_name) as project_name @@ -111,7 +125,8 @@ async def fetch_operations_dashboard( fact_rows = await conn.fetch( f""" select fact.post_id, fact.case_kind_code, fact.fact_type_code, - fact.value_text, fact.evidence_text, fact.fact_ordinal + fact.value_text, fact.evidence_text, fact.evidence_post_id, + fact.fact_ordinal from operations_case_fact fact join source_post post on post.post_id = fact.post_id where {visible} @@ -128,6 +143,7 @@ async def fetch_operations_dashboard( "fact_type_label": FACT_TYPE_LABELS[row["fact_type_code"]], "value_text": row["value_text"], "evidence_text": row["evidence_text"], + "evidence_post_id": str(row["evidence_post_id"]), } ) total = int(metrics["total_post_count"]) @@ -139,6 +155,7 @@ async def fetch_operations_dashboard( "external_post_count": external, "external_percent": external * 100 / total if total else 0.0, "pending_analysis_count": int(metrics["pending_analysis_count"]), + "failed_analysis_count": int(metrics["failed_analysis_count"]), "cases": [ { "post_id": str(row["post_id"]), @@ -147,6 +164,7 @@ async def fetch_operations_dashboard( "project_name": row["project_name"], "summary_text": row["summary_text"], "evidence_text": row["evidence_text"], + "evidence_post_id": str(row["evidence_post_id"]), "occurred_at": row["occurred_at"].isoformat(), "facts": facts.get((str(row["post_id"]), row["case_kind_code"]), []), } diff --git a/backend/app/post_chat_ingestion.py b/backend/app/post_chat_ingestion.py index 5f9832c41..4ca9e2f2d 100644 --- a/backend/app/post_chat_ingestion.py +++ b/backend/app/post_chat_ingestion.py @@ -8,7 +8,7 @@ not that the requesting account may see both. `gather_global_chat_sources` (Global Ask, no starting post) also expands -its single best keyword match through the same `post_lineage_edge` +its single best persisted-embedding match through the same `post_lineage_edge` neighbors, so an answer speaks to a connected timeline rather than one isolated snapshot -- it does not have a starting post to run the Knowledge Graph's indirect random-walk expansion from, only the lineage @@ -18,7 +18,6 @@ from __future__ import annotations import asyncio -import re from dataclasses import dataclass from datetime import date, datetime from typing import Any, Callable, Iterable @@ -27,6 +26,7 @@ import asyncpg from lineageweave.ask_time_axis import row_matches_time_range, time_axis_evidence_fact +from lineageweave.embedding_client import EmbeddingClient, NullEmbeddingClient from lineageweave.image_content import ImageContentClient, NullImageContentClient from lineageweave.knowledge_graph import ( NODE_POST, @@ -44,12 +44,10 @@ normalize_chat_question, ) from lineageweave.post_content_normalization import normalize_post_body -from lineageweave.temporal_expressions import ( - TEMPORAL_STOPWORDS, - resolve_korean_relative_time, -) +from lineageweave.temporal_expressions import resolve_korean_relative_time from .knowledge_graph import hydrate_related_nodes, load_visible_subgraph +from .post_eligibility import SOURCE_POST_ELIGIBILITY_SQL from lineageweave.ontology import ontology_annotations @@ -168,7 +166,6 @@ async def _graph_facts_for_posts( ("source_project_name", "source project name"), ) -_GLOBAL_ASK_TERM_PATTERN = re.compile(r"[^\W_]+(?:-[^\W_]+)*", re.UNICODE) _POST_CHAT_SOURCE_LIMIT = 8 # Korean relative-time words ("어제", "오늘", ...) name a KST calendar day, @@ -394,6 +391,7 @@ async def gather_global_chat_sources( authorized_corporate_entity_ids: Iterable[str] = (), authorized_process_unit_ids: Iterable[str] = (), vision_client: ImageContentClient | None = None, + embedding_client: EmbeddingClient | None = None, *, question: str | None = None, limit: int = 4, @@ -414,133 +412,96 @@ async def gather_global_chat_sources( or no expression at all applies no date filter. Cited sources name which clock matched (ADR 0202). - The source set is intentionally bounded until retrieval/reranking is - needed for a much larger corpus; every selected body still uses the same - image normalization and persisted graph evidence as post-scoped chat. + Candidates are ranked by the maximum cosine similarity between the + question embedding and each post's persisted semantic-unit embeddings. + The embedding model and dimension must match exactly. An unavailable + channel or incomplete persisted vectors returns no source instead of + falling back to lexical matching. """ if limit <= 0: return [] if vision_client is None: vision_client = NullImageContentClient() - # A relative-time expression ("어제", "작년 이맘때쯤", ...) narrows the - # candidate window by event time (fallback: created_at) below; it must - # not also become a near-meaningless literal keyword search term - # (see TEMPORAL_STOPWORDS). + if embedding_client is None: + embedding_client = NullEmbeddingClient() resolved_time_range = resolve_korean_relative_time( question or "", today=today or _seoul_today() ) - search_terms = tuple( - dict.fromkeys( - token.casefold() - for token in _GLOBAL_ASK_TERM_PATTERN.findall(question or "") - if len(token) >= 2 - and token.casefold() - not in { - "which", - "what", - "where", - "when", - "who", - "why", - "how", - "the", - "this", - "that", - "posts", - "post", - "글", - "게시글", - "질문", - "관련", - "확인되는", - "핵심", - "사실", - "무엇", - "무엇인가요", - "인가요", - } - # A Korean particle (은/는/이/가/에/의/도/쯤/...) attaches directly - # to a time word with no space ("어제는", "지난주에"), so the - # tokenizer above yields one token that a bare `in` check against - # TEMPORAL_STOPWORDS never matches -- check by prefix instead. - and not any(token.startswith(stopword) for stopword in TEMPORAL_STOPWORDS) - ) - )[:8] - # A post whose title names the exact thing asked about is a far more - # specific match than one that only shares a generic term (a common - # word, or a hit buried in a 16KB body prefix); weighting every match - # equally and then falling back on created_at desc as the only - # tiebreak let recency crowd out relevance -- a year-old post whose - # title is an exact company-name match lost to four newer, only - # loosely related posts in a live reproduction of this bug. - _MATCH_WEIGHT = {"title": 3.0, "body": 1.0, "source_field": 1.0} - candidate_scores: dict[str, float] = {} - for term in search_terms: - candidate_rows = await conn.fetch( - """ - select post_id, matched_in - from ( - (select post_id, coalesce(event_occurred_at, created_at) as event_clock, - 'title' as matched_in - from source_post - where post_title ilike '%' || $1 || '%' - and ($2::date is null or (coalesce(event_occurred_at, created_at) at time zone 'Asia/Seoul')::date >= $2) - and ($3::date is null or (coalesce(event_occurred_at, created_at) at time zone 'Asia/Seoul')::date <= $3) - limit 32) - union all - (select post_id, coalesce(event_occurred_at, created_at) as event_clock, - 'body' as matched_in - from source_post - where lower(left(source_post_search_text(post_body), 16384)) - like '%' || lower($1) || '%' - and ($2::date is null or (coalesce(event_occurred_at, created_at) at time zone 'Asia/Seoul')::date >= $2) - and ($3::date is null or (coalesce(event_occurred_at, created_at) at time zone 'Asia/Seoul')::date <= $3) - limit 32) - union all - (select post_id, coalesce(event_occurred_at, created_at) as event_clock, - 'body' as matched_in - from source_post - where to_tsvector('simple', source_post_search_text(post_body)) - @@ plainto_tsquery('simple', $1) - and ($2::date is null or (coalesce(event_occurred_at, created_at) at time zone 'Asia/Seoul')::date >= $2) - and ($3::date is null or (coalesce(event_occurred_at, created_at) at time zone 'Asia/Seoul')::date <= $3) - limit 32) - union all - (select post_id, coalesce(event_occurred_at, created_at) as event_clock, - 'source_field' as matched_in - from source_post - where concat_ws(' ', source_system_code, source_record_key, - source_author_code, source_author_name, - source_company_code, source_company_name, - source_process_unit_code, source_process_unit_name, - source_sales_pool_code, source_sales_pool_name, - source_customer_code, source_customer_name, - source_project_code, source_project_name) - ilike '%' || $1 || '%' - and ($2::date is null or (coalesce(event_occurred_at, created_at) at time zone 'Asia/Seoul')::date >= $2) - and ($3::date is null or (coalesce(event_occurred_at, created_at) at time zone 'Asia/Seoul')::date <= $3) - limit 32) - ) matches - order by event_clock desc, post_id desc - limit 32 - """, - term, - resolved_time_range[0] if resolved_time_range else None, - resolved_time_range[1] if resolved_time_range else None, + if not (question and question.strip() and embedding_client.available): + return [] + try: + question_vector = await asyncio.to_thread(embedding_client.embed, question) + except (OSError, RuntimeError, ValueError): + return [] + if not question_vector: + return [] + embedding_model_code = embedding_client.resolved_model + if not embedding_model_code: + return [] + question_norm = sum(value * value for value in question_vector) ** 0.5 + if question_norm == 0.0: + return [] + # Safe SQL: the only interpolation is the repository-owned eligibility + # expression; all request and model values remain asyncpg parameters. + candidate_rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" + with question_vector as ( + select ordinality - 1 as dimension_index, dimension_value + from unnest($1::double precision[]) with ordinality + as vector(dimension_value, ordinality) + ), unit_similarity as ( + select unit.post_id, embedding.post_content_embedding_id, + sum(value.dimension_value * question.dimension_value) + / nullif( + sqrt(sum(value.dimension_value * value.dimension_value)) * $2, + 0 + ) as cosine_similarity + from source_post post + join post_content_unit unit on unit.post_id = post.post_id + join post_content_embedding embedding + on embedding.post_content_unit_id = unit.post_content_unit_id + join post_content_embedding_value value + on value.post_content_embedding_id = embedding.post_content_embedding_id + join question_vector question + on question.dimension_index = value.dimension_index + where embedding.embedding_model_code = $3 + and embedding.embedding_dimension_count = cardinality($1::double precision[]) + and (post.visibility_code = 'public' + or (post.corporate_entity_id::text = any($4::text[]) + and (cardinality($5::text[]) = 0 + or post.process_unit_id::text = any($5::text[])))) + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} + and ($6::date is null or (coalesce(post.event_occurred_at, post.created_at) at time zone 'Asia/Seoul')::date >= $6) + and ($7::date is null or (coalesce(post.event_occurred_at, post.created_at) at time zone 'Asia/Seoul')::date <= $7) + group by unit.post_id, embedding.post_content_embedding_id + having count(*) = cardinality($1::double precision[]) ) - for row in candidate_rows: - post_id = str(row["post_id"]) - candidate_scores[post_id] = candidate_scores.get(post_id, 0.0) + _MATCH_WEIGHT[row["matched_in"]] - candidate_ids = sorted(candidate_scores, key=lambda post_id: candidate_scores[post_id], reverse=True) - - # A keyword match only proves one post's text is relevant -- the - # account asking almost always wants to know what happened before and - # after that event too, not just this one snapshot. Expand the single - # best match through its direct Event Lineage neighbors + select similarity.post_id, max(similarity.cosine_similarity) as semantic_score, + max(coalesce(post.event_occurred_at, post.created_at)) as event_clock + from unit_similarity similarity + join source_post post on post.post_id = similarity.post_id + group by similarity.post_id + order by semantic_score desc, event_clock desc, similarity.post_id desc + limit $8 + """, + question_vector, + question_norm, + embedding_model_code, + list(authorized_corporate_entity_ids), + list(authorized_process_unit_ids), + resolved_time_range[0] if resolved_time_range else None, + resolved_time_range[1] if resolved_time_range else None, + limit, + ) + candidate_ids = [str(row["post_id"]) for row in candidate_rows] + candidate_id_set = frozenset(candidate_ids) + + # One semantic match is still only one event snapshot. Expand the + # best-matching post through its direct Event Lineage neighbors # (`post_lineage_edge`, `lineageweave.reconstruct`'s output), mirroring # `find_linked_post_ids`'s `.direct` set used by the post-scoped chat - # flow. Only the top match is expanded -- expanding every keyword hit - # would let a loosely related term drag in an unrelated lineage chain. + # flow. Only the top match is expanded so lower-ranked semantic candidates + # cannot each pull a separate lineage chain into the bounded context. lineage_neighbor_ids: list[str] = [] lineage_anchor_id = candidate_ids[0] if candidate_ids else None if lineage_anchor_id: @@ -553,7 +514,7 @@ async def gather_global_chat_sources( { str(row["other_id"]) for row in lineage_rows - if str(row["other_id"]) not in candidate_scores + if str(row["other_id"]) not in candidate_id_set } ) candidate_ids = list( @@ -563,8 +524,10 @@ async def gather_global_chat_sources( candidate_ids = [] lineage_neighbor_id_set = frozenset(lineage_neighbor_ids) - rows = await conn.fetch( - """ + # Safe SQL: the only interpolation is the repository-owned eligibility + # expression; all request and identity values remain asyncpg parameters. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select post_id, post_title, post_body, visibility_code, corporate_entity_id, process_unit_id, source_system_code, source_record_key, source_author_code, source_author_name, source_company_code, source_company_name, source_process_unit_code, @@ -577,6 +540,8 @@ async def gather_global_chat_sources( or (corporate_entity_id::text = any($1::text[]) and (cardinality($2::text[]) = 0 or process_unit_id::text = any($2::text[])))) + and source_post.post_id = any($3::uuid[]) + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')} and ($5::date is null or (coalesce(event_occurred_at, created_at) at time zone 'Asia/Seoul')::date >= $5) and ($6::date is null or (coalesce(event_occurred_at, created_at) at time zone 'Asia/Seoul')::date <= $6) order by array_position($3::uuid[], post_id) nulls last, diff --git a/backend/app/post_content_worker.py b/backend/app/post_content_worker.py index bb08242f1..825c35b77 100644 --- a/backend/app/post_content_worker.py +++ b/backend/app/post_content_worker.py @@ -26,12 +26,16 @@ transition_post_content_job, ) from backend.app.operations_case_ingestion import persist_operations_cases +from backend.app.post_chat_ingestion import gather_chat_sources from lineageweave.embedding_client import EmbeddingClient from lineageweave.http_client import HttpClientError from lineageweave.image_content import ImageContentClient from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata from lineageweave.observability import record_server_failure, traced -from lineageweave.operations_case_analysis import ContextualOrchestratorOperationsCaseAnalysisClient +from lineageweave.operations_case_analysis import ( + ContextualOrchestratorOperationsCaseAnalysisClient, + OperationsEvidenceSource, +) from lineageweave.post_content_normalization import normalize_post_body from lineageweave.post_content_persistence import persist_post_content from lineageweave.post_structure import PostStructureClient @@ -45,6 +49,40 @@ _UNEXPECTED_FAILURE_DETAIL = "post-content provider operation failed; retry the ingestion job" +async def _operations_evidence_sources( + pool: asyncpg.Pool, + post_id: str, + focal_row: asyncpg.Record, + vision_client: ImageContentClient, +) -> tuple[OperationsEvidenceSource, ...]: + """Reuse authorized lineage/semantic chat retrieval for case inference.""" + focal_entity = str(focal_row["corporate_entity_id"]) + focal_process = focal_row.get("process_unit_id") + + def can_see(row: asyncpg.Record) -> bool: + """Keep linked private evidence inside the focal entity and PU scope.""" + return row["visibility_code"] == "public" or ( + str(row["corporate_entity_id"]) == focal_entity + and row.get("process_unit_id") == focal_process + ) + + async with pool.acquire() as conn: + sources = await gather_chat_sources(conn, post_id, can_see, vision_client) + return tuple( + OperationsEvidenceSource( + source.post_id, + source.post_title, + source.post_body + + ( + "\nPersisted semantic evidence:\n" + "\n".join(source.evidence_facts) + if source.evidence_facts + else "" + ), + ) + for source in sources + ) + + async def _stream_tail(client: redis.Redis) -> str: """Start after historical wake-ups; the normalized ledger drives recovery.""" with traced( @@ -302,10 +340,12 @@ async def process_post_content_job( ) if row.get(name) is not None and str(row[name]).strip() ) + evidence_sources = await _operations_evidence_sources( + pool, post_id, row, vision_client + ) cases = await asyncio.to_thread( case_client.analyze, - str(row["post_title"]), - normalized.text, + evidence_sources, context, ) async with pool.acquire() as conn: diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index b4d0aa342..fc46a79a4 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -156,6 +156,11 @@ / "migrations" / "0201_lineage_pair_judgment.sql" ) +_TEPP_LINEAGE_ANCHOR_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0207_lineage_weight_tepp_anchor.sql" +) _LEFTOVER_OBSERVED_EXPECTED_MIGRATION = ( Path(__file__).resolve().parents[2] / "migrations" @@ -327,6 +332,7 @@ def seeded_db(demo_analyst_token): cur.execute(_INTERVAL_RELATION_MIGRATION.read_text()) cur.execute(_CHANNEL_WEIGHT_UNION_MIGRATION.read_text()) cur.execute(_PAIR_JUDGMENT_MIGRATION.read_text()) + cur.execute(_TEPP_LINEAGE_ANCHOR_MIGRATION.read_text()) # Product reconstruction fails closed without an ACTIVATED # estimate (ADR 0200 points 1+3); this synthetic fixture set # under the authorized anchor stands in for a fast-mlsirm @@ -338,14 +344,14 @@ def seeded_db(demo_analyst_token): " anchor_method_code, source_snapshot_sha256, sample_pair_count, " " knowledge_cutoff) values " "('channel_set_deterministic', 'temporal', 0.5, " - " '00000000-0000-0000-0000-000000000001', 'test_fixture', 'test', " - " 'unanchored_internal_structure', repeat('a', 64), 600, now()), " + " '00000000-0000-0000-0000-000000000001', 'mls2plm_expected_information', 'test', " + " 'tepp_lineage_criterion_v1', repeat('a', 64), 600, '2026-01-12T00:00:00Z'), " "('channel_set_deterministic', 'secondary_key', 0.34, " - " '00000000-0000-0000-0000-000000000001', 'test_fixture', 'test', " - " 'unanchored_internal_structure', repeat('a', 64), 600, now()), " + " '00000000-0000-0000-0000-000000000001', 'mls2plm_expected_information', 'test', " + " 'tepp_lineage_criterion_v1', repeat('a', 64), 600, '2026-01-12T00:00:00Z'), " "('channel_set_deterministic', 'text', 0.16, " - " '00000000-0000-0000-0000-000000000001', 'test_fixture', 'test', " - " 'unanchored_internal_structure', repeat('a', 64), 600, now())" + " '00000000-0000-0000-0000-000000000001', 'mls2plm_expected_information', 'test', " + " 'tepp_lineage_criterion_v1', repeat('a', 64), 600, '2026-01-12T00:00:00Z')" ) cur.execute(_LEFTOVER_OBSERVED_EXPECTED_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_RANK_MIGRATION.read_text()) @@ -420,6 +426,45 @@ def seeded_db(demo_analyst_token): (subject,), ) account_id = cur.fetchone()[0] + cur.execute( + """ + with snapshot as ( + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (repeat('a', 64), 'synthetic-anchor-v1', + '2026-01-11T23:00:00Z', '2026-01-11T23:30:00Z') + returning analysis_source_snapshot_id + ), tepp_run as ( + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, + requested_by_account_id, idempotency_key, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + select analysis_source_snapshot_id, 'analysis_run_tepp', %s, + 'synthetic-lineage-anchor', '2026-01-12T00:00:00Z', + 'tepp-lineage-criterion-v1', repeat('b', 64), + repeat('c', 40), '2026-01-12T00:30:00Z' + from snapshot + returning analysis_run_id + ), tepp_result as ( + insert into analysis_run_tepp_result + (analysis_run_id, remote_run_id, result_json, result_sha256) + select analysis_run_id, 'synthetic-tepp-anchor', '{}'::jsonb, repeat('d', 64) + from tepp_run + returning analysis_run_id + ) + insert into lineage_weight_tepp_anchor + (estimation_run_id, tepp_analysis_run_id, anchor_kind_code, + anchor_contract_version, source_snapshot_sha256, knowledge_cutoff, + criterion_validity_status_code, validated_pair_count) + select '00000000-0000-0000-0000-000000000001', analysis_run_id, + 'lineage_pair_criterion', 1, repeat('a', 64), + '2026-01-12T00:00:00Z', 'accepted', 600 + from tepp_result + """, + (account_id,), + ) cur.execute( "insert into account_affiliation (user_account_id, corporate_entity_id) values (%s, %s)", (account_id, own_corp_id), diff --git a/backend/tests/test_similar_voc_api.py b/backend/tests/test_similar_voc_api.py new file mode 100644 index 000000000..54e83dc9c --- /dev/null +++ b/backend/tests/test_similar_voc_api.py @@ -0,0 +1,148 @@ +"""Focused API contract tests for live Similar VOC evidence.""" + +from contextlib import asynccontextmanager +import asyncio +from datetime import datetime, timezone +from types import SimpleNamespace + +from backend.app import main +from lineageweave.similar_voc import SimilarVocEvidence + + +class _Connection: + def __init__(self, rows): + self.rows = rows + self.query = "" + self.args = () + + async def fetch(self, query, *args): + self.query = query + self.args = args + return self.rows + + +class _Pool: + def __init__(self, rows): + self.connection = _Connection(rows) + + @asynccontextmanager + async def acquire(self): + yield self.connection + + +def test_similar_voc_adjudicates_visible_semantic_candidates(monkeypatch) -> None: + """The live endpoint omits an ABAC-hidden candidate and exposes no score.""" + focal = { + "post_id": "focal", + "post_title": "Current VOC", + "post_body": "Current seal failed.", + } + visible = { + "post_id": "prior", + "post_title": "Prior VOC", + "post_body": "Prior seal failed. Replaced gasket.", + "visibility_code": "private", + "corporate_entity_id": "corp-a", + "process_unit_id": "unit-a", + "occurred_at": datetime(2026, 8, 20, tzinfo=timezone.utc), + } + hidden = {**visible, "post_id": "hidden", "visibility_code": "private", "corporate_entity_id": "corp-b"} + + async def load_visible_post(*_args): + return focal + + class Client: + def analyze(self, *_args): + return SimilarVocEvidence( + "prior", "Equivalent seal failure", "Current seal failed.", + "Prior seal failed.", None, ("Replaced gasket.",), + ) + + monkeypatch.setattr(main, "_load_visible_post", load_visible_post) + monkeypatch.setattr(main, "_similar_voc_client", Client) + account = SimpleNamespace(corporate_entity_ids={"corp-a"}, process_unit_ids={"unit-a"}) + + pool = _Pool([visible, hidden]) + payload = asyncio.run(main.read_similar_voc("focal", 0, account, pool)) + + assert [item["post_id"] for item in payload["items"]] == ["prior"] + assert "score" not in payload["items"][0] + assert payload["items"][0]["action_history"] == ("Replaced gasket.",) + assert payload["next_offset"] is None + assert "process_unit_id::text = any($3::text[])" in pool.connection.query + assert pool.connection.args == ("focal", ["corp-a"], ["unit-a"], 0, 9) + + +def test_similar_voc_pages_orchestrator_work(monkeypatch) -> None: + """One request adjudicates only one bounded page and exposes continuation.""" + focal = {"post_id": "focal", "post_title": "Current", "post_body": "Current issue."} + + async def load_visible_post(*_args): + return focal + + calls: list[str] = [] + + class Client: + def analyze(self, _title, _body, candidate_id, _candidate_title, candidate_body): + calls.append(candidate_id) + return SimilarVocEvidence( + candidate_id, "Equivalent issue", "Current issue.", candidate_body, + None, (), + ) + + rows = [ + { + "post_id": f"prior-{index}", + "post_title": f"Prior {index}", + "post_body": f"Prior issue {index}.", + "visibility_code": "public", + "corporate_entity_id": "corp-a", + "process_unit_id": "unit-a", + "occurred_at": datetime(2026, 8, 20, tzinfo=timezone.utc), + } + for index in range(9) + ] + monkeypatch.setattr(main, "_load_visible_post", load_visible_post) + monkeypatch.setattr(main, "_similar_voc_client", Client) + account = SimpleNamespace(corporate_entity_ids={"corp-a"}, process_unit_ids={"unit-a"}) + + payload = asyncio.run(main.read_similar_voc("focal", 16, account, _Pool(rows))) + + assert len(calls) == 8 + assert payload["next_offset"] == 24 + + +def test_similar_voc_keeps_success_when_one_adjudication_fails(monkeypatch) -> None: + """One provider failure does not discard evidence from sibling candidates.""" + focal = {"post_id": "focal", "post_title": "Current", "post_body": "Current issue."} + + async def load_visible_post(*_args): + return focal + + class Client: + def analyze(self, _title, _body, candidate_id, _candidate_title, candidate_body): + if candidate_id == "failed": + raise OSError("synthetic provider failure") + return SimilarVocEvidence( + candidate_id, "Equivalent issue", "Current issue.", candidate_body, None, () + ) + + rows = [ + { + "post_id": candidate_id, + "post_title": candidate_id, + "post_body": f"{candidate_id} issue.", + "visibility_code": "public", + "corporate_entity_id": "corp-a", + "process_unit_id": "unit-a", + "occurred_at": datetime(2026, 8, 20, tzinfo=timezone.utc), + } + for candidate_id in ("failed", "succeeded") + ] + monkeypatch.setattr(main, "_load_visible_post", load_visible_post) + monkeypatch.setattr(main, "_similar_voc_client", Client) + account = SimpleNamespace(corporate_entity_ids={"corp-a"}, process_unit_ids={"unit-a"}) + + payload = asyncio.run(main.read_similar_voc("focal", 0, account, _Pool(rows))) + + assert [item["post_id"] for item in payload["items"]] == ["succeeded"] diff --git a/docs/adr/0047-global-ask-semantic-retrieval.md b/docs/adr/0047-global-ask-semantic-retrieval.md index 06d15fbcc..d7a0954b0 100644 --- a/docs/adr/0047-global-ask-semantic-retrieval.md +++ b/docs/adr/0047-global-ask-semantic-retrieval.md @@ -10,11 +10,19 @@ find a post while Ask Agent could not. ## Decision -Global Ask candidate retrieval searches the same authorized source context as -the board: raw source hints, source record identity, project mentions, stored -roles, cataloged Keyman mentions, title, and normalized body. The retrieved -posts carry their raw source fields and persisted project/role/Keyman facts -into the contextual-orchestrator prompt with column/table provenance. +Global Ask embeds the complete natural-language question once through +contextual-orchestrator and ranks authorized posts by the maximum raw cosine +similarity against their persisted semantic-unit embeddings. Query and unit +vectors must have the same configured embedding model and dimension. No token +extraction, keyword matching, lexical weighting, similarity threshold, or +locally invented channel weight participates in candidate selection. + +The retrieved posts carry their raw source fields and persisted +project/role/Keyman facts into the contextual-orchestrator prompt with +column/table provenance. These facts enrich grounded answering; they do not +become keyword retrieval signals. If the embedding channel or a complete +matching-model vector is unavailable, retrieval returns no evidence rather +than falling back to lexical search. Raw source fields remain `hint_only`; the prompt explicitly distinguishes them from resolved ontology assertions. The existing ABAC filter is applied before @@ -22,9 +30,10 @@ semantic evidence is loaded, and the bounded source limit remains in place. ## Consequences -- Ask Agent can answer evidence-grounded questions when the relevant project - or identity is not repeated in the body. +- Ask Agent retrieves by semantic-unit meaning without a keyword rule. - A source hint can retrieve a post but cannot silently bind a customer, project, PU, or Keyman. - The orchestrator receives more useful evidence while still receiving only authorized, bounded source documents. +- Missing semantic measurement fails closed and cannot silently change the + retrieval method. diff --git a/docs/adr/0090-global-ask-lineage-timeline-expansion.md b/docs/adr/0090-global-ask-lineage-timeline-expansion.md index da98d9388..e2ddf30a6 100644 --- a/docs/adr/0090-global-ask-lineage-timeline-expansion.md +++ b/docs/adr/0090-global-ask-lineage-timeline-expansion.md @@ -8,7 +8,8 @@ ADR 0047 gave Global Ask's retrieve step the same source-context search surface as the board (raw source hints, project mentions, roles, Keyman -mentions, title, body). That step ranks and returns keyword-matched posts, +mentions, title, body). The current ADR 0047 revision ranks persisted +semantic-unit embeddings against the complete question, but it never touches `post_lineage_edge` -- the Event-Lineage relation `lineageweave.reconstruct` already persists, and the same relation the post-scoped chat flow (`gather_chat_sources`) already expands through for a @@ -16,30 +17,28 @@ single known starting post. A relevance-correct top match is still one snapshot. A live reproduction asking about a specific real event got an accurate answer about that one -post and nothing about what led up to it or what happened next, even after -match-specificity ranking (title/body/source-field weighting) was already -fixed to stop recency from crowding out the right post. The account asking +post and nothing about what led up to it or what happened next. The account asking almost always wants the event's place in a sequence, not an isolated record. ## Decision -After `gather_global_chat_sources` ranks candidates by match specificity, +After `gather_global_chat_sources` ranks candidates by persisted semantic-unit +embedding similarity, it expands only the single top-ranked match through its direct `post_lineage_edge` neighbors (parent and child), mirroring the `.direct` set `find_linked_post_ids` already computes for the post-scoped flow. The expansion: -- Is bounded to the top match only. Expanding every keyword hit was +- Is bounded to the top match only. Expanding every semantic candidate was rejected -- a loosely related term matching a second post would drag an unrelated lineage chain into the model's context for no benefit. - Never bypasses ABAC. Lineage-neighbor ids are merged into the same candidate set the existing visibility filter (`can_see_post`) already runs over; nothing lineage-adjacent is shown without passing that check. -- Is additive to the existing bounded source `limit`, not a replacement - for it -- the limit grows by exactly the number of lineage neighbors - found, so lineage expansion cannot silently starve the keyword-matched - candidates of their own slots. +- Shares the existing bounded source `limit`. The anchor and its direct + neighbors take the first slots and lower-ranked semantic candidates fill + any remaining slots; the returned source count never exceeds `limit`. - Tags each expanded source with an explicit `Event Lineage: reconstructed timeline neighbor of post_id=...` evidence fact, and only when the anchor post itself is visible -- an expanded neighbor must never cite an @@ -52,12 +51,12 @@ sequence around it. ## Considered alternatives -- Expand every keyword-matched candidate's lineage neighbors, not just the +- Expand every semantically ranked candidate's lineage neighbors, not just the top one: rejected for the reason above -- unbounded relevance drift into the prompt. -- Increase `limit` and let the ranking naturally surface neighbors if they - also match the search terms: rejected -- a genuine lineage predecessor or - successor frequently shares no keyword with the question at all (a +- Increase `limit` and let the ranking naturally surface neighbors: rejected + -- a genuine lineage predecessor or successor can express a different event + in the sequence (a Kick-off Meeting and its follow-up rarely repeat the same terms), so ranking alone cannot be relied on to surface it. @@ -65,9 +64,7 @@ sequence around it. - Global Ask answers can now speak to a connected sequence of records around its best match, not only that match's own content. -- The source budget is no longer a fixed constant per request; callers - reading `limit` as an upper bound on retrieved posts must account for the - lineage-expansion addition. +- `limit` remains the hard upper bound after lineage expansion. - Global Ask still has no persisted multi-turn conversation state, so there is no long-context-compression problem yet. Recursive dialogue summarization (Wang et al., 2023) is recorded in diff --git a/docs/adr/0150-korean-relative-time-retrieval.md b/docs/adr/0150-korean-relative-time-retrieval.md index 1e2d805b2..b61e3a0a6 100644 --- a/docs/adr/0150-korean-relative-time-retrieval.md +++ b/docs/adr/0150-korean-relative-time-retrieval.md @@ -8,11 +8,8 @@ A question like "어제 무슨 일이 있었나요?" ("what happened yesterday?") names a time window the reader already has in mind. Before this decision, -`gather_global_chat_sources`'s keyword retrieval (ADR 0047) had no way to -use that window: "어제" only ever became a literal search token against -post titles and bodies, indistinguishable from any other two-character -term. A fresh, unrelated post that happened to rank highest on unrelated -keyword overlap could outrank the post the reader actually meant. +`gather_global_chat_sources` had no way to use that window. A fresh, +unrelated post could outrank the post the reader actually meant. ## Decision @@ -29,17 +26,16 @@ retrieval behavior as finding no expression at all. `gather_global_chat_sources` applies the resolved window as an additional event-time bound on its final ABAC-filtered candidate query (ADR 0202: `coalesce(event_occurred_at, created_at)`), additive to the existing -keyword-match ranking -- it narrows the already-ranked candidate set, it -does not replace ranking with a date filter. Cited sources name which -clock matched. Matched temporal literals are excluded from keyword-term -extraction (`TEMPORAL_STOPWORDS`) so a resolved expression does not also -become a near-meaningless literal search term. +semantic-unit embedding ranking -- it narrows the already-ranked candidate +set, it does not replace ranking with a date filter. Cited sources name which +clock matched. The complete question is embedded once; no temporal-token +removal or keyword extraction occurs. ## Considered alternatives -- Send the raw question to an LLM to extract a date range: rejected for the - same reason ADR 0047's keyword step avoids ungrounded LLM inference at - the retrieval boundary -- a hallucinated date range would silently +- Send the raw question to an LLM to extract a date range: rejected because + ungrounded LLM inference at the retrieval boundary could hallucinate a + date range that would silently narrow (or widen) the candidate set with no way for the reader to verify it, and every extra provider round-trip is retrieval latency the reader pays before seeing an answer. @@ -55,9 +51,9 @@ become a near-meaningless literal search term. term itself acting as retrieval noise. - The resolver is locale-specific (Korean only); a question in another supported UI locale (ADR on i18n scope, `frontend/src/i18n.ts`) that - names a relative time in that language still falls back to keyword-only - retrieval. Extending to additional locales is a follow-up, not required - by this decision. + names a relative time in that language receives semantic retrieval without + a date bound. Extending deterministic date resolution to additional locales + is a follow-up, not required by this decision. - `today` is always passed explicitly by the caller (server-local date); the resolver itself never reads the wall clock, keeping it a pure, trivially unit-testable function. diff --git a/docs/adr/0200-channel-weight-reconciliation.md b/docs/adr/0200-channel-weight-reconciliation.md index b2cbe1fdd..84712338d 100644 --- a/docs/adr/0200-channel-weight-reconciliation.md +++ b/docs/adr/0200-channel-weight-reconciliation.md @@ -109,9 +109,8 @@ argument. `estimation_method_code`, `estimator_version`, `anchor_method_code`, `source_snapshot_sha256`, `sample_pair_count`, `knowledge_cutoff`). The loader requires an exact active-channel match AND single-run - provenance integrity AND an authorized anchor method code - (`unanchored_internal_structure` joins the authorized set under - point 3's labeling duty). One migration with rollbacks lands the + provenance integrity AND the sole authorized anchor method code + (`tepp_lineage_criterion_v1`, per ADR 0205). One migration with rollbacks lands the union on whichever predecessor schema a database has. 5. **Queued judge scoring.** The llm channel's pair scoring moves to the repository's durable queue idiom (`post_content_queue` / diff --git a/docs/adr/0205-tepp-lineage-anchor.md b/docs/adr/0205-tepp-lineage-anchor.md index ac1a3a8c1..711027a86 100644 --- a/docs/adr/0205-tepp-lineage-anchor.md +++ b/docs/adr/0205-tepp-lineage-anchor.md @@ -33,6 +33,12 @@ vector activates only when one normalized `lineage_weight_tepp_anchor` row: cutoff, and validated pair count as every weight in the vector; and 4. matches the TEPP analysis run's immutable snapshot and cutoff exactly. +When that accepted artifact is persisted, the same transaction promotes only +the fast-mlsirm rows whose estimation-run identity, expected-information +method, snapshot, cutoff, and pair count exactly match the artifact. A partial +or mismatched candidate remains inactive; there is no operator-authored anchor +label or second promotion path. + The RFC 3339 request preserves the database cutoff's fractional-second precision; truncating it would make an otherwise valid exact anchor permanently unavailable. diff --git a/docs/adr/0206-evidence-operations-dashboard.md b/docs/adr/0206-evidence-operations-dashboard.md index ecbfade38..3f9f5505d 100644 --- a/docs/adr/0206-evidence-operations-dashboard.md +++ b/docs/adr/0206-evidence-operations-dashboard.md @@ -30,6 +30,9 @@ provenance. explicit fallback, matching ADR 0202. The response names that clock. 3. Every count is authorization-filtered before aggregation. The API returns both event count and distinct post count; neither substitutes for the other. + Analysis-pending and ingestion-failed post counts are disjoint: a failed + current job is shown as retryable failure, never hidden inside the pending + count or interpreted as a negative classification. 4. Extend the existing post-summary semantic workflow through contextual-orchestrator with a schema-validated case analysis. It classifies zero or more case kinds (`claim_investigation`, `rebid_handover`, @@ -53,6 +56,13 @@ provenance. When the focal post lacks an answer, the orchestrator follows authorized Event Lineage and semantic project evidence before concluding the fact is absent from the authorized corpus. + The analysis input reuses the post-chat source assembler: focal post first, + then bounded Event Lineage and semantic-neighborhood posts after the same + corporate-entity/process-unit ABAC check. Every classification and fact + persists its evidence post id and the SHA-256 of the exact numbered input + document. A span that does not occur in that identified document rejects + the whole provider response; linked evidence is never rewritten as focal + post evidence. 8. Claim-investigation and rebid/handover panels include positively classified cases and show extracted answers plus cited spans. A required answer that the source does not support is stored as an explicit missing fact, so the @@ -65,7 +75,16 @@ provenance. source-supported improvement action. Its Dashboard flow is As-Is evidence to To-Be action: rebid history retrieval, originating-order/specification reverse tracing, repeated-issue grouping, and design-improvement return. - Similarity alone never establishes that two issues are the same type. + Similarity alone never establishes that two issues are the same type. The + per-post Similar VOC view uses visible `repeat_issue` classifications only + as a semantic candidate pool, then requires contextual-orchestrator to + adjudicate each pair with verbatim evidence from both records. Results are + displayed by source event time, not a similarity score. It does not reuse + Event Lineage channel weights, and it does not invoke RankWeave without a + separately authorized Similar-VOC measurement contract. Candidate + adjudication is paged in eight-record resource batches with an explicit + continuation offset; the page boundary caps request fan-out but does not + discard older candidates or become a relevance threshold. 11. The Dashboard uses existing design tokens and native HTML controls. Tables and ordered journey steps remain usable without color, with visible focus, keyboard activation, responsive overflow, and reduced-motion support. diff --git a/docs/adr/0208-externalize-local-mathematical-compute.md b/docs/adr/0208-externalize-local-mathematical-compute.md new file mode 100644 index 000000000..42a5a0591 --- /dev/null +++ b/docs/adr/0208-externalize-local-mathematical-compute.md @@ -0,0 +1,116 @@ +# ADR 0208 — Externalize local mathematical computation + +**Decision status:** Accepted +**Date:** 2026-08-25 +**Amends:** ADR 0003, ADR 0024, ADR 0064, ADR 0084, ADR 0132, ADR 0145, +ADR 0148, ADR 0167, ADR 0168, ADR 0182, ADR 0185, ADR 0200, ADR 0201, and +ADR 0205 + +## Context + +LineageWeave's product boundary says that it reconstructs, authorizes, +persists, and presents evidence but does not own calibrated estimation. The +current exact head nevertheless contains Python implementations of IRT report +fitting and scoring, expected-information channel weights, residual SVD and +Gabriel coordinates, embedding cosine, graph random-walk ranking, score +normalization, and RRF contribution arithmetic. Calling a Rust-backed Python +package does not remove the local arithmetic that prepares, transforms, or +interprets its numerical result. + +The ecosystem product boundaries are already sufficient: + +- TEPP's approved PRD owns multilingual temporal and relational measurement, + shared-latent topic identity, trajectories, uncertainty, and event lineage. +- fast-mlsirm's PRD owns reusable IRT/LSIRM estimation, prediction, + diagnostics, recovery, multilevel and multiple-membership computation. +- RankWeave owns retrieval fusion, ranking, evaluation, comparison, and + policy selection. Its calculation core must itself move behind a Rust + CPU/GPU implementation before LineageWeave treats a new result as governed + numerical evidence. + +LineageWeave has no standalone canonical PRD file on this exact head. Until +one lands, `ARCHITECTURE.md` and the accepted ADR set are the product baseline; +this absence remains a product-documentation gap, not permission to infer a +different responsibility. + +## Decision + +1. **No new local numerical model.** LineageWeave adds no Python + mathematical, statistical, psychometric, ranking, fusion, optimization, + matrix-factorization, graph-centrality, or similarity implementation. +2. **Owner by construct.** TEPP owns temporal/topic/event/trajectory + measurement. fast-mlsirm owns psychometric estimation, item information, + expected responses, residual interaction maps, uncertainty, recovery, and + multilevel/multiple-membership post importance. RankWeave owns retrieval + fusion, ranking metrics, contribution evidence, comparisons, and policy + selection. A construct is not moved merely to obtain a preferred language. +3. **Rust execution contract.** New or migrated owner computation executes in + the owner's Rust core with GPU acceleration when supported and a + deterministic multithreaded CPU path. Python may be a generated binding or + transport adapter only; it may not reproduce a formula. +4. **Consumer-only LineageWeave.** This repository retains request/envelope + validation, ABAC filtering, immutable input/output digests, run and model + versions, knowledge cutoff, provenance persistence, and UI projection. + Missing, malformed, non-converged, mixed-snapshot, or unsupported results + fail closed. It never repairs, normalizes, estimates, or substitutes a + numerical result. +5. **No big-bang rewrite.** Existing local computation is frozen as named + migration debt in + `docs/doctoring/python-mathematical-compute-boundary-audit.md`. Each owner + contract lands and proves recovery/equivalence before the corresponding + LineageWeave implementation is deleted. Existing behavior is not relabeled + as compliant while it remains local. +6. **Independent TEPP anchor.** Event-Lineage channel-weight activation keeps + ADR 0205's exact TEPP anchor requirement. fast-mlsirm may estimate weights + conditional on that accepted independent anchor; it does not manufacture + the criterion. +7. **No heuristic exception.** Candidate windows, score floors, token overlap, + string similarity, or equal weights are not promoted to measurement. + Operational bounds may remain only as disclosed resource limits and may + not determine a scientific score or ground truth. + +## Stacked delivery order + +1. Owner PRs publish versioned request/result schemas, model identity, + convergence/uncertainty evidence, input digest, and deterministic recovery + tests: TEPP first, fast-mlsirm second, RankWeave third. +2. A LineageWeave contract-only PR adds strict clients and provenance tables; + no UI activates from an unpersisted envelope. +3. A shadow-validation PR compares owner outputs with frozen synthetic + fixtures and records aggregate, non-identifying evidence. +4. Separate deletion PRs remove `channel_weight_estimation.py`, numerical + portions of `period_report.py` and `leftover_pairs.py`, local cosine/RWR, + and local ranking contribution/normalization code after their owner path is + accepted. +5. The final PR removes NumPy/fast-mlsirm/RankWeave calculation imports from + LineageWeave, updates architecture/PRD/ADRs, and makes the transition guard + require an empty debt inventory. + +## Consequences + +The Dashboard may show TEPP topics and fast-mlsirm importance only from exact, +persisted owner artifacts. It can explain the source posts, memberships, +levels, time window, model version, uncertainty, and provenance, but cannot +recalculate or rank them locally. During migration, affected capabilities +remain explicitly legacy or unavailable rather than presenting local results +as Rust/GPU-backed. + +## References (APA 7th) + +Fox, J.-P., & Glas, C. A. W. (2001). Bayesian estimation of a multilevel item +response model using Gibbs sampling. *Psychometrika, 66*(2), 271–288. +https://doi.org/10.1007/BF02294839 + +Gabriel, K. R. (1971). The biplot graphic display of matrices with application +to principal component analysis. *Biometrika, 58*(3), 453–467. +https://doi.org/10.1093/biomet/58.3.453 + +Jeon, M., Jin, I. H., Schweinberger, M., & Baugh, S. (2021). Mapping unobserved +item-respondent interactions: A latent space item response model with +interaction map. *Psychometrika, 86*(2), 378–403. +https://doi.org/10.1007/s11336-021-09762-5 + +Roberts, M. E., Stewart, B. M., & Tingley, D. (2019). stm: An R package for +structural topic models. *Journal of Statistical Software, 91*(2), 1–40. +https://doi.org/10.18637/jss.v091.i02 + diff --git a/docs/adr/0210-temporal-topic-context-influence-dashboard.md b/docs/adr/0210-temporal-topic-context-influence-dashboard.md new file mode 100644 index 000000000..e9be55d48 --- /dev/null +++ b/docs/adr/0210-temporal-topic-context-influence-dashboard.md @@ -0,0 +1,236 @@ +# ADR 0210: TEPP temporal topics and fast-mlsirm context influence + +- Status: Accepted +- Implementation maturity: producer-contract required; consumer projection not yet shipped +- Date: 2026-08-25 +- Depends on: ADR 0132 (TEPP topic-lineage boundary), ADR 0206 (operations Dashboard) +- Upstream authorities: TEPP ADR 0012; fast-mlsirm ADR 0002 and ADR 0007 + +## Context + +The operations Dashboard must show how topics evolve through Event Lineage and +which posts materially influence a topic's fitted state at business-unit, +process-unit (PU), team, and person levels. A lexical cluster, one topic model +per time bin, raw topic proportion, engagement count, or hand-authored weighted +sum cannot answer that question. Those approaches lose stable topic identity, +ignore multiple membership, understate dependence, or silently redefine +"important". + +TEPP's approved PRD and ADR 0012 already own Temporal Relational Shared-Latent +Topic Measurement (TRSL-TM): global topic identities, event time, explicit +document relations, weighted cross-classified memberships, posterior +uncertainty, and topic activity over time. LineageWeave therefore consumes a +TEPP artifact; it does not fit or label topics locally. + +For this surface, **important post** has one exact statistical meaning: +case-deletion influence on a fitted topic-by-context parameter. For topic +`k`, context dimension `l`, and post `d`, fast-mlsirm reports + +\[ +D_{dkl}=(\hat\psi_{kl,-d}-\hat\psi_{kl})^\top +I_{kl}(\hat\psi)(\hat\psi_{kl,-d}-\hat\psi_{kl}), +\] + +where `I` is the same fitted model's observed-information block and +`psi[-d]` is the estimate after removing that post's complete observation. +This is a multilevel case-deletion diagnostic, not business value, causal +impact, author performance, or an outlier-removal instruction (Shi & Chen, +2008). It is selected because it is defined by the fitted likelihood and +observed information, so no arbitrary cross-level weights or score constants +are introduced. + +## Product requirements (PRD) + +1. The Dashboard presents TEPP topics on one event-time axis with stable topic + identity and explicit active, dormant, and reactivated states. Topic + birth/split/merge/retirement appears only when the TEPP artifact explicitly + supplies that lineage event. +2. Selecting a topic shows separate business-unit, PU, team, and person views. + A post may belong to more than one context in the same dimension and to + contexts in several dimensions. The UI never flattens those assignments + into a single owner. +3. Each level lists posts by fast-mlsirm case-deletion influence `D[d,k,l]`, + with exact value, uncertainty/diagnostic status, source event time, topic + state, membership provenance, and a link to the authorized source post. + No score threshold is applied. Equal values remain ties; deterministic + source time and post identity order only stabilize rendering and do not + break the statistical tie. +4. Copy names the estimand as **model influence**. It must not say business + importance, performance, causality, risk, or priority unless a separately + validated outcome model establishes that construct. +5. Pending, failed, non-converged, unidentified, incomplete-membership, + CPU/GPU-parity-failed, or contract-mismatched runs render an actionable + unavailable state. LineageWeave never fills them with keyword search, + engagement counts, RankWeave output, default weights, or a local estimate. +6. All rows are authorization-filtered before topic/context aggregation. A + hidden source post contributes neither a displayed rank nor an exact value + that could disclose its influence. +7. The topic view is a Dashboard section, not a new external-information + board. It reuses the existing GNB destination and post-detail navigation. + +## Technical requirements (TRD) + +### TEPP producer contract + +The accepted TEPP result schema must include: + +- immutable model-run, source-snapshot SHA-256, knowledge cutoff, model/schema + version, event clock, and posterior-draw identity; +- global topic identity and activity interval, plus explicit lineage event and + provenance when present; +- per-post posterior logistic-normal topic coordinates or plausible values, + not a hard topic label derived from a threshold; +- Event Lineage/document-relation edges admitted by the TEPP run; +- versioned, time-valid business-unit, PU, team, and person membership edges + with source-derived weights and evidence. A missing weight is unavailable; + equal membership is never invented. + +LineageWeave verifies the exact snapshot and cutoff before persisting a 3NF +projection. It does not inspect TEPP's private tables or reinterpret posterior +coordinates. + +TEPP protected main currently exposes `tepp.trsl_topic_lineage.v1`, a +digest-bound CPU-`f64` artifact containing fitted forward sequence edges and +aggregate counts. That is real producer progress, but it does not contain the +per-post posterior coordinates/plausible values or dimension-qualified +membership evidence required by this decision. LineageWeave must reject that +schema for the context-influence surface rather than reconstruct the omitted +inputs from its association-strength field. + +### fast-mlsirm producer contract + +fast-mlsirm owns a versioned `topic_context_influence` estimand over TEPP +posterior plausible values. It jointly retains topic, event time, and the four +dimension-qualified multiple-membership designs. Rust owns likelihood, +gradients, observed information, deletion refits, posterior-draw combination, +and influence arithmetic. The CPU `f64` path is the numerical reference; +GPU execution is a Rust device path and must pass identification-aware parity. +Python may validate and marshal only. + +The result envelope contains the exact TEPP run/snapshot/cutoff, fast-mlsirm +version and code revision, estimand/schema version, backend/precision, +convergence and identification diagnostics, posterior-draw coverage, context +membership fingerprint, post/topic/context identities, `D[d,k,l]`, and its +uncertainty evidence. A result for one context dimension cannot be copied to +another dimension. + +### LineageWeave consumer and persistence + +Use normalized objects such as `topic_model_run`, `topic_definition`, +`topic_activity_interval`, `topic_lineage_relation`, `topic_post_coordinate`, +`topic_context_membership`, `topic_influence_run`, and +`topic_post_context_influence`. Large result tables are partitioned by tenant +and modeled-period identity rather than one global time partition. Foreign +keys bind every influence row to the exact TEPP and fast-mlsirm artifacts. + +The API returns only persisted accepted rows after ABAC. It returns exact +ties, producer diagnostics, and provenance rather than computing or +renormalizing scores. The frontend renders an exact-value table alongside the +temporal topic view, uses text/pattern as well as color for topic state, and +supports keyboard, touch, reduced motion, narrow viewports, and screen readers. + +```mermaid +sequenceDiagram + participant Source as Authorized source snapshot + participant TEPP + participant MLS as fast-mlsirm Rust CPU/GPU + participant LW as LineageWeave projection + participant UI as Dashboard + Source->>TEPP: snapshot + cutoff + Event Lineage + memberships + TEPP-->>MLS: versioned posterior topic plausible values + MLS->>MLS: multilevel multiple-membership fit + MLS->>MLS: post deletion refits + observed-information D + MLS-->>LW: accepted topic_context_influence envelope + LW->>LW: exact contract, cutoff, digest, diagnostics, ABAC validation + LW-->>UI: temporal topics + level-specific tied influence rows +``` + +## Verification and acceptance + +The feature is not release-ready until all of the following are protected-main +evidence rather than a local or contract-only claim: + +1. TEPP simulation recovers known global topic identity, temporal prevalence, + relation effects, dormancy/reactivation, and cross-classified membership + effects with reported bias, RMSE, interval coverage, and posterior-draw + diagnostics; relation-aware splits prove no future leakage. +2. fast-mlsirm simulation recovers known context effects and ranks known + injected influential posts by the declared deletion estimand. Tests include + nested, crossed, weighted multiple-membership, time-varying membership, + sparse/unbalanced levels, missing observations, exact ties, and masked or + jointly influential cases. Correlation alone is not acceptance evidence. +3. Rust CPU worker-count determinism and CPU/GPU parity pass on the same + estimand. A GPU test proves actual device execution; fallback is explicit. +4. Contract tests reject wrong snapshot/cutoff/model/schema, missing posterior + draws, invented membership weights, non-convergence, unidentified + information blocks, non-finite influence, mixed producer runs, and partial + result sets. +5. Integration tests prove 3NF foreign-key integrity, idempotent replay, hot- + partition distribution, pre-aggregation ABAC, and no hidden-post leakage. +6. Storybook and browser screenshots cover populated, ties, dormant/reactivated, + multiple-membership, unavailable, narrow, dark, reduced-motion, keyboard, + and touch scenes. The exact-value table remains usable without the chart. +7. Public docstring, production line/branch, interaction, design-token, i18n, + and edge-case coverage remain 100% under repository gates. + +## Alternatives considered + +1. **LineageWeave fits a local dynamic topic model.** Rejected because TEPP + owns the temporal/relational posterior and measurement contract. +2. **Rank by posterior topic share, recency, engagement, or a weighted sum.** + Rejected because it ignores contextual influence or invents a construct and + weights. RankWeave may present an independently authorized retrieval rank, + but it is not this measurement. +3. **Use fast-mlsirm's current crossed binary kernel unchanged.** Rejected + because thresholding TEPP posterior coordinates into binary responses + discards uncertainty and changes the estimand. The producer must expose the + versioned topic-context influence contract above. +4. **Call the diagnostic business impact.** Rejected. Statistical influence + measures sensitivity of fitted topic/context parameters, not causal or + economic value. + +## Consequences + +The user receives a precise, reproducible answer to “which posts shape this +topic at this organizational level?” without arbitrary weights. Activation +depends on two upstream protected contracts and full recovery evidence; until +then the Dashboard truthfully shows why the result is unavailable rather than +inventing a ranking. + +## References (APA 7th) + +American Educational Research Association, American Psychological +Association, & National Council on Measurement in Education. (2014). +*Standards for educational and psychological testing*. American Educational +Research Association. + +Blei, D. M., & Lafferty, J. D. (2006). Dynamic topic models. In *Proceedings +of the 23rd International Conference on Machine Learning* (pp. 113–120). +Association for Computing Machinery. https://doi.org/10.1145/1143844.1143859 + +Browne, W. J., Goldstein, H., & Rasbash, J. (2001). Multiple membership +multiple classification (MMMC) models. *Statistical Modelling, 1*(2), +103–124. https://doi.org/10.1177/1471082X0100100202 + +Fox, J.-P., & Glas, C. A. W. (2001). Bayesian estimation of a multilevel IRT +model using Gibbs sampling. *Psychometrika, 66*(2), 271–288. +https://doi.org/10.1007/BF02294839 + +Jin, I. H., Jeon, M., Schweinberger, M., Yun, J., & Lin, L. (2022). +Multilevel network item response modelling for discovering differences +between innovation and regular school systems in Korea. *Journal of the Royal +Statistical Society: Series C (Applied Statistics), 71*(5), 1225–1244. +https://doi.org/10.1111/rssc.12569 + +Molenaar, D., & Jeon, M. (2026). Regularized joint maximum likelihood +estimation of latent space item response models. *Psychometrika, 91*(1), +335–359. https://doi.org/10.1017/psy.2025.10068 + +Shi, L., & Chen, G. (2008). Case deletion diagnostics in multilevel models. +*Journal of Multivariate Analysis, 99*(9), 1860–1877. +https://doi.org/10.1016/j.jmva.2008.01.023 + +Zhang, D. C., & Lauw, H. (2022). Dynamic topic models for temporal document +networks. In *Proceedings of the 39th International Conference on Machine +Learning* (pp. 26281–26292). PMLR. +https://proceedings.mlr.press/v162/zhang22n.html diff --git a/docs/adr/README.md b/docs/adr/README.md index 6aca500f1..39c7c8bd3 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -18,6 +18,8 @@ decision from them. | [`storybook-inventory.md`](../storybook-inventory.md) | [0118](0118-uiux-standard-guide-v3-design-overhaul.md), [0184](0184-ontology-provenance-explorer.md) | | [`POSTGRESQL_CONCURRENCY_REFERENCES.md`](../doctoring/POSTGRESQL_CONCURRENCY_REFERENCES.md) | [0204](0204-analysis-run-short-transaction-delivery.md) | | Evidence operations Dashboard (`/`) | [0206](0206-evidence-operations-dashboard.md) | +| [`temporal-topic-context-influence-research.md`](../temporal-topic-context-influence-research.md) | [0210](0210-temporal-topic-context-influence-dashboard.md) | +| [`python-mathematical-compute-boundary-audit.md`](../doctoring/python-mathematical-compute-boundary-audit.md) | [0208](0208-externalize-local-mathematical-compute.md) | [0011](0011-prov-o-standard-relations.md) and [0065](0065-prov-o-provenance-boundary.md) cite the dated W3C PROV-O and PROV-DM Recommendations (https://www.w3.org/TR/2013/REC-prov-o-20130430/ and https://www.w3.org/TR/2013/REC-prov-dm-20130430/). diff --git a/docs/doctoring/python-mathematical-compute-boundary-audit.md b/docs/doctoring/python-mathematical-compute-boundary-audit.md new file mode 100644 index 000000000..f1dbeee83 --- /dev/null +++ b/docs/doctoring/python-mathematical-compute-boundary-audit.md @@ -0,0 +1,64 @@ +# Python mathematical-compute boundary audit + +**Exact-head audit date:** 2026-08-25 +**Normative decision:** [ADR 0208](../adr/0208-externalize-local-mathematical-compute.md) + +This inventory names migration debt; it is not evidence that the current +Python paths satisfy the Rust/GPU requirement. + +## Product-boundary sources read + +- LineageWeave `ARCHITECTURE.md` and accepted ADRs 0003, 0132, 0145, + 0200, 0201, and 0205. This exact head has no standalone canonical PRD. +- TEPP `docs/product/prd-v0.4-approved.md`, whose approved TRSL-TM scope + owns temporal, relational, multilingual, topic, event, and trajectory + measurement. +- fast-mlsirm `docs/PRD.md`, whose reusable library scope owns + multilevel/contextual/longitudinal psychometric estimation, diagnostics, + recovery, and versioned artifacts rather than hosted product storage. +- RankWeave `README.md` and `ARCHITECTURE.md`. Its exact head has no PRD; + those files define the current fusion/ranking/evaluation responsibility. + A canonical RankWeave PRD is required before expanding that contract. + +| Current LineageWeave path | Local computation | Owner | Consumer replacement | Principal callers / tests | +|---|---|---|---|---| +| `lineageweave/channel_weight_estimation.py` | dichotomization, synthetic simulation, MLS2PLM input construction, expected item information and normalization | fast-mlsirm, conditional on TEPP anchor | versioned anchored-weight artifact; strict digest/convergence validation | estimation scripts, seed/server/rebuild paths; `tests/test_channel_weight_estimation.py`, estimator-script tests | +| `lineageweave/period_report.py` | response matrix, GRM/GPCM fit/FIPC/EAP, likelihood, category expectation, information ordering | fast-mlsirm | period-measurement artifact with item bank, scores, uncertainty, diagnostics | report ingestion and demo seed; period-report and report API tests | +| `lineageweave/leftover_pairs.py` | residual matrix, complete-case selection, SVD/Gabriel coordinates, distances, reconstruction, axis shares | fast-mlsirm | residual-interaction artifact with observed/expected identity and coverage | `period_report.py`, report ingestion/seed; `tests/test_leftover_pairs.py`, report tests | +| `lineageweave/embedding_client.py` and `backend/app/post_chat_ingestion.py` | cosine similarity, vector norms, maximum semantic score | RankWeave retrieval-score contract | ranked evidence envelope over ABAC-visible semantic units | reconstruction text channel and Global Ask retrieval; embedding/post-chat tests | +| `lineageweave/knowledge_graph.py` | random walk with restart, convergence delta, adaptive relevance cutoff | RankWeave graph-ranking contract | ranked-node artifact with contribution and convergence evidence | related-person/entity API paths; knowledge-graph tests | +| `lineageweave/reconstruct.py` | channel-weight renormalization, candidate-score fusion and minimum-score decision | RankWeave fusion; TEPP supplies independent lineage criterion | accepted edge-ranking artifact; LineageWeave persists selected edge and channel provenance | lineage rebuild/start/seed/server; reconstruct, persistence, API tests | +| `lineageweave/rankweave_client.py` | channel construction, token overlap, RRF weights and contribution arithmetic | RankWeave | strict ranking artifact exposing owner-computed contributions | `/api/rankings`, frontend Rankings; `tests/test_rankweave_client.py` and frontend tests | + +`lineageweave/post_evaluation.py` imports fast-mlsirm only for its published +judge contract and `to_irt_row` projection. It performs no fitted numerical +estimation, but remains in the transition guard because any direct owner-package +import must be reviewed before LineageWeave's final wire-only state. + +Validation-only uses of `math.isfinite` and database aggregation are not model +ownership and remain. Date ordering, counts, pagination, authorization, schema +validation, and presentation formatting also remain LineageWeave concerns. + +## Required owner contracts + +- **TEPP:** temporal-relational topic identity and Event-Lineage criterion + artifacts, with snapshot/cutoff, posterior uncertainty, evidence status, + lineage transitions, and deterministic Rust CPU/GPU execution evidence. +- **fast-mlsirm:** anchored channel information; GRM/GPCM fit, score and item + information; Gabriel residual interaction map; and topic-conditional + multiple-membership multilevel importance for business unit, PU, team, and + person, with recovery/RMSE and coverage evidence. +- **RankWeave:** Rust-backed similarity, graph ranking, fusion, contribution, + evaluation, and policy-selection artifacts. Its present Python calculation + core is the correct product owner but not the final execution architecture. + +## Persistence and UI blast radius + +Owner envelopes require normalized run/artifact tables keyed by analysis run, +owner contract version, model version, source snapshot SHA-256, knowledge +cutoff, and authorization scope. Topic, membership, level-specific importance, +uncertainty, and source-post evidence occupy separate child rows; arrays or +labels do not replace foreign keys. Dashboard and post detail endpoints read +only accepted persisted rows and preserve source-post ABAC. Storybook covers +accepted, pending, failed, stale-digest, non-converged, hidden-evidence, and +multiple-membership cases before UI activation. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 4df379e4e..692cbccaa 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,6 +1,73 @@ # Product & Technical Gap Baseline -> Audit snapshot: 2026-08-25 18:00 KST (refreshed by the autonomous merge +> Dashboard delivery snapshot: 2026-08-25 19:55 KST. Protected `main` was +> `3f4734806bdc7ef5843f36c7dbbcceb62cd51b9e`. This local branch is not +> protected-main release evidence. + +## Operations Dashboard PRD/TRD traceability + +| Requirement | Evidence contract | Delivery state | +|---|---|---| +| Claim cause delay: order, specification change, originating order, sales pool, Event/post counts | ADR 0206; contextual-orchestrator case classification with cited spans; Event Lineage context | Candidate implementation; authenticated runtime acceptance pending | +| Rebid/handover: discussion, counterparties, our owner, decisions, Event/post counts | ADR 0206; normalized case facts plus persisted summary actions/roles | Candidate implementation; corpus backfill pending | +| External information count/rate and sales/project relation | ADR 0206; semantic `external_information` classification inside Dashboard GNB | Candidate implementation; no separate Board by product decision | +| Project-specific journey | Explicit source/semantic project membership plus event-time ordering | Candidate API and ordered journey UI implemented; authenticated runtime acceptance pending | +| Repeat issue to design improvement | `repeat_issue`, `issue_pattern`, and `improvement_action` cited facts | Candidate semantic contract; design-system connector acceptance pending | +| Natural-language Ask with evidence, report, alert, MCP | Persisted semantic-unit embeddings plus versioned delivery/resource contract | Candidate implementation uses whole-question embedding retrieval with no lexical fallback; authenticated runtime acceptance pending | +| Similar VOC, customer cohort, prior action | Persisted repeat-issue candidate semantics plus orchestrator pair adjudication and extractive evidence | Candidate live post endpoint and post-detail UI implemented; authenticated runtime acceptance pending | +| TEPP independent Event Lineage anchor | Accepted, persisted TEPP criterion bound to exact snapshot/cutoff before fast-mlsirm activation | Consumer PR #606 is on protected main; TEPP producer PR #237 remains open, so no end-to-end accepted artifact is release evidence yet | + +### Technical contract and flow + +```mermaid +sequenceDiagram + participant Source as Authorized source_post + participant CO as contextual-orchestrator + participant Case as operations_case_* (3NF) + participant TEPP as TEPP criterion run + participant MLS as fast-mlsirm + participant API as Dashboard/Ask API + Source->>CO: semantic units + lineage + ontology context + CO-->>Case: cases, cited facts, session provenance + Source->>TEPP: versioned snapshot and independent criterion + TEPP-->>MLS: exact accepted anchor only + MLS-->>API: anchored vector or unavailable + Case-->>API: ABAC-filtered evidence and counts +``` + +Security/operability: every aggregation applies `post_read` plus row-level +corporate-entity visibility before counting; source-body digests invalidate +stale inference; provider errors persist no positive/negative result; PII +remains authorized at the UI boundary and is excluded from telemetry. The +tables use composite keys and bounded kind-first indexes; production hot-path +acceptance still requires `EXPLAIN (ANALYZE, BUFFERS)` on an anonymized runtime +snapshot. + +### Exact-head UI audit + +The `f0b96029` Storybook build was rendered at 1440×1100 and 402×1200 with +synthetic evidence; `416fd19d` changes only post-navigation request isolation. +Desktop inspection showed all four case kinds, five non-conflated metrics, +project-journey ordering, cited facts, and evidence actions without horizontal +card overflow. Narrow inspection showed two-column metrics, readable cards and +44px-class actions; the project journey remains intentionally horizontally +scrollable. No identifying runtime record or screenshot is committed. The +`EvidenceReady`, `NarrowViewport`, `AnalysisPendingAndMissingEvidence`, +`AnalysisFailed`, and `LoadError` scenes cover the ADR 0206 state inventory. +Authenticated authorized-corpus acceptance remains separate and may return +only aggregate, non-identifying evidence to this repository. + +### Exact open-PR boundary + +At this snapshot there were 7 open PRs and 17 open issues. Exact observed heads +were `#616 10d289cd`, `#615 dadf2ccd`, `#614 416fd19d`, `#613 024fb6ab`, +`#612 2c50d8b6`, `#579 b782e5f7`, and `#387 d6b74f53`. All were blocked on +hosted gates and/or independent review. These observations are not merge +readiness. Re-fetch exact heads, +unresolved threads, checks, approvals, rulesets, and merge SHA before any +lifecycle claim. + +> Audit snapshot: 2026-08-25 19:55 KST (refreshed by the autonomous merge > loop). This repository records synthetic fixtures and aggregate, > non-identifying runtime evidence only. Open PRs and local checks are not > protected-default-branch release evidence. Identifying post identifiers, @@ -9,29 +76,32 @@ ## 1. Exact-head and governance evidence -The protected default branch was `3d6d7188a3ae299ffef77eb991032268a4c2160d` -when this baseline was refreshed. The live queue contained 5 open PRs and 17 +The protected default branch was `3f4734806bdc7ef5843f36c7dbbcceb62cd51b9e` +when this baseline was refreshed. The live queue contained 7 open PRs and 17 open issues. The exact-head inventory below supersedes older per-PR snapshots elsewhere in this document; those older rows remain useful historical delivery context only. | PR | Exact observed head | Merge/check state at this snapshot | | ---: | --- | --- | -| #609 | `820fe648` | exact-main follow-up that renders #493's focused isolation reasons with localized next actions and an accepted ADR; backend/documentation, focused UI, i18n, lint, and build checks passed locally, auto-merge is enabled, and hosted gates plus independent review remain outstanding | -| #606 | `61fd631c` | retires the internally anchored channel-weight vector in favor of the exact TEPP#237 criterion-validity contract and contains the former #607 operations-dashboard stack; the head now conflicts with protected `main` and must be composed without losing either capability | -| #579 | `a8e9ef9e` | interaction-map coordinates remain open and now conflict with protected `main`; prior branch tests and screenshot audits do not transfer across the required composition | -| #490 | `73413d0b` | broad historical workspace branch is conflicting with failing checks; decompose/restack rather than replaying its 931-commit merge wholesale | -| #387 | `ab5cf345` | channel evidence plus Ask reconstruction-profile preservation is mergeable on the current head, but a current changes-requested decision remains; resolve valid findings and obtain independent exact-head re-review before enabling auto-merge | +| #616 | `10d289cd` | narrows global Ask to exact resolved embedding identity; hosted gates and independent review remain required | +| #615 | `dadf2ccd` | removes a resolved calendar placeholder's duplicate live-region role; hosted gates and independent review remain required | +| #614 | `416fd19d` | Dashboard/Ask/Similar-VOC, disjoint failed-analysis metrics, and post-isolated pagination are composed with current main; same-head hosted gates and independent review remain required | +| #613 | `024fb6ab` | exact-head baseline refresh awaits hosted gates and independent review; its snapshot does not supersede this later live inventory | +| #612 | `2c50d8b6` | focused graph navigation no longer falls back to stale global graph state; hosted gates and independent review remain required | +| #579 | `b782e5f7` | interaction-map coordinate persistence is composed with protected main; hosted gates and independent review remain required | +| #387 | `d6b74f53` | channel evidence and delivery repairs are composed, but independent exact-head review remains required | No row above is merge evidence. Immediately before any lifecycle action, re-fetch the head, unresolved threads, formal reviews, rulesets, and same-head check conclusions. In particular, queued checks are infrastructure state and do not transfer evidence from an earlier SHA. -PR #607 merged as `61fd631c7bb3c57113fd19763c2c43161eeb2824` -into #606's non-default branch. That stack merge preserves its implementation -but is not protected-`main` evidence; only #606's eventual protected merge can -deliver the combined TEPP-anchor and operations-dashboard work. +PR #607 first merged as `61fd631c7bb3c57113fd19763c2c43161eeb2824` +into #606's non-default branch. PR #606 subsequently passed the protected gate, +so the combined TEPP-consumer and operations-dashboard implementation is now +on `main`; the still-open TEPP producer PR #237 keeps end-to-end anchor +acceptance unavailable. PR #604 was closed unmerged after its exact OIDC repair was composed into #605; its green or pending checks are not delivery evidence. PR #482 merged as @@ -53,9 +123,10 @@ longer part of the open queue. That merge also left a standalone conflict marker and duplicated stale tail in `CLAUDE.md`; #594 repaired it through protected `main` as `241be2dddf657f854cb8be54fe11d4ef48d37976`. -Protected main now calls the ADR 0109 OIDC return helpers before -`signinRedirect`; #600 and #605 delivered the repair through the protected -gate. The authenticated-only `accessToken` narrowing remains present. +Protected main now contains the ADR 0109 OIDC return restoration from #605, +including fragment preservation and storage fallback. The #606 dashboard +landing must additionally route `?post=` deep links to the Board; that focused +regression is part of the current candidate and is not delivery evidence yet. Three systemic gates currently dominate the queue: @@ -99,11 +170,11 @@ Recent protected-default-branch delivery evidence (squash merges onto | PR | Merged (UTC) | Delivered | | ---: | --- | --- | -| #468 | 2026-08-25 08:44 | fast-mlsirm, Keyverse, orchestrator, and TEPP integration boundaries | -| #493 | 2026-08-25 08:44 | ABAC-safe focused Event Lineage isolation reasons in the backend; buyer copy continues on #609 | -| #600 | 2026-08-25 08:44 | non-identifying live gap baseline and ADR 0109 login-helper repair | -| #605 | 2026-08-25 08:44 | modal focus containment/refocus, readable evidence labels, and OIDC return context | -| #608 | 2026-08-25 08:43 | Naruon projection wired into Workspace Calendar | +| #468 | 2026-08-25 08:44 | fast-mlsirm, Keyverse, contextual-orchestrator, and TEPP integration boundaries | +| #493 | 2026-08-25 08:44 | evidence-grounded Event Lineage isolation reasons | +| #600 | 2026-08-25 08:44 | then-current exact-head product/technical baseline | +| #605 | 2026-08-25 08:44 | dialog focus order, evidence readability, and OIDC return-context restoration | +| #608 | 2026-08-25 08:43 | Naruon projection consumed by Workspace Calendar | | #603 | 2026-08-25 07:24 | short analysis-run transactions, session advisory locking, package-marker/privacy repair, and provider-work lease release | | #602 | 2026-08-25 07:24 | post-detail modal semantics, Escape close, initial focus, and opener restoration; navigation-refocus edge case continues on #605 | | #582 | 2026-08-25 07:24 | bounded batched cited-lineage graph fetch | @@ -279,7 +350,7 @@ this file per §3.5 of the prior snapshot). | #280 | Full project-lifecycle history and handover intervals | Tracked with issue #284; no active delivery PR confirmed | | #284 | Authoritative lifecycle ingestion and idempotent reconciliation | No active delivery PR confirmed | | #289 | Activate the optional lineage LLM channel through a bounded asynchronous rebuild | #434 | -| #336 | Replace pseudo-CalDAV feed with a Naruon-owned calendar projection | Contract on `main` (#355); Buyer consume wiring in `feat/naruon-calendar-buyer-wiring-v2170` | +| #336 | Replace pseudo-CalDAV feed with a Naruon-owned calendar projection | Contract on `main` (#355); operator consume wiring in historical branch `feat/naruon-calendar-buyer-wiring-v2170` | | #338 | Evidence-bounded email/project lineage contract for Naruon consumption | #355 | | #341 | Heterogeneous ontology and provenance explorer separate from Event Lineage | #349 | | #358 | Batch reauthorize persisted post-Ask evidence without N+1 queries | Ask stack | @@ -291,8 +362,8 @@ this file per §3.5 of the prior snapshot). | Gap | Current evidence | Acceptance requirement | | --- | --- | --- | -| Protected release | 5 open PRs at snapshot; #609 is mergeable with auto-merge, #606/#579/#490 conflict, and #387 retains changes requested | Terminal exact-head checks, no unresolved threads, independent exact-head approvals, protected squash-merge SHA | -| Evidence-grounded operations workspace | The former #607 implements persisted operational cases, an authenticated dashboard, citation-first actions, report/alert delivery metadata, tokens, Storybook, and regression coverage inside #606; authenticated production backfill and the similar-VOC live endpoint remain unavailable | Clear #606 normally on its exact combined head, perform authenticated desktop/mobile acceptance with aggregate evidence, and confirm a protected merge SHA | +| Protected release | 8 open PRs at snapshot; the queue is split between mergeable exact-main heads and older conflicting work, while the former #607 is preserved inside #606 | Terminal exact-head checks, no unresolved threads, independent exact-head approvals, protected squash-merge SHA | +| Evidence-grounded operations workspace | #606 delivered the initial consumer and Dashboard to protected `main`; #614 adds governed semantic Ask, live Similar VOC, disjoint pending/failed analysis metrics, full Storybook state inventory, and current desktop/mobile screenshot evidence. Authorized-corpus backfill acceptance remains unavailable | Clear #614 normally on its exact head, perform authenticated authorized-corpus acceptance with aggregate evidence, and confirm a protected merge SHA | | Shared frontend gate | The ADR 0109 login repair is on protected `main`; eight older branches carried the defect and received the same verified repair this loop (#521–#560) | Keep every future branch cut from post-repair bases; re-verify with frontend lint/test/build before push | | Identifying baseline regression | `main` gap file listed real post identifiers; separately, closed #506 and pre-existing public history contain a private runtime source-table identifier, while current `main` and #507 trees are clean | Land this non-identifying rewrite, then coordinate ADR 0001 history remediation with security/privacy owners; do not reproduce the value, force-push, or delete evidence ad hoc | | Authorized-corpus runtime | Repository tests use synthetic fixtures; private records remain outside git | Authenticated runtime validation returning only aggregate, non-identifying evidence | @@ -301,13 +372,13 @@ this file per §3.5 of the prior snapshot). | Event and project semantics | Multi-project mentions, project-bound actions, 5W1H, requester/processor, and semantic relations exist in ADR 0036/0052/0100/0111/0129 and active stacks | Aggregate authenticated evidence must show distinct projects and events, explicit requester/processor and real R&R, normalized relative time, and product/entity relations without promoting attendance or co-occurrence | | Knowledge Graph readability | The black evidence-node root cause is an undefined-token fallback; the design-token repair and long-label/evidence-table coverage are present on #490, not protected `main` | Deliver the token repair through protected `main`, then verify light/dark contrast, keyboard graph navigation, full labels, and evidence tables in the authenticated rendered surface | | Source-code lookup UX | Source state/detail codes remain evidence-bearing machine values and current detail presentation is dense | Catalog-backed display labels with raw-code provenance, compact 5W1H/source-detail hierarchy, keyboard access, and no unsupported customer/project binding | -| Calendar / Naruon | #355 delivered the projection contract; v2.17.0 wires Buyer consume without forwarding the end-user token. Naruon producer, provider/consumer fixtures, and protected merge remain open (#336) | Verify observed events against the published schema without invented events; keep commitments available when the channel is unwired | +| Calendar / Naruon | #355 delivered the projection contract; v2.17.0 wires operator consumption without forwarding the end-user token. Naruon producer, provider/consumer fixtures, and protected merge remain open (#336) | Verify observed events against the published schema without invented events; keep commitments available when the channel is unwired | | SKOS organization aliases | Catalog binding and chip caption live on #480 / #482 | One catalog row per corroborated org; companion caption is hint-only until bound | | Event Lineage evidence | Channel evidence and Allen relations live on #387 / #484 | Persist channel scores, explain them in the popup, never invent a fused score | -| Scientific measurement | Durable accepted TEPP receipts are protected (`main`); #606 replaces the internally anchored channel-weight activation with the exact TEPP-owned criterion-validity contract from TEPP#237, while #468 binds fast-mlsirm/Keyverse/orchestrator/TEPP integration tests and fails closed on upstream probability-axis drift. Neither cross-repository PR is protected delivery yet. #387 removes inferred/default persistence weights, but several older reconstruction tests still pass hand-authored numeric weight dictionaries; those constants are not estimator evidence | Land TEPP#237 before or together with #606 through their protected gates, then continue replacing remaining reconstruction-test constants with provenance-bearing fast-mlsirm estimates over synthetic fixtures; tests unrelated to fusion must bypass weighting entirely, as #484 does. Retain true-parameter RMSE recovery as the acceptance bar | +| Scientific measurement | Durable accepted TEPP receipts are protected; TEPP #237 head `9b96c1ae` publishes the strict independent criterion-anchor contract, while LineageWeave #614 accepts only an exact accepted snapshot/cutoff/run/pair-count match and fails closed on duplicate sets. Neither current head is protected delivery yet. #387 removes inferred/default persistence weights, but several older reconstruction tests still pass hand-authored numeric dictionaries that are not estimator evidence | Land TEPP #237 before #614 through their protected gates, then replace remaining reconstruction-test constants with provenance-bearing fast-mlsirm estimates over synthetic fixtures. Retain true-parameter RMSE recovery as the acceptance bar | | Asynchronous authorization | Protected `main` rebuilds Global Ask worker scope after the bearer token leaves the request; #468 now persists exact Keyverse organization/process-unit scope in 3NF child tables and intersects it with current affiliations | Land #468 through the protected gate; prove a second affiliation and a revoked process unit cannot widen delayed-job evidence | | Planned-facility intent | Planned-facility relationship intent rides on open #490 (`d0cad030`), whose earlier stack-only merges were not protected delivery | Settle #490 exact-head checks plus independent approval, then land through protected `main` before a release claim | -| Accessibility and responsive UX | #602 and protected-main #605 deliver modal semantics, selected-post refocus, collapsed/hidden/inert/CSS-invisible focus exclusion, and readable evidence separators | Complete screen-reader and authenticated Playwright acceptance on one exact release head | +| Accessibility and responsive UX | #602 delivered base post-detail modal semantics; #605 adds selected-post refocus, collapsed/hidden/inert/CSS-invisible focus exclusion across both modal types, readable evidence separators, focused tests, and desktop/mobile Storybook screenshots | Land #605 through the protected gate, then complete screen-reader and authenticated Playwright acceptance on the exact release head | | Design tokens and repeated objects | Token extraction started; sanitized Figma Event Lineage desktop/mobile frames exist, while other repeated product surfaces remain incomplete | Tokens in CSS + Storybook stories for board, popup, DAG, Ask, calendar, forms, charts; same-viewport Figma/runtime visual comparison before release | | External integrations | Search, Zotero, calendar, Keyverse, orchestrator, RankWeave, ThreadWeave, TEPP, disksage, wardnet | Provider conformance, failure/reconciliation behavior, and provenance-bearing integration evidence | | MSA / modular reuse | LineageWeave must run standalone and as a consumer of org packages | Do not reimplement RankWeave/TEPP/orchestrator/ThreadWeave/Keyverse; fix upstream and PR there | @@ -393,7 +464,7 @@ review latency are never blockers — keep working while they settle. (#518–#564) merges in ascending order. 3. Keep the shared ADR 0109 repair verified on #521–#560 heads (done this loop; frontend lint/test/build passed locally before each push). -4. After PRs drain below a handful, resume buyer-visible gaps from §5 in +4. After PRs drain below a handful, resume operator-visible gaps from §5 in leverage order: Event Lineage evidence (#387/#274), Naruon calendar (#355/#336), SKOS aliases (#480/#482), ontology explorer (#349/#341). 5. Rename remaining `[Buyer Gap]` issue titles to neutral product-object @@ -416,7 +487,7 @@ review latency are never blockers — keep working while they settle. - Orchestrator / paper-grounded models: ADR 0015, ADR 0076 (Fugu, TRINITY, Conductor) - Ontology / PROV-O / SKOS: ADR 0004, ADR 0011, issue #372 - Analysis runs / TEPP: ADR 0013–0023, issue #79 / #277 -- Calendar / Naruon: issues #336 / #338, PR #355, Buyer consume v2.17.0 +- Calendar / Naruon: issues #336 / #338, PR #355, operator consumption v2.17.0 - Ask Agent: issues #269–#272, #358–#363 Citations in doctoring and ADRs use APA 7th. Do not invent a heuristic where diff --git a/docs/storybook-inventory.md b/docs/storybook-inventory.md index b77187547..85db06359 100644 --- a/docs/storybook-inventory.md +++ b/docs/storybook-inventory.md @@ -1,11 +1,11 @@ # Storybook inventory Open the catalog after `cd frontend && pnpm run storybook`. Each story is a -buyer-facing control you can click before changing product CSS. +operator-facing control you can click before changing product CSS. -| Story | Buyer next action | Token / module | +| Story | Operator next action | Token / module | |---|---|---| -| `Workspace/OperationsDashboard` | Compare Event and post counts, inspect external-information coverage, then open the cited source behind a claim, handover, or repeat-issue fact. Evidence-ready and narrow-viewport scenes are required. | `--color-dashboard-*`, `OperationsDashboard` | +| `Workspace/OperationsDashboard` | Compare Event and post counts, inspect external-information coverage, then open the cited source behind a claim, handover, or repeat-issue fact. `EvidenceReady`, `NarrowViewport`, `AnalysisPendingAndMissingEvidence`, `AnalysisFailed`, and `LoadError` cover populated, mobile, unavailable-evidence, analysis-pending, retryable failure, and transport-error states. | `--color-dashboard-*`, `OperationsDashboard` | | `Post/SimilarVocPanel` | Compare ontology/semantic similar VOC and prior action evidence, then open the source; unavailable states show no fabricated TEPP theta or weight. | `SimilarVocPanel.css`, `SimilarVocPanel` | | `Evidence/CitationChip` | Click a cited title to open that source post. | `--color-chip-border`, `--radius-chip`, `CitationChip` | | `Evidence/OrganizationAliasChip` | Click a cataloged org; the parenthetical is the unique corroborated SKOS companion. | `--color-chip-border`, `--radius-chip`, `OrganizationAliasChip` | diff --git a/docs/temporal-topic-context-influence-research.md b/docs/temporal-topic-context-influence-research.md new file mode 100644 index 000000000..05a80c68a --- /dev/null +++ b/docs/temporal-topic-context-influence-research.md @@ -0,0 +1,72 @@ +# Temporal topic and context-influence evidence brief + +## Research question + +How can LineageWeave present time-aware, lineage-aware topics and identify the +posts that influence topic estimates at business-unit, PU, team, and person +levels without keyword rules or arbitrary weights? + +## Search and source selection + +- Concepts: dynamic topic models, temporal document networks, multilevel IRT, + multiple-membership multiple-classification, latent-space IRT, and + multilevel case-deletion diagnostics. +- Priority: peer-reviewed primary papers, official proceedings, accepted + author manuscripts, TEPP's approved PRD/ADR, and fast-mlsirm's normative + ADR/research register. +- Excluded as authorities: review-only pages, vendor summaries, lexical topic + matching, engagement ranking, and methods that do not preserve posterior, + time, relation, or membership identity. + +## Findings + +### Temporal topic identity and document relations + +Blei and Lafferty (2006) establish state-space topic evolution rather than +independent time-bin models. Zhang and Lauw (2022) jointly model temporal +document topics and network structure; this directly supports consuming +explicit Event Lineage as relational evidence rather than matching topic +labels after fitting. TEPP PRD v0.4 and ADR 0012 combine those concerns in the +TRSL-TM producer boundary, including global topic identity, posterior +coordinates, multiple clocks, relations, and cross-classified membership. + +The evidence does not establish that every relation is causal or that a +reactivated topic is newly born. Those states and lineage events must arrive +from a versioned TEPP result. + +### Multilevel and multiple-membership measurement + +Fox and Glas (2001) show why latent rather than observed scores should be +modeled jointly with cluster effects and measurement error. Browne, Goldstein, +and Rasbash (2001) define crossed and weighted multiple-membership structures. +Jin et al. (2022) demonstrate a multilevel network item-response model that +can expose differences missed by conventional multilevel models. These papers +support distinct business-unit, PU, team, and person dimensions with explicit +time-valid membership; they do not support inferring equal weights when the +source has none. + +### “Important post” estimand + +Shi and Chen (2008) define case-deletion diagnostics at multiple levels for +fixed and random parameters. ADR 0210 therefore gives importance the bounded +name **model influence** and defines it as observed-information-scaled change +in the topic-by-context estimate after deleting the complete post +observation. This answers sensitivity of the fitted model, not business value +or causality. Molenaar and Jeon (2026) support recovery-tested regularized JML +for latent-space IRT, but do not by themselves validate this product-specific +construct; the fast-mlsirm producer must implement and recover the exact +versioned influence estimand before LineageWeave activates it. + +## Architecture consequence + +LineageWeave is a strict consumer. TEPP owns temporal/relational topic +posterior arithmetic. fast-mlsirm owns multilevel multiple-membership fitting, +observed information, deletion refits, posterior-draw combination, and CPU/GPU +parity in Rust. LineageWeave persists exact accepted artifacts, applies ABAC, +and renders tied exact values; it adds no threshold, fallback score, or local +numerical formula. + +## Primary sources (APA 7th) + +See [ADR 0210](adr/0210-temporal-topic-context-influence-dashboard.md#references-apa-7th) +for the full APA 7 bibliography and exact architecture mapping. diff --git a/frontend/src/App.css b/frontend/src/App.css index d2ff10630..a0fd54c09 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -1159,6 +1159,22 @@ color: var(--color-text-heading); } +.dashboard-period-form { + align-items: end; + display: flex; + flex-wrap: wrap; + gap: var(--space-control-gap); + padding: var(--space-panel-block); +} + +.dashboard-period-form label { + display: grid; + gap: var(--space-control-gap); + font-weight: 700; +} + +.dashboard-period-form input { min-height: 44px; } + .operations-dashboard-heading { display: flex; align-items: end; @@ -1175,7 +1191,7 @@ .dashboard-metrics { display: grid; - grid-template-columns: repeat(4, minmax(0, 1fr)); + grid-template-columns: repeat(5, minmax(0, 1fr)); margin: 1.5rem 0; border: 1px solid var(--color-border); } diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 1fb7233fd..e311d5f02 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -119,6 +119,7 @@ describe("App, authenticated", () => { organizationAliases?: boolean; askLineageGraph?: boolean; askImageCitation?: boolean; + askDelivery?: boolean; lineageIsolationReason?: "comparison_candidates_available" | "no_comparison_group"; }): ReturnType & { releaseMe: () => void; releasePostOne: () => void } { const statusLabel: Record = { @@ -1160,6 +1161,9 @@ describe("App, authenticated", () => { ); } const postOneUrl = new URL(url, "https://backend.test"); + if (postOneUrl.pathname === "/api/posts/post-1/similar-voc") { + return Promise.resolve(jsonResponse({ items: [] })); + } if (postOneUrl.pathname === "/api/posts/post-1") { const asOf = postOneUrl.searchParams.get("as_of"); return postOneReady.then(() => @@ -1788,6 +1792,21 @@ describe("App, authenticated", () => { truncated: false, } : { nodes: [], edges: [], truncated: false }, + delivery: options?.askDelivery ? { + contract_version: "1.0", + report: { + media_type: "text/markdown", + body: "Answer", + source_documents: [{ + post_id: "post-2", title: "Linked post", api_path: "/api/posts/post-2", + resource_uri: "lineageweave://posts/post-2", evidence_facts: [], + }], + }, + alert: { + trigger_code: "cited_evidence_changed", delivery_status_code: "not_subscribed", + eligible: true, watched_resource_uris: ["lineageweave://posts/post-2"], + }, + } : undefined, }, }), ); @@ -1943,6 +1962,20 @@ describe("App, authenticated", () => { expect(screen.queryByText(/ontology_iri|contextual_orchestrator/i)).not.toBeInTheDocument(); }); + it("localizes Ask delivery copy instead of rendering Korean literals in English", async () => { + stubBackend({ askDelivery: true }); + render(); + expect(await screen.findByRole("button", { name: "View post: Public post" })).toBeInTheDocument(); + await userEvent.click(screen.getByRole("button", { name: "Ask Agent" })); + await userEvent.type(screen.getByRole("textbox", { name: "Ask a question" }), "Which project?"); + await userEvent.click(screen.getByRole("button", { name: "Ask" })); + + expect(await screen.findByRole("complementary", { name: "Report · alert · MCP" })).toHaveTextContent( + "1 evidence documents are linked to this report.", + ); + expect(screen.queryByText(/근거 문서/)).not.toBeInTheDocument(); + }); + it("renders every cited lineage thread as its own git-branch-style graph", async () => { stubBackend({ askLineageGraph: true }); render(); @@ -3059,6 +3092,45 @@ describe("App, authenticated", () => { expect(screen.queryByText("Pricing renegotiation: revised quote sent")).not.toBeInTheDocument(); }); + it("drops a prior post's in-flight similar-VOC page after navigation", async () => { + const backend = stubBackend(); + const original = backend.getMockImplementation() as ( + input: RequestInfo | URL, + init?: RequestInit, + ) => Promise; + let releasePage!: (response: Response) => void; + const deferredPage = new Promise((resolve) => { releasePage = resolve; }); + backend.mockImplementation((...args) => { + const requestUrl = new URL(String(args[0]), "https://backend.test"); + if (requestUrl.pathname === "/api/posts/post-1/similar-voc") { + if (requestUrl.searchParams.get("offset") === "50") return deferredPage; + return Promise.resolve(jsonResponse({ + items: [{ + post_id: "prior-1", post_title: "Prior evidence", issue_summary: "Prior issue", + focal_evidence_text: "Current evidence", candidate_evidence_text: "Prior evidence", + customer_cohort_text: null, action_history: [], occurred_at: "2025-12-01T00:00:00Z", + }], + next_offset: 50, + })); + } + return original(args[0] as RequestInfo | URL, args[1] as RequestInit | undefined); + }); + render(); + await userEvent.click(await screen.findByRole("button", { name: /open report post: public post/i })); + await userEvent.click(await screen.findByRole("button", { name: "이전 VOC 더 보기" })); + await userEvent.click((await screen.findAllByLabelText("Open post: Linked post"))[0]); + await screen.findByText("The evidence panel should show exactly this text."); + releasePage(jsonResponse({ + items: [{ + post_id: "stale-prior", post_title: "Stale prior VOC", issue_summary: "Stale issue", + focal_evidence_text: "Stale current", candidate_evidence_text: "Stale prior", + customer_cohort_text: null, action_history: [], occurred_at: "2025-11-01T00:00:00Z", + }], + next_offset: null, + })); + await waitFor(() => expect(screen.queryByText("Stale prior VOC")).not.toBeInTheDocument()); + }, 15_000); + it("opens an accepted ranking hit without inventing a fused score", async () => { stubBackend({ rankings: { diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 1c3b53ed2..921c04672 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -35,6 +35,7 @@ import { fetchPostSummary, fetchPostTickets, fetchPostVocEvidence, + fetchSimilarVoc, fetchPeriodComparison, fetchPeriodReportIndex, fetchPeriodReports, @@ -82,6 +83,7 @@ import { type RelatedNode, type RelatedNodeType, type VocEvidence, + type SimilarVocItem, fetchTenantConfig, } from "./api"; import { CitationChip } from "./components/CitationChip"; @@ -92,6 +94,7 @@ import { LineageEntityPicker } from "./components/LineageEntityPicker"; import { OntologyExplorer } from "./components/OntologyExplorer"; import { AskEvidenceLayerPopup } from "./components/AskEvidenceLayerPopup"; import { PopupCloseButton } from "./components/PopupCloseButton"; +import { SimilarVocPanel } from "./components/SimilarVocPanel"; import { chatEvidenceKindLabel } from "./evidenceKindLabels"; import { WorkspaceNav, type WorkspaceDestination } from "./components/WorkspaceNav"; import { OperationsDashboard } from "./components/OperationsDashboard"; @@ -1809,6 +1812,13 @@ function PostDetailPopup({ const [lineage, setLineage] = useState(null); const [affiliateTrees, setAffiliateTrees] = useState(null); const [vocEvidence, setVocEvidence] = useState(null); + const [similarVoc, setSimilarVoc] = useState(null); + const [similarVocError, setSimilarVocError] = useState(null); + const [similarVocNextOffset, setSimilarVocNextOffset] = useState(null); + const [similarVocLoadingMore, setSimilarVocLoadingMore] = useState(false); + const similarVocLoadingMoreRef = useRef(false); + const similarVocScopeRef = useRef({ postId }); + if (similarVocScopeRef.current.postId !== postId) similarVocScopeRef.current = { postId }; const [evaluation, setEvaluation] = useState(null); const [focusPerson, setFocusPerson] = useState<{ personId: string; personName: string } | null>(null); const [focusEntity, setFocusEntity] = useState<{ entityId: string; entityName: string } | null>(null); @@ -1907,6 +1917,11 @@ function PostDetailPopup({ setLineage(null); setAffiliateTrees(null); setVocEvidence(null); + setSimilarVoc(null); + setSimilarVocError(null); + setSimilarVocNextOffset(null); + setSimilarVocLoadingMore(false); + similarVocLoadingMoreRef.current = false; setEvaluation(null); setFocusPerson(null); setFocusEntity(null); @@ -1963,6 +1978,17 @@ function PostDetailPopup({ .then((r) => setAffiliateTrees(r.trees)) .catch(() => setAffiliateTrees([])); fetchPostVocEvidence(accessToken, postId).then(setVocEvidence).catch(() => setVocEvidence(null)); + fetchSimilarVoc(accessToken, postId) + .then((result) => { + if (disposed) return; + setSimilarVoc(result.items); + setSimilarVocNextOffset(result.next_offset); + }) + .catch(() => { + if (disposed) return; + setSimilarVoc([]); + setSimilarVocError("유사 VOC 판정을 사용할 수 없습니다. 잠시 후 다시 확인하세요."); + }); return () => { disposed = true; if (contentPollTimer !== undefined) window.clearTimeout(contentPollTimer); @@ -2468,6 +2494,36 @@ function PostDetailPopup({ }} /> + onSelectPost?.(candidatePostId)} + loadingMore={similarVocLoadingMore} + onLoadMore={similarVocNextOffset === null ? null : () => { + if (similarVocLoadingMoreRef.current) return; + const requestScope = similarVocScopeRef.current; + similarVocLoadingMoreRef.current = true; + setSimilarVocLoadingMore(true); + setSimilarVocError(null); + fetchSimilarVoc(accessToken, postId, similarVocNextOffset) + .then((result) => { + if (similarVocScopeRef.current !== requestScope) return; + setSimilarVoc((current) => [...(current ?? []), ...result.items]); + setSimilarVocNextOffset(result.next_offset); + }) + .catch(() => { + if (similarVocScopeRef.current === requestScope) { + setSimilarVocError("이전 VOC를 더 불러오지 못했습니다. 다시 시도하세요."); + } + }) + .finally(() => { + if (similarVocScopeRef.current !== requestScope) return; + similarVocLoadingMoreRef.current = false; + setSimilarVocLoadingMore(false); + }); + }} + /> +
@@ -4800,13 +4856,15 @@ function AskAgentPanel({ {answer.answer_text ?

{answer.answer_text}

: null} {answer.next_action ?

{t(answer.next_action)}

: null} {answer.delivery ? ( -