diff --git a/.env.example b/.env.example index 06cb82d91..7282c5a2e 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,8 @@ # Copy to .env to override. Every value below already has a working # default baked into docker-compose.yml (see ${VAR:-default} references) -- # `docker compose up` succeeds from a clean checkout with no .env file at -# all. These defaults are throwaway local-dev-only credentials, not +# all for the default profile. The optional MCP profile requires measured +# quota inputs below. Other defaults are throwaway local-dev-only credentials, not # production secrets; see docs/adr/0001-demo-identity-and-data-boundary.md. # Host ports deliberately avoid each service's own default (5432, 6379, @@ -27,6 +28,12 @@ OIDC_AUDIENCE=lineageweave-api BACKEND_PORT=18420 +# Optional authenticated MCP profile. The quota pair is mandatory when the +# profile is enabled and must come from that deployment's k6 capacity evidence. +MCP_ALLOWED_ORIGINS= +MCP_RATE_LIMIT_REQUESTS= +MCP_RATE_LIMIT_WINDOW_SECONDS= + # Optional. Empty = every LLM/vision channel is unavailable (Null client, # dropped and renormalized -- never a placeholder score). Point these at a # running contextual-orchestrator to turn the channels on. diff --git a/CHANGELOG.d/2.18.1-current-contract-mcp-global-ask.md b/CHANGELOG.d/2.18.1-current-contract-mcp-global-ask.md new file mode 100644 index 000000000..54200561a --- /dev/null +++ b/CHANGELOG.d/2.18.1-current-contract-mcp-global-ask.md @@ -0,0 +1,6 @@ +### Added + +- Added an authenticated Streamable HTTP MCP adapter that queues and reads the + same durable Global Ask jobs as REST, with exact-resource OAuth, bounded + pre-auth request admission, owner/affiliation scope preservation, and a + fail-closed distributed quota whose capacity inputs are deployment evidence. diff --git a/CHANGELOG.d/2.20.0-project-ontology-neighborhood.md b/CHANGELOG.d/2.20.0-project-ontology-neighborhood.md new file mode 100644 index 000000000..5b86074dc --- /dev/null +++ b/CHANGELOG.d/2.20.0-project-ontology-neighborhood.md @@ -0,0 +1,22 @@ +# 2.20.0 — Evidence-backed Project ontology neighborhoods + +- Added canonical Project nodes and `mentionsProject` relations to the bounded + ontology API, JSON-LD/CSV projections, and accessible explorer. +- Bound Project labels and evidence to the request cutoff and sealed cursor + snapshot; hidden or unavailable evidence remains absent. +- Scoped unresolved Project candidate identifiers to their evidence Post so + equal names cannot silently merge across records. +- Unified RDF and JSON-LD node IRI encoding for multilingual Project keys. +- Corrected JSON-LD edge semantics to emit both the direct assertion and its + evidence-bearing RDF reification. +- Aligned JSON-LD system and validity time with the API contract and exposed + exact validity/evidence values in the accessible explorer table. +- Made synthetic channel-weight seeding record one explicit cutoff/estimate + instant, avoiding transaction-start clock violations during long test runs. +- Included process-unit scope in Post-focus authorization rows so private + ontology neighborhoods fail closed instead of crashing during ABAC checks. +- Reconciled the Project ontology work with the Global Ask provenance, public + verification, semantic rewrite, knowledge-cutoff, and MCP stack. +- Added the deterministic joined-row RDF projector used by SHACL acceptance, + including the direct relation, complete reified statement, evidence, + confidence, creation time, and PROV derivation. diff --git a/Makefile b/Makefile index 62e1b3198..d68780720 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: up down logs smoke seed ps load-http +.PHONY: up down logs smoke seed ps load-http load-mcp # Keep provider credentials outside the repository. Compose interpolation must # read the same home env file as the orchestrator container's env_file. @@ -35,4 +35,12 @@ seed: load-http: @test -n "$${LINEAGEWEAVE_VUS:-}" || { echo "LINEAGEWEAVE_VUS is required" >&2; exit 1; } @test -n "$${LINEAGEWEAVE_DURATION:-}" || { echo "LINEAGEWEAVE_DURATION is required" >&2; exit 1; } - k6 run --vus "$${LINEAGEWEAVE_VUS}" --duration "$${LINEAGEWEAVE_DURATION}" scripts/k6_http_e2e.js + @test -n "$${LINEAGEWEAVE_REQUEST_TIMEOUT:-}" || { echo "LINEAGEWEAVE_REQUEST_TIMEOUT is required" >&2; exit 1; } + k6 run -e REQUEST_TIMEOUT="$${LINEAGEWEAVE_REQUEST_TIMEOUT}" --vus "$${LINEAGEWEAVE_VUS}" --duration "$${LINEAGEWEAVE_DURATION}" scripts/k6_http_e2e.js + +# Authenticated MCP measurement with operator-supplied observation bounds. +load-mcp: + @test -n "$${LINEAGEWEAVE_VUS:-}" || { echo "LINEAGEWEAVE_VUS is required" >&2; exit 1; } + @test -n "$${LINEAGEWEAVE_DURATION:-}" || { echo "LINEAGEWEAVE_DURATION is required" >&2; exit 1; } + @test -n "$${LINEAGEWEAVE_REQUEST_TIMEOUT:-}" || { echo "LINEAGEWEAVE_REQUEST_TIMEOUT is required" >&2; exit 1; } + k6 run -e REQUEST_TIMEOUT="$${LINEAGEWEAVE_REQUEST_TIMEOUT}" --vus "$${LINEAGEWEAVE_VUS}" --duration "$${LINEAGEWEAVE_DURATION}" scripts/k6_mcp_e2e.js diff --git a/README.md b/README.md index 22633f79d..5f1480b40 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ carrying `corp_code` / `pu_code` as token claims -- these are throwaway local-dev credentials in a locally-run realm, never the org's real Keyverse tenant (see ADR 0001 for why). -Host ports (15432, 16379, 18080, 18420) deliberately avoid each service's +Host ports (15432, 16379, 18080, 18001, 18420) deliberately avoid each service's own default -- a dev machine commonly already runs its own Postgres/Redis/local server on those. Override via `.env` (copy `.env.example`) or inline if even those collide, e.g. @@ -156,6 +156,18 @@ make seed # scripts/seed_demo_data.py: inserts synthetic corp/account/post curl http://localhost:18420/healthz ``` +The optional authenticated MCP resource server submits and reads the same +durable Global Ask jobs as REST. Enable it only with quota values established +by the deployment's k6 capacity evidence; the service intentionally has no +guessed request/window defaults: + +```bash +MCP_RATE_LIMIT_REQUESTS= \ +MCP_RATE_LIMIT_WINDOW_SECONDS= \ +docker compose --profile mcp up mcp +# Streamable HTTP resource: http://localhost:18001/mcp +``` + `GET /api/posts`, `GET /api/posts/{post_id}`, `GET /api/posts/{post_id}/keymen`, `GET /api/keymen/{person_id}/related`, `GET /api/posts/{post_id}/affiliate-tree`, diff --git a/backend/app/auth.py b/backend/app/auth.py index e34c4c668..41085d7e3 100644 --- a/backend/app/auth.py +++ b/backend/app/auth.py @@ -125,8 +125,10 @@ def has_permission(self, permission_code: str) -> bool: return permission_code in self.permission_codes -def _decode_access_token(token: str, settings: Settings) -> dict: - """Validate signature, issuer, resource audience, time claims, and subject.""" +def decode_access_token( + token: str, settings: Settings, *, audience: str | None = None +) -> dict: + """Validate a token for the REST or an explicit resource audience.""" required_claims = ["exp", "sub"] if settings.keyverse_claim_binding_required: required_claims.insert(1, "iat") @@ -136,7 +138,7 @@ def _decode_access_token(token: str, settings: Settings) -> dict: key=_signing_key(settings, token), algorithms=["RS256"], issuer=settings.oidc_issuer, - audience=settings.oidc_audience, + audience=settings.oidc_audience if audience is None else audience, leeway=settings.oidc_clock_skew_seconds, options={"require": required_claims}, ) @@ -150,6 +152,11 @@ def _decode_access_token(token: str, settings: Settings) -> dict: return claims +def _decode_access_token(token: str, settings: Settings) -> dict: + """Validate a REST bearer token against the configured API audience.""" + return decode_access_token(token, settings) + + def _keyverse_account_claims(claims: dict) -> tuple[str, str, list[str]]: """Return Keyverse's atomic account scope, rejecting ambiguous wire shapes.""" organization = claims.get("org") @@ -177,13 +184,10 @@ def _keyverse_account_claims(claims: dict) -> tuple[str, str, list[str]]: return organization, workspace, [role.strip() for role in roles] -async def get_current_account( - credentials: HTTPAuthorizationCredentials = Depends(_bearer_scheme), - pool: asyncpg.Pool = Depends(get_pool), +async def resolve_current_account( + pool: asyncpg.Pool, claims: dict, settings: Settings ) -> CurrentAccount: - """Resolve the bearer token to a provisioned ``user_account`` row.""" - settings = load_settings() - claims = _decode_access_token(credentials.credentials, settings) + """Resolve verified claims to database-owned scope and permissions.""" subject = claims["sub"] keyverse_scope = ( _keyverse_account_claims(claims) @@ -269,3 +273,13 @@ async def get_current_account( process_unit_ids=frozenset(str(row["process_unit_id"]) for row in process_rows), permission_codes=frozenset(row["permission_code"] for row in permission_rows), ) + + +async def get_current_account( + credentials: HTTPAuthorizationCredentials = Depends(_bearer_scheme), + pool: asyncpg.Pool = Depends(get_pool), +) -> CurrentAccount: + """Resolve the bearer token to a provisioned ``user_account`` row.""" + settings = load_settings() + claims = _decode_access_token(credentials.credentials, settings) + return await resolve_current_account(pool, claims, settings) diff --git a/backend/app/config.py b/backend/app/config.py index 827441648..a49bd5390 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -6,7 +6,7 @@ import math import os -from dataclasses import dataclass +from dataclasses import dataclass, field # Hard ceiling on one Global Ask job's answer computation, shared with the # worker in global_ask_queue.py so config validation and execution can never @@ -65,6 +65,16 @@ class Settings: naruon_calendar_service_token: str rankweave_disabled: bool ontology_source_cursor_secret: str + mcp_resource_url: str = "http://localhost:18001/mcp" + mcp_audience: str = "http://localhost:18001/mcp" + mcp_required_scopes: list[str] = field(default_factory=list) + mcp_allowed_hosts: list[str] = field( + default_factory=lambda: ["localhost:*", "127.0.0.1:*", "mcp:8001"] + ) + mcp_allowed_origins: list[str] = field(default_factory=list) + mcp_max_request_bytes: int = 65_536 + mcp_rate_limit_requests: int | None = None + mcp_rate_limit_window_seconds: int | None = None @property def keycloak_jwks_uri(self) -> str: @@ -83,7 +93,9 @@ def _validated_answer_timeout(raw: str) -> float: try: value = float(raw) except ValueError as exc: - raise ValueError("ORCHESTRATOR_ANSWER_TIMEOUT_SECONDS must be a number") from exc + raise ValueError( + "ORCHESTRATOR_ANSWER_TIMEOUT_SECONDS must be a number" + ) from exc if not math.isfinite(value) or not 0 < value < GLOBAL_ASK_JOB_DEADLINE_SECONDS: raise ValueError( "ORCHESTRATOR_ANSWER_TIMEOUT_SECONDS must be a finite number greater" @@ -92,6 +104,20 @@ def _validated_answer_timeout(raw: str) -> float: return value +def _optional_positive_int(name: str) -> int | None: + """Parse an optional positive deployment integer without inventing a default.""" + raw = os.environ.get(name, "").strip() + if not raw: + return None + try: + value = int(raw, 10) + except ValueError as exc: + raise ValueError(f"{name} must be a base-10 integer") from exc + if value <= 0: + raise ValueError(f"{name} must be positive") + return value + + def load_settings() -> Settings: """Read Settings from the environment, with local-dev defaults only.""" keycloak_base_url = os.environ.get("KEYCLOAK_BASE_URL", "http://localhost:18080") @@ -103,7 +129,9 @@ def load_settings() -> Settings: keyverse_issuer = os.environ.get("KEYVERSE_ISSUER", "").strip() generic_oidc_issuer = os.environ.get("OIDC_ISSUER", "").strip() external_oidc = bool(keyverse_issuer or generic_oidc_issuer) - oidc_issuer = (keyverse_issuer or generic_oidc_issuer or keycloak_issuer).rstrip("/") + oidc_issuer = (keyverse_issuer or generic_oidc_issuer or keycloak_issuer).rstrip( + "/" + ) oidc_client_id = ( os.environ.get("KEYVERSE_CLIENT_ID", "").strip() or os.environ.get("OIDC_CLIENT_ID", "").strip() @@ -119,9 +147,13 @@ def load_settings() -> Settings: "do not infer a resource-server audience from the browser client id" ) oidc_audience = configured_audience or "lineageweave-api" - oidc_discovery_uri = os.environ.get("KEYVERSE_DISCOVERY_URI", "").strip() or os.environ.get( - "OIDC_DISCOVERY_URI", "" + mcp_resource_url = os.environ.get( + "MCP_RESOURCE_URL", "http://localhost:18001/mcp" ).strip() + oidc_discovery_uri = ( + os.environ.get("KEYVERSE_DISCOVERY_URI", "").strip() + or os.environ.get("OIDC_DISCOVERY_URI", "").strip() + ) if not oidc_discovery_uri: discovery_base = oidc_issuer if external_oidc else keycloak_base_url oidc_discovery_uri = ( @@ -161,7 +193,9 @@ def load_settings() -> Settings: keyverse_claim_binding_required=bool(keyverse_issuer), frontend_origins=[ origin.strip() - for origin in os.environ.get("FRONTEND_ORIGINS", "http://localhost:5173").split(",") + for origin in os.environ.get( + "FRONTEND_ORIGINS", "http://localhost:5173" + ).split(",") if origin.strip() ], orchestrator_base_url=os.environ.get("ORCHESTRATOR_BASE_URL", ""), @@ -178,9 +212,33 @@ def load_settings() -> Settings: naruon_calendar_service_token=os.environ.get( "NARUON_CALENDAR_SERVICE_TOKEN", "" ).strip(), - rankweave_disabled=os.environ.get("RANKWEAVE_DISABLED", "") - .strip() - .lower() + rankweave_disabled=os.environ.get("RANKWEAVE_DISABLED", "").strip().lower() in {"1", "true", "yes", "on"}, - ontology_source_cursor_secret=os.environ.get("ONTOLOGY_SOURCE_CURSOR_SECRET", "").strip(), + ontology_source_cursor_secret=os.environ.get( + "ONTOLOGY_SOURCE_CURSOR_SECRET", "" + ).strip(), + mcp_resource_url=mcp_resource_url, + mcp_audience=os.environ.get("MCP_AUDIENCE", mcp_resource_url).strip(), + mcp_required_scopes=[ + item.strip() + for item in os.environ.get("MCP_REQUIRED_SCOPES", "").split(",") + if item.strip() + ], + mcp_allowed_hosts=[ + item.strip() + for item in os.environ.get( + "MCP_ALLOWED_HOSTS", "localhost:*,127.0.0.1:*,mcp:8001" + ).split(",") + if item.strip() + ], + mcp_allowed_origins=[ + item.strip() + for item in os.environ.get("MCP_ALLOWED_ORIGINS", "").split(",") + if item.strip() + ], + mcp_max_request_bytes=_optional_positive_int("MCP_MAX_REQUEST_BYTES") or 65_536, + mcp_rate_limit_requests=_optional_positive_int("MCP_RATE_LIMIT_REQUESTS"), + mcp_rate_limit_window_seconds=_optional_positive_int( + "MCP_RATE_LIMIT_WINDOW_SECONDS" + ), ) diff --git a/backend/app/global_ask_queue.py b/backend/app/global_ask_queue.py index c7e570d81..9bffd8502 100644 --- a/backend/app/global_ask_queue.py +++ b/backend/app/global_ask_queue.py @@ -21,7 +21,7 @@ import logging import time from collections.abc import Callable -from datetime import date +from datetime import date, datetime from typing import Any import asyncpg @@ -29,14 +29,29 @@ from fastapi import HTTPException, status from lineageweave.ask_delivery import build_ask_delivery +from lineageweave.claim_verification import ( + CLAIM_NOT_ENOUGH_INFORMATION, + VERIFICATION_COMPLETED, + VERIFICATION_NO_PUBLIC_CLAIMS, + VERIFICATION_SKIPPED, + VERIFICATION_UNAVAILABLE, + ClaimVerificationClient, + ClaimVerificationResult, + NullClaimVerificationClient, + public_claim_candidates, +) from lineageweave.embedding_client import EmbeddingClient, NullEmbeddingClient from lineageweave.http_client import HttpClientError from lineageweave.observability import record_server_failure from lineageweave.post_chat import ( + ChatSourceDocument, PostChatClient, + ask_grounding_status, cited_post_evidence, cited_post_summaries, + historical_body_limitations, ) +from lineageweave.semantic_query import NullSemanticQueryClient, SemanticQueryClient from lineageweave.temporal_expressions import resolve_korean_relative_time from .config import GLOBAL_ASK_JOB_DEADLINE_SECONDS @@ -46,6 +61,7 @@ _seoul_today, cited_post_images, gather_global_chat_sources, + prepare_global_question_embedding, ) GLOBAL_ASK_STREAM_KEY = "global_ask_request_stream" @@ -92,6 +108,8 @@ async def enqueue_global_ask_job( *, requesting_account_id: str, question_text: str, + verify_external_requested: bool, + knowledge_cutoff: datetime | None, corporate_entity_ids: frozenset[str], process_unit_ids: frozenset[str], ) -> str: @@ -104,11 +122,15 @@ async def enqueue_global_ask_job( async with conn.transaction(): job_id = await conn.fetchval( """ - insert into global_ask_job (requesting_account_id, question_text) - values ($1, $2) returning global_ask_job_id + insert into global_ask_job + (requesting_account_id, question_text, verify_external_requested, + knowledge_cutoff) + values ($1, $2, $3, $4) returning global_ask_job_id """, requesting_account_id, question_text, + verify_external_requested, + knowledge_cutoff, ) await conn.executemany( """ @@ -141,6 +163,56 @@ async def enqueue_global_ask_job( return str(job_id) +def _verification_next_action(status_code: str) -> str: + """Name the next evidence action without promoting web results to authority.""" + + return { + VERIFICATION_SKIPPED: "Enable public verification to check eligible public claims.", + VERIFICATION_UNAVAILABLE: "Configure public search and contextual-orchestrator, then retry.", + VERIFICATION_NO_PUBLIC_CLAIMS: "Inspect the internal cited posts; no public claim was eligible.", + VERIFICATION_COMPLETED: "Inspect public evidence separately before any governed graph review.", + CLAIM_NOT_ENOUGH_INFORMATION: "Collect stronger authoritative evidence before accepting the claim.", + }.get(status_code, "Inspect the authorized cited posts and their evidence.") + + +async def _verify_public_claims( + question: str, + sources: list[ChatSourceDocument], + cited_post_ids: list[str], + *, + verify_external: bool, + client: ClaimVerificationClient, +) -> tuple[str, tuple[ClaimVerificationResult, ...]]: + """Verify only cited claims explicitly marked safe for public egress.""" + + if not verify_external: + return VERIFICATION_SKIPPED, () + cited_ids = frozenset(cited_post_ids) + claims = tuple( + claim + for claim in public_claim_candidates(sources, question) + if set(claim.source_post_ids).issubset(cited_ids) + ) + if not claims: + return VERIFICATION_NO_PUBLIC_CLAIMS, () + if not client.available: + return VERIFICATION_UNAVAILABLE, () + try: + results = tuple( + await asyncio.gather( + *(asyncio.to_thread(client.verify, claim) for claim in claims) + ) + ) + except Exception: + _logger.exception("public claim verification is unavailable") + return VERIFICATION_UNAVAILABLE, () + return VERIFICATION_COMPLETED, tuple( + result + for result in results + if set(result.source_post_ids).issubset(cited_ids) + ) + + async def load_job_visibility( conn: asyncpg.Connection, job_id: str, account_id: str ) -> tuple[set[str], set[str], bool, bool]: @@ -215,6 +287,10 @@ async def compute_global_ask_answer( process_scope_limited: bool, chat_client: PostChatClient, embedding_client: EmbeddingClient | None = None, + semantic_query_client: SemanticQueryClient | None = None, + verify_external: bool = False, + claim_verification_client: ClaimVerificationClient | None = None, + knowledge_cutoff: datetime | None = None, ) -> dict[str, Any]: """Assemble one complete Ask answer payload from authorized evidence. @@ -237,6 +313,23 @@ def can_see(row: asyncpg.Record) -> bool: today = _seoul_today() try: + search_phrases = (question_text,) + rewriter = semantic_query_client or NullSemanticQueryClient() + if rewriter.available: + try: + search_phrases = await asyncio.to_thread(rewriter.rewrite, question_text) + except Exception as exc: + # Query rewriting is an optional recall channel. Any provider + # or envelope defect retains the original authorized query; + # cancellation remains outside Exception and still propagates. + log_provider_unavailable("global_ask_query_rewrite", exc) + question_embedding = ( + None + if knowledge_cutoff is not None + else await prepare_global_question_embedding( + question_text, embedding_client or NullEmbeddingClient() + ) + ) async with pool.acquire() as conn: sources = await gather_global_chat_sources( conn, @@ -244,8 +337,11 @@ def can_see(row: asyncpg.Record) -> bool: corporate_entity_ids, process_unit_ids, question=question_text, + search_phrases=search_phrases, + question_embedding=question_embedding, today=today, - embedding_client=embedding_client, + embedding_client=NullEmbeddingClient(), + knowledge_cutoff=knowledge_cutoff, ) except Exception as exc: log_internal_fault("global_ask", exc) @@ -254,22 +350,51 @@ def can_see(row: asyncpg.Record) -> bool: status.HTTP_503_SERVICE_UNAVAILABLE, "Ask Agent is unavailable: authorized evidence could not be assembled", ) from exc - if not sources: + cutoff_text = knowledge_cutoff.isoformat() if knowledge_cutoff else None + grounding_status = ask_grounding_status(sources, cutoff_text) + limitations = historical_body_limitations(sources) if knowledge_cutoff else [] + usable_sources = ( + [source for source in sources if not source.historical_body_unavailable] + if knowledge_cutoff + else sources + ) + verification_client = claim_verification_client or NullClaimVerificationClient() + if not usable_sources: + verification_status, external_claims = await _verify_public_claims( + question_text, + usable_sources, + [], + verify_external=verify_external, + client=verification_client, + ) delivery = build_ask_delivery("", (), ()) return { "answer_text": "", "cited_post_ids": [], "cited_posts": [], - "source_post_ids": [], + "source_post_ids": [source.post_id for source in sources], "cited_post_evidence": [], "lineage_graph": {"nodes": [], "edges": [], "truncated": False}, "cited_post_images": [], - "next_action": "No authorized source posts are available for this question.", + "external_verification_status": verification_status, + "external_claims": [claim.to_payload() for claim in external_claims], + "next_action": ( + "Review unavailable historical channels before relying on this cutoff answer." + if limitations + else "No authorized source posts are available for this question." + ), "delivery": delivery, + "knowledge_cutoff": cutoff_text, + "grounding_status": grounding_status, + "limitations": limitations, } try: answer = await asyncio.to_thread( - chat_client.answer, _temporally_grounded_question(question_text, today=today), sources + chat_client.answer, + _temporally_grounded_question( + question_text, today=today, knowledge_cutoff=knowledge_cutoff + ), + usable_sources, ) except (HttpClientError, OSError) as exc: # Known transport/provider failure. Same generic 503 text on every @@ -300,11 +425,29 @@ def can_see(row: asyncpg.Record) -> bool: "Ask Agent is unavailable: contextual-orchestrator could not complete the answer", ) from exc cited_ids = list(answer.cited_post_ids) - async with pool.acquire() as conn: - lineage_graph = await lineage_graphs_for_posts(conn, can_see, cited_ids) - images = await cited_post_images(conn, cited_ids) - cited_posts = cited_post_summaries(sources, cited_ids) - cited_evidence = cited_post_evidence(sources, cited_ids) + verification_status, external_claims = await _verify_public_claims( + question_text, + usable_sources, + cited_ids, + verify_external=verify_external, + client=verification_client, + ) + if knowledge_cutoff is None: + async with pool.acquire() as conn: + lineage_graph = await lineage_graphs_for_posts(conn, can_see, cited_ids) + images = await cited_post_images(conn, cited_ids) + else: + lineage_graph = {"nodes": [], "edges": [], "truncated": False} + images = [] + cited_posts = cited_post_summaries(usable_sources, cited_ids) + cited_evidence = cited_post_evidence(usable_sources, cited_ids) + next_action = _verification_next_action(verification_status) + if knowledge_cutoff is not None: + next_action = ( + "Review unavailable historical channels before relying on this cutoff answer." + if limitations + else "Compare these cutoff-grounded citations with live evidence next." + ) return { "answer_text": answer.answer_text, "cited_post_ids": cited_ids, @@ -314,10 +457,21 @@ def can_see(row: asyncpg.Record) -> bool: "source_post_ids": [source.post_id for source in sources], "lineage_graph": lineage_graph, "delivery": build_ask_delivery(answer.answer_text, cited_posts, cited_evidence), + "external_verification_status": verification_status, + "external_claims": [claim.to_payload() for claim in external_claims], + "next_action": next_action, + "knowledge_cutoff": cutoff_text, + "grounding_status": grounding_status, + "limitations": limitations, } -def _temporally_grounded_question(question_text: str, *, today: date | None = None) -> str: +def _temporally_grounded_question( + question_text: str, + *, + today: date | None = None, + knowledge_cutoff: datetime | None = None, +) -> str: """Restate a resolved relative-time window inside the question. Retrieval already scopes sources to the resolved window, but the @@ -329,19 +483,26 @@ def _temporally_grounded_question(question_text: str, *, today: date | None = No """ today = today or _seoul_today() window = resolve_korean_relative_time(question_text, today=today) - if window is None: - return question_text - start_date, end_date = window - # Phrasing matters: an earlier clause that only named the window was - # read by the model as the reference point ("now"), which re-subtracted - # the offset and looked for events seven further months back. Anchor - # today's date and equate the expression to the window outright. - return ( - f"{question_text}\n(오늘은 {today.isoformat()}입니다. 질문의 상대 시점 표현은 " - f"{start_date.isoformat()}부터 {end_date.isoformat()}까지의 기간을 가리킵니다. " - "제공된 소스 게시물은 모두 이 기간에 작성된 것이므로, 이 기간의 일을 " - "이 소스들로 답하십시오.)" - ) + grounded = question_text + if window is not None: + start_date, end_date = window + # Phrasing matters: an earlier clause that only named the window was + # read by the model as the reference point ("now"), which re-subtracted + # the offset and looked for events seven further months back. Anchor + # today's date and equate the expression to the window outright. + grounded = ( + f"{question_text}\n(오늘은 {today.isoformat()}입니다. 질문의 상대 시점 표현은 " + f"{start_date.isoformat()}부터 {end_date.isoformat()}까지의 기간을 가리킵니다. " + "제공된 소스 게시물은 모두 이 기간에 작성된 것이므로, 이 기간의 일을 " + "이 소스들로 답하십시오.)" + ) + if knowledge_cutoff is not None: + grounded += ( + f"\n(Knowledge cutoff: {knowledge_cutoff.isoformat()}. Every numbered " + "source body is the retained revision available by this cutoff. Do not " + "claim that later evidence was known at the cutoff.)" + ) + return grounded async def process_global_ask_job( @@ -350,6 +511,10 @@ async def process_global_ask_job( job_id: str, chat_factory: Callable[[], PostChatClient], embedding_factory: Callable[[], EmbeddingClient] = NullEmbeddingClient, + semantic_query_factory: Callable[[], SemanticQueryClient] = NullSemanticQueryClient, + claim_verification_factory: Callable[ + [], ClaimVerificationClient + ] = NullClaimVerificationClient, ) -> None: """Claim, answer, and settle one Ask job. @@ -363,7 +528,8 @@ async def process_global_ask_job( """ update global_ask_job set job_status_code = $2, updated_at = now() where global_ask_job_id = $1 and job_status_code = $3 - returning requesting_account_id, question_text + returning requesting_account_id, question_text, verify_external_requested, + knowledge_cutoff """, job_id, RUNNING, @@ -397,6 +563,10 @@ async def process_global_ask_job( process_scope_limited=process_scope_limited, chat_client=chat_client, embedding_client=embedding_factory(), + semantic_query_client=semantic_query_factory(), + verify_external=bool(row["verify_external_requested"]), + claim_verification_client=claim_verification_factory(), + knowledge_cutoff=row["knowledge_cutoff"], ), timeout=JOB_DEADLINE_SECONDS, ) @@ -511,6 +681,8 @@ async def consume_global_ask_stream_once( last_id: str, chat_factory: Callable[[], PostChatClient], embedding_factory: Callable[[], EmbeddingClient] = NullEmbeddingClient, + semantic_query_factory: Callable[[], SemanticQueryClient] = NullSemanticQueryClient, + claim_verification_factory: Callable[[], ClaimVerificationClient] = NullClaimVerificationClient, limiter: asyncio.Semaphore | None = None, tasks: set[asyncio.Task] | None = None, ) -> str: @@ -535,6 +707,8 @@ async def consume_global_ask_stream_once( job_id=job_id, chat_factory=chat_factory, embedding_factory=embedding_factory, + semantic_query_factory=semantic_query_factory, + claim_verification_factory=claim_verification_factory, ) else: await limiter.acquire() @@ -544,6 +718,8 @@ async def consume_global_ask_stream_once( job_id=job_id, chat_factory=chat_factory, embedding_factory=embedding_factory, + semantic_query_factory=semantic_query_factory, + claim_verification_factory=claim_verification_factory, limiter=limiter, ) ) @@ -560,6 +736,8 @@ async def _process_and_release( job_id: str, chat_factory: Callable[[], PostChatClient], embedding_factory: Callable[[], EmbeddingClient], + semantic_query_factory: Callable[[], SemanticQueryClient], + claim_verification_factory: Callable[[], ClaimVerificationClient], limiter: asyncio.Semaphore, ) -> None: """Run one dispatched job and free its concurrency slot afterwards.""" @@ -569,6 +747,8 @@ async def _process_and_release( job_id=job_id, chat_factory=chat_factory, embedding_factory=embedding_factory, + semantic_query_factory=semantic_query_factory, + claim_verification_factory=claim_verification_factory, ) finally: limiter.release() @@ -590,6 +770,8 @@ async def run_global_ask_worker( *, chat_factory: Callable[[], PostChatClient], embedding_factory: Callable[[], EmbeddingClient] = NullEmbeddingClient, + semantic_query_factory: Callable[[], SemanticQueryClient] = NullSemanticQueryClient, + claim_verification_factory: Callable[[], ClaimVerificationClient] = NullClaimVerificationClient, ) -> None: """Run the at-least-once Ask consumer with periodic queued-row recovery.""" last_id = await _stream_tail(client) @@ -609,6 +791,8 @@ async def run_global_ask_worker( last_id=last_id, chat_factory=chat_factory, embedding_factory=embedding_factory, + semantic_query_factory=semantic_query_factory, + claim_verification_factory=claim_verification_factory, limiter=limiter, tasks=tasks, ) diff --git a/backend/app/global_ask_service.py b/backend/app/global_ask_service.py new file mode 100644 index 000000000..a8f398a7c --- /dev/null +++ b/backend/app/global_ask_service.py @@ -0,0 +1,92 @@ +"""Shared durable Global Ask application service for REST and MCP.""" + +from __future__ import annotations + +import json +from typing import Any +from uuid import UUID + +import asyncpg +import redis.asyncio as redis +from fastapi import HTTPException, status + +from backend.app.auth import CurrentAccount +from backend.app.global_ask_queue import enqueue_global_ask_job +from backend.app.source_post_revision import parse_as_of_clock + + +async def submit_global_ask( + *, + pool: asyncpg.Pool, + valkey: redis.Redis, + account: CurrentAccount, + question: str, + verify_external: bool, + knowledge_cutoff: str | None, + service_available: bool, +) -> dict[str, Any]: + """Validate and enqueue one durable owner-scoped Global Ask job.""" + if not account.has_permission("post_read"): + raise HTTPException(status.HTTP_403_FORBIDDEN, "post_read permission required") + normalized_question = question.strip() + if not normalized_question: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_CONTENT, "question is required" + ) + cutoff = None + if knowledge_cutoff is not None: + try: + cutoff = parse_as_of_clock(knowledge_cutoff) + except ValueError as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_CONTENT, + "knowledge_cutoff must be an ISO-8601 timestamp", + ) from exc + async with pool.acquire() as conn: + if cutoff is not None and cutoff > await conn.fetchval("select now()"): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_CONTENT, + "knowledge_cutoff must be at or before the database clock", + ) + if not service_available: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Ask Agent is unavailable. Ask an administrator to configure the analysis service, then retry.", + ) + job_id = await enqueue_global_ask_job( + conn, + valkey, + requesting_account_id=account.user_account_id, + question_text=normalized_question, + verify_external_requested=verify_external, + knowledge_cutoff=cutoff, + corporate_entity_ids=account.corporate_entity_ids, + process_unit_ids=account.process_unit_ids, + ) + return {"ask_job_id": job_id, "job_status_code": "queued"} + + +async def read_global_ask_job( + *, pool: asyncpg.Pool, account: CurrentAccount, ask_job_id: UUID +) -> dict[str, Any]: + """Read one owner's durable Global Ask status and persisted result.""" + if not account.has_permission("post_read"): + raise HTTPException(status.HTTP_403_FORBIDDEN, "post_read permission required") + async with pool.acquire() as conn: + row = await conn.fetchrow( + "select requesting_account_id, job_status_code, answer_payload," + " failure_detail from global_ask_job where global_ask_job_id = $1", + ask_job_id, + ) + if row is None or str(row["requesting_account_id"]) != account.user_account_id: + raise HTTPException(status.HTTP_404_NOT_FOUND, "ask job not found") + body: dict[str, Any] = { + "ask_job_id": str(ask_job_id), + "job_status_code": row["job_status_code"], + } + if row["job_status_code"] == "succeeded" and row["answer_payload"] is not None: + payload = row["answer_payload"] + body["answer"] = json.loads(payload) if isinstance(payload, str) else payload + if row["job_status_code"] == "failed": + body["failure_detail"] = row["failure_detail"] + return body diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index ee240cd34..538ded900 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -14,13 +14,16 @@ import math import re from collections import defaultdict -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime from typing import Any import asyncpg -from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL +from backend.app.post_eligibility import ( + SOURCE_POST_ELIGIBILITY_SQL, + source_post_scope_sql, +) from lineageweave.adjudication_client import AdjudicationClient from lineageweave.interval_relation import ( INTERVAL_RELATION_LABELS, @@ -47,6 +50,15 @@ {"tepp_lineage_criterion_v1"} ) +_LINEAGE_LANDING_SQL = ( + "select post_id, post_title, voc_type_code, visibility_code, " + "corporate_entity_id, process_unit_id, thread_group_key, created_at " + "from source_post where {eligibility} and {visibility} " + "order by created_at desc, post_id desc limit $3" +).format( + eligibility=SOURCE_POST_ELIGIBILITY_SQL.format(alias="source_post"), + visibility=source_post_scope_sql("source_post"), +) def estimated_weight_channels(llm: AdjudicationClient | None) -> set[str]: """Return the channels that one live reconstruction can actually use.""" @@ -512,7 +524,7 @@ def _interval_payload(row: Mapping[str, Any]) -> dict[str, Any]: async def _fetch_visible_lineage_rows(conn: asyncpg.Connection, can_see_post): """One ABAC-filtered ``source_post`` scan plus one edge-table read.""" - posts = await conn.fetch( + posts = await conn.fetch( # nosemgrep "select post_id, post_title, voc_type_code, visibility_code, " "corporate_entity_id, process_unit_id, thread_group_key, created_at " f"from source_post where {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}" @@ -525,6 +537,37 @@ async def _fetch_visible_lineage_rows(conn: asyncpg.Connection, can_see_post): return visible_all, edge_rows +async def _fetch_lineage_landing_rows( + conn: asyncpg.Connection, + corporate_entity_ids: Sequence[str], + process_unit_ids: Sequence[str], + limit: int, +): + """Fetch only the authorized, bounded landing projection in PostgreSQL.""" + # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + # `_LINEAGE_LANDING_SQL` is a module constant; all runtime values below + # use asyncpg bind parameters. No caller-controlled SQL is interpolated. + posts = await conn.fetch( # nosemgrep + _LINEAGE_LANDING_SQL, + list(corporate_entity_ids), + list(process_unit_ids), + limit + 1, + ) + visible = list(posts[:limit]) + visible_ids = [str(row["post_id"]) for row in visible] + edge_rows = ( + await conn.fetch( + "select parent_post_id, child_post_id, fused_score, interval_relation_code " + "from post_lineage_edge where parent_post_id = any($1::uuid[]) " + "and child_post_id = any($1::uuid[])", + visible_ids, + ) + if visible_ids + else [] + ) + return visible, edge_rows, len(posts) > limit + + def _undirected_neighbors(edge_rows) -> dict[str, set[str]]: neighbors: dict[str, set[str]] = {} for edge in edge_rows: @@ -658,6 +701,8 @@ async def visible_lineage_graph( limit: int = _LINEAGE_GRAPH_NODE_LIMIT, focus_post_id: str | None = None, include_isolated: bool = False, + corporate_entity_ids: Sequence[str] | None = None, + process_unit_ids: Sequence[str] | None = None, ) -> dict[str, Any]: """ABAC-filtered graph bounded for the browser's initial viewport. @@ -665,16 +710,27 @@ async def visible_lineage_graph( individual posts for complete lineage, while this landing projection keeps only the newest ``limit`` visible nodes and edges between them. """ - visible_all, edge_rows = await _fetch_visible_lineage_rows(conn, can_see_post) + optimized_landing = ( + focus_post_id is None + and corporate_entity_ids is not None + and process_unit_ids is not None + ) + if optimized_landing: + visible, edge_rows, truncated = await _fetch_lineage_landing_rows( + conn, corporate_entity_ids, process_unit_ids, limit + ) + visible_all = visible + else: + visible_all, edge_rows = await _fetch_visible_lineage_rows(conn, can_see_post) - if focus_post_id is None: + if focus_post_id is None and not optimized_landing: visible = sorted( visible_all, key=lambda row: (row["created_at"], str(row["post_id"])), reverse=True, )[:limit] truncated = len(visible_all) > len(visible) - else: + elif focus_post_id is not None: focus_id = str(focus_post_id) neighbors = _undirected_neighbors(edge_rows) allowed = {str(row["post_id"]) for row in visible_all} diff --git a/backend/app/main.py b/backend/app/main.py index 08ff32428..6457bbde1 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -34,6 +34,11 @@ from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel +from lineageweave.claim_verification import ( + NullClaimVerificationClient, + SearxngOrchestratedClaimVerificationClient, +) + from backend.app.activity_stream import ( create_valkey_client, get_valkey, @@ -81,9 +86,9 @@ ) from backend.app.five_w1h_ingestion import load_five_w1h_slots from backend.app.global_ask_queue import ( - enqueue_global_ask_job, run_global_ask_worker, ) +from backend.app.global_ask_service import read_global_ask_job, submit_global_ask from backend.app.issue_ticket_ingestion import ( create_ticket, fetch_ticket_post_id, @@ -137,7 +142,7 @@ publish_post_content_event, ) from backend.app.post_content_worker import run_post_content_worker -from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL +from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL, source_post_visible from backend.app.post_evaluation_ingestion import ( fetch_post_evaluation, ingest_post_evaluation, @@ -148,7 +153,7 @@ require_summary_source_body, ) from backend.app.ranking_ingestion import load_visible_ranking_posts -from backend.app.relation_verification_ingestion import verify_post_relations +from backend.app.relation_verification_ingestion import verify_post_relations_from_pool from backend.app.report_ingestion import ( GROUPING_KINDS, fetch_period_comparison, @@ -237,6 +242,10 @@ SearxngRelationVerificationClient, ) from lineageweave.semantic_hints import customer_hint_trust, format_semantic_hints +from lineageweave.semantic_query import ( + ContextualOrchestratorSemanticQueryClient, + NullSemanticQueryClient, +) _POST_READ = "post_read" _POST_ADMIN = "post_admin" @@ -295,6 +304,8 @@ async def lifespan(app: FastAPI): timeout=load_settings().orchestrator_answer_timeout_seconds ), embedding_factory=_embedding_client, + semantic_query_factory=_semantic_query_client, + claim_verification_factory=_claim_verification_client_factory, ) ) app.state.global_ask_worker = global_ask_worker @@ -371,6 +382,28 @@ def _relation_verification_client(): return SearxngRelationVerificationClient(base_url=settings.searxng_base_url) +def _claim_verification_client(): + """Return the public-evidence verifier, or its unavailable null channel.""" + + settings = load_settings() + if not ( + settings.searxng_base_url + and settings.orchestrator_base_url + and settings.orchestrator_api_key + ): + return NullClaimVerificationClient() + return SearxngOrchestratedClaimVerificationClient( + settings.searxng_base_url, + settings.orchestrator_base_url, + settings.orchestrator_api_key, + ) + + +def _claim_verification_client_factory(): + """Resolve the verifier late so runtime overrides reach the worker.""" + return _claim_verification_client() + + def _organization_name_resolution_client(): """Live orchestrator client when configured; otherwise the unavailable null.""" settings = load_settings() @@ -495,6 +528,16 @@ def _embedding_client(): ) +def _semantic_query_client(): + """Build the orchestrator query rewriter, or an unavailable channel.""" + settings = load_settings() + if not (settings.orchestrator_base_url and settings.orchestrator_api_key): + return NullSemanticQueryClient() + return ContextualOrchestratorSemanticQueryClient( + settings.orchestrator_base_url, settings.orchestrator_api_key + ) + + def _post_evaluation_client(): """Live judge client when configured; otherwise the unavailable null.""" settings = load_settings() @@ -524,14 +567,8 @@ def _rankweave_client(): def _can_see_post(account: CurrentAccount, post: asyncpg.Record) -> bool: """ABAC: public rows are visible; private rows require the bound local scope.""" - if post["visibility_code"] == "public": - return True - return ( - str(post["corporate_entity_id"]) in account.corporate_entity_ids - and ( - not account.process_unit_ids - or str(post["process_unit_id"]) in account.process_unit_ids - ) + return source_post_visible( + post, account.corporate_entity_ids, account.process_unit_ids ) @@ -1251,6 +1288,8 @@ async def read_lineage_graph( lambda row: _can_see_post(account, row), limit=limit, focus_post_id=post_id, + corporate_entity_ids=account.corporate_entity_ids, + process_unit_ids=account.process_unit_ids, ) @@ -2310,28 +2349,25 @@ async def verify_post_entity_relationships( status.HTTP_503_SERVICE_UNAVAILABLE, "Relation verification is unavailable: set SEARXNG_BASE_URL", ) - async with pool.acquire() as conn: - try: - verified = await verify_post_relations( - conn, - client, - post_id, - visible_corporate_entity_ids=account.corporate_entity_ids, - ) - except (HttpClientError, OSError) as exc: - # verify_post_relations() deliberately raises on a failed search - # (a failed search is not "searched and found nothing" -- see - # its docstring); this is the one caller, so it is the right - # place to turn that into a clean 503 instead of a raw 500. - raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Relation verification is unavailable: the search provider did not respond", - ) from exc - except Exception as exc: # noqa: BLE001 - provider boundary is fail-closed. - raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Relation verification is unavailable: the search provider did not respond", - ) from exc + try: + verified = await verify_post_relations_from_pool( + pool, + client, + post_id, + visible_corporate_entity_ids=account.corporate_entity_ids, + ) + except (HttpClientError, OSError) as exc: + # A failed search is not "searched and found nothing"; turn the + # provider failure into a clean 503 rather than persisting a miss. + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Relation verification is unavailable: the search provider did not respond", + ) from exc + except Exception as exc: # noqa: BLE001 - provider boundary is fail-closed. + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Relation verification is unavailable: the search provider did not respond", + ) from exc await publish_activity_event( valkey, post_id, @@ -2960,6 +2996,8 @@ class GlobalAskRequest(BaseModel): """JSON body for the buyer's source-grounded Global Ask Agent.""" question: str + verify_external: bool = False + knowledge_cutoff: str | None = None @app.get("/api/posts/{post_id}/chat") @@ -3101,26 +3139,15 @@ async def ask_agent( still fails fast on the states that cannot ever succeed (blank question, missing permission, unconfigured orchestrator). """ - question = request.question.strip() - if not question: - raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, "question is required") - _require_post_read(account) - if not _post_chat_client().available: - raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Ask Agent is unavailable. Ask an administrator to configure the analysis service, " - "then retry.", - ) - async with pool.acquire() as conn: - job_id = await enqueue_global_ask_job( - conn, - valkey, - requesting_account_id=account.user_account_id, - question_text=question, - corporate_entity_ids=account.corporate_entity_ids, - process_unit_ids=account.process_unit_ids, - ) - return {"ask_job_id": job_id, "job_status_code": "queued"} + return await submit_global_ask( + pool=pool, + valkey=valkey, + account=account, + question=request.question, + verify_external=request.verify_external, + knowledge_cutoff=request.knowledge_cutoff, + service_available=_post_chat_client().available, + ) @app.get("/api/ask/jobs/{ask_job_id}") @@ -3134,25 +3161,7 @@ async def read_ask_job( Owner-scoped: another account's job id reads as absent (404, not 403) so job ids do not leak their existence across accounts. """ - _require_post_read(account) - async with pool.acquire() as conn: - row = await conn.fetchrow( - "select requesting_account_id, job_status_code, answer_payload," - " failure_detail from global_ask_job where global_ask_job_id = $1", - ask_job_id, - ) - if row is None or str(row["requesting_account_id"]) != account.user_account_id: - raise HTTPException(status.HTTP_404_NOT_FOUND, "ask job not found") - body: dict[str, Any] = { - "ask_job_id": str(ask_job_id), - "job_status_code": row["job_status_code"], - } - if row["job_status_code"] == "succeeded" and row["answer_payload"] is not None: - payload = row["answer_payload"] - body["answer"] = json.loads(payload) if isinstance(payload, str) else payload - if row["job_status_code"] == "failed": - body["failure_detail"] = row["failure_detail"] - return body + return await read_global_ask_job(pool=pool, account=account, ask_job_id=ask_job_id) class PostBookmarkRequest(BaseModel): @@ -3386,7 +3395,12 @@ async def derive_post_commitment( # Friday" in a January post must resolve to that January, not to the # Friday after the operator clicked Derive. reference_date = post["created_at"].date().isoformat() - commitment = client.extract(post["post_title"], normalized_body, reference_date) + commitment = await asyncio.to_thread( + client.extract, + post["post_title"], + normalized_body, + reference_date, + ) except (HttpClientError, KeyError, OSError, TypeError, ValueError, RuntimeError) as exc: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, @@ -3600,7 +3614,8 @@ async def read_calendar( settings = load_settings() if window_start is None or window_end is None: window_start, window_end = default_calendar_window(datetime.now(timezone.utc)) - naruon = load_observed_calendar_events( + naruon = await asyncio.to_thread( + load_observed_calendar_events, build_workspace_naruon_client( settings.naruon_calendar_base_url, settings.naruon_calendar_service_token, diff --git a/backend/app/mcp_admission.py b/backend/app/mcp_admission.py new file mode 100644 index 000000000..ccd73536f --- /dev/null +++ b/backend/app/mcp_admission.py @@ -0,0 +1,131 @@ +"""Bound MCP request bodies before OAuth and JSON decoding.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence + +from starlette.types import ASGIApp, Message, Receive, Scope, Send + + +class BoundedRequestBodyApp: + """Reject ambiguous or oversized MCP POST bodies before parsing.""" + + def __init__(self, app: ASGIApp, *, maximum_bytes: int) -> None: + """Wrap ``app`` with one positive finite body-size limit.""" + if maximum_bytes <= 0: + raise ValueError("maximum_bytes must be positive") + self._app = app + self._maximum_bytes = maximum_bytes + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + """Apply admission to HTTP POST and pass other traffic unchanged.""" + if scope["type"] != "http" or scope.get("method") != "POST": + await self._app(scope, receive, send) + return + + content_lengths = _header_values(scope, b"content-length") + transfer_encodings = _header_values(scope, b"transfer-encoding") + declared_length = _parse_content_length( + content_lengths, transfer_encodings, self._maximum_bytes + ) + if declared_length is _INVALID_LENGTH: + await _send_error(send, 400, "mcp_invalid_content_length") + return + if isinstance(declared_length, int) and declared_length > self._maximum_bytes: + await _send_error(send, 413, "mcp_request_too_large") + return + + body = bytearray() + while True: + message = await receive() + message_type = message.get("type") + if message_type == "http.disconnect": + await _send_error(send, 400, "mcp_request_disconnected") + return + if message_type != "http.request": + await _send_error(send, 400, "mcp_invalid_request_body") + return + chunk = message.get("body", b"") + if not isinstance(chunk, bytes): + await _send_error(send, 400, "mcp_invalid_request_body") + return + if len(body) + len(chunk) > self._maximum_bytes: + await _send_error(send, 413, "mcp_request_too_large") + return + body.extend(chunk) + if not message.get("more_body", False): + break + + if isinstance(declared_length, int) and declared_length != len(body): + await _send_error(send, 400, "mcp_content_length_mismatch") + return + + replayed = False + + async def replay_receive() -> Message: + """Replay the admitted body once, then preserve client lifecycle events.""" + nonlocal replayed + if replayed: + return await receive() + replayed = True + return {"type": "http.request", "body": bytes(body), "more_body": False} + + await self._app(scope, replay_receive, send) + + +class _InvalidLength: + """Sentinel distinguishing an invalid length from an absent one.""" + + +_INVALID_LENGTH = _InvalidLength() + + +def _header_values(scope: Scope, name: bytes) -> tuple[bytes, ...]: + """Return every raw value for one case-insensitive request header.""" + return tuple( + value + for header_name, value in scope.get("headers", []) + if header_name.lower() == name + ) + + +def _parse_content_length( + content_lengths: Sequence[bytes], + transfer_encodings: Sequence[bytes], + maximum_bytes: int, +) -> int | None | _InvalidLength: + """Return an unambiguous nonnegative length, absence, or invalid sentinel.""" + if len(content_lengths) > 1 or (content_lengths and transfer_encodings): + return _INVALID_LENGTH + if not content_lengths: + return None + try: + decoded = content_lengths[0].decode("ascii") + except UnicodeDecodeError: + return _INVALID_LENGTH + if not decoded or not decoded.isdecimal(): + return _INVALID_LENGTH + normalized = decoded.lstrip("0") or "0" + maximum = str(maximum_bytes) + if len(normalized) > len(maximum) or ( + len(normalized) == len(maximum) and normalized > maximum + ): + return maximum_bytes + 1 + return int(normalized, 10) + + +async def _send_error(send: Send, status_code: int, error_code: str) -> None: + """Send one bounded payload-safe admission error response.""" + body = json.dumps( + {"error_code": error_code}, ensure_ascii=True, separators=(",", ":") + ).encode("ascii") + headers = [ + (b"content-type", b"application/json"), + (b"content-length", str(len(body)).encode("ascii")), + (b"cache-control", b"no-store"), + ] + await send( + {"type": "http.response.start", "status": status_code, "headers": headers} + ) + await send({"type": "http.response.body", "body": body}) diff --git a/backend/app/mcp_auth.py b/backend/app/mcp_auth.py new file mode 100644 index 000000000..7bff0ae08 --- /dev/null +++ b/backend/app/mcp_auth.py @@ -0,0 +1,67 @@ +"""OAuth resource-server token verification for LineageWeave MCP.""" + +from __future__ import annotations + +import asyncio +from functools import partial +from typing import Any + +from fastapi import HTTPException +from mcp.server.auth.provider import AccessToken, TokenVerifier + +from backend.app.auth import decode_access_token +from backend.app.config import Settings + + +def _scopes_from_claim(claim: Any) -> list[str]: + """Normalize string or array scope claims without inventing scopes.""" + if isinstance(claim, str): + return [scope for scope in claim.split() if scope] + if isinstance(claim, list): + return [scope for scope in claim if isinstance(scope, str) and scope] + return [] + + +class KeyverseMcpTokenVerifier(TokenVerifier): + """Validate a JWT for the exact configured MCP resource audience.""" + + def __init__(self, settings: Settings) -> None: + """Retain immutable identity and MCP audience settings.""" + self._settings = settings + + async def verify_token(self, token: str) -> AccessToken | None: + """Return MCP access metadata for a valid token; otherwise fail closed.""" + if not self._settings.mcp_audience.strip(): + return None + try: + claims = await asyncio.to_thread( + partial( + decode_access_token, + token, + self._settings, + audience=self._settings.mcp_audience, + ) + ) + except HTTPException: + return None + subject = claims.get("sub") + client_id = claims.get("azp") or claims.get("client_id") + expires_at = claims.get("exp") + if ( + not isinstance(subject, str) + or not subject + or not isinstance(client_id, str) + or not client_id + ): + return None + return AccessToken( + token=token, + client_id=client_id, + scopes=_scopes_from_claim(claims.get("scope")), + expires_at=int(expires_at) + if isinstance(expires_at, (int, float)) + else None, + resource=self._settings.mcp_audience, + subject=subject, + claims=claims, + ) diff --git a/backend/app/mcp_rate_limit.py b/backend/app/mcp_rate_limit.py new file mode 100644 index 000000000..fd4e60e49 --- /dev/null +++ b/backend/app/mcp_rate_limit.py @@ -0,0 +1,69 @@ +"""Valkey-backed quota for authenticated, provisioned MCP accounts.""" + +from __future__ import annotations + +import hashlib +from typing import Any + +from backend.app.activity_stream import create_valkey_client + +_SCRIPT = """ +local count = redis.call('INCR', KEYS[1]) +if count == 1 then redis.call('EXPIRE', KEYS[1], ARGV[1]) end +local ttl = redis.call('TTL', KEYS[1]) +return {count, ttl} +""" + + +class McpRateLimitExceeded(Exception): + """The account exhausted its current shared window.""" + + def __init__(self, retry_after_seconds: int) -> None: + super().__init__("MCP account rate limit exceeded") + self.retry_after_seconds = retry_after_seconds + + +class McpRateLimiterUnavailable(Exception): + """The shared limiter could not make an authoritative decision.""" + + +class ValkeyMcpRateLimiter: + """Consume one atomic fixed-window quota entry in shared Valkey.""" + + def __init__(self, client: Any, *, request_limit: int, window_seconds: int) -> None: + self._client = client + self._request_limit = request_limit + self._window_seconds = window_seconds + + async def consume(self, user_account_id: str) -> None: + """Consume one provisioned account request or fail closed.""" + digest = hashlib.sha256(user_account_id.encode("utf-8")).hexdigest() + key = f"lineageweave:mcp-rate-limit:v1:{digest}" + try: + result = await self._client.eval(_SCRIPT, 1, key, self._window_seconds) + count, ttl = int(result[0]), int(result[1]) + except Exception as exc: + raise McpRateLimiterUnavailable( + "shared MCP rate limiter unavailable" + ) from exc + if count < 1 or ttl < 0: + raise McpRateLimiterUnavailable( + "shared MCP rate limiter returned invalid state" + ) + if count > self._request_limit: + raise McpRateLimitExceeded(max(1, min(ttl, self._window_seconds))) + + async def close(self) -> None: + """Close the underlying Valkey client.""" + await self._client.aclose() + + +def build_mcp_rate_limiter( + valkey_url: str, request_limit: int, window_seconds: int +) -> ValkeyMcpRateLimiter: + """Build the limiter from validated deployment settings.""" + return ValkeyMcpRateLimiter( + create_valkey_client(valkey_url), + request_limit=request_limit, + window_seconds=window_seconds, + ) diff --git a/backend/app/mcp_server.py b/backend/app/mcp_server.py new file mode 100644 index 000000000..9fa71de2a --- /dev/null +++ b/backend/app/mcp_server.py @@ -0,0 +1,350 @@ +"""Authenticated Streamable HTTP MCP adapter for durable Global Ask.""" + +from __future__ import annotations + +import json +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlsplit +from uuid import UUID + +from mcp.server import MCPServer +from mcp.server.auth.middleware.auth_context import get_access_token +from mcp.server.auth.provider import AccessToken, TokenVerifier +from mcp.server.auth.settings import AuthSettings +from mcp.server.mcpserver import Context +from mcp.server.transport_security import ( + TransportSecurityMiddleware, + TransportSecuritySettings, +) +from mcp.shared.exceptions import MCPError +from mcp.types import ToolAnnotations +from pydantic import AnyHttpUrl +from starlette.middleware.cors import CORSMiddleware +from starlette.requests import Request +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from backend.app.activity_stream import create_valkey_client +from backend.app.auth import CurrentAccount, resolve_current_account +from backend.app.config import Settings, load_settings +from backend.app.db import create_pool +from backend.app.global_ask_service import ( + read_global_ask_job as read_global_ask_job_service, +) +from backend.app.global_ask_service import ( + submit_global_ask as submit_global_ask_service, +) +from backend.app.mcp_admission import BoundedRequestBodyApp +from backend.app.mcp_auth import KeyverseMcpTokenVerifier +from backend.app.mcp_rate_limit import ( + McpRateLimiterUnavailable, + McpRateLimitExceeded, + ValkeyMcpRateLimiter, +) +from lineageweave.post_chat import ( + ContextualOrchestratorPostChatClient, + NullPostChatClient, +) + + +@dataclass +class McpAppContext: + """Long-lived dependencies shared by MCP tool calls.""" + + pool: Any + valkey: Any + limiter: ValkeyMcpRateLimiter + service_available: bool + settings: Settings + + +class PreAuthTransportSecurityApp: + """Reject hostile Host and Origin metadata before OAuth processing.""" + + def __init__(self, app: ASGIApp, settings: TransportSecuritySettings) -> None: + """Wrap an ASGI app with the SDK transport validator.""" + self._app = app + self._security = TransportSecurityMiddleware(settings) + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + """Validate HTTP transport metadata and pass non-HTTP traffic through.""" + if scope["type"] != "http": + await self._app(scope, receive, send) + return + request = Request(scope, receive=receive) + rejection = await self._security.validate_request( + request, is_post=request.method == "POST" + ) + if rejection is not None: + if request.headers.get("origin") is not None: + rejection.headers.add_vary_header("Origin") + await rejection(scope, receive, send) + return + await self._app(scope, receive, send) + + +class McpRetryAfterHeaderApp: + """Expose a bounded retry delay only for exhausted authenticated quota.""" + + def __init__(self, app: ASGIApp) -> None: + """Wrap the SDK response serializer.""" + self._app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + """Add Retry-After from the serialized quota error before headers commit.""" + response_start: Message | None = None + + async def send_with_retry(message: Message) -> None: + """Hold response start until the first MCP event reveals its status.""" + nonlocal response_start + if message.get("type") == "http.response.start": + response_start = message + return + if response_start is not None: + retry_after = _quota_retry_after(message.get("body", b"")) + headers = [ + (name, value) + for name, value in response_start.get("headers", []) + if name.lower() != b"retry-after" + ] + if retry_after is not None: + headers.append( + (b"retry-after", str(retry_after).encode("ascii")) + ) + await send({**response_start, "headers": headers}) + response_start = None + await send(message) + + await self._app(scope, receive, send_with_retry) + + +def _quota_retry_after(body: bytes) -> int | None: + """Read the bounded retry delay from an exact MCP JSON-RPC error event.""" + for line in body.splitlines(): + if not line.startswith(b"data:"): + continue + try: + payload = json.loads(line.removeprefix(b"data:").strip()) + error = payload["error"] + retry_after = error["data"]["retry_after_seconds"] + except (json.JSONDecodeError, KeyError, TypeError): + continue + if error.get("code") == -31929 and type(retry_after) is int and retry_after > 0: + return retry_after + return None + + +def _validate_mcp_settings(settings: Settings) -> tuple[int, int]: + """Require exact origins and measured deployment quota parameters.""" + if not settings.mcp_audience.strip(): + raise ValueError("MCP_AUDIENCE must name the exact MCP resource") + for origin in settings.mcp_allowed_origins: + parsed = urlsplit(origin) + if ( + origin in {"*", "null"} + or parsed.scheme not in {"http", "https"} + or not parsed.netloc + or parsed.username is not None + or parsed.password is not None + or parsed.path + or parsed.query + or parsed.fragment + ): + raise ValueError( + "MCP_ALLOWED_ORIGINS entries must be exact HTTP(S) origins" + ) + if ( + settings.mcp_rate_limit_requests is None + or settings.mcp_rate_limit_window_seconds is None + ): + raise ValueError( + "MCP_RATE_LIMIT_REQUESTS and MCP_RATE_LIMIT_WINDOW_SECONDS must be set from measured capacity" + ) + return settings.mcp_rate_limit_requests, settings.mcp_rate_limit_window_seconds + + +PoolFactory = Callable[[str], Awaitable[Any]] +ValkeyFactory = Callable[[str], Any] +LimiterFactory = Callable[[Any, int, int], ValkeyMcpRateLimiter] +AccountResolver = Callable[[Any, dict, Settings], Awaitable[CurrentAccount]] +AccessTokenProvider = Callable[[], AccessToken | None] + + +def _build_limiter(client: Any, requests: int, window: int) -> ValkeyMcpRateLimiter: + """Build the production shared limiter from validated inputs.""" + return ValkeyMcpRateLimiter(client, request_limit=requests, window_seconds=window) + + +async def _account( + ctx: Context[McpAppContext, Any], + *, + access_token_provider: AccessTokenProvider, + account_resolver: AccountResolver, +) -> CurrentAccount: + """Resolve the authenticated token to one provisioned database account.""" + token = access_token_provider() + if token is None or not token.subject or not isinstance(token.claims, dict): + raise PermissionError("authenticated MCP principal is unavailable") + dependencies = ctx.request_context.lifespan_context + account = await account_resolver( + dependencies.pool, token.claims, dependencies.settings + ) + if not account.has_permission("post_read"): + raise PermissionError("post_read permission required") + try: + await dependencies.limiter.consume(account.user_account_id) + except McpRateLimitExceeded as exc: + raise MCPError( + -31929, + "mcp_rate_limit_exceeded", + {"retry_after_seconds": exc.retry_after_seconds}, + ) from exc + except McpRateLimiterUnavailable as exc: + raise MCPError(-31930, "mcp_rate_limiter_unavailable") from exc + return account + + +def build_mcp_server( + settings: Settings | None = None, + *, + pool_factory: PoolFactory = create_pool, + valkey_factory: ValkeyFactory = create_valkey_client, + limiter_factory: LimiterFactory = _build_limiter, + token_verifier: TokenVerifier | None = None, + account_resolver: AccountResolver = resolve_current_account, + access_token_provider: AccessTokenProvider = get_access_token, +) -> MCPServer[McpAppContext]: + """Build the authenticated MCP server over the current durable Ask contract.""" + resolved = settings or load_settings() + request_limit, window_seconds = _validate_mcp_settings(resolved) + + @asynccontextmanager + async def lifespan(_: MCPServer) -> AsyncIterator[McpAppContext]: + """Open and close process-wide database and quota clients.""" + pool = await pool_factory(resolved.database_url) + valkey = valkey_factory(resolved.valkey_url) + limiter = limiter_factory(valkey, request_limit, window_seconds) + chat_client = ( + ContextualOrchestratorPostChatClient( + base_url=resolved.orchestrator_base_url, + api_key=resolved.orchestrator_api_key, + ) + if resolved.orchestrator_base_url and resolved.orchestrator_api_key + else NullPostChatClient() + ) + try: + yield McpAppContext(pool, valkey, limiter, chat_client.available, resolved) + finally: + try: + await limiter.close() + finally: + await pool.close() + + server = MCPServer( + "lineageweave", + title="LineageWeave", + description="Authenticated provenance-bearing lineage intelligence.", + version="2.18.0", + lifespan=lifespan, + token_verifier=token_verifier or KeyverseMcpTokenVerifier(resolved), + auth=AuthSettings( + issuer_url=AnyHttpUrl(resolved.oidc_issuer), + resource_server_url=AnyHttpUrl(resolved.mcp_resource_url), + required_scopes=resolved.mcp_required_scopes, + ), + ) + + @server.tool( + title="Submit Global Ask", + description="Queue a question against the caller's authorized LineageWeave evidence.", + annotations=ToolAnnotations( + read_only_hint=False, idempotent_hint=False, open_world_hint=True + ), + ) + async def submit_global_ask( + question: str, + ctx: Context[McpAppContext, Any], + verify_external: bool = False, + knowledge_cutoff: str | None = None, + ) -> dict[str, Any]: + """Queue one current-contract Global Ask job without blocking transport.""" + dependencies = ctx.request_context.lifespan_context + account = await _account( + ctx, + access_token_provider=access_token_provider, + account_resolver=account_resolver, + ) + return await submit_global_ask_service( + pool=dependencies.pool, + valkey=dependencies.valkey, + account=account, + question=question, + verify_external=verify_external, + knowledge_cutoff=knowledge_cutoff, + service_available=dependencies.service_available, + ) + + @server.tool( + title="Read Global Ask Job", + description="Read a queued Global Ask job owned by the authenticated caller.", + annotations=ToolAnnotations( + read_only_hint=True, idempotent_hint=True, open_world_hint=False + ), + ) + async def read_global_ask_job( + ask_job_id: str, ctx: Context[McpAppContext, Any] + ) -> dict[str, Any]: + """Read one current-contract Global Ask job and its persisted answer.""" + account = await _account( + ctx, + access_token_provider=access_token_provider, + account_resolver=account_resolver, + ) + try: + parsed_job_id = UUID(ask_job_id) + except ValueError as exc: + raise ValueError("ask_job_id must be a UUID") from exc + return await read_global_ask_job_service( + pool=ctx.request_context.lifespan_context.pool, + account=account, + ask_job_id=parsed_job_id, + ) + + return server + + +def build_mcp_http_app(server: MCPServer[McpAppContext], settings: Settings) -> ASGIApp: + """Build exact-origin, byte-bounded Streamable HTTP outside OAuth.""" + _validate_mcp_settings(settings) + security = TransportSecuritySettings( + enable_dns_rebinding_protection=True, + allowed_hosts=settings.mcp_allowed_hosts, + allowed_origins=settings.mcp_allowed_origins, + ) + sdk_app = server.streamable_http_app(transport_security=security) + cors_app = CORSMiddleware( + McpRetryAfterHeaderApp(sdk_app), + allow_origins=settings.mcp_allowed_origins, + allow_methods=["GET", "POST", "DELETE"], + allow_headers=[ + "Accept", + "Authorization", + "Content-Type", + "Last-Event-ID", + "MCP-Protocol-Version", + "Mcp-Session-Id", + ], + expose_headers=["MCP-Protocol-Version", "Mcp-Session-Id", "WWW-Authenticate"], + allow_credentials=False, + ) + return PreAuthTransportSecurityApp( + BoundedRequestBodyApp(cors_app, maximum_bytes=settings.mcp_max_request_bytes), + security, + ) + + +_settings = load_settings() +mcp = build_mcp_server(_settings) +app = build_mcp_http_app(mcp, _settings) diff --git a/backend/app/ontology_neighborhood_ingestion.py b/backend/app/ontology_neighborhood_ingestion.py index b9d6d4900..cbcfef440 100644 --- a/backend/app/ontology_neighborhood_ingestion.py +++ b/backend/app/ontology_neighborhood_ingestion.py @@ -18,11 +18,14 @@ visible_team_mention_post_ids, ) from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL +from lineageweave.post_summary import parse_project_candidate_node_id from lineageweave.knowledge_graph import ( NODE_CORPORATE_ENTITY, NODE_PERSON, NODE_POST, + NODE_PROJECT, NODE_TEAM, + EDGE_MENTION_PROJECT, ) from lineageweave.ontology_neighborhood import ( DEFAULT_MAXIMUM_DEPTH, @@ -122,28 +125,78 @@ async def visible_post_ids_for_focus( focus_node_type_code: str, focus_node_id: str, can_see_post: Callable[[asyncpg.Record], bool], + *, + knowledge_cutoff: datetime | None = None, + snapshot_at: datetime | None = None, ) -> list[str]: """Visible evidence posts that authorize the requested focus node.""" if focus_node_type_code == NODE_POST: # Safe SQL: eligibility is an immutable schema fragment; id is bound. row = await conn.fetchrow( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli f""" - select post_id, visibility_code, corporate_entity_id + select post_id, visibility_code, corporate_entity_id, process_unit_id from source_post where post_id = $1 and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')} + and ($2::timestamptz is null or created_at <= $2::timestamptz) + and ($3::timestamptz is null or created_at <= $3::timestamptz) """, focus_node_id, + knowledge_cutoff, + snapshot_at, ) if row is None: return [] return [str(row["post_id"])] if can_see_post(row) else [] + candidate_post_ids: list[str] | None = None if focus_node_type_code == NODE_PERSON: - return await visible_mention_post_ids(conn, focus_node_id, can_see_post) - if focus_node_type_code == NODE_CORPORATE_ENTITY: - return await visible_affiliation_post_ids(conn, focus_node_id, can_see_post) - if focus_node_type_code == NODE_TEAM: - return await visible_team_mention_post_ids(conn, focus_node_id, can_see_post) + candidate_post_ids = await visible_mention_post_ids(conn, focus_node_id, can_see_post) + elif focus_node_type_code == NODE_CORPORATE_ENTITY: + candidate_post_ids = await visible_affiliation_post_ids(conn, focus_node_id, can_see_post) + elif focus_node_type_code == NODE_TEAM: + candidate_post_ids = await visible_team_mention_post_ids(conn, focus_node_id, can_see_post) + if candidate_post_ids is not None: + if knowledge_cutoff is None and snapshot_at is None: + return candidate_post_ids + rows = await conn.fetch( + """ + select post_id + from source_post + where post_id = any($1::uuid[]) + and ($2::timestamptz is null or created_at <= $2::timestamptz) + and ($3::timestamptz is null or created_at <= $3::timestamptz) + """, + candidate_post_ids, + knowledge_cutoff, + snapshot_at, + ) + admitted = {str(row["post_id"]) for row in rows} + return [post_id for post_id in candidate_post_ids if post_id in admitted] + if focus_node_type_code == NODE_PROJECT: + project_post_id, project_key = parse_project_candidate_node_id(focus_node_id) + # Safe SQL: eligibility is an immutable schema fragment and the alias is + # fixed here; all request-derived values remain asyncpg parameters. + project_posts_sql = f""" + select post.post_id, post.visibility_code, post.corporate_entity_id, + post.process_unit_id + from post_project_mention mention + join source_post post on post.post_id = mention.post_id + where mention.post_id = $1::uuid + and mention.project_key = $2 + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} + and ($3::timestamptz is null + or greatest(post.created_at, mention.created_at) <= $3::timestamptz) + and ($4::timestamptz is null + or greatest(post.created_at, mention.created_at) <= $4::timestamptz) + """ + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + project_posts_sql, + project_post_id, + project_key, + knowledge_cutoff, + snapshot_at, + ) + return [str(row["post_id"]) for row in rows if can_see_post(row)] raise OntologyNeighborhoodError("unknown_node_type", f"unknown node type {focus_node_type_code!r}") @@ -151,8 +204,11 @@ async def _visible_post_ids_by_nodes( conn: asyncpg.Connection, node_keys: set[tuple[str, str]], can_see_post: Callable[[asyncpg.Record], bool], + *, + knowledge_cutoff: datetime | None = None, + snapshot_at: datetime | None = None, ) -> dict[tuple[str, str], list[str]]: - """Load evidence visibility for all endpoint nodes in four bounded queries. + """Load evidence visibility for all endpoint nodes in five bounded queries. The neighborhood can contain many endpoints. Grouping ids by node type preserves the same ABAC predicate as the single-node readers while @@ -168,28 +224,34 @@ async def _visible_post_ids_by_nodes( NODE_POST, """ select post.post_id, post.visibility_code, - post.corporate_entity_id, post.post_id as node_id + post.corporate_entity_id, post.process_unit_id, + post.post_id as node_id from source_post post where post.post_id = any($1::uuid[]) and {eligibility} + and ($2::timestamptz is null or post.created_at <= $2::timestamptz) + and ($3::timestamptz is null or post.created_at <= $3::timestamptz) """, ), ( NODE_PERSON, """ select post.post_id, post.visibility_code, - post.corporate_entity_id, mention.person_id as node_id + post.corporate_entity_id, post.process_unit_id, + mention.person_id as node_id from combined_post_person_mention mention join source_post post on post.post_id = mention.post_id where mention.person_id = any($1::uuid[]) and {eligibility} + and ($2::timestamptz is null or post.created_at <= $2::timestamptz) + and ($3::timestamptz is null or post.created_at <= $3::timestamptz) """, ), ( NODE_CORPORATE_ENTITY, """ select distinct post.post_id, post.visibility_code, - post.corporate_entity_id, + post.corporate_entity_id, post.process_unit_id, affiliation.affiliated_corporate_entity_id as node_id from person_affiliation affiliation join combined_post_person_mention mention @@ -197,25 +259,48 @@ async def _visible_post_ids_by_nodes( join source_post post on post.post_id = mention.post_id where affiliation.affiliated_corporate_entity_id = any($1::uuid[]) and {eligibility} + and ($2::timestamptz is null or post.created_at <= $2::timestamptz) + and ($3::timestamptz is null or post.created_at <= $3::timestamptz) union select distinct post.post_id, post.visibility_code, - post.corporate_entity_id, + post.corporate_entity_id, post.process_unit_id, org_mention.corporate_entity_id as node_id from post_organization_mention org_mention join source_post post on post.post_id = org_mention.post_id where org_mention.corporate_entity_id = any($1::uuid[]) and {eligibility} + and ($2::timestamptz is null or post.created_at <= $2::timestamptz) + and ($3::timestamptz is null or post.created_at <= $3::timestamptz) """, ), ( NODE_TEAM, """ select post.post_id, post.visibility_code, - post.corporate_entity_id, mention.team_id as node_id + post.corporate_entity_id, post.process_unit_id, + mention.team_id as node_id from post_team_mention mention join source_post post on post.post_id = mention.post_id where mention.team_id = any($1::uuid[]) and {eligibility} + and ($2::timestamptz is null or post.created_at <= $2::timestamptz) + and ($3::timestamptz is null or post.created_at <= $3::timestamptz) + """, + ), + ( + NODE_PROJECT, + """ + select post.post_id, post.visibility_code, + post.corporate_entity_id, post.process_unit_id, + mention.post_id::text || '/' || mention.project_key as node_id + from post_project_mention mention + join source_post post on post.post_id = mention.post_id + where mention.post_id::text || '/' || mention.project_key = any($1::text[]) + and {eligibility} + and ($2::timestamptz is null + or greatest(post.created_at, mention.created_at) <= $2::timestamptz) + and ($3::timestamptz is null + or greatest(post.created_at, mention.created_at) <= $3::timestamptz) """, ), ) @@ -224,7 +309,7 @@ async def _visible_post_ids_by_nodes( if not ids: continue query = template.format(eligibility=SOURCE_POST_ELIGIBILITY_SQL.format(alias="post")) - rows = await conn.fetch(query, ids) + rows = await conn.fetch(query, ids, knowledge_cutoff, snapshot_at) for row in rows: try: raw_node_id = row["node_id"] @@ -248,7 +333,15 @@ async def _visible_post_ids_by_nodes( async def focus_catalog_exists( conn: asyncpg.Connection, focus_node_type_code: str, focus_node_id: str ) -> bool: - """True when the focus id exists in the governed catalog.""" + """True when the focus id exists in its governed relational source.""" + if focus_node_type_code == NODE_PROJECT: + project_post_id, project_key = parse_project_candidate_node_id(focus_node_id) + row = await conn.fetchrow( + "select 1 from post_project_mention where post_id = $1::uuid and project_key = $2", + project_post_id, + project_key, + ) + return row is not None if not _is_uuid(focus_node_id): return False if focus_node_type_code == NODE_POST: @@ -286,7 +379,8 @@ async def _load_facts( edge.target_node_type_code, edge.target_node_id::text as target_node_id, edge.edge_type_code, - min(post.created_at) as available_at, + 'truth_observed'::text as truth_status_code, + greatest(edge.created_at, min(post.created_at)) as available_at, array_agg(evidence.evidence_post_id::text order by evidence.evidence_post_id) as evidence_ids from knowledge_graph_edge edge @@ -296,11 +390,28 @@ async def _load_facts( on post.post_id = evidence.evidence_post_id where evidence.evidence_post_id = any($1::uuid[]) and ($6::timestamptz is null or post.created_at <= $6::timestamptz) + and ($6::timestamptz is null or edge.created_at <= $6::timestamptz) and ($7::timestamptz is null or post.created_at <= $7::timestamptz) and ($7::timestamptz is null or edge.created_at <= $7::timestamptz) group by edge.source_node_type_code, edge.source_node_id, edge.target_node_type_code, edge.target_node_id, edge.edge_type_code + union all + select 'node_post'::text as source_node_type_code, + mention.post_id::text as source_node_id, + 'node_project'::text as target_node_type_code, + mention.post_id::text || '/' || mention.project_key as target_node_id, + 'edge_mention_project'::text as edge_type_code, + 'truth_proposed'::text as truth_status_code, + greatest(post.created_at, mention.created_at) as available_at, + array[mention.post_id::text] as evidence_ids + from post_project_mention mention + join source_post post on post.post_id = mention.post_id + where mention.post_id = any($1::uuid[]) + and ($6::timestamptz is null + or greatest(post.created_at, mention.created_at) <= $6::timestamptz) + and ($7::timestamptz is null + or greatest(post.created_at, mention.created_at) <= $7::timestamptz) ), reachable(node_type_code, node_id, depth) as ( values ($2::text, $3::text, 0) union @@ -326,6 +437,7 @@ async def _load_facts( candidate.target_node_type_code, candidate.target_node_id, candidate.edge_type_code, + candidate.truth_status_code, candidate.available_at, candidate.evidence_ids, min(reachable.depth) as hop_depth @@ -341,6 +453,7 @@ async def _load_facts( candidate.target_node_type_code, candidate.target_node_id, candidate.edge_type_code, + candidate.truth_status_code, candidate.available_at, candidate.evidence_ids ) @@ -349,6 +462,7 @@ async def _load_facts( target_node_type_code, target_node_id, edge_type_code, + truth_status_code, available_at, evidence_ids, hop_depth @@ -386,6 +500,10 @@ async def _load_facts( facts: list[NeighborhoodFact] = [] source_keys_by_edge: dict[tuple[str, str, str, str, str], OntologySourceKey] = {} for row in page_rows: + try: + truth_status_code = row["truth_status_code"] + except (KeyError, IndexError): + truth_status_code = "truth_observed" fact = fact_from_knowledge_graph_edge( source_node_type_code=row["source_node_type_code"], source_node_id=str(row["source_node_id"]), @@ -394,7 +512,12 @@ async def _load_facts( edge_type_code=row["edge_type_code"], recorded_at=row["available_at"], evidence_references=tuple(row["evidence_ids"] or ()), - provenance_reference="knowledge_graph_edge", + provenance_reference=( + "post_project_mention" + if row["edge_type_code"] == EDGE_MENTION_PROJECT + else "knowledge_graph_edge" + ), + truth_status_code=truth_status_code, ) try: hop_depth = row["hop_depth"] @@ -461,7 +584,11 @@ async def _load_skos_facts( async def _load_labels( - conn: asyncpg.Connection, facts: list[NeighborhoodFact] + conn: asyncpg.Connection, + facts: list[NeighborhoodFact], + *, + knowledge_cutoff: datetime | None = None, + snapshot_at: datetime | None = None, ) -> dict[tuple[str, str], str]: """Load only non-empty buyer-visible labels for fact endpoints.""" ids_by_type = _node_ids_by_type(facts) @@ -469,6 +596,7 @@ async def _load_labels( post_ids = ids_by_type[NODE_POST] corp_ids = ids_by_type[NODE_CORPORATE_ENTITY] team_ids = ids_by_type[NODE_TEAM] + project_ids = ids_by_type[NODE_PROJECT] labels: dict[tuple[str, str], str] = {} if person_ids: for row in await conn.fetch( @@ -499,6 +627,40 @@ async def _load_labels( ): if row["team_name"]: labels[(NODE_TEAM, str(row["team_id"]))] = str(row["team_name"]) + if project_ids: + evidence_post_ids = sorted( + { + post_id + for fact in facts + for post_id in fact.evidence_references + if fact.source_node_type_code == NODE_PROJECT + or fact.target_node_type_code == NODE_PROJECT + } + ) + if evidence_post_ids: + for row in await conn.fetch( + """ + select mention.post_id::text || '/' || mention.project_key as node_id, + mention.project_name as display_label + from post_project_mention mention + join source_post post on post.post_id = mention.post_id + where mention.post_id::text || '/' || mention.project_key = any($1::text[]) + and mention.post_id = any($2::uuid[]) + and ($3::timestamptz is null + or greatest(post.created_at, mention.created_at) <= $3::timestamptz) + and ($4::timestamptz is null + or greatest(post.created_at, mention.created_at) <= $4::timestamptz) + group by mention.post_id, mention.project_key, mention.project_name + """, + project_ids, + evidence_post_ids, + knowledge_cutoff, + snapshot_at, + ): + if row["display_label"]: + labels[(NODE_PROJECT, str(row["node_id"]))] = str( + row["display_label"] + ) return labels @@ -623,16 +785,19 @@ async def visible_ontology_neighborhood( ) if not focus_node_id or focus_node_id.strip() != focus_node_id: raise OntologyNeighborhoodError("invalid_focus_id", "focus node id is empty or malformed") - if not _is_uuid(focus_node_id): - raise OntologyNeighborhoodError("invalid_focus_id", "focus node id is not a UUID") - focus_node_id = str(UUID(focus_node_id)) + if focus_node_type_code == NODE_PROJECT: + try: + parse_project_candidate_node_id(focus_node_id) + except ValueError as exc: + raise OntologyNeighborhoodError( + "invalid_focus_id", "project focus id is not a post-scoped candidate id" + ) from exc + else: + if not _is_uuid(focus_node_id): + raise OntologyNeighborhoodError("invalid_focus_id", "focus node id is not a UUID") + focus_node_id = str(UUID(focus_node_id)) if not await focus_catalog_exists(conn, focus_node_type_code, focus_node_id): raise OntologyNeighborhoodError("unknown_node_type", "focus node not found") - visible_post_ids = await visible_post_ids_for_focus( - conn, focus_node_type_code, focus_node_id, can_see_post - ) - if not visible_post_ids: - raise OntologyNeighborhoodError("focus_not_visible", "focus node is not visible") secret = source_cursor_secret_from_env(source_cursor_secret) snapshot_at = datetime.now(timezone.utc) after_key: OntologySourceKey | None = None @@ -660,6 +825,16 @@ async def visible_ontology_neighborhood( after_key = source_cursor_claims.last_key elif cursor is not None and not cursor.startswith("after:"): raise OntologyNeighborhoodError("malformed_cursor", "cursor must be an opaque after: or source token") + visible_post_ids = await visible_post_ids_for_focus( + conn, + focus_node_type_code, + focus_node_id, + can_see_post, + knowledge_cutoff=knowledge_cutoff, + snapshot_at=snapshot_at, + ) + if not visible_post_ids: + raise OntologyNeighborhoodError("focus_not_visible", "focus node is not visible") fact_window = await _load_facts( conn, visible_post_ids, @@ -686,7 +861,8 @@ async def visible_ontology_neighborhood( if not endpoint_keys: break visible_by_node = await _visible_post_ids_by_nodes( - conn, endpoint_keys, can_see_post + conn, endpoint_keys, can_see_post, + knowledge_cutoff=knowledge_cutoff, snapshot_at=snapshot_at, ) candidate_post_ids = loaded_post_ids | { post_id @@ -724,7 +900,8 @@ async def visible_ontology_neighborhood( } if endpoint_keys: visible_by_node = await _visible_post_ids_by_nodes( - conn, endpoint_keys, can_see_post + conn, endpoint_keys, can_see_post, + knowledge_cutoff=knowledge_cutoff, snapshot_at=snapshot_at, ) frozen_posts = sorted(loaded_post_ids) if source_cursor_claims is not None: @@ -781,7 +958,10 @@ async def visible_ontology_neighborhood( # Continuation pages can introduce endpoints absent from the first # window. Rebuild the authorization cache for the actual page before # discarding unseen relations. - visible_by_node = await _visible_post_ids_by_nodes(conn, endpoint_keys, can_see_post) + visible_by_node = await _visible_post_ids_by_nodes( + conn, endpoint_keys, can_see_post, + knowledge_cutoff=knowledge_cutoff, snapshot_at=snapshot_at, + ) corp_ids = [ fact.source_node_id if fact.source_node_type_code == NODE_CORPORATE_ENTITY else fact.target_node_id for fact in facts @@ -799,7 +979,10 @@ async def visible_ontology_neighborhood( } missing_parent_keys = {key for key in parent_keys if key not in visible_by_node} if missing_parent_keys: - parent_visible = await _visible_post_ids_by_nodes(conn, missing_parent_keys, can_see_post) + parent_visible = await _visible_post_ids_by_nodes( + conn, missing_parent_keys, can_see_post, + knowledge_cutoff=knowledge_cutoff, snapshot_at=snapshot_at, + ) visible_by_node.update(parent_visible) hidden_node_keys: set[str] = set() authorized_facts: list[NeighborhoodFact] = [] @@ -829,7 +1012,12 @@ async def visible_ontology_neighborhood( if authorized: authorized_facts.append(fact) facts = authorized_facts - labels = await _load_labels(conn, facts) + labels = await _load_labels( + conn, + facts, + knowledge_cutoff=knowledge_cutoff, + snapshot_at=snapshot_at, + ) if hasattr(conn, "fetchval"): if focus_node_type_code == NODE_POST: title = await conn.fetchval("select post_title from source_post where post_id = $1", focus_node_id) @@ -848,7 +1036,7 @@ async def visible_ontology_neighborhood( ) if name: labels[(NODE_CORPORATE_ENTITY, focus_node_id)] = name - else: + elif focus_node_type_code == NODE_TEAM: name = await conn.fetchval( "select team_name from cataloged_team where team_id = $1", focus_node_id ) diff --git a/backend/app/post_chat_ingestion.py b/backend/app/post_chat_ingestion.py index 4ca9e2f2d..4a208c13c 100644 --- a/backend/app/post_chat_ingestion.py +++ b/backend/app/post_chat_ingestion.py @@ -18,6 +18,7 @@ from __future__ import annotations import asyncio +import math from dataclasses import dataclass from datetime import date, datetime from typing import Any, Callable, Iterable @@ -26,6 +27,7 @@ import asyncpg from lineageweave.ask_time_axis import row_matches_time_range, time_axis_evidence_fact +from lineageweave.claim_verification import GlobalAskSourceDocument from lineageweave.embedding_client import EmbeddingClient, NullEmbeddingClient from lineageweave.image_content import ImageContentClient, NullImageContentClient from lineageweave.knowledge_graph import ( @@ -36,6 +38,7 @@ random_walk_with_restart, select_related_nodes, ) +from lineageweave.ontology import all_declared_lookup_codes, ontology_annotations from lineageweave.post_chat import ( CANONICAL_CHAT_QUESTION, CANONICAL_COMMITMENT_QUESTION, @@ -44,11 +47,13 @@ normalize_chat_question, ) from lineageweave.post_content_normalization import normalize_post_body +from lineageweave.rankweave_client import RankWeaveNotAvailable, build_rankweave_client from lineageweave.temporal_expressions import resolve_korean_relative_time +from .config import load_settings from .knowledge_graph import hydrate_related_nodes, load_visible_subgraph from .post_eligibility import SOURCE_POST_ELIGIBILITY_SQL -from lineageweave.ontology import ontology_annotations +from .source_post_revision import fetch_known_at_revisions @dataclass(frozen=True) @@ -78,21 +83,25 @@ async def _normalize_post_body_text( async def _graph_facts_for_posts( conn: asyncpg.Connection, visible_post_ids: list[str], -) -> tuple[str, ...]: - """Render persisted, ontology-annotated graph facts for visible posts. + knowledge_cutoff: datetime | None = None, +) -> dict[str, tuple[str, ...]]: + """Render graph facts under each visible post that evidences them. The evidence join is deliberate: a graph edge without a visible evidence post must never enter an LLM prompt. This is the chat-side trust boundary - in addition to the post-level ABAC check. + in addition to the post-level ABAC check. Keeping the evidence-post mapping + also prevents a fact evidenced by one source from being rendered beneath a + different source and then cited as though that source supported it. """ - if not visible_post_ids: - return () + if not visible_post_ids or knowledge_cutoff is not None: + return {} edge_rows = await conn.fetch( """ select edge.source_node_type_code, edge.source_node_id, edge.target_node_type_code, edge.target_node_id, edge.edge_type_code, edge.edge_weight, - array_agg(distinct evidence.evidence_post_id::text) as evidence_post_ids + array_agg(distinct evidence.evidence_post_id::text + order by evidence.evidence_post_id::text) as evidence_post_ids from knowledge_graph_edge edge join knowledge_graph_edge_evidence evidence on evidence.knowledge_graph_edge_id = edge.knowledge_graph_edge_id @@ -107,7 +116,23 @@ async def _graph_facts_for_posts( visible_post_ids, ) if not edge_rows: - return () + return {} + + visible_post_id_set = frozenset(visible_post_ids) + edge_rows = [ + row + for row in edge_rows + if not ( + row["source_node_type_code"] == NODE_POST + and str(row["source_node_id"]) not in visible_post_id_set + ) + and not ( + row["target_node_type_code"] == NODE_POST + and str(row["target_node_id"]) not in visible_post_id_set + ) + ] + if not edge_rows: + return {} endpoint_keys = { node_key(row["source_node_type_code"], str(row["source_node_id"])) @@ -125,7 +150,8 @@ async def _graph_facts_for_posts( for item in hydrated } - facts: list[str] = [] + facts_by_post: dict[str, list[str]] = {} + fact_count = 0 for row in edge_rows: source_type = row["source_node_type_code"] source_id = str(row["source_node_id"]) @@ -140,13 +166,20 @@ async def _graph_facts_for_posts( edge_name = row["edge_type_code"] if ontology_iri: edge_name = f"{edge_name} ({ontology_iri})" - evidence_ids = ",".join(sorted(str(value) for value in row["evidence_post_ids"])) - facts.append( + fact_prefix = ( f'{source_type} "{source["label"]}" ' f'--{edge_name}--> {target_type} "{target["label"]}" ' - f"[evidence_post_id={evidence_ids}]" ) - return tuple(dict.fromkeys(facts)) + for evidence_post_id in row["evidence_post_ids"]: + post_id = str(evidence_post_id) + post_facts = facts_by_post.setdefault(post_id, []) + fact = f"{fact_prefix}[evidence_post_id={post_id}]" + if fact not in post_facts: + post_facts.append(fact) + fact_count += 1 + if fact_count >= 64: + return {key: tuple(value) for key, value in facts_by_post.items()} + return {key: tuple(value) for key, value in facts_by_post.items()} _SOURCE_HINT_FIELDS = ( @@ -197,7 +230,9 @@ def _source_hint_facts(row: Any) -> tuple[str, ...]: async def _semantic_facts_for_posts( - conn: asyncpg.Connection, post_ids: list[str] + conn: asyncpg.Connection, + post_ids: list[str], + knowledge_cutoff: datetime | None = None, ) -> dict[str, tuple[str, ...]]: """Load persisted project/role/Keyman facts for already-visible posts.""" if not post_ids: @@ -213,6 +248,7 @@ async def _semantic_facts_for_posts( || ' [provenance=post_project_mention]' as fact from post_project_mention where post_id = any($1::uuid[]) + and ($2::timestamptz is null or created_at <= $2) union all select post_id::text as post_id, 'actor: ' || left(actor_name, 200) @@ -221,6 +257,7 @@ async def _semantic_facts_for_posts( || ' [provenance=post_summary_role]' as fact from post_summary_role where post_id = any($1::uuid[]) + and $2::timestamptz is null union all select mention.post_id::text as post_id, 'Keyman mention: ' || left(person.person_name, 200) @@ -229,9 +266,11 @@ async def _semantic_facts_for_posts( from post_person_mention mention join cataloged_person person on person.person_id = mention.person_id where mention.post_id = any($1::uuid[]) + and $2::timestamptz is null order by post_id, fact """, post_ids, + knowledge_cutoff, ) facts: dict[str, list[str]] = {} for row in rows: @@ -322,14 +361,6 @@ async def gather_chat_sources( this_post["post_body"], vision_client, ) - sources = [ - ChatSourceDocument( - source_id, - this_post["post_title"], - normalized_body, - evidence_facts=_source_hint_facts(this_post) + semantic_facts.get(source_id, ()), - ) - ] linked = await find_linked_post_ids(conn, post_id) candidate_ids = [ @@ -337,7 +368,17 @@ async def gather_chat_sources( *sorted(linked.indirect), ][:_POST_CHAT_CANDIDATE_LIMIT] if not candidate_ids: - return sources + graph_facts = await _graph_facts_for_posts(conn, [source_id]) + return [ + ChatSourceDocument( + source_id, + this_post["post_title"], + normalized_body, + graph_facts=graph_facts.get(source_id, ()), + evidence_facts=_source_hint_facts(this_post) + + semantic_facts.get(source_id, ()), + ) + ] rows = await conn.fetch( "select post_id, post_title, post_body, visibility_code, corporate_entity_id, process_unit_id, " @@ -363,13 +404,15 @@ async def gather_chat_sources( semantic_facts = await _semantic_facts_for_posts(conn, visible_source_ids) graph_facts = await _graph_facts_for_posts(conn, visible_source_ids) - sources[0] = ChatSourceDocument( - sources[0].post_id, - sources[0].post_title, - sources[0].post_body, - graph_facts=graph_facts, - evidence_facts=sources[0].evidence_facts, - ) + sources = [ + ChatSourceDocument( + source_id, + this_post["post_title"], + normalized_body, + graph_facts=graph_facts.get(source_id, ()), + evidence_facts=_source_hint_facts(this_post) + semantic_facts.get(source_id, ()), + ) + ] for row in visible_rows: normalized_body = await _normalize_post_body_text(row["post_body"], vision_client) sources.append( @@ -377,6 +420,7 @@ async def gather_chat_sources( str(row["post_id"]), row["post_title"], normalized_body, + graph_facts=graph_facts.get(str(row["post_id"]), ()), evidence_facts=_source_hint_facts(row) + semantic_facts.get(str(row["post_id"]), ()), ) @@ -385,6 +429,71 @@ async def gather_chat_sources( return sources +async def prepare_global_question_embedding( + question: str, + embedding_client: EmbeddingClient, +) -> tuple[list[float], str, float] | None: + """Resolve one question embedding without holding a database connection.""" + if not question.strip() or not embedding_client.available: + return None + try: + question_vector = await asyncio.to_thread(embedding_client.embed, question) + except (OSError, RuntimeError, ValueError): + return None + return _validated_question_embedding( + question_vector, embedding_client.resolved_model + ) + + +def _validated_question_embedding( + question_vector: list[float], embedding_model_code: str | None +) -> tuple[list[float], str, float] | None: + """Return a finite, non-zero embedding envelope or fail closed.""" + if ( + not question_vector + or not embedding_model_code + or any(not math.isfinite(value) for value in question_vector) + ): + return None + question_norm = math.sqrt(sum(value * value for value in question_vector)) + if not math.isfinite(question_norm) or question_norm == 0.0: + return None + return question_vector, embedding_model_code, question_norm + + +def _ontology_lookup_codes_in_question(question: str) -> list[str]: + """Return ontology lookup codes whose complete canonical IRI is cited.""" + folded_question = question.casefold() + matched: list[str] = [] + for lookup_code in sorted(all_declared_lookup_codes()): + ontology_iri = ontology_annotations(lookup_code).get("ontology_iri") + if ontology_iri and ontology_iri.casefold() in folded_question: + matched.append(lookup_code) + return matched + + +def _fuse_global_candidate_ids( + embedding_ids: list[str], evidence_ids: list[str], limit: int +) -> list[str]: + """Fuse two owned rank lists with RankWeave parameter-free RRF.""" + if not embedding_ids: + return evidence_ids[:limit] + if not evidence_ids: + return embedding_ids[:limit] + channels = {"embedding": embedding_ids, "evidence": evidence_ids} + titles_by_id = { + post_id: post_id + for post_id in dict.fromkeys([*embedding_ids, *evidence_ids]) + } + try: + fused = build_rankweave_client( + disabled=load_settings().rankweave_disabled + ).fuse_rankings(channels, titles_by_id) + except RankWeaveNotAvailable: + return embedding_ids[:limit] + return [item.post_id for item in fused.items[:limit]] + + async def gather_global_chat_sources( conn: asyncpg.Connection, can_see_post: Callable[[asyncpg.Record], bool], @@ -394,8 +503,11 @@ async def gather_global_chat_sources( embedding_client: EmbeddingClient | None = None, *, question: str | None = None, + search_phrases: tuple[str, ...] | None = None, + question_embedding: tuple[list[float], str, float] | None = None, limit: int = 4, today: date | None = None, + knowledge_cutoff: datetime | None = None, ) -> list[ChatSourceDocument]: """Assemble a bounded, ABAC-filtered source set for Global Ask. @@ -412,11 +524,13 @@ async def gather_global_chat_sources( or no expression at all applies no date filter. Cited sources name which clock matched (ADR 0202). - Candidates are ranked by the maximum cosine similarity between the - question embedding and each post's persisted semantic-unit embeddings. - The embedding model and dimension must match exactly. An unavailable - channel or incomplete persisted vectors returns no source instead of - falling back to lexical matching. + Embedding candidates use maximum cosine similarity with exact model and + dimension agreement. Persisted semantic/KG evidence remains available + when that channel is unavailable; title/body lexical fallback does not. + A cutoff instead retrieves retained revisions plus timestamped project and + ontology-edge evidence. Current-only embeddings, roles, Keymen, graph + labels, lineage, images, and source hints are excluded rather than + back-projected into history. """ if limit <= 0: return [] @@ -427,23 +541,95 @@ async def gather_global_chat_sources( resolved_time_range = resolve_korean_relative_time( question or "", today=today or _seoul_today() ) - if not (question and question.strip() and embedding_client.available): + if not (question and question.strip()): return [] - try: - question_vector = await asyncio.to_thread(embedding_client.embed, question) - except (OSError, RuntimeError, ValueError): - return [] - if not question_vector: - return [] - embedding_model_code = embedding_client.resolved_model - if not embedding_model_code: - return [] - question_norm = sum(value * value for value in question_vector) ** 0.5 - if question_norm == 0.0: + retrieval_phrases = search_phrases or (question,) + supplied_question_embedding = question_embedding is not None + if knowledge_cutoff is not None: + question_embedding = None + supplied_question_embedding = False + if question_embedding is None and knowledge_cutoff is None: + question_embedding = await prepare_global_question_embedding( + question, embedding_client + ) + validated_embedding = ( + _validated_question_embedding(question_embedding[0], question_embedding[1]) + if question_embedding is not None + else None + ) + embedding_enabled = validated_embedding is not None + if supplied_question_embedding and not embedding_enabled: return [] + question_vector, embedding_model_code, question_norm = validated_embedding or ( + [], + "", + 1.0, + ) # Safe SQL: the only interpolation is the repository-owned eligibility # expression; all request and model values remain asyncpg parameters. - candidate_rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + if knowledge_cutoff is not None: + candidate_rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" + with evidence_query as ( + select websearch_to_tsquery('simple', phrase) as terms + from unnest($1::text[]) as phrase + ), evidence_post_candidates as ( + select revision.post_id + from source_post_revision revision, evidence_query query + where revision.written_at <= $2 + and (revision.superseded_at is null or revision.superseded_at > $2) + and to_tsvector( + 'simple', + coalesce(revision.post_title, '') || ' ' || + coalesce(revision.post_body, '') + ) @@ query.terms + union + select project.post_id + from post_project_mention project, evidence_query query + where project.created_at <= $2 + and to_tsvector( + 'simple', + coalesce(project.project_name, '') || ' ' || + coalesce(project.evidence_text, '') || ' ' || + coalesce(project.ontology_iri, '') + ) @@ query.terms + union + select evidence.evidence_post_id + from knowledge_graph_edge edge + join knowledge_graph_edge_evidence evidence + on evidence.knowledge_graph_edge_id = edge.knowledge_graph_edge_id + where edge.created_at <= $2 + and edge.edge_type_code = any($3::text[]) + ) + select 'evidence'::text as candidate_channel, candidate.post_id, + row_number() over ( + order by coalesce(post.event_occurred_at, post.created_at) desc, + candidate.post_id desc + ) as channel_rank + from evidence_post_candidates candidate + join source_post post on post.post_id = candidate.post_id + where post.created_at <= $2 + and (post.visibility_code = 'public' + or (post.corporate_entity_id::text = any($4::text[]) + and (cardinality($5::text[]) = 0 + or post.process_unit_id::text = any($5::text[])))) + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} + and ($6::date is null or (coalesce(post.event_occurred_at, post.created_at) at time zone 'Asia/Seoul')::date >= $6) + and ($7::date is null or (coalesce(post.event_occurred_at, post.created_at) at time zone 'Asia/Seoul')::date <= $7) + order by channel_rank + limit $8 + """, + list(retrieval_phrases), + knowledge_cutoff, + _ontology_lookup_codes_in_question(question), + list(authorized_corporate_entity_ids), + list(authorized_process_unit_ids), + resolved_time_range[0] if resolved_time_range else None, + resolved_time_range[1] if resolved_time_range else None, + limit, + ) + else: + candidate_rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli f""" with question_vector as ( select ordinality - 1 as dimension_index, dimension_value @@ -464,7 +650,8 @@ async def gather_global_chat_sources( on value.post_content_embedding_id = embedding.post_content_embedding_id join question_vector question on question.dimension_index = value.dimension_index - where embedding.embedding_model_code = $3 + where $11::boolean + and embedding.embedding_model_code = $3 and embedding.embedding_dimension_count = cardinality($1::double precision[]) and (post.visibility_code = 'public' or (post.corporate_entity_id::text = any($4::text[]) @@ -475,14 +662,146 @@ async def gather_global_chat_sources( and ($7::date is null or (coalesce(post.event_occurred_at, post.created_at) at time zone 'Asia/Seoul')::date <= $7) group by unit.post_id, embedding.post_content_embedding_id having count(*) = cardinality($1::double precision[]) + ), embedding_candidates as ( + select similarity.post_id, + max(similarity.cosine_similarity) as semantic_score, + max(coalesce(post.event_occurred_at, post.created_at)) as event_clock + from unit_similarity similarity + join source_post post on post.post_id = similarity.post_id + group by similarity.post_id + order by semantic_score desc, event_clock desc, similarity.post_id desc + limit $8 + ), evidence_query as ( + select websearch_to_tsquery('simple', phrase) as terms + from unnest($9::text[]) as phrase + ), matching_nodes as ( + select 'node_person'::text as node_type_code, person.person_id as node_id + from cataloged_person person, evidence_query query + where to_tsvector( + 'simple', + coalesce(person.person_name, '') || ' ' || + coalesce(person.last_known_job_title, '') + ) @@ query.terms + union + select 'node_corporate_entity', entity.corporate_entity_id + from corporate_entity entity, evidence_query query + where to_tsvector( + 'simple', + coalesce(entity.corporate_entity_code, '') || ' ' || + coalesce(entity.entity_name, '') + ) @@ query.terms + union + select 'node_team', team.team_id + from cataloged_team team, evidence_query query + where to_tsvector( + 'simple', + coalesce(team.team_name, '') || ' ' || + coalesce(team.affiliated_organization_name, '') + ) @@ query.terms + union + select 'node_post', endpoint.post_id + from source_post endpoint, evidence_query query + where to_tsvector('simple', coalesce(endpoint.post_title, '')) @@ query.terms + and (endpoint.visibility_code = 'public' + or (endpoint.corporate_entity_id::text = any($4::text[]) + and (cardinality($5::text[]) = 0 + or endpoint.process_unit_id::text = any($5::text[])))) + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='endpoint')} + ), matching_edges as ( + select edge.knowledge_graph_edge_id + from knowledge_graph_edge edge + join common_lookup_value lookup + on lookup.lookup_code = edge.edge_type_code + cross join evidence_query query + where to_tsvector( + 'simple', + coalesce(lookup.lookup_code, '') || ' ' || + coalesce(lookup.lookup_label, '') + ) @@ query.terms + union + select edge.knowledge_graph_edge_id + from knowledge_graph_edge edge + where edge.edge_type_code = any($10::text[]) + union + select edge.knowledge_graph_edge_id + from knowledge_graph_edge edge + join matching_nodes node + on node.node_type_code = edge.source_node_type_code + and node.node_id = edge.source_node_id + union + select edge.knowledge_graph_edge_id + from knowledge_graph_edge edge + join matching_nodes node + on node.node_type_code = edge.target_node_type_code + and node.node_id = edge.target_node_id + ), evidence_post_candidates as ( + select project.post_id + from post_project_mention project, evidence_query query + where to_tsvector( + 'simple', + coalesce(project.project_name, '') || ' ' || + coalesce(project.evidence_text, '') || ' ' || + coalesce(project.ontology_iri, '') + ) @@ query.terms + union + select role.post_id + from post_summary_role role, evidence_query query + where to_tsvector( + 'simple', + coalesce(role.actor_name, '') || ' ' || + coalesce(role.responsibility, '') || ' ' || + coalesce(role.affiliated_organization_name, '') + ) @@ query.terms + union + select mention.post_id + from combined_post_person_mention mention + join cataloged_person person on person.person_id = mention.person_id + cross join evidence_query query + where to_tsvector( + 'simple', + coalesce(person.person_name, '') || ' ' || + coalesce(person.last_known_job_title, '') + ) @@ query.terms + union + select mention.post_id + from combined_post_person_mention mention + join person_affiliation affiliation + on affiliation.person_id = mention.person_id + cross join evidence_query query + where to_tsvector( + 'simple', + coalesce(affiliation.affiliated_organization_name, '') || ' ' || + coalesce(affiliation.role_title, '') + ) @@ query.terms + union + select evidence.evidence_post_id + from matching_edges edge + join knowledge_graph_edge_evidence evidence + on evidence.knowledge_graph_edge_id = edge.knowledge_graph_edge_id + ), authorized_evidence_candidates as ( + select candidate.post_id, + max(coalesce(post.event_occurred_at, post.created_at)) as event_clock + from evidence_post_candidates candidate + join source_post post on post.post_id = candidate.post_id + where (post.visibility_code = 'public' + or (post.corporate_entity_id::text = any($4::text[]) + and (cardinality($5::text[]) = 0 + or post.process_unit_id::text = any($5::text[])))) + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} + and ($6::date is null or (coalesce(post.event_occurred_at, post.created_at) at time zone 'Asia/Seoul')::date >= $6) + and ($7::date is null or (coalesce(post.event_occurred_at, post.created_at) at time zone 'Asia/Seoul')::date <= $7) + group by candidate.post_id + order by event_clock desc, candidate.post_id desc + limit $8 ) - select similarity.post_id, max(similarity.cosine_similarity) as semantic_score, - max(coalesce(post.event_occurred_at, post.created_at)) as event_clock - from unit_similarity similarity - join source_post post on post.post_id = similarity.post_id - group by similarity.post_id - order by semantic_score desc, event_clock desc, similarity.post_id desc - limit $8 + select 'embedding'::text as candidate_channel, post_id, + row_number() over (order by semantic_score desc, event_clock desc, post_id desc) as channel_rank + from embedding_candidates + union all + select 'evidence', post_id, + row_number() over (order by event_clock desc, post_id desc) as channel_rank + from authorized_evidence_candidates + order by candidate_channel, channel_rank """, question_vector, question_norm, @@ -492,8 +811,21 @@ async def gather_global_chat_sources( resolved_time_range[0] if resolved_time_range else None, resolved_time_range[1] if resolved_time_range else None, limit, + list(retrieval_phrases), + _ontology_lookup_codes_in_question(question), + embedding_enabled, + ) + embedding_candidate_ids: list[str] = [] + evidence_candidate_ids: list[str] = [] + for row in candidate_rows: + channel = str(row.get("candidate_channel") or "embedding") + target = ( + evidence_candidate_ids if channel == "evidence" else embedding_candidate_ids + ) + target.append(str(row["post_id"])) + candidate_ids = _fuse_global_candidate_ids( + embedding_candidate_ids, evidence_candidate_ids, limit ) - candidate_ids = [str(row["post_id"]) for row in candidate_rows] candidate_id_set = frozenset(candidate_ids) # One semantic match is still only one event snapshot. Expand the @@ -504,7 +836,7 @@ async def gather_global_chat_sources( # cannot each pull a separate lineage chain into the bounded context. lineage_neighbor_ids: list[str] = [] lineage_anchor_id = candidate_ids[0] if candidate_ids else None - if lineage_anchor_id: + if lineage_anchor_id and knowledge_cutoff is None: lineage_rows = await conn.fetch( "select child_post_id as other_id from post_lineage_edge where parent_post_id = $1 " "union select parent_post_id as other_id from post_lineage_edge where child_post_id = $1", @@ -520,7 +852,7 @@ async def gather_global_chat_sources( candidate_ids = list( dict.fromkeys([lineage_anchor_id, *lineage_neighbor_ids, *candidate_ids[1:]]) )[:limit] - else: + elif not lineage_anchor_id: candidate_ids = [] lineage_neighbor_id_set = frozenset(lineage_neighbor_ids) @@ -534,7 +866,7 @@ async def gather_global_chat_sources( source_process_unit_name, source_sales_pool_code, source_sales_pool_name, source_customer_code, source_customer_name, source_project_code, source_project_name, - created_at, event_occurred_at + created_at, updated_at, event_occurred_at from source_post where (visibility_code = 'public' or (corporate_entity_id::text = any($1::text[]) @@ -542,6 +874,7 @@ async def gather_global_chat_sources( or process_unit_id::text = any($2::text[])))) and source_post.post_id = any($3::uuid[]) and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')} + and ($7::timestamptz is null or source_post.created_at <= $7) and ($5::date is null or (coalesce(event_occurred_at, created_at) at time zone 'Asia/Seoul')::date >= $5) and ($6::date is null or (coalesce(event_occurred_at, created_at) at time zone 'Asia/Seoul')::date <= $6) order by array_position($3::uuid[], post_id) nulls last, @@ -554,6 +887,7 @@ async def gather_global_chat_sources( limit, resolved_time_range[0] if resolved_time_range else None, resolved_time_range[1] if resolved_time_range else None, + knowledge_cutoff, ) visible_rows = [ row @@ -562,33 +896,74 @@ async def gather_global_chat_sources( ][:limit] visible_ids = [str(row["post_id"]) for row in visible_rows] anchor_is_visible = lineage_anchor_id in visible_ids - semantic_facts = await _semantic_facts_for_posts(conn, visible_ids) - graph_facts = (await _graph_facts_for_posts(conn, visible_ids))[:16] + revisions = await fetch_known_at_revisions(conn, visible_ids, knowledge_cutoff) if knowledge_cutoff else {} + semantic_facts = await _semantic_facts_for_posts(conn, visible_ids, knowledge_cutoff) + graph_facts = await _graph_facts_for_posts(conn, visible_ids, knowledge_cutoff) + remaining_graph_facts = 16 time_filter_active = resolved_time_range is not None sources: list[ChatSourceDocument] = [] for index, row in enumerate(visible_rows): - normalized_body = await _normalize_post_body_text(row["post_body"], vision_client) + post_id = str(row["post_id"]) + revision = revisions.get(post_id) if knowledge_cutoff else None + historical_body_unavailable = knowledge_cutoff is not None and revision is None + source_title = ( + revision["post_title"] + if revision is not None + else ("Historical body unavailable" if historical_body_unavailable else row["post_title"]) + ) + source_body = revision["post_body"] if revision is not None else ( + "" if historical_body_unavailable else row["post_body"] + ) + normalized_body = await _normalize_post_body_text(source_body, vision_client) if len(normalized_body) > 4000: normalized_body = ( normalized_body[:4000] + "\n[Source body truncated for Global Ask; open the cited post for the full body.]" ) - post_id = str(row["post_id"]) lineage_fact = ( (f"Event Lineage: reconstructed timeline neighbor of post_id={lineage_anchor_id}",) if post_id in lineage_neighbor_id_set and anchor_is_visible else () ) + post_graph_facts = graph_facts.get(post_id, ())[:remaining_graph_facts] + remaining_graph_facts -= len(post_graph_facts) + source_type = ( + GlobalAskSourceDocument + if row["visibility_code"] == "public" + else ChatSourceDocument + ) + source_arguments: dict[str, Any] = {} + if source_type is GlobalAskSourceDocument: + source_arguments["external_claim_facts"] = ( + semantic_facts.get(post_id, ()) + post_graph_facts + ) sources.append( - ChatSourceDocument( + source_type( post_id, - row["post_title"], + source_title, normalized_body, - graph_facts=graph_facts if index == 0 else (), - evidence_facts=_source_hint_facts(row) + graph_facts=post_graph_facts, + evidence_facts=( + () if knowledge_cutoff is not None else _source_hint_facts(row) + ) + semantic_facts.get(post_id, ()) + lineage_fact + time_axis_evidence_fact(row, time_filter_active=time_filter_active), + source_post_revision_id=( + revision["source_post_revision_id"] if revision is not None else None + ), + evidence_available_at=(revision["written_at"] if revision is not None else None), + knowledge_cutoff=(knowledge_cutoff.isoformat() if knowledge_cutoff else None), + live_changed_after_cutoff=( + knowledge_cutoff is not None and row["updated_at"] > knowledge_cutoff + ), + historical_body_unavailable=historical_body_unavailable, + unavailable_channels=( + ("historical_body", "semantic_role", "semantic_keyman", "knowledge_graph", "lineage", "image") + if historical_body_unavailable + else (("semantic_role", "semantic_keyman", "knowledge_graph", "lineage", "image") if knowledge_cutoff else ()) + ), + **source_arguments, ) ) return sources diff --git a/backend/app/post_eligibility.py b/backend/app/post_eligibility.py index 54e0a551b..2dd7db2c2 100644 --- a/backend/app/post_eligibility.py +++ b/backend/app/post_eligibility.py @@ -1,4 +1,6 @@ -"""Shared source-post eligibility SQL for analysis-facing evidence reads.""" +"""Shared source-post eligibility and visibility contracts.""" + +from collections.abc import Collection, Mapping SOURCE_CONTEXT_COLUMNS = ( "source_author_code", @@ -45,3 +47,26 @@ def source_context_missing_sql(alias: str) -> str: missing_context=source_context_missing_sql("{alias}"), present_context=source_context_present_sql("real_post"), ) + + +def source_post_scope_sql(alias: str) -> str: + """Return the shared ABAC SQL using entity ``$1`` and process-unit ``$2``.""" + return ( + f"({alias}.visibility_code = 'public' or " + f"({alias}.corporate_entity_id::text = any($1::text[]) and " + f"(cardinality($2::text[]) = 0 or " + f"{alias}.process_unit_id::text = any($2::text[]))))" + ) + + +def source_post_visible( + post: Mapping[str, object], + corporate_entity_ids: Collection[str], + process_unit_ids: Collection[str], +) -> bool: + """Apply the same public-or-bound-scope ABAC contract outside SQL.""" + if post["visibility_code"] == "public": + return True + return str(post["corporate_entity_id"]) in corporate_entity_ids and ( + not process_unit_ids or str(post["process_unit_id"]) in process_unit_ids + ) diff --git a/backend/app/relation_verification_ingestion.py b/backend/app/relation_verification_ingestion.py index ad93729a4..2f230d226 100644 --- a/backend/app/relation_verification_ingestion.py +++ b/backend/app/relation_verification_ingestion.py @@ -8,6 +8,7 @@ from __future__ import annotations +import asyncio from collections.abc import Sequence from dataclasses import dataclass @@ -26,6 +27,13 @@ class VerifiedRelation: verification_evidence_post_id: str | None +@dataclass(frozen=True) +class _PendingRelation: + counterparty_entity_name: str + relationship_label: str + internal_evidence_post_id: str | None + + async def _find_internal_evidence_post( conn: asyncpg.Connection, post_id: str, @@ -124,7 +132,11 @@ async def verify_post_relations( row["relationship_label"], visible_corporate_entity_ids, ) - result = client.verify(row["counterparty_entity_name"], row["relationship_label"]) + result = await asyncio.to_thread( + client.verify, + row["counterparty_entity_name"], + row["relationship_label"], + ) await conn.execute( """ update post_counterparty_entity @@ -149,3 +161,76 @@ async def verify_post_relations( ) ) return verified + + +async def verify_post_relations_from_pool( + pool: asyncpg.Pool, + client: RelationVerificationClient, + post_id: str, + visible_corporate_entity_ids: Sequence[str] = (), +) -> list[VerifiedRelation]: + """Verify relations without reserving a DB connection during web I/O.""" + async with pool.acquire() as conn: + rows = await conn.fetch( + """ + select c.counterparty_entity_name, v.lookup_label as relationship_label + from post_counterparty_entity c + join common_lookup_value v on v.lookup_code = c.relationship_type_code + where c.post_id = $1 and c.verification_status_code = 'verify_pending' + order by c.counterparty_entity_name + """, + post_id, + ) + pending = [ + _PendingRelation( + str(row["counterparty_entity_name"]), + str(row["relationship_label"]), + await _find_internal_evidence_post( + conn, + post_id, + row["counterparty_entity_name"], + row["relationship_label"], + visible_corporate_entity_ids, + ), + ) + for row in rows + ] + + verified = [] + for relation in pending: + result = await asyncio.to_thread( + client.verify, + relation.counterparty_entity_name, + relation.relationship_label, + ) + verified.append( + VerifiedRelation( + relation.counterparty_entity_name, + result.status_code, + result.evidence_url, + relation.internal_evidence_post_id, + ) + ) + + persisted = [] + async with pool.acquire() as conn, conn.transaction(): + for relation in verified: + update_status = await conn.execute( + """ + update post_counterparty_entity + set verification_status_code = $3, + verification_evidence_url = $4, + verification_evidence_post_id = $5, + verification_checked_at = now() + where post_id = $1 and counterparty_entity_name = $2 + and verification_status_code = 'verify_pending' + """, + post_id, + relation.counterparty_entity_name, + relation.verification_status_code, + relation.verification_evidence_url, + relation.verification_evidence_post_id, + ) + if update_status == "UPDATE 1": + persisted.append(relation) + return persisted diff --git a/backend/app/report_ingestion.py b/backend/app/report_ingestion.py index 4539710d6..f01c15ae0 100644 --- a/backend/app/report_ingestion.py +++ b/backend/app/report_ingestion.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import re from collections import defaultdict from datetime import datetime, timezone @@ -555,7 +556,8 @@ async def rebuild_period_reports( previous = await load_previous_group_mean(conn, kind, grouping_key, period_code) if previous is not None: previous_means[grouping_key] = previous - bank_report, scored = score_groups_on_shared_metric( + bank_report, scored = await asyncio.to_thread( + score_groups_on_shared_metric, groups, item_bank=item_bank, previous_means=previous_means, diff --git a/backend/app/source_post_revision.py b/backend/app/source_post_revision.py index 489f6f486..3b51e09d3 100644 --- a/backend/app/source_post_revision.py +++ b/backend/app/source_post_revision.py @@ -72,7 +72,7 @@ async def fetch_known_at_revision( cutoff label when no revision covers the clock. """ row = await conn.fetchrow( - "select post_title, post_body, written_at " + "select source_post_revision_id, post_title, post_body, written_at " "from source_post_revision " "where post_id = $1 " "and written_at <= $2 " @@ -85,8 +85,45 @@ async def fetch_known_at_revision( if row is None: return None return { + "source_post_revision_id": str(row["source_post_revision_id"]), "post_title": row["post_title"], "post_body": row["post_body"], "written_at": _iso(row["written_at"]), "as_of": _iso(as_of), } + + +async def fetch_known_at_revisions( + conn: "asyncpg.Connection", + post_ids: list[str], + as_of: datetime, +) -> dict[str, dict[str, str]]: + """Batch-load the retained revision covering ``as_of`` for each post. + + Missing posts stay absent so callers can report an honest historical-body + limitation without substituting the live title or body. + """ + + if not post_ids: + return {} + rows = await conn.fetch( + "select distinct on (post_id) post_id, source_post_revision_id, " + "post_title, post_body, written_at " + "from source_post_revision " + "where post_id = any($1::uuid[]) " + "and written_at <= $2 " + "and (superseded_at is null or superseded_at > $2) " + "order by post_id, written_at desc", + post_ids, + as_of, + ) + return { + str(row["post_id"]): { + "source_post_revision_id": str(row["source_post_revision_id"]), + "post_title": row["post_title"], + "post_body": row["post_body"], + "written_at": _iso(row["written_at"]), + "as_of": _iso(as_of), + } + for row in rows + } diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 45ed5d9d4..892c8231a 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -28,7 +28,6 @@ from lineageweave.http_client import HttpClientError, get_json, post_form from lineageweave.knowledge_graph import knowledge_graph_edges_for_post -from lineageweave.post_chat import ChatSourceDocument from lineageweave.post_summary import POST_SUMMARY_CONTRACT_VERSION _POSTGRES_ADMIN_DSN = os.environ.get( @@ -192,6 +191,21 @@ / "migrations" / "0203_global_ask_authorization_scope.sql" ) +_GLOBAL_ASK_EVIDENCE_SEARCH_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0210_global_ask_evidence_search_indexes.sql" +) +_GLOBAL_ASK_PUBLIC_VERIFICATION_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0218_global_ask_public_verification.sql" +) +_GLOBAL_ASK_KNOWLEDGE_CUTOFF_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0212_global_ask_knowledge_cutoff.sql" +) _LEFTOVER_MAP_AXIS_MIGRATION = ( Path(__file__).resolve().parents[2] / "migrations" @@ -296,6 +310,9 @@ def seeded_db(demo_analyst_token): db_dsn = _POSTGRES_ADMIN_DSN.rsplit("/", 1)[0] + f"/{db_name}" conn = psycopg2.connect(db_dsn) + # CREATE INDEX CONCURRENTLY in the production migration stream is + # intentionally applied outside a transaction (psql -X per file). + conn.autocommit = True try: with conn.cursor() as cur: cur.execute(_MIGRATION_PATH.read_text()) @@ -366,6 +383,9 @@ def seeded_db(demo_analyst_token): cur.execute(_LEFTOVER_MAP_COVERAGE_MIGRATION.read_text()) cur.execute(_GLOBAL_ASK_JOB_MIGRATION.read_text()) cur.execute(_GLOBAL_ASK_SCOPE_MIGRATION.read_text()) + cur.execute(_GLOBAL_ASK_EVIDENCE_SEARCH_MIGRATION.read_text()) + cur.execute(_GLOBAL_ASK_KNOWLEDGE_CUTOFF_MIGRATION.read_text()) + cur.execute(_GLOBAL_ASK_PUBLIC_VERIFICATION_MIGRATION.read_text()) cur.execute(_EVENT_OCCURRED_AT_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_AXIS_MIGRATION.read_text()) cur.execute(_CHANNEL_EVIDENCE_MIGRATION.read_text()) @@ -412,6 +432,8 @@ def seeded_db(demo_analyst_token): "('prov_agent_type', 'prov_organization', 'Organization'), " "('prov_agent_type', 'prov_team', 'Team')" ) + conn.commit() + conn.autocommit = False cur.execute( "insert into corporate_entity (corporate_entity_code, entity_name, entity_level_code) " "values ('TEST-GROUP', 'Test Group', 'group') returning corporate_entity_id" @@ -569,7 +591,11 @@ def _seed_analysis_run( ) other_account_id = cur.fetchone()[0] visible_run_id = _seed_analysis_run( - "0" * 64, + # The TEPP anchor above already owns the all-``a`` digest. + # Keep each synthetic snapshot distinct so the database's + # content-addressed uniqueness constraint is exercised rather + # than tripping during fixture setup. + "8" * 64, "visible-own-corp", account_id, "analysis_scope_corporate_entity", @@ -3946,20 +3972,12 @@ class _FailingAskClient: def answer(self, question: str, sources) -> object: raise Exception("raw-global-provider-secret") - async def _source(*_args, **_kwargs): - return [ - ChatSourceDocument( - seeded_db["own_private_post_id"], "Authorized source", "Evidence" - ) - ] - - monkeypatch.setattr("backend.app.global_ask_queue.gather_global_chat_sources", _source) monkeypatch.setattr("backend.app.main._post_chat_client", lambda **_kwargs: _FailingAskClient()) headers = {"Authorization": f"Bearer {demo_analyst_token}"} submitted = client.post( "/api/ask", - json={"question": "What happened in this global failure case?"}, + json={"question": "Public post"}, headers=headers, ) assert submitted.status_code == 202 @@ -5060,6 +5078,27 @@ def test_ask_rejects_an_empty_question(client, demo_analyst_token, seeded_db) -> assert response.status_code == 422 +def test_ask_rejects_invalid_or_future_knowledge_cutoffs( + client, demo_analyst_token, seeded_db +) -> None: + """The HTTP trust boundary accepts only a valid clock no later than DB now.""" + + headers = {"Authorization": f"Bearer {demo_analyst_token}"} + invalid = client.post( + "/api/ask", + json={"question": "What was known?", "knowledge_cutoff": "not-a-clock"}, + headers=headers, + ) + future = client.post( + "/api/ask", + json={"question": "What was known?", "knowledge_cutoff": "2999-01-01T00:00:00Z"}, + headers=headers, + ) + + assert invalid.status_code == 422 + assert future.status_code == 422 + + def test_ask_is_unavailable_without_orchestrator_credentials( client, demo_analyst_token, seeded_db, monkeypatch ) -> None: @@ -5104,18 +5143,10 @@ def answer(self, question, sources): # noqa: ARG002 - contract shape cited_post_ids=(sources[0].post_id,), ) - async def _source(*_args, **_kwargs): - return [ - ChatSourceDocument( - seeded_db["own_private_post_id"], "Authorized source", "Evidence" - ) - ] - - monkeypatch.setattr("backend.app.global_ask_queue.gather_global_chat_sources", _source) monkeypatch.setattr("backend.app.main._post_chat_client", lambda **_kwargs: _FakeChatClient()) headers = {"Authorization": f"Bearer {demo_analyst_token}"} submitted = client.post( - "/api/ask", json={"question": "What happened with the public post?"}, headers=headers + "/api/ask", json={"question": "Public post"}, headers=headers ) assert submitted.status_code == 202 job_id = submitted.json()["ask_job_id"] @@ -5137,6 +5168,86 @@ async def _source(*_args, **_kwargs): assert "lineage_graph" in answer and "cited_post_images" in answer +def test_ask_public_verification_is_opt_in_and_separate_from_post_citations( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """A cited public semantic claim can be refuted without changing its post id.""" + + import time as _time + + from lineageweave import claim_verification as cv + from lineageweave.post_chat import ChatAnswer + + class _FakeChatClient: + available = True + + def answer(self, question, sources): # noqa: ARG002 - contract shape + return ChatAnswer("Internal answer.", (sources[0].post_id,)) + + class _FakeVerificationClient: + available = True + + def verify(self, claim): + return cv.ClaimVerificationResult( + claim.claim_text, + claim.claim_kind, + cv.CLAIM_REFUTED, + "The selected public evidence conflicts with the claim.", + claim.source_post_ids, + ( + cv.ExternalEvidenceDocument( + "Public evidence", + "https://example.com/public-evidence", + "The published record describes a conflicting state.", + ), + ), + ) + + with closing(psycopg2.connect(seeded_db["dsn"])) as conn, conn.cursor() as cur: + cur.execute( + """ + insert into post_project_mention + (post_id, project_key, project_name, evidence_text, confidence, + ontology_iri, extraction_method) + values (%s, 'synthetic-apollo', 'Apollo', 'Public project evidence', + 1.0, 'https://contextualwisdomlab.github.io/LineageWeave/ontology#Project', + 'synthetic_test') + """, + (seeded_db["public_post_id"],), + ) + conn.commit() + + monkeypatch.setattr("backend.app.main._post_chat_client", lambda **_kwargs: _FakeChatClient()) + monkeypatch.setattr( + "backend.app.main._claim_verification_client", + lambda: _FakeVerificationClient(), + ) + headers = {"Authorization": f"Bearer {demo_analyst_token}"} + submitted = client.post( + "/api/ask", + json={"question": "Apollo", "verify_external": True}, + headers=headers, + ) + assert submitted.status_code == 202 + job_id = submitted.json()["ask_job_id"] + + deadline = _time.monotonic() + 30 + body: dict = {} + while _time.monotonic() < deadline: + body = client.get(f"/api/ask/jobs/{job_id}", headers=headers).json() + if body["job_status_code"] in ("succeeded", "failed"): + break + _time.sleep(0.25) + + assert body.get("job_status_code") == "succeeded", body + answer = body["answer"] + assert answer["source_post_ids"] == [seeded_db["public_post_id"]] + assert answer["external_verification_status"] == cv.VERIFICATION_COMPLETED + assert answer["external_claims"][0]["status_code"] == cv.CLAIM_REFUTED + assert answer["cited_post_ids"] == [seeded_db["public_post_id"]] + assert "https://example.com/public-evidence" not in answer["cited_post_ids"] + + def test_ask_job_reads_are_owner_scoped( client, demo_analyst_token, seeded_db, monkeypatch ) -> None: diff --git a/backend/tests/test_config.py b/backend/tests/test_config.py index e3c182a0b..a826fc013 100644 --- a/backend/tests/test_config.py +++ b/backend/tests/test_config.py @@ -36,15 +36,26 @@ def test_oidc_clock_skew_is_bounded(monkeypatch) -> None: raise AssertionError("clock skew above the bound must be rejected") -def test_tepp_transport_url_defaults_empty_and_is_not_a_score(monkeypatch) -> None: - """Missing TEPP_TRANSPORT_URL keeps the channel dropped.""" +def test_tepp_transport_defaults_empty_and_preserve_runtime_credentials(monkeypatch) -> None: + """Missing TEPP transport config drops the channel; a key stays runtime-only.""" monkeypatch.delenv("TEPP_TRANSPORT_URL", raising=False) - assert load_settings().tepp_transport_url == "" + monkeypatch.delenv("TEPP_API_KEY", raising=False) + settings = load_settings() + assert settings.tepp_transport_url == "" + assert settings.tepp_api_key == "" monkeypatch.setenv("TEPP_TRANSPORT_URL", "https://tepp.example/v1/analysis-runs") - monkeypatch.setenv("TEPP_API_KEY", "runtime-only-secret") + monkeypatch.setenv("TEPP_API_KEY", "runtime-test-key") settings = load_settings() assert settings.tepp_transport_url == "https://tepp.example/v1/analysis-runs" - assert settings.tepp_api_key == "runtime-only-secret" + assert settings.tepp_api_key == "runtime-test-key" + + +def test_tepp_api_key_is_runtime_only(monkeypatch) -> None: + """TEPP authentication comes from the process boundary, never source.""" + monkeypatch.delenv("TEPP_API_KEY", raising=False) + assert load_settings().tepp_api_key == "" + monkeypatch.setenv("TEPP_API_KEY", "runtime-only-test-value") + assert load_settings().tepp_api_key == "runtime-only-test-value" def test_keyverse_issuer_overrides_local_keycloak_and_uses_oidc_discovery(monkeypatch) -> None: diff --git a/docker-compose.yml b/docker-compose.yml index e2990df32..d0a2422aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -175,6 +175,7 @@ services: ORCHESTRATOR_API_KEY: ${ORCHESTRATOR_API_KEY:-${CONTEXTUAL_ORCHESTRATOR_TOKEN:-lineageweave-orchestrator-dev-only}} SEARXNG_BASE_URL: http://searxng:8080 TEPP_TRANSPORT_URL: ${TEPP_TRANSPORT_URL:-} + TEPP_API_KEY: ${TEPP_API_KEY:-} CALDAV_BASE_URL: ${CALDAV_BASE_URL:-} NARUON_CALENDAR_BASE_URL: ${NARUON_CALENDAR_BASE_URL:-} NARUON_CALENDAR_SERVICE_TOKEN: ${NARUON_CALENDAR_SERVICE_TOKEN:-} @@ -198,6 +199,58 @@ services: searxng: condition: service_healthy + mcp: + profiles: ["mcp"] + build: + context: . + dockerfile: backend/Dockerfile + command: ["uvicorn", "backend.app.mcp_server:app", "--host", "0.0.0.0", "--port", "8001"] + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-lineageweave}:${POSTGRES_PASSWORD:-lineageweave_dev_only}@postgres:5432/${POSTGRES_DB:-lineageweave} + KEYCLOAK_BASE_URL: http://keycloak:8080 + KEYCLOAK_ISSUER: http://localhost:${KEYCLOAK_PORT:-18080}/realms/lineageweave-demo + KEYCLOAK_REALM: lineageweave-demo + KEYVERSE_ISSUER: ${KEYVERSE_ISSUER:-} + KEYVERSE_CLIENT_ID: ${KEYVERSE_CLIENT_ID:-} + KEYVERSE_AUDIENCE: ${KEYVERSE_AUDIENCE:-} + KEYVERSE_DISCOVERY_URI: ${KEYVERSE_DISCOVERY_URI:-} + KEYVERSE_JWKS_URI: ${KEYVERSE_JWKS_URI:-} + OIDC_ISSUER: ${OIDC_ISSUER:-} + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-} + OIDC_AUDIENCE: lineageweave-api + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-} + OIDC_JWKS_URI: ${OIDC_JWKS_URI:-} + OIDC_CLOCK_SKEW_SECONDS: ${OIDC_CLOCK_SKEW_SECONDS:-5} + VALKEY_URL: redis://valkey:6379/0 + ORCHESTRATOR_BASE_URL: ${ORCHESTRATOR_BASE_URL:-http://orchestrator:8000} + ORCHESTRATOR_API_KEY: ${ORCHESTRATOR_API_KEY:-${CONTEXTUAL_ORCHESTRATOR_TOKEN:-lineageweave-orchestrator-dev-only}} + # Local Keycloak mints this exact fixed audience. Production Keyverse + # deployments configure both values together outside this demo stack. + MCP_RESOURCE_URL: http://localhost:18001/mcp + MCP_AUDIENCE: http://localhost:18001/mcp + MCP_ALLOWED_HOSTS: localhost:*,127.0.0.1:*,mcp:8001 + MCP_ALLOWED_ORIGINS: ${MCP_ALLOWED_ORIGINS:-} + MCP_MAX_REQUEST_BYTES: ${MCP_MAX_REQUEST_BYTES:-65536} + # No guessed quota: operators must supply values justified by the k6 + # capacity artifact for their deployment before enabling this profile. + MCP_RATE_LIMIT_REQUESTS: ${MCP_RATE_LIMIT_REQUESTS:-} + MCP_RATE_LIMIT_WINDOW_SECONDS: ${MCP_RATE_LIMIT_WINDOW_SECONDS:-} + ports: + - "18001:8001" + depends_on: + postgres: + condition: service_healthy + database_migration: + condition: service_completed_successfully + orchestrator: + condition: service_healthy + keycloak: + condition: service_started + valkey: + condition: service_healthy + backend: + condition: service_started + frontend: build: context: ./frontend diff --git a/docker/keycloak/realm-export.json b/docker/keycloak/realm-export.json index be9826ea4..98d04881b 100644 --- a/docker/keycloak/realm-export.json +++ b/docker/keycloak/realm-export.json @@ -33,6 +33,16 @@ "access.token.claim": "true" } }, + { + "name": "lineageweave-mcp-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "config": { + "included.custom.audience": "http://localhost:18001/mcp", + "id.token.claim": "false", + "access.token.claim": "true" + } + }, { "name": "corp-code", "protocol": "openid-connect", diff --git a/docs/adr/0039-global-ask-agent-source-boundary.md b/docs/adr/0039-global-ask-agent-source-boundary.md index c56a8005f..889c0b34f 100644 --- a/docs/adr/0039-global-ask-agent-source-boundary.md +++ b/docs/adr/0039-global-ask-agent-source-boundary.md @@ -16,7 +16,13 @@ control as the product feature. `source_post` rows. Each row is rechecked with the requesting account's `post_read` RBAC and post ABAC predicate before its normalized body enters the context. Persisted Knowledge Graph facts and embedded image normalization use -the existing chat pipeline. The answer is produced only by +the existing chat pipeline. Each Knowledge Graph fact remains attached only +to the visible source post recorded as its evidence; facts are never collected +under the first candidate merely because that post appears first in the prompt. +When a graph endpoint is itself a post, that endpoint must also belong to the +same authorized source window before its label can be hydrated. A visible +evidence post never makes a hidden or out-of-window endpoint post visible. +The answer is produced only by `ContextualOrchestratorPostChatClient`, and citations resolve to the returned source post ids and titles. diff --git a/docs/adr/0047-global-ask-semantic-retrieval.md b/docs/adr/0047-global-ask-semantic-retrieval.md index d7a0954b0..5a6e03722 100644 --- a/docs/adr/0047-global-ask-semantic-retrieval.md +++ b/docs/adr/0047-global-ask-semantic-retrieval.md @@ -13,16 +13,33 @@ find a post while Ask Agent could not. Global Ask embeds the complete natural-language question once through contextual-orchestrator and ranks authorized posts by the maximum raw cosine similarity against their persisted semantic-unit embeddings. Query and unit -vectors must have the same configured embedding model and dimension. No token -extraction, keyword matching, lexical weighting, similarity threshold, or -locally invented channel weight participates in candidate selection. +vectors must have the same configured embedding model and dimension. -The retrieved posts carry their raw source fields and persisted -project/role/Keyman facts into the contextual-orchestrator prompt with -column/table provenance. These facts enrich grounded answering; they do not -become keyword retrieval signals. If the embedding channel or a complete -matching-model vector is unavailable, retrieval returns no evidence rather -than falling back to lexical search. +Persisted project, role/responsibility/affiliation, Keyman, Knowledge Graph +edge/endpoint-label, and ontology-IRI evidence is a second candidate-nomination +channel. PostgreSQL `websearch_to_tsquery('simple', ...)` runs against GIN +expression indexes on the normalized owning tables; it does not copy evidence +into a denormalized search table. A complete canonical ontology IRI in the +question maps through the published lookup-code annotation. A Knowledge Graph +match nominates only `knowledge_graph_edge_evidence.evidence_post_id`, never an +endpoint post merely because that post labels a node. + +Both owned rank lists are bounded independently after the same SQL +visibility, corporate/process scope, source-eligibility, and event-time +predicates. RankWeave combines them with Cormack, Clarke, and Buettcher's +(2009) parameter-free reciprocal rank fusion. No token extractor, similarity +threshold, hand-authored channel preference, or locally invented weight is +allowed. The existing final source-row query and `can_see_post` callback remain +a second authorization check. If RankWeave cannot combine two present +channels, the new evidence channel is dropped and the embedding ranking +remains; a sole available channel needs no fusion. + +The retrieved posts carry their raw source fields and persisted semantic/KG +facts into the contextual-orchestrator prompt with column/table provenance. +Candidate nomination does not make a fact authoritative and does not bypass +the evidence-post mapping. If the embedding channel or a complete +matching-model vector is unavailable, retrieval may use only the persisted +evidence channel; it never falls back to title/body lexical search. Raw source fields remain `hint_only`; the prompt explicitly distinguishes them from resolved ontology assertions. The existing ABAC filter is applied before @@ -31,9 +48,19 @@ semantic evidence is loaded, and the bounded source limit remains in place. ## Consequences - Ask Agent retrieves by semantic-unit meaning without a keyword rule. +- A term present only in normalized semantic, Knowledge Graph, endpoint-label, + or ontology evidence can nominate its authorized evidence post. - A source hint can retrieve a post but cannot silently bind a customer, project, PU, or Keyman. - The orchestrator receives more useful evidence while still receiving only authorized, bounded source documents. - Missing semantic measurement fails closed and cannot silently change the retrieval method. + +## References + +Cormack, G. V., Clarke, C. L. A., & Buettcher, S. (2009). Reciprocal rank +fusion outperforms Condorcet and individual rank learning methods. In +*Proceedings of the 32nd International ACM SIGIR Conference on Research and +Development in Information Retrieval* (pp. 758–759). Association for +Computing Machinery. https://doi.org/10.1145/1571941.1572114 diff --git a/docs/adr/0067-visual-region-vision-agent.md b/docs/adr/0067-visual-region-vision-agent.md index 735ef24dc..cf0dd2292 100644 --- a/docs/adr/0067-visual-region-vision-agent.md +++ b/docs/adr/0067-visual-region-vision-agent.md @@ -34,6 +34,14 @@ instruction such as “extract Keyman” as source-post content. Existing posts are reprocessed only by an explicit operator backfill with selected post IDs; the buyer read path never starts an unbounded VISION job. +The locator contract does not carry an authoritative assertion that a set of +boxes exhausts the source image. LineageWeave therefore requests parent-image +evidence whenever it accepts any proper subregion, even when the returned +rectangles appear to tile the normalized plane. It does not estimate coverage +from sampled points or implement rectangle-union arithmetic locally. A single +explicit `(0, 0, 1, 1)` locator result is treated as the parent image rather +than persisted as a decomposed region. + ## Consequences - Search can attribute a hit to a document image and a specific visual panel. diff --git a/docs/adr/0184-ontology-provenance-explorer.md b/docs/adr/0184-ontology-provenance-explorer.md index 530238687..5b9c254af 100644 --- a/docs/adr/0184-ontology-provenance-explorer.md +++ b/docs/adr/0184-ontology-provenance-explorer.md @@ -5,7 +5,7 @@ **Figma:** File ID `1Su3lDRmiZdcUs47t1QwIX` **Issue:** [#341](https://github.com/ContextualWisdomLab/LineageWeave/issues/341) -**Context:** The workspace DAG is reconstructed Event Lineage (post/record nodes and inferred parent-to-child links). The formal LineageWeave ontology also defines heterogeneous instance types (`Post`, `Person`, `CorporateEntity`, `Team`) and properties (`mentions`, `affiliatedWith`, `coMentionedWith`, SKOS broader). Calling Event Lineage an ontology graph overstates what that surface renders. PR #330 remains the Event Lineage readability slice and must not become a mixed lineage/ontology graph. +**Context:** The workspace DAG is reconstructed Event Lineage (post/record nodes and inferred parent-to-child links). The formal LineageWeave ontology also defines heterogeneous instance types (`Post`, `Person`, `CorporateEntity`, `Team`, `Project`) and properties (`mentions`, `mentionsProject`, `affiliatedWith`, `coMentionedWith`, SKOS broader). Calling Event Lineage an ontology graph overstates what that surface renders. PR #330 remains the Event Lineage readability slice and must not become a mixed lineage/ontology graph. Project projection is completed by [ADR 0222](0222-project-nodes-in-ontology-neighborhood.md). **Decision:** @@ -19,9 +19,13 @@ value is omitted from JSON-LD and represented as `null` in the typed API; edge truth and edge availability never fill a node field. 5. SKOS broader is projected from `corporate_entity.parent_entity_id`. OWL class subsumption is schema, not an instance neighborhood edge, and fails closed. -6. `knowledge_cutoff` binds `available_time` (`min(source_post.created_at)` of supporting evidence). Current-only facts without a time contract stay out of an as-of response. +6. `knowledge_cutoff` binds `available_time`. An evidence-backed graph edge is + available at the later of its creation and its earliest supporting source; + a project mention is available at the later of its mention and source + creation. Current-only facts without a time contract stay out of an as-of + response. 7. The workspace surface extends the existing Keyman/evidence panel with **Inspect ontology neighborhood**. It is not a second GNB destination. -8. Node type uses shape plus text (never color alone). Every edge carries both endpoint type codes and IDs, so heterogeneous catalogs remain unambiguous even if UUIDs collide. Keyboard users can select every visible node and edge. The graph SVG has no enclosing ARIA `img`. Exact-value table, CSV, JSON-LD, and print expose the same authorized visible graph. +8. Node type uses shape plus text (never color alone). Every edge carries both endpoint type codes and IDs, so heterogeneous catalogs remain unambiguous even if UUIDs collide. Keyboard users can select every visible node and edge. The graph SVG has no enclosing ARIA `img`; native browser text layout wraps complete node labels instead of truncating or estimating character widths. Exact-value table, CSV, JSON-LD, and print expose the same authorized visible graph. JSON-LD emits the source-to-target property assertion directly and describes its evidence-bearing edge as an RDF reified statement with exact `rdf:subject`, `rdf:predicate`, and `rdf:object`; it does not make the edge resource itself the relationship subject. JSON-LD represents system time with `prov:generatedAtTime` and non-null validity bounds as OWL-Time `time:Instant` values using `time:inXSDDateTimeStamp`; it omits unavailable bounds rather than inventing them. 9. Synthetic Storybook frames cover desktop, narrow exact-value-first, node drawer, edge drawer, legend, empty, truncated, denied, stale, and rejected states. No confidential Figma content enters the repository. Storybook inventory records the implementation surface; frame IDs are not copied from the confidential design file (ADR 0002). **Consequences:** diff --git a/docs/adr/0213-global-ask-embedding-pool-release.md b/docs/adr/0213-global-ask-embedding-pool-release.md new file mode 100644 index 000000000..753605f05 --- /dev/null +++ b/docs/adr/0213-global-ask-embedding-pool-release.md @@ -0,0 +1,41 @@ +# ADR 0213 — Global Ask embeds before acquiring a pooled connection + +**Decision status:** Accepted +**Date:** 2026-08-25 +**Related:** [0204](0204-analysis-run-short-transaction-delivery.md) + +## Context + +The authenticated k6 HTTP exercise found ordinary post and Event Lineage +reads waiting while Global Ask jobs called the external embedding provider. +`compute_global_ask_answer` acquired an asyncpg connection before +`gather_global_chat_sources` called that provider, so provider latency could +occupy every slot in the shared ten-connection pool. Moving the call to a +thread kept the event loop responsive but did not release the pool resource. + +## Decision + +Resolve and validate the question embedding before acquiring an asyncpg +connection. Acquire the pool only for the bounded persisted-vector query and +release it before answer generation. An unavailable, empty, unbound, or +zero-norm embedding disables the embedding channel. Persisted semantic and +Knowledge Graph evidence retrieval remains available; LineageWeave does not +substitute lexical retrieval, a local model, or an invented vector. + +The same boundary applies to future provider work: a provider call must not +run inside a pooled-connection context unless one atomic database operation +requires it and an ADR records that exception. + +## Consequences + +- Embedding latency cannot exhaust the shared HTTP database pool. +- Authorization predicates and persisted model/dimension matching remain in + the database query and are unchanged. +- A regression test observes the pool state at the embedding boundary. +- Capacity remains environment-specific; k6 observations do not create an + uncited concurrency or latency threshold. + +## References + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18.6 documentation: +19.4 resource consumption*. https://www.postgresql.org/docs/18/runtime-config-resource.html diff --git a/docs/adr/0215-global-ask-public-claim-verification.md b/docs/adr/0215-global-ask-public-claim-verification.md new file mode 100644 index 000000000..ca70f566e --- /dev/null +++ b/docs/adr/0215-global-ask-public-claim-verification.md @@ -0,0 +1,64 @@ +# ADR 0215: Global Ask verifies eligible public claims outside internal authority + +## Status + +Accepted + +## Context + +ADR 0047 lets normalized semantic and Knowledge Graph evidence nominate an +authorized source post. Nomination and an internal citation do not establish +that a real-world claim is publicly corroborated. Conversely, sending private +post bodies, people facts, measurement payloads, or source hints to a public +search service would cross the authorization boundary. + +FEVER distinguishes supported, refuted, and not-enough-information judgments +and requires cited evidence for the first two. PROV-O requires internal source +evidence, external retrieval evidence, and the verification activity to remain +distinguishable. SearXNG's current Search API supports bounded JSON results from +`GET /search` when that output format is enabled. + +## Decision + +Public verification is explicit opt-in and defaults to false. The choice is +persisted on the asynchronous `global_ask_job`; the worker never reconstructs +consent from later state. + +Only a source post whose persisted `visibility_code` is `public` receives the +`GlobalAskSourceDocument` egress capability. Eligible claims are limited to +project/ontology assertions and non-person Knowledge Graph relations already +carried by a cited public source. Private sources, Keyman/person facts, raw +source hints, source bodies, TEPP artifacts, fast-mlsirm artifacts, prompts, +credentials, and uncited facts never form a public query. + +SearXNG retrieves at most five bounded snippets for at most four claims. Result +URLs must be HTTP(S), must not be search pages, localhost, `.local`, or literal +non-global addresses, and are never fetched by LineageWeave. The untrusted +snippets cross contextual-orchestrator with `mode="verify"` and +`reasoning_effort="auto"`. A supported or refuted response without selected +evidence is downgraded to not enough information. + +External URLs remain `external_claims[].evidence`; internal post identifiers +remain `cited_post_ids`. Verification never mutates ontology, Knowledge Graph, +Event Lineage, TEPP, or fast-mlsirm state. Unconfigured or failed retrieval is +an explicit unavailable state, not a negative judgment. + +## Consequences + +- Readers can request public corroboration without exporting private evidence. +- Conflicting evidence is visible without changing internal graph authority. +- Async HTTP responsiveness is retained; public retrieval runs in the worker. +- Search snippets remain evidence inputs, not trusted instructions or facts. + +## References + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV +ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ + +SearXNG. (2026). *Search API*. https://docs.searxng.org/dev/search_api.html + +Thorne, J., Vlachos, A., Christodoulopoulos, C., & Mittal, A. (2018). FEVER: A +large-scale dataset for fact extraction and verification. In *Proceedings of +the 2018 Conference of the North American Chapter of the Association for +Computational Linguistics: Human Language Technologies* (Vol. 1, pp. 809–819). +Association for Computational Linguistics. https://doi.org/10.18653/v1/N18-1074 diff --git a/docs/adr/0216-global-ask-knowledge-cutoff.md b/docs/adr/0216-global-ask-knowledge-cutoff.md new file mode 100644 index 000000000..484fc4403 --- /dev/null +++ b/docs/adr/0216-global-ask-knowledge-cutoff.md @@ -0,0 +1,50 @@ +# ADR 0216: Global Ask uses retained revisions at a knowledge cutoff + +## Status + +Accepted + +## Context + +Global Ask previously answered only from live source bodies. Filtering posts by +their creation clock does not establish what body or derived semantic evidence +was available at an earlier instant. PROV-O distinguishes an entity from its +specializations and derivations, while OWL-Time defines instants and intervals; +therefore an as-of answer needs a recorded revision interval, not a rewritten +live body presented as historical evidence. + +## Decision + +`POST /api/ask` accepts an optional `knowledge_cutoff` instant no later than the +database clock. The async job persists that instant. Retrieval applies ABAC, +source eligibility, creation/event time, and the cutoff before its candidate +limit, then substitutes the `source_post_revision` whose half-open availability +interval contains the cutoff. + +When no retained revision covers the instant, the response records +`historical_body_unavailable` and does not send the live body to +contextual-orchestrator. Current-only role, Keyman, graph-label, embedding, +image, and Event Lineage projections are excluded until their stores expose a +compatible system-time contract. Timestamped project and ontology-edge +evidence may nominate a post only when their recorded creation time is not +later than the cutoff; the answer still cites the retained source revision. + +Responses expose the cutoff, full/partial grounding status, retained revision +identity and availability time, later-live-change status, and limitations. +Omitting the cutoff preserves the existing live request and response behavior. + +## Consequences + +- A historical answer cannot silently quote a later rewrite. +- Missing historical bodies and semantic channels remain explicit limitations. +- Historical graph/image projections stay unavailable instead of being + reconstructed from current state. +- MCP parity remains a separate delivery requirement on the shared Ask contract. + +## References + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV +ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ + +Cox, S., & Little, C. (Eds.). (2020). *Time ontology in OWL*. World Wide Web +Consortium. https://www.w3.org/TR/owl-time/ diff --git a/docs/adr/0217-evidence-constrained-semantic-query-rewrite.md b/docs/adr/0217-evidence-constrained-semantic-query-rewrite.md new file mode 100644 index 000000000..d65745e3d --- /dev/null +++ b/docs/adr/0217-evidence-constrained-semantic-query-rewrite.md @@ -0,0 +1,59 @@ +# ADR 0217: Evidence-constrained semantic query rewriting + +## Status + +Accepted + +## Context + +Global Ask already fuses embedding and persisted semantic/Knowledge Graph +candidate lists through RankWeave. Its database-native evidence channel passed +the complete conversational question to PostgreSQL +`websearch_to_tsquery('simple', ...)`; retained generic words could therefore +turn a valid semantic fact into a miss. Local stop-word lists, term weights, or +language-specific rules would be uncalibrated heuristics. + +Ma et al. (2023) show that query rewriting can improve retrieval-augmented +language-model retrieval. Their result supports a rewrite stage, but it does +not authorize LineageWeave to invent synonyms, translations, or factual +expansions. ADR 0076 also assigns model discovery, structured synthesis, and +reasoning allocation to contextual-orchestrator. + +## Decision + +Before acquiring a database connection, the asynchronous Global Ask worker may +ask contextual-orchestrator for a strict structured list of retrieval phrases. +Every returned phrase must be a non-empty, exact substring of the submitted +question. The client rejects invented terms, translations, case changes, +non-text values, empty lists, and more than 32 phrases. + +Each accepted phrase becomes its own parameterized PostgreSQL +`websearch_to_tsquery('simple', phrase)`. Matching candidates are unioned and +deduplicated before the existing authorization, eligibility, event-time, +knowledge-cutoff, bounded-channel, and RankWeave fusion boundaries. No phrase +receives a local score or weight. + +An unavailable or invalid rewrite retains the original complete question. It +does not fabricate a phrase, silently broaden access, or disable the embedding +channel. The fallback is an honest lower-recall compatibility path. + +## Consequences + +- Natural-language framing no longer has to occur in persisted ontology or + semantic evidence for an exact question phrase to nominate that evidence. +- The rewrite cannot add a fact absent from the user's question. +- Provider latency stays outside the asyncpg pool, and Global Ask remains an + asynchronous job rather than an interactive blocking request. +- Runtime multilingual recall remains a release-evidence requirement; unit + tests prove the contract and authorization-preserving SQL shape only. + +## References + +Ma, X., Gong, Y., He, P., Zhao, H., & Duan, N. (2023). Query rewriting in +retrieval-augmented large language models. In *Proceedings of the 2023 +Conference on Empirical Methods in Natural Language Processing* (pp. +5303–5315). Association for Computational Linguistics. +https://doi.org/10.18653/v1/2023.emnlp-main.322 + +PostgreSQL Global Development Group. (2026). *Text search functions and +operators*. https://www.postgresql.org/docs/current/functions-textsearch.html diff --git a/docs/adr/0218-current-contract-mcp-global-ask.md b/docs/adr/0218-current-contract-mcp-global-ask.md new file mode 100644 index 000000000..2b2e15f60 --- /dev/null +++ b/docs/adr/0218-current-contract-mcp-global-ask.md @@ -0,0 +1,90 @@ +# ADR 0218: MCP Global Ask submits and reads the durable current Ask contract + +## Status + +Accepted + +## Context + +The protected product exposes Global Ask as a durable asynchronous job. Its +current contract includes account and process-unit scope snapshots, revocation +intersection, evidence-constrained semantic rewriting, explicit public-claim +verification opt-in, retained revisions at a knowledge cutoff, limitations, +and provenance-bearing citations. Historical MCP work implemented a separate +synchronous Ask pipeline on a non-default stack. Reintroducing that pipeline +would let REST and MCP disagree about authorization, time, retrieval, and +verification. + +Remote MCP clients also cross a distinct Streamable HTTP trust boundary. The +MCP transport specification requires Origin validation to prevent DNS +rebinding. OAuth protected-resource metadata and audience-restricted tokens +prevent a token issued for one resource from becoming authority at another. +Browser preflight and request-body admission must therefore happen before +OAuth, JSON parsing, database acquisition, quota consumption, or tool +invocation. + +## Decision + +1. LineageWeave exposes two MCP tools over Streamable HTTP: + `submit_global_ask` queues the same durable job as `POST /api/ask`, and + `read_global_ask_job` returns the same owner-scoped state and settled payload + as `GET /api/ask/jobs/{id}`. MCP does not recreate answer computation. +2. Submission accepts `question`, `verify_external`, and optional + `knowledge_cutoff`. Shared application-service functions own blank-question, + permission, orchestrator-availability, ISO-8601, database-clock, scope + snapshot, and enqueue behavior for both transports. +3. Reading preserves the stored answer payload without a second semantic, + citation, public-verification, or cutoff interpretation. Another account's + job remains indistinguishable from an absent job. +4. Keyverse issues an MCP-resource audience. The resource server validates + issuer, signature, expiry, audience, required scope, and the existing + provisioned LineageWeave account/affiliation contract before a tool runs. + OAuth protected-resource metadata follows RFC 9728 and advertises this exact + resource identifier. +5. An outer admission boundary validates Host and every present Origin, + answers only exact configured browser preflights, rejects ambiguous or + oversized framing while streaming, and replays admitted bytes once. It + exposes browser-readable MCP session/protocol and `WWW-Authenticate` + headers. No-Origin non-browser clients remain supported. +6. A shared Valkey counter consumes one quota unit only after token and + provisioned-account resolution. The key contains a SHA-256 account digest, + never a bearer token or display identifier. Limiter failure is explicitly + unavailable; it never falls back to a process-local counter. The request + limit and window are mandatory positive deployment inputs established by + measured capacity policy, not library defaults. Exhaustion returns the + actual bounded window remainder in structured MCP data and `Retry-After`. +7. MCP runs as a dedicated Compose service and reuses the existing PostgreSQL, + Valkey, Keyverse, contextual-orchestrator, semantic retrieval, and worker + boundaries. It does not add a database, provider call, model selector, or + LineageWeave-local scheduler. + +## Consequences + +- REST, MCP, UI polling, reports, and alerts read one persisted answer contract. +- MCP submission remains responsive while multi-minute orchestration stays in + the existing worker. +- Preflight, hostile transport metadata, malformed framing, and oversized + bodies cannot consume authentication, database, worker, or quota capacity. +- Deployment must supply an evidence-backed quota policy; missing policy fails + startup instead of silently choosing a rule of thumb. +- The historical MCP stacks remain reusable implementation evidence, not + protected-main delivery or a second product contract. + +## References + +Campbell, B., Bradley, J., & Tschofenig, H. (2020). *Resource indicators for +OAuth 2.0* (RFC 8707). Internet Engineering Task Force. +https://doi.org/10.17487/RFC8707 + +Jones, M., Hunt, P., & Parecki, A. (2025). *OAuth 2.0 protected resource +metadata* (RFC 9728). Internet Engineering Task Force. +https://doi.org/10.17487/RFC9728 + +Lodderstedt, T., Bradley, J., Labunets, A., & Fett, D. (2025). *Best current +practice for OAuth 2.0 security* (RFC 9700). Internet Engineering Task Force. +https://doi.org/10.17487/RFC9700 + +Model Context Protocol. (2025). *Transports: Streamable HTTP* (Specification +2025-06-18). +https://modelcontextprotocol.io/specification/2025-06-18/basic/transports + diff --git a/docs/adr/0222-project-nodes-in-ontology-neighborhood.md b/docs/adr/0222-project-nodes-in-ontology-neighborhood.md new file mode 100644 index 000000000..900b76e8e --- /dev/null +++ b/docs/adr/0222-project-nodes-in-ontology-neighborhood.md @@ -0,0 +1,83 @@ +# ADR 0222: Project nodes in the ontology neighborhood + +**Status:** Accepted +**Date:** 2026-08-26 +**Extends:** [ADR 0036](0036-semantic-project-and-keyman-evidence.md), [ADR 0184](0184-ontology-provenance-explorer.md) +**Figma:** File ID `1Su3lDRmiZdcUs47t1QwIX` + +## Context + +The published vocabulary and PRD define `Project` and `mentionsProject`, and +`post_project_mention` already preserves a normalized lexical project key, visible +label, confidence, evidence phrase, ontology IRI, extraction method, and +creation time. The bounded ontology API nevertheless recognizes only Post, +Person, CorporateEntity, and Team. A customer can therefore see project +evidence in a post summary but cannot traverse the same governed assertion in +the ontology neighborhood. That contradicts PRD-FR-2 across PostgreSQL, RDF, +API, and UI. + +## Decision + +1. Register `node_project` and `edge_mention_project` in the governed lookup + vocabulary and map them to canonical `:Project` and `:mentionsProject` + terms. PostgreSQL remains the source of truth; no second project store is + created. +2. `project_key` is a normalized lexical candidate key, not a resolved Project + identity. A Project candidate node id is the exact + `/` pair. The API validates both components + and never merges same-named candidates from different posts. A future + source/tenant-scoped Project catalog may resolve multiple candidates to one + identity through a separate evidence-backed decision; this projection does + not perform that resolution or mint a cross-post identity. + RDF and JSON-LD serialize the pair through the same UTF-8 percent-encoded + canonical node IRI, so multilingual keys cannot denote two resources across + export formats. +3. Each visible `post_project_mention` projects one Post `mentionsProject` + Project fact. Its availability time is the later of source-post creation + and mention persistence. The fact is `truth_proposed`, never observed or + authoritative, because contextual-orchestrator extraction remains a + reviewable semantic candidate. +4. Authorization, source eligibility, knowledge cutoff, snapshot time, + traversal bounds, and cursor ordering apply in the same SQL source window + as every Knowledge Graph fact. A Project focus and every expanded Project + endpoint are authorized only by eligible visible evidence posts. +5. The candidate's source-preserved `project_name` is its display label. Hidden + rows cannot select or alter the label. +6. The UI uses a text-labeled diamond and exposes the same assertion through + the graph, exact-value table, CSV, JSON-LD, print, and evidence drawer. +7. `project_project_mention_rdf` is the deterministic DB-row projection for a + joined `source_post` / `post_project_mention` record. It emits the direct + `mentionsProject` triple and the complete reified `ProjectMention` + subject/predicate/object chain, evidence, confidence, creation time, and + PROV derivation. It performs no database access and creates no mutable RDF + store; callers must still apply authorization before supplying a row. + +## Consequences + +- Project evidence becomes traversable without copying or promoting it. +- Project candidate IDs are intentionally the one composite ontology node + identifier; all other governed node types keep their UUID validation. +- Same-name candidates remain separate until an evidence-backed Project + catalog resolves them. This preserves uncertainty instead of collapsing + unrelated work into one Project node. +- Confidence and evidence text remain on `post_project_mention` and its + summary projection. Adding them to the neighborhood edge needs a separate + typed API decision; this change does not invent edge scores. +- SHACL acceptance now exercises the production row projector rather than a + hand-authored ProjectMention graph that could drift from application IRIs. + +## Verification + +- Ontology/lookup round-trip and SHACL publication tests. +- Assembler and ingestion tests for proposed truth, cutoff-safe availability, + visible-label conflict behavior, and canonical focus validation. +- Frontend interaction, i18n, Storybook build, and desktop/mobile screenshot + review with a synthetic Project node. + +## References + +Cyganiak, R., Wood, D., & Lanthaler, M. (Eds.). (2014). *RDF 1.1 concepts and abstract syntax*. World Wide Web Consortium. https://www.w3.org/TR/rdf11-concepts/ + +Knublauch, H., & Kontokostas, D. (Eds.). (2017). *Shapes constraint language (SHACL)*. World Wide Web Consortium. https://www.w3.org/TR/shacl/ + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ diff --git a/docs/adr/0230-pnpm-build-script-allowlist.md b/docs/adr/0230-pnpm-build-script-allowlist.md new file mode 100644 index 000000000..8433fb4e6 --- /dev/null +++ b/docs/adr/0230-pnpm-build-script-allowlist.md @@ -0,0 +1,36 @@ +# ADR 0230: pnpm build-script allowlist + +**Status:** Accepted +**Date:** 2026-08-26 + +## Context + +The pinned pnpm 11 install fails closed when a dependency requests a lifecycle +build script without an explicit project decision. The pnpm workspace already +allows Vite's locked `esbuild` dependency, but the Docker dependency layer +copied only `package.json` and `pnpm-lock.yaml`. Consequently the exact-head +container build could not see the policy and stopped with +`ERR_PNPM_IGNORED_BUILDS` before compiling the application. + +## Decision + +Allow only the already-locked `esbuild` package through pnpm's `allowBuilds` +workspace setting, and copy that policy into the dependency-install layer of +the frontend image. Keep the exact package-manager and lockfile pins. Do not +enable build scripts globally or add a Docker-only bypass. +Exclude local build and test output from the image context. + +## Consequences + +- Local, CI, and container installs apply one source-controlled policy. +- Any new dependency build script remains rejected until separately reviewed + and explicitly named. + +## Verification + +- `corepack pnpm install --frozen-lockfile` +- exact-head frontend container build + +## References + +pnpm. (2026). *Settings: allowBuilds*. https://pnpm.io/settings#allowbuilds diff --git a/docs/adr/0231-external-lineage-arithmetic-authority.md b/docs/adr/0231-external-lineage-arithmetic-authority.md new file mode 100644 index 000000000..c520cbeb5 --- /dev/null +++ b/docs/adr/0231-external-lineage-arithmetic-authority.md @@ -0,0 +1,66 @@ +# ADR 0231: External lineage arithmetic authority + +**Status:** Accepted +**Date:** 2026-08-26 + +## Context + +LineageWeave is the authorization, provenance, orchestration, persistence, and +evidence-navigation product. It is not the owner of mathematical or +psychometric computation. The current reconstruction path still calculates +temporal, secondary-key, and text scores, renormalizes channel weights, applies +a fixed candidate window of 50, and applies a fixed fused-score floor of 0.3. +Those local defaults are neither an estimator result nor an owning-library +contract, so they cannot remain a production authority. + +## Decision + +LineageWeave will retain source admission, knowledge-cutoff enforcement, +opaque evidence references, run orchestration, provenance persistence, and UI. +It will consume versioned, provenance-bearing results from these owners: + +- TEPP owns temporal-event and psychometric measurement in Rust. +- fast-mlsirm owns multilevel psychometric estimation and estimated weights in + Rust. +- RankWeave owns retrieval fusion, contribution evidence, evaluation, and + policy selection; its calculation core must satisfy the ecosystem Rust-core + requirement before it becomes the production lineage arithmetic boundary. +- ThreadWeave owns deterministic reference-thread assembly. + +The external result contract must carry the immutable implementation revision, +model or policy revision, input-evidence digest, availability/knowledge cutoff, +active channels, estimated parameters, and limitations. LineageWeave validates +and persists that envelope; it does not recompute it. Missing or incompatible +owners make reconstruction unavailable. There is no Python, heuristic, fixed +threshold, fixed window, or hand-authored-weight fallback. + +## Migration + +1. Publish the owning Rust calculation contracts and synthetic consumer + fixtures without database or provider access. +2. Add a fail-closed LineageWeave adapter and exact-envelope persistence. +3. Run parity tests only as migration evidence; do not retain a second engine. +4. Remove local scoring, weight renormalization, thresholding, and candidate + window arithmetic after the released owner is pinned. + +Until step 4, the existing local path is legacy demo behavior and not +production calculation evidence. + +## Consequences + +- LineageWeave remains independently useful for governed evidence navigation, + while calculation authorities remain independently releasable modules. +- A provider outage is visible as unavailable rather than an invented score. +- The previously merged external-lineage contract from stack PR #343 must not + be resurrected unchanged because it imports local reconstruction arithmetic. + +## References + +ContextualWisdomLab. (2026a). *fast-mlsirm product requirements* [Software +documentation]. https://github.com/ContextualWisdomLab/fast-mlsirm + +ContextualWisdomLab. (2026b). *RankWeave architecture* [Software +documentation]. https://github.com/ContextualWisdomLab/RankWeave + +ContextualWisdomLab. (2026c). *TEPP product requirements* [Software +documentation]. https://github.com/ContextualWisdomLab/TEPP diff --git a/docs/adr/README.md b/docs/adr/README.md index eadef2874..83e56345c 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -16,9 +16,14 @@ decision from them. | [`PROV_O_IMPLEMENTATION_MATRIX.md`](../PROV_O_IMPLEMENTATION_MATRIX.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`ONTOLOGY_NAMESPACE_INVENTORY.md`](../doctoring/ONTOLOGY_NAMESPACE_INVENTORY.md) | [0207](0207-repository-case-ontology-namespace-canonical.md), [0157](0157-public-ontology-namespace-identity.md) | | [`image-content-schema.md`](../image-content-schema.md) | [0066](0066-position-preserving-image-content.md) | -| [`storybook-inventory.md`](../storybook-inventory.md) | [0118](0118-uiux-standard-guide-v3-design-overhaul.md), [0184](0184-ontology-provenance-explorer.md) | -| [`POSTGRESQL_CONCURRENCY_REFERENCES.md`](../doctoring/POSTGRESQL_CONCURRENCY_REFERENCES.md) | [0204](0204-analysis-run-short-transaction-delivery.md) | -| [`operability/http-concurrency-evidence.md`](../operability/http-concurrency-evidence.md) | [0204](0204-analysis-run-short-transaction-delivery.md), [0212](0212-single-query-authorized-post-filter-options.md) | +| [`storybook-inventory.md`](../storybook-inventory.md) | [0118](0118-uiux-standard-guide-v3-design-overhaul.md), [0184](0184-ontology-provenance-explorer.md), [0222](0222-project-nodes-in-ontology-neighborhood.md) | +| [`POSTGRESQL_CONCURRENCY_REFERENCES.md`](../doctoring/POSTGRESQL_CONCURRENCY_REFERENCES.md) | [0204](0204-analysis-run-short-transaction-delivery.md), [0213](0213-global-ask-embedding-pool-release.md) | +| [`GLOBAL_ASK_PUBLIC_VERIFICATION_REFERENCES.md`](../doctoring/GLOBAL_ASK_PUBLIC_VERIFICATION_REFERENCES.md) | [0215](0215-global-ask-public-claim-verification.md) | +| [`GLOBAL_ASK_KNOWLEDGE_CUTOFF_REFERENCES.md`](../doctoring/GLOBAL_ASK_KNOWLEDGE_CUTOFF_REFERENCES.md) | [0216](0216-global-ask-knowledge-cutoff.md) | +| [`GLOBAL_ASK_QUERY_REWRITE_REFERENCES.md`](../doctoring/GLOBAL_ASK_QUERY_REWRITE_REFERENCES.md) | [0217](0217-evidence-constrained-semantic-query-rewrite.md) | +| [`MCP_GLOBAL_ASK_REFERENCES.md`](../doctoring/MCP_GLOBAL_ASK_REFERENCES.md) | [0218](0218-current-contract-mcp-global-ask.md) | +| [`operability/http-concurrency-evidence.md`](../operability/http-concurrency-evidence.md) | [0204](0204-analysis-run-short-transaction-delivery.md), [0212](0212-single-query-authorized-post-filter-options.md), [0213](0213-global-ask-embedding-pool-release.md) | +| [`operability/mcp-concurrency-evidence.md`](../operability/mcp-concurrency-evidence.md) | [0218](0218-current-contract-mcp-global-ask.md) | | Evidence operations Dashboard (`/`) | [0206](0206-evidence-operations-dashboard.md) | | [`temporal-topic-context-influence-research.md`](../temporal-topic-context-influence-research.md) | [0210](0210-temporal-topic-context-influence-dashboard.md) | | [`python-mathematical-compute-boundary-audit.md`](../doctoring/python-mathematical-compute-boundary-audit.md) | [0208](0208-externalize-local-mathematical-compute.md) | diff --git a/docs/doctoring/GLOBAL_ASK_KNOWLEDGE_CUTOFF_REFERENCES.md b/docs/doctoring/GLOBAL_ASK_KNOWLEDGE_CUTOFF_REFERENCES.md new file mode 100644 index 000000000..1929cae21 --- /dev/null +++ b/docs/doctoring/GLOBAL_ASK_KNOWLEDGE_CUTOFF_REFERENCES.md @@ -0,0 +1,18 @@ +# Global Ask knowledge-cutoff references + +Supporting research register for ADR 0216. The ADR is normative. + +## Adopted standards + +Cox, S., & Little, C. (Eds.). (2020). *Time ontology in OWL*. World Wide Web +Consortium. https://www.w3.org/TR/owl-time/ + +Adopted use: model the caller's cutoff as an instant and revision availability +as a half-open interval. LineageWeave does not infer an instant when none was +recorded. + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV +ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ + +Adopted use: keep the retained revision and later live entity distinguishable, +and retain the revision identifier alongside each historical citation. diff --git a/docs/doctoring/GLOBAL_ASK_PUBLIC_VERIFICATION_REFERENCES.md b/docs/doctoring/GLOBAL_ASK_PUBLIC_VERIFICATION_REFERENCES.md new file mode 100644 index 000000000..18bfaf147 --- /dev/null +++ b/docs/doctoring/GLOBAL_ASK_PUBLIC_VERIFICATION_REFERENCES.md @@ -0,0 +1,23 @@ +# Global Ask public-verification research register + +ADR 0215 adopts three distinct contracts: + +- FEVER supplies the evidence-dependent `supported`, `refuted`, and + `not_enough_information` outcome model. +- W3C PROV-O keeps internal source evidence, external web evidence, and the + verification activity separate. +- SearXNG's Search API defines the bounded JSON retrieval transport; public + instance defaults are not assumed. + +## APA 7 references + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV +ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ + +SearXNG. (2026). *Search API*. https://docs.searxng.org/dev/search_api.html + +Thorne, J., Vlachos, A., Christodoulopoulos, C., & Mittal, A. (2018). FEVER: A +large-scale dataset for fact extraction and verification. In *Proceedings of +the 2018 Conference of the North American Chapter of the Association for +Computational Linguistics: Human Language Technologies* (Vol. 1, pp. 809–819). +Association for Computational Linguistics. https://doi.org/10.18653/v1/N18-1074 diff --git a/docs/doctoring/GLOBAL_ASK_QUERY_REWRITE_REFERENCES.md b/docs/doctoring/GLOBAL_ASK_QUERY_REWRITE_REFERENCES.md new file mode 100644 index 000000000..5250a921d --- /dev/null +++ b/docs/doctoring/GLOBAL_ASK_QUERY_REWRITE_REFERENCES.md @@ -0,0 +1,19 @@ +# Global Ask query-rewrite references + +This register supports ADR 0217. It records research and standards inputs; the +ADR is the normative decision. + +Ma, X., Gong, Y., He, P., Zhao, H., & Duan, N. (2023). Query rewriting in +retrieval-augmented large language models. In *Proceedings of the 2023 +Conference on Empirical Methods in Natural Language Processing* (pp. +5303–5315). Association for Computational Linguistics. +https://doi.org/10.18653/v1/2023.emnlp-main.322 + +PostgreSQL Global Development Group. (2026). *Text search functions and +operators*. https://www.postgresql.org/docs/current/functions-textsearch.html + +Adoption note: LineageWeave adopts a query-rewrite stage, not unconstrained +query expansion. The repository accepts only phrases copied exactly from the +question and applies them as separately parameterized PostgreSQL text-search +queries. contextual-orchestrator owns structured generation, model discovery, +and reasoning allocation; RankWeave retains ranking fusion ownership. diff --git a/docs/doctoring/MCP_GLOBAL_ASK_REFERENCES.md b/docs/doctoring/MCP_GLOBAL_ASK_REFERENCES.md new file mode 100644 index 000000000..9d51e8408 --- /dev/null +++ b/docs/doctoring/MCP_GLOBAL_ASK_REFERENCES.md @@ -0,0 +1,31 @@ +# MCP Global Ask standards register + +Supporting research for [ADR 0218](../adr/0218-current-contract-mcp-global-ask.md). +The ADR is normative; this register records why each external standard is in +scope. + +| Source | Adopted contract | +|---|---| +| MCP Streamable HTTP 2025-06-18 | Validate every present Origin, authenticate remote connections, and carry each JSON-RPC message in a new POST request. | +| RFC 9728 | Publish protected-resource metadata at the resource-derived well-known location and keep the advertised resource identifier exact. | +| RFC 8707 | Bind the Keyverse access token audience to the MCP resource identifier. | +| RFC 9700 | Apply current OAuth security best practice rather than treating bearer possession as cross-resource authority. | + +## References — APA 7th + +Campbell, B., Bradley, J., & Tschofenig, H. (2020). *Resource indicators for +OAuth 2.0* (RFC 8707). Internet Engineering Task Force. +https://doi.org/10.17487/RFC8707 + +Jones, M., Hunt, P., & Parecki, A. (2025). *OAuth 2.0 protected resource +metadata* (RFC 9728). Internet Engineering Task Force. +https://doi.org/10.17487/RFC9728 + +Lodderstedt, T., Bradley, J., Labunets, A., & Fett, D. (2025). *Best current +practice for OAuth 2.0 security* (RFC 9700). Internet Engineering Task Force. +https://doi.org/10.17487/RFC9700 + +Model Context Protocol. (2025). *Transports: Streamable HTTP* (Specification +2025-06-18). +https://modelcontextprotocol.io/specification/2025-06-18/basic/transports + diff --git a/docs/ontology/lineageweave-kg.ttl b/docs/ontology/lineageweave-kg.ttl index 24aeabdba..0aeb57f9a 100644 --- a/docs/ontology/lineageweave-kg.ttl +++ b/docs/ontology/lineageweave-kg.ttl @@ -411,6 +411,7 @@ # imported grouping fields: a post may mention a project without carrying a # project field, and the mention keeps evidence/confidence for review. :Project a owl:Class ; + :lookupCode "node_project" ; rdfs:label "Project"@en ; rdfs:comment "A business project referred to by a source post."@en . @@ -423,8 +424,10 @@ rdfs:comment "An evidence-backed, RDF-reified assertion that a post refers to a project; rdf:subject identifies the post, rdf:predicate is :mentionsProject, and rdf:object identifies the project."@en . :mentionsProject a owl:ObjectProperty ; + :lookupCode "edge_mention_project" ; rdfs:domain :Post ; - rdfs:range :Project . + rdfs:range :Project ; + rdfs:label "mentions project"@en . :projectEvidence a owl:DatatypeProperty ; rdfs:domain :ProjectMention ; diff --git a/docs/operability/http-concurrency-evidence.md b/docs/operability/http-concurrency-evidence.md index ae3684142..657bdd3bf 100644 --- a/docs/operability/http-concurrency-evidence.md +++ b/docs/operability/http-concurrency-evidence.md @@ -20,7 +20,8 @@ window that match the environment under review: ```bash make up KEYCLOAK_ADMIN_PASSWORD=admin_dev_only make seed -k6 run --vus --duration \ +k6 run -e REQUEST_TIMEOUT= \ + --vus --duration \ scripts/k6_http_e2e.js ``` @@ -29,6 +30,10 @@ Pass `BACKEND_URL`, `KEYCLOAK_URL`, `KEYCLOAK_REALM`, `KEYCLOAK_CLIENT_ID`, harness at another authorized synthetic environment. Never run repository performance evidence against identifying production records. +`REQUEST_TIMEOUT` is mandatory because an unbounded request hid the first +observed saturation behind k6's graceful-stop window. It is an operator-declared +observation boundary, not a product latency threshold. + ## Interpret the output k6 reports observed request counts, failure rate, and duration distributions. @@ -56,8 +61,57 @@ or shared-runner result to a product guarantee. Figma and screenshot review do not apply: this is a non-UI HTTP load harness. +## Exact-head synthetic verification record + +On 2026-08-26, an isolated Compose stack built from PR #663 commit +`be361f10` completed an authenticated 4-VU, 30-second run against 27 synthetic +`source_post` rows. The run completed 5,537 iterations and 16,613 HTTP +requests; all 16,611 endpoint checks passed and k6 recorded no HTTP failures. +Ask enqueue averaged 11.88 ms. Ask polling averaged 13.61 ms, with 21.46 ms +p95 and 156.69 ms maximum. The combined post/lineage reader metric averaged +19.57 ms, with 31.39 ms p95 and 198.64 ms maximum. Overall HTTP duration +averaged 17.59 ms with 29.25 ms p95, at 183.44 iterations and 550.39 requests +per second. + +The host exposed 10 logical CPUs and 32 GiB RAM; Compose imposed no explicit +backend CPU or memory limit. This exact-head observation verifies concurrent +responsiveness for the small synthetic fixture and the asynchronous Ask +enqueue/poll path. It does not represent authorized production volume, +establish capacity, isolate a causal bottleneck, or establish an SLO. + ## Current-main verification record +On 2026-08-25, the follow-up change at `a700374e` was exercised against the +authorized local Compose PostgreSQL/Keycloak/Valkey/orchestrator stack after +all schema migrations and index builds had completed. Only aggregate evidence +was retained: the database held 43,189 source posts. Ten-second authenticated +observations used the same endpoint mix and reported zero HTTP errors at 1, +10, and 25 VUs. Before the bounded-lineage query, HTTP median/p95/p99 and +throughput were 809.03 ms/6.18 s/6.22 s and 0.618 requests/s at 1 VU; +4.63 s/20.10 s/20.46 s and 1.531 requests/s at 10 VUs; and +25.35 s/33.59 s/36.30 s and 1.046 requests/s at 25 VUs. The 25-VU observation +completed six iterations. + +The same observations after moving the landing lineage ABAC, ordering, node +bound, and edge bound into PostgreSQL were 179.52 ms/3.43 s/4.17 s and 1.067 +requests/s at 1 VU; 1.88 s/20.80 s/21.04 s and 1.487 requests/s at 10 VUs; +and 22.03 s/29.78 s/31.38 s and 2.411 requests/s at 25 VUs. The 25-VU +observation completed 25 iterations. The 10-VU tail did not improve, so this +evidence does not establish a latency SLO or a product capacity ceiling. It +does establish that repeatedly loading all visible posts and all lineage edges +before applying the 500-node contract was avoidable work; the remaining tail +requires endpoint-tagged traces and database-pool telemetry before another +cause is assigned. + +An exact-code-head 4-VU, 60-second confirmation at `a700374e` completed 36 +iterations and 110 HTTP requests with zero failed checks or requests. Overall +HTTP median/p95/p99 were 392.15 ms/8.82 s/9.68 s at 1.644 requests/s. The +combined posts/lineage read median/p95/p99 were 3.21 s/9.14 s/9.89 s; Ask poll +median/p95/p99 were 41.39 ms/413.16 ms/462.65 ms. All 36 iterations observed +the Ask lifecycle state. This confirms asynchronous Ask polling remained +responsive in that observation while also preserving the remaining reader-tail +gap; it is not a deployment SLO. + On 2026-08-25, a worktree based on protected-main commit `48f013a2` passed `k6 inspect` for this script. A fresh Compose project did not reach an application-ready state: the build was stopped @@ -68,8 +122,64 @@ HTTP latency distribution was produced and no application bottleneck is claimed. This is local build-environment evidence only. Re-run the command above on an application-ready stack to obtain the product measurement. +The next application-ready exercise on protected-main `d7d5eeb3` exposed two +failures before a capacity distribution could be accepted. A clean backend +process could not start because `Settings` omitted the already-consumed +`tepp_api_key`, and the replay database lacked the non-idempotent 0203 Global +Ask scope tables. After repairing those startup and replay contracts, the k6 +setup completed, but its authenticated read batch overlapped migration replay: +PostgreSQL was still building the 0035 trigram index with a `DataFileRead` wait, +and the not-yet-reached 0140 migration meant Event Lineage correctly failed on +its absent interval column. This run therefore cannot attribute read latency to +Global Ask and is not a valid steady-state capacity exercise. + +Independent code-path diagnosis did confirm that Global Ask resolved its +external question embedding inside `pool.acquire()`. ADR 0213 moves that call +before acquisition and adds a regression check that observes zero held pool +slots during embedding. With one virtual user, a 10-second observation, and a +declared 20-second request window, the post-fix branch then observed Ask enqueue +at 3.11 seconds and Ask polling at 1.31 seconds while both reads failed under +that incomplete migration state (one reached the 20-second request boundary; +combined read duration averaged 14.13 seconds). This is replay-in-progress +failure evidence, not a steady-state capacity result or product latency claim. +Re-run only after migration replay completes. + +A subsequent exact-head run reached the 0140 interval migration but still was +not steady state: replay stopped at migration 0165 because its queue table and +indexes lacked the ADR 0166 replay guards, so migration 0174's edge-signal +table was absent. With one virtual user, a 15-second observation, and the same +20-second request window, Ask enqueue averaged 125.05 milliseconds, Ask polls +averaged 123.41 milliseconds, and posts succeeded, but all four Event Lineage +reads failed on that absent table. The branch now makes migration 0165 +idempotent and regression-checks both Global Ask migrations. These values are +diagnostic evidence only. + +After replaying the repaired 0165–0205 range to completion, a four-VU, +30-second observation with the declared 20-second request window completed 13 +iterations and all 39 endpoint checks without an HTTP failure. Ask enqueue was +57.32 milliseconds, Ask polling averaged 359.91 milliseconds (p95 969.66 +milliseconds), and the combined posts/Event-Lineage read distribution averaged +5.75 seconds (p95 11.88 seconds, maximum 12.36 seconds). A second four-VU, +15-second diagnostic run also completed every endpoint check; concurrent +`pg_stat_activity` samples repeatedly observed the authorized filter-option, +post-list, and lineage-page queries as active, including `MessageQueueSend` and +one temporary-buffer write. This identifies the measured database work to +profile next; it does not by itself assign causality or establish an SLO. + ## Older-image diagnostic observation +On 2026-08-26, the same non-exact local Compose boundary completed an +authenticated 4-VU, 30-second run over 43,189 aggregate synthetic +`source_post` rows: 87 full iterations, 263 HTTP requests, and 261/261 endpoint +checks succeeded. Ask enqueue was 57.25 ms. Ask polling was 56.83 ms mean, +233.37 ms p95, and 765.24 ms maximum. The combined post/lineage reader metric +was 842.14 ms mean, 1.53 s p95, and 2.80 s maximum. The host exposed 10 logical +CPUs and 32 GiB RAM; Compose imposed no explicit backend CPU or memory limit. +The backend container came from image `sha256:28234aa5db0e` created on +2026-08-24, not the current PR head. These distributions show that concurrent +readers remained responsive on that image; they do not validate an exact-head +regression, identify a causal bottleneck, or establish an SLO. + On 2026-08-25, an application-ready local Compose stack configured with four worker VUs completed zero full iterations in two observations. In the second 30-second observation, Ask enqueue took 2.69 seconds, the maximum completed diff --git a/docs/operability/mcp-concurrency-evidence.md b/docs/operability/mcp-concurrency-evidence.md new file mode 100644 index 000000000..5c7290bfd --- /dev/null +++ b/docs/operability/mcp-concurrency-evidence.md @@ -0,0 +1,42 @@ +# MCP concurrency evidence + +This supporting record is governed by [ADR 0218](../adr/0218-current-contract-mcp-global-ask.md). +It reports an observation, not an SLO or production capacity claim. + +## 2026-08-26 synthetic isolated-Compose observation + +The candidate containing the request-lifecycle repair was run in an isolated +Compose project with synthetic fixtures only. The MCP service used the +operator-declared diagnostic quota envelope of 1,000 authenticated tool calls +per 60 seconds; this value is not a deployment recommendation. The committed +`scripts/k6_mcp_e2e.js` initialized an MCP session per VU, submitted one durable +Global Ask job, and concurrently read that job through the MCP tool contract. + +```shell +REQUEST_TIMEOUT=20s k6 run --vus 5 --duration 5s scripts/k6_mcp_e2e.js +``` + +| Observation | Result | +| --- | ---: | +| Completed iterations | 628 | +| Interrupted iterations | 0 | +| HTTP requests | 642 | +| HTTP request failures | 0 | +| Successful MCP Ask-read checks | 628 / 628 | +| Iteration rate | 115.30467/s | +| Initialize duration, average / p95 / maximum | 37.46 / 55.76 / 57.39 ms | +| Submit duration | 34.13 ms | +| Read duration, average / p95 / maximum | 38.96 / 79.96 / 268.31 ms | + +The first live initialization exposed a transport defect: the bounded-body +middleware manufactured `http.disconnect` immediately after replaying the +admitted body, so the streaming response ended incomplete. The shared +middleware now replays the body once and then delegates subsequent lifecycle +messages to the real client receive channel. Focused admission and MCP contract +tests pass, and the repeated live run completed without the incomplete-response +error. + +This workstation result proves only that the declared synthetic workload +completed on this candidate. Representative infrastructure telemetry and an +approved quota/SLO decision remain required before a production capacity +claim. diff --git a/docs/product-requirements.md b/docs/product-requirements.md index 75cba0410..0d456f2ae 100644 --- a/docs/product-requirements.md +++ b/docs/product-requirements.md @@ -45,6 +45,8 @@ edge exposes the same authorized endpoints and evidence through API and UI. - Project typed Post, Person, CorporateEntity, Team, Project, and governed relationship evidence from PostgreSQL without creating a second mutable source of truth. +- Keep name-derived Project candidates scoped to their evidence Post until a + governed catalog resolution supplies a stable cross-record identity. - Preserve truth status, valid/system time, provenance, and evidence references. - Validate DB-to-RDF projections with SHACL, including complete reified @@ -75,9 +77,14 @@ dangling endpoints fail closed; fixed input produces stable page boundaries. - Apply authorization/time/process scope before ranking and again before response delivery. - Keep internal post citations separate from external public citations. +- Interpret natural-language retrieval through contextual-orchestrator while + accepting only literal question phrases; do not invent local stop-word, + expansion, scoring, or weighting rules. Acceptance: a semantic-only term can retrieve an authorized unit; private -content never becomes an external query or citation. +content never becomes an external query or citation; and multilingual +conversational framing cannot suppress a persisted fact named by an exact +question phrase. ### PRD-FR-5 — Evidence operations @@ -90,6 +97,51 @@ content never becomes an external query or citation. Acceptance: each state tells the user the next valid action and never displays stale evidence from a previously opened post. +### PRD-FR-5A — Opt-in public claim verification + +- Persist an explicit per-question opt-in before any external search begins. +- Nominate only cited, public semantic/KG facts; source bodies, private facts, + personal facts, and measurement outputs never become external queries. +- Retrieve bounded public evidence through SearXNG and adjudicate through + contextual-orchestrator's verification mode. +- Report supported, refuted, and not-enough-information outcomes without + promoting public pages to internal ontology authority. +- Keep external URLs visually and structurally separate from authorized + internal post citations. + +Acceptance: leaving the control off causes no public request; hidden or +uncited facts cause no public request; unavailable services fail closed; and +each displayed public judgment retains its originating internal evidence IDs. + +### PRD-FR-5B — Knowledge-cutoff Global Ask + +- Persist the optional cutoff with the asynchronous request and reject a future + instant against the database clock. +- Apply authorization, eligibility, and cutoff filters before candidate limits, + then cite the retained source revision available at that instant. +- Never replace a missing historical body or semantic channel with current + state; expose the limitation and later-live-change status. +- Preserve the live contract when no cutoff is supplied. + +Acceptance: a later rewrite never appears in a cutoff answer; an uncovered +revision is explicitly unavailable; and API and rendered citations identify +the retained revision and full/partial grounding state. + +### PRD-FR-5C — Authenticated MCP Global Ask + +- Expose asynchronous submission and owner-scoped job reading over MCP while + reusing the REST application service and persisted answer payload. +- Validate the exact MCP resource audience, provisioned account, permission, + affiliation scope, Host, Origin, and bounded request body before a tool runs. +- Consume one distributed quota unit only for an admitted authenticated tool + call; preflight and rejected admission consume none. +- Require deployment-supplied, load-evidence-backed quota parameters and fail + closed when shared Valkey cannot decide. + +Acceptance: MCP and REST produce the same scope snapshot, verification opt-in, +knowledge cutoff, status, citations, and limitations; cross-account reads are +404-equivalent; and exhaustion returns the bounded actual retry interval. + ### PRD-FR-6 — Measurement boundary - Consume TEPP accepted/completed wire contracts and fast-mlsirm outputs; do @@ -166,10 +218,10 @@ A release claim requires one exact protected-main head that proves: ## 7. Traceability - Product/data boundary: ADR 0001, ADR 0089. -- Asynchronous delivery and database-pool isolation: ADR 0204. +- Asynchronous delivery and database-pool isolation: ADR 0204, ADR 0213. - Knowledge Graph, ontology, and provenance: ADR 0004, ADR 0011, ADR 0065, - ADR 0184, ADR 0207. -- Semantic units and retrieval: ADR 0047, ADR 0062, ADR 0102. + ADR 0184, ADR 0207, ADR 0222. +- Semantic units and retrieval: ADR 0047, ADR 0062, ADR 0102, ADR 0217. - LLM/model boundary: ADR 0070, ADR 0072, ADR 0076, ADR 0079. - Measurement: ADR 0003, ADR 0145, ADR 0200, ADR 0205. - UX and publication: ADR 0118, ADR 0159. diff --git a/docs/storybook-inventory.md b/docs/storybook-inventory.md index 7ec497477..81ab3a8af 100644 --- a/docs/storybook-inventory.md +++ b/docs/storybook-inventory.md @@ -9,13 +9,15 @@ operator-facing control you can click before changing product CSS. | `Post/SimilarVocPanel` | Compare ontology/semantic similar VOC and prior action evidence, then open the source; unavailable states show no fabricated TEPP theta or weight. | `SimilarVocPanel.css`, `SimilarVocPanel` | | `Evidence/CitationChip` | Click a cited title to open that source post. | `--color-chip-border`, `--radius-chip`, `CitationChip` | | `Evidence/OrganizationAliasChip` | Click a cataloged org; the parenthetical is the unique corroborated SKOS companion. | `--color-chip-border`, `--radius-chip`, `OrganizationAliasChip` | +| `Evidence/OntologyExplorer` | Distinguish Event Lineage from typed ontology facts, inspect Post/Person/Organization/Team/Project shapes, token-backed secondary cues, and truth labels, then open authorized evidence. The named exact-values region supports keyboard scrolling; `LongLabelsAndEvidenceTable` proves complete labels wrap without character-count truncation. Desktop, narrow, drawers, legend/filter, empty, truncated, partial, denied, stale, and rejected scenes cover ADR 0184/0222 states. | `OntologyExplorer`, `ontologyLayout`, `--ontology-node-*-fill`, `--color-table-border` | | `AnalysisRun/CutoffKnownBody` | Read the cutoff-known sentence, then compare it with the live body below. | `--color-accent-border`, `--space-panel-block`, `--radius-panel`, `CutoffKnownBody` | | `Analysis/LineageEntityPicker` | Choose which corp to reconstruct, then click Request a lineage reconstruction. | `--space-control-gap`, `--size-control-min`, `--radius-control`, `LineageEntityPicker` | | `Admin/AdminPanel` | Change the tenant brand name, then verify the saved or failed state before leaving settings. | `--surface`, `--border`, `--space-panel-block`, `AdminPanel` | | `Lineage/LineageDag` | Open a reconstructed connection to read its inferred channel scores and Allen interval relation, or open the current branch node; compare empty, single-branch, grouped/forked, mobile-scroll, ungrouped, and long-title states before changing graph CSS. On narrow viewports, swipe the named viewport or focus it and use arrow keys to inspect the full lineage. | `--color-accent-background`, `--radius-control`, `--surface`, `--border`, `--color-focus-border`, `--size-control-min`, `LineageDag` | | `Chrome/PopupCloseButton` | Close the evidence panel or post popup. | `--space-close-inset`, `--font-size-close`, `PopupCloseButton` | | `Workspace/WorkspaceCalendar` | Read observed Naruon events, or open a commitment to land on that post. Fail-closed copy stays `이 범위의 일정을 아직 받을 수 없습니다`. | `--color-chip-border`, `WorkspaceCalendar`, `EvidenceStatusMark` | -| `Evidence/OntologyExplorer` | Distinguish Post, Person, Organization, and Team by shape and text, use the token-backed surface as a secondary cue, then open the exact-value table or cited evidence. Compare desktop, narrow, drawer, empty, truncated, denied, stale, and rejected states. | `--ontology-node-*-fill`, `OntologyExplorer` | +| `Ask Agent/Public claim verification` | Compare supported, refuted, and not-enough-information states; open only the external evidence link, then review the separate internal citation before changing governed graph state. | `--space-panel-block`, `--space-control-gap`, `--color-border`, `--size-control-min`, `PublicClaimVerification` | +| `Ask Agent/Knowledge cutoff` | Exercise partial historical grounding, retained-revision provenance, later-live-change disclosure, and the narrow viewport before relying on a historical answer. | Native `datetime-local`, `--space-panel-block`, `--space-control-gap`, `--color-border`, `--size-control-min` | Repeated web objects must use `frontend/src/styles/tokens.css` and a module under `frontend/src/components/`. Do not add a second Node package manager; diff --git a/frontend/.dockerignore b/frontend/.dockerignore new file mode 100644 index 000000000..d3e118508 --- /dev/null +++ b/frontend/.dockerignore @@ -0,0 +1,5 @@ +node_modules +dist +storybook-static +test-results +playwright-report diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 30420c475..5f2e1eb9a 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,7 +1,7 @@ FROM node:24-slim@sha256:3638d9a6fe4030bd716be989438248074489337ba3275657f93595428be4fc03 AS build WORKDIR /app RUN corepack enable -COPY package.json pnpm-lock.yaml ./ +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ RUN pnpm install --frozen-lockfile COPY . . # Vite bakes VITE_* vars in at build time, not runtime -- build args let diff --git a/frontend/src/App.css b/frontend/src/App.css index b4b65dfc8..901b17b1a 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -843,6 +843,42 @@ color: var(--badge-status-prediction-text); } +.ask-agent-form { + display: grid; + gap: var(--space-control-gap); + max-width: 42rem; +} + +.ask-agent-field { + display: grid; + gap: 0.35rem; +} + +.ask-agent-field textarea, +.ask-agent-field input { + box-sizing: border-box; + width: 100%; + min-height: var(--size-control-min); + border: 1px solid var(--color-border); + border-radius: var(--radius-control); + background: var(--color-background); + color: var(--color-text); + font: inherit; + padding: 0.65rem; +} + +.ask-agent-checkbox { + display: inline-flex; + align-items: center; + gap: 0.5rem; + min-height: var(--size-control-min); +} + +.ask-agent-form > .btn-primary { + justify-self: start; + min-height: var(--size-control-min); +} + .keyman-select { background: none; border: none; @@ -1152,6 +1188,28 @@ stroke-width: 1.5; } +.ontology-node-label { + pointer-events: none; +} + +.ontology-node-label div { + color: var(--color-text-heading); + display: flex; + flex-direction: column; + font-size: 0.8rem; + line-height: 1.15; + overflow-wrap: anywhere; + text-shadow: + -1px -1px var(--color-background), + 1px -1px var(--color-background), + -1px 1px var(--color-background), + 1px 1px var(--color-background); +} + +.ontology-node-label small { + color: var(--color-text); +} + .ontology-node-post { fill: var(--ontology-node-post-fill); } @@ -1168,6 +1226,10 @@ fill: var(--ontology-node-team-fill); } +.ontology-node-project { + fill: var(--ontology-node-project-fill); +} + .ontology-node-generic { fill: var(--ontology-node-generic-fill); } @@ -1182,7 +1244,6 @@ .ontology-node text.ontology-node-type { font-size: 0.7rem; - fill: var(--color-text); } .ontology-node-selected { diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 76ff51dec..fbba1d9f2 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -94,6 +94,7 @@ import { CutoffKnownBody } from "./components/CutoffKnownBody"; import { LineageEntityPicker } from "./components/LineageEntityPicker"; import { OntologyExplorer } from "./components/OntologyExplorer"; import { AskEvidenceLayerPopup } from "./components/AskEvidenceLayerPopup"; +import { PublicClaimVerification } from "./components/PublicClaimVerification"; import { PopupCloseButton } from "./components/PopupCloseButton"; import { SimilarVocPanel } from "./components/SimilarVocPanel"; import { chatEvidenceKindLabel } from "./evidenceKindLabels"; @@ -4813,7 +4814,7 @@ function CustomerMasterPanel({ ); } -function AskAgentPanel({ +export function AskAgentPanel({ accessToken, onOpenPost, }: { @@ -4824,7 +4825,13 @@ function AskAgentPanel({ const [answer, setAnswer] = useState(null); const [error, setError] = useState(null); const [asking, setAsking] = useState(false); + const [verifyExternal, setVerifyExternal] = useState(false); + const [knowledgeCutoff, setKnowledgeCutoff] = useState(""); const [evidenceLayerPostId, setEvidenceLayerPostId] = useState(null); + const now = new Date(); + const localKnowledgeCutoffMax = new Date( + now.getTime() - now.getTimezoneOffset() * 60_000, + ).toISOString().slice(0, 16); async function handleAsk() { const normalized = question.trim(); @@ -4832,7 +4839,14 @@ function AskAgentPanel({ setAsking(true); setError(null); try { - setAnswer(await askAgent(accessToken, normalized)); + setAnswer( + await askAgent( + accessToken, + normalized, + verifyExternal, + knowledgeCutoff ? new Date(knowledgeCutoff).toISOString() : undefined, + ), + ); } catch (err) { setAnswer(null); setError(orchestratorUnavailableMessage(err, t("Ask Agent"))); @@ -4847,23 +4861,59 @@ function AskAgentPanel({

{t("Ask Agent")}

{t("Questions use authorized posts and their evidence.")}

{error ?

{error}

: null} -