From ef6f5a5ffcb467bd935dc1e53acc0029669b0bd7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:04:45 -0700 Subject: [PATCH 1/7] feat: enforce Korean UI standards on top of latest PR stack (#347) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: queue post-content analysis durably * fix(import): allow repeated source lookup keys * fix(docs): assign post-content ADR a unique number * feat: add keyverse-bound mcp api key management * revert: keep MCP key management out of buyer root stack The prior commit added a second local MCP credential authority to #258 even though authenticated MCP Global Ask is owned by draft PR #270. Restore the exact pre-injection Buyer tree without rewriting history; the isolated commit remains available for separate evaluation against #270's OAuth resource-server contract. * security: document safe composed SQL * fix: preserve OIDC post deep links * ci: verify audited safe-SQL suppressions * ci: harden audited safe-SQL repair anchors * fix(ci): remove self-modifying SQL repair workflow * test(security): reject dynamic asyncpg statement construction * fix: render resolved post structure for ordinary paragraphs * fix: hide source offsets from image accessibility labels * fix: requeue incomplete post embeddings * fix: align static SQL contract with safe composition * ci: repair PR 258 static SQL review findings * ci: repair PR 258 one-shot template * fix: pass embedding model to backend * fix(ci): remove self-modifying PR 258 repair workflow * fix(ci): remove PR 258 workflow template * fix(security): bind customer-hint SQL to a static statement * fix(security): precompile demo-scope SQL statements * fix(security): precompile relationship-network SQL * fix(security): precompile Keyman backfill selectors * fix(security): precompile summary backfill selectors * test: apply image-region embedding migration in API fixture * fix(security): precompile content backfill selectors * fix(security): move synthetic cleanup identifiers into PostgreSQL * test(security): require static reviewed asyncpg statements * fix(backfill): preserve process-unit semantic hint name * fix: wire provider embeddings through orchestrator * fix: retain embedding provider URL during startup * fix: keep orchestrator provider URL for embedding startup * chore: stage exact SQL review repair payload * fix(security): keep relationship-network SQL literal * fix(security): keep demo-scope SQL literal * fix(security): keep customer-hint SQL literal * fix(security): keep Keyman selector SQL literal * fix(security): use one literal summary selector * fix(security): use one literal content selector * fix: requeue incomplete post content jobs * fix(security): keep synthetic cleanup SQL literal * test(security): require literal reviewed SQL statements * fix: keep ADR numbers unique and remove repair artifact * feat: queue real post content backfill through valkey * chore: configure project test import path * fix(security): keep queue backfill SQL literal * test(security): cover queue backfill SQL boundary * perf: parallelize visual region evidence * fix: require a majority of tokens for relation corroboration corroborating_evidence_url treated a single distinctive name token as enough corroboration for an organization's real-world footprint. An invented name containing an ordinary dictionary word (e.g. "Fictitious", "Nonexistent") could match unrelated search results on that one word alone, flipping a genuinely uncorroborated relation to corroborated. Now requires a majority of a multi-token name's tokens to co-occur in the same result; single-token names fall back to that one token since there is no majority to require. Also renumbers migrations/0012_role_responsibility_agent_type.sql to 0060 -- it collided with the already-mainlined 0012_report_leftover_pair.sql -- and updates every reference plus the migrate.sh replay allowlist. Co-Authored-By: Claude Sonnet 5 * test(frontend): match embedded image accessibility label * docs: ground orchestration policy in primary sources * style: tokenize repeated colors and add dark-mode badge variants App.css hardcoded ~13 color sites (lineage-link accent colors, actor-type badges, verification-status badges, a subtle border) that duplicated the same values across the file with zero dark-mode coverage -- the pastel badge backgrounds would have stayed light in dark mode, clashing against the dark chrome. Promotes them to semantic tokens in styles/tokens.css with light and dark variants, following the file's existing --color-accent-* pattern, and points every App.css occurrence at the new custom properties. Co-Authored-By: Claude Sonnet 5 * docs(adr): record buyer surface Figma file * docs+test: ADR and regression test for badge/accent tokens Adds ADR 0099 documenting the design-token pass (Figma File ID 1Su3lDRmiZdcUs47t1QwIX, reusing ADR 0002's already-cleared boundary) and a CHANGELOG.d fragment. Adds frontend/src/styles/tokens.test.ts so a future edit can't silently drop the dark-mode half of a badge token or reintroduce the retired inline hex colors. Also drops a duplicate 0060_* case arm in migrate.sh's replay allowlist left over from a concurrent edit. Co-Authored-By: Claude Sonnet 5 * fix: keep 0060 migration allowlist entry as its own case arm My earlier cleanup merged 0060_* into the 0012-0050 case arm, but test_migration_replay_window_includes_post_content_queue asserts the exact substring "0050_*)" is present -- merging broke that literal match. Restores the separate appended arm, matching the pattern already used for 0051_*/0052_*. Co-Authored-By: Claude Sonnet 5 * fix(security): audit every reviewed SQL composition site * fix: align embedded image accessibility assertion * fix: recover post content queue leases * fix: allow slow structured post adjudication * fix: bound post content queue retries * fix: align integration fixtures with current contracts * feat: persist requester and processor evidence * docs: record product gap baseline * fix: keep summaries available when enrichment times out * docs: align product gap evidence with current head * fix: fence stale post-content worker leases * test: apply action migration to projection fixtures * fix: stop failed evidence panels from loading forever * docs: record bounded summary refresh evidence * fix: document audited asyncpg queries * docs: track latest protected-head audit * docs: record entity-resolution PR correction * fix: preserve semantic source unit boundaries * fix: preserve oidc deep links across callback storage * fix: render buyer-facing image evidence * fix: label unavailable image evidence * docs: avoid oidc adr number collision * docs: avoid buyer image adr number collision * fix: bind summary actions to source-grounded projects * test: cover project-bound summary action round trip * docs: record buyer gap checkpoint evidence * docs: align image evidence ADR heading * docs: align OIDC ADR heading * fix: preserve legacy action evidence delimiters * docs: refresh buyer gap audit checkpoint * docs: record current project action head * feat: bind summary events to source-grounded projects * docs: record full checkpoint verification * fix(summary): normalize project keys before persistence * docs: refresh exact-head buyer gap audit * test(summary): cover project key sentinel normalization * test(image): cover source placement and OCR rendering * fix: preserve stale summary buyer continuity * fix: keep post content recovery alive * test(summary): cover stale continuity and retry * feat: add explicit post content retry * test(queue): cover explicit retry rejection * docs(queue): complete retry command docstrings * fix(ontology): align person mention direction with KG triples * docs: record audited safe SQL gate * fix: reconcile backfill content with job ledger * fix(ui): preserve indentation after mixed content * docs: record post content recovery evidence * docs(ui): clarify indentation matching boundary * test(ui): cover missing source unit fallback * fix: serialize per-post vision region analysis * fix(ui): preserve adjacent table boundaries * fix: preserve honest image evidence rendering * fix: separate source whitespace from explicit structure * docs: record source indentation buyer gap * fix: retain partial image regions * docs: record partial image region gap * test: cover partial parent vision failure * feat: enrich semantic hints from source catalogs * fix: reject non-decomposed vision regions * docs: refresh protected gate baseline * docs: record corrected image region evidence * fix: trim source catalog lookup codes * docs: record catalog hint boundary * docs: record exact-head validation evidence * docs: record live browser acceptance * docs: refresh live gate topology * fix: reject unbounded visual locator regions * docs: refresh exact-head product gap audit * test: cover visual region fallback branches * docs: refresh exact-head product gap audit * test: cover visual region fallback branches * docs: record current audit head * docs: repair stacked audit traceability * fix: fall back from malformed visual locator output * docs: refresh stacked PR exact heads * docs: record audit refresh head * docs: refresh visual locator exact-head evidence * style: normalize merged branch whitespace * docs: refresh current visual stack state * docs: record visual stack audit head * docs: align visual stack gate states * fix: preserve semantic footnote structure * docs: record footnote structure checkpoint * fix: classify OOXML footnote containers * fix: distinguish footnote citations from definitions * docs: record footnote citation guard * docs: record fresh runtime aggregate evidence * docs: align runtime evidence head * docs: record embedding provenance guard * docs: align stacked audit head * test: keep invalid locator fallback coordinate-free * docs: keep PR 320 status under its checkpoint * docs: refresh PR 324 exact-head evidence * fix: refresh content after summary ingestion * fix: bound OIDC return state parsing * fix: restrict synthetic cleanup to demo scopes * test: cover empty synthetic cleanup * fix: validate buyer image sources at render boundary * docs: record current exact-head gap checkpoint * docs: retain current buyer stack roots * fix: safely handle malformed region locator output * fix: expose unexpected content channel defects * fix: expose recoverable content batch failures * fix: safely handle malformed region locator output * docs: record current exact-head review gates * docs: align orchestrator runtime pin * docs: refresh current queue roots * docs: record buyer-stack restack * docs: record MCP key boundary closure * style: use deferred region result annotation * feat: enforce Korean UI standards on top of latest PR stack - Re-apply UI·UX Standard Guide Ver.3.0 layout (app-shell, login-card, footer) - Update i18n translation dictionary to match standard Korean vocabulary (조회, 티켓 작성, 로그인 등) - Preserve existing i18n localization hooks instead of hardcoding - Preserve all stacked PR functionality (BuyerNav, LLM extraction fixes, backend parsing) * feat: parameterize brand name for future admin configuration - Replaced hardcoded 'LineageWeave' and 'BRAND' with state. - Prepared for admin mode integration where tenant/brand name can be dynamically fetched. * feat: add admin screen for tenant configuration - Added Admin destination to BuyerNav - Created AdminPanel for configuring brandName dynamically - Wired up AdminPanel rendering in App.tsx - Added i18n support for new admin terms * feat: backend API and DB connection for admin tenant settings - Created migration 0103_tenant_settings for persistent brand name storage - Added GET/PATCH /api/settings endpoints in backend main.py - Wired App.tsx and AdminPanel.tsx to fetch and update via actual API * fix: revert bad chunking sed from uiux restack * fix: rename duplicate ADR 0117 to 0118 * docs: correct UI UX ADR number * feat: close extraction gaps (5W1H, events, Keymen) and update baseline --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .env.example | 21 +- .gitignore | 2 +- AGENTS.md | 159 +- ARCHITECTURE.md | 58 +- CHANGELOG.d/0.71.2-leftover-pairs.md | 4 +- .../2.10.0-leftover-pairs-and-rankings.md | 13 - ...0.1-analysis-run-accessible-next-action.md | 5 - .../2.10.2-embedded-image-html-parser.md | 5 - CHANGELOG.d/2.10.3-stale-hidden-run-list.md | 7 - ...retention-purge-reconstruction-children.md | 6 - .../2.11.0-customer-group-tree-searxng.md | 7 - CHANGELOG.d/2.12.0-persistable-tepp-result.md | 8 - ...2.12.1-tepp-accepted-transport-evidence.md | 9 - CHANGELOG.d/2.12.2-tepp-accepted-clocks.md | 7 - CHANGELOG.d/2.12.3-late-demo-cutoff-post.md | 6 - .../2.12.6-buyer-image-source-safety.md | 6 + CHANGELOG.d/2.12.6-buyer-main-port.md | 11 + CHANGELOG.d/2.12.6-oidc-deep-link-safety.md | 7 + CHANGELOG.d/2.12.6-oidc-resource-audience.md | 8 + CHANGELOG.d/2.12.7-safe-sql-audit.md | 6 + CHANGELOG.d/2.13.0-buyer-gnb-i18n.md | 7 + CHANGELOG.d/2.13.1-badge-color-tokens.md | 7 + CHANGELOG.d/2.13.1-mixed-body-indentation.md | 7 + CHANGELOG.d/2.13.1-partial-image-regions.md | 5 + CHANGELOG.d/2.13.1-separate-source-tables.md | 7 + CHANGELOG.d/2.13.1-source-indent-semantics.md | 7 + CHANGELOG.d/2.21.0-synthetic-cleanup-scope.md | 4 + CHANGELOG.d/2.21.1-orchestrator-pin-doc.md | 6 + CHANGELOG.md | 14 + CLAUDE.md | 54 +- Makefile | 12 +- README.md | 16 +- add_translations.py | 48 + ...breviation_tree_corroboration_ingestion.py | 149 - backend/app/analysis_run_ingestion.py | 160 +- backend/app/analysis_run_outbox.py | 1 - backend/app/analysis_run_start.py | 221 +- backend/app/analysis_run_worker.py | 83 + backend/app/auth.py | 102 +- backend/app/config.py | 97 +- backend/app/corporate_entity_ingestion.py | 74 +- backend/app/customer_group_tree_ingestion.py | 88 - backend/app/customer_hint_ingestion.py | 167 ++ backend/app/demo_scope.py | 83 + backend/app/entity_relationship_ingestion.py | 195 +- backend/app/five_w1h_ingestion.py | 52 + backend/app/issue_ticket_ingestion.py | 7 +- backend/app/keyman_ingestion.py | 9 +- backend/app/knowledge_graph.py | 81 +- backend/app/lineage_ingestion.py | 57 +- backend/app/main.py | 2023 +++++++++++-- backend/app/post_chat_ingestion.py | 447 ++- backend/app/post_content_queue.py | 454 +++ backend/app/post_content_worker.py | 331 +++ backend/app/post_eligibility.py | 45 + backend/app/post_summary_ingestion.py | 206 +- .../app/relation_verification_ingestion.py | 74 + backend/app/report_ingestion.py | 168 +- backend/tests/test_api.py | 1032 +++++-- backend/tests/test_auth_jwks.py | 175 ++ backend/tests/test_config.py | 51 + backend/tests/test_oidc_audience.py | 43 + docker-compose.yml | 82 +- docker/contextual-orchestrator/Dockerfile | 26 + docker/contextual-orchestrator/agents.json | 20 + docker/contextual-orchestrator/start.py | 115 + docker/keycloak/realm-export.json | 10 + docker/postgres-init/Dockerfile | 54 +- docker/postgres-init/migrate.sh | 29 + docs/PROV_O_IMPLEMENTATION.md | 3 + docs/PROV_O_IMPLEMENTATION_MATRIX.md | 3 + docs/adr/0002-figma-access-boundary.md | 6 + .../0003-fast-mlsirm-report-integration.md | 2 +- docs/adr/0004-knowledge-graph-ontology.md | 5 +- ...0006-role-responsibility-agent-ontology.md | 2 +- docs/adr/0009-cross-post-actor-identity.md | 9 +- .../0013-normalized-analysis-run-registry.md | 11 +- docs/adr/0014-authorized-analysis-run-read.md | 43 +- ...016-analysis-run-knowledge-cutoff-posts.md | 12 +- docs/adr/0018-related-nodes-team-org-walk.md | 3 - docs/adr/0020-analysis-run-retention-purge.md | 9 - docs/adr/0022-authorized-tepp-start.md | 3 +- ...d => 0024-rankweave-fusion-fail-closed.md} | 7 +- docs/adr/0028-keyverse-oidc-provider.md | 40 + docs/adr/0029-zotero-local-reproducibility.md | 47 + .../0030-external-llm-gateway-environment.md | 100 + docs/adr/0031-embedded-image-html-parser.md | 78 - docs/adr/0032-bounded-browser-projections.md | 37 + ...retention-purge-reconstruction-children.md | 62 - ...r-group-tree-abbreviation-corroboration.md | 76 - docs/adr/0033-vision-raster-normalization.md | 41 + docs/adr/0034-persistable-tepp-result.md | 107 - ...team-multiple-membership-period-reports.md | 29 + docs/adr/0035-project-period-reports.md | 25 + .../0035-tepp-accepted-transport-evidence.md | 135 - ...36-semantic-project-and-keyman-evidence.md | 53 + .../adr/0037-buyer-gnb-and-product-surface.md | 59 + docs/adr/0038-calendar-source-contract.md | 37 + .../0039-global-ask-agent-source-boundary.md | 34 + docs/adr/0040-source-state-provenance.md | 35 + ...1-source-context-vs-authorization-scope.md | 43 + ...42-source-hints-before-customer-binding.md | 36 + docs/adr/0043-indexed-body-search-boundary.md | 25 + ...0044-source-author-account-keyman-hints.md | 42 + .../0045-orchestrator-credential-source.md | 41 + docs/adr/0046-source-record-identity.md | 44 + .../adr/0047-global-ask-semantic-retrieval.md | 30 + ...d => 0048-persist-lsirm-leftover-pairs.md} | 6 +- ...-ui.md => 0049-leftover-pair-report-ui.md} | 6 +- ...> 0050-seed-period-report-analysis-run.md} | 2 +- docs/adr/0051-source-named-hints.md | 36 + ...52-plain-orchestrator-semantic-evidence.md | 66 + docs/adr/0053-related-post-body-evidence.md | 31 + .../adr/0054-author-linked-keyman-evidence.md | 29 + .../0055-board-semantic-project-evidence.md | 29 + ...ce-body-availability-and-reconciliation.md | 50 + ...mmutable-source-identity-reconciliation.md | 41 + .../adr/0058-source-publication-state-gate.md | 47 + .../0059-unknown-publication-state-surface.md | 31 + docs/adr/0060-source-voc-type-vocabulary.md | 31 + ...-post-body-character-reference-decoding.md | 33 + docs/adr/0062-semantic-unit-embedding.md | 42 + docs/adr/0063-third-normal-form-bookmarks.md | 40 + ...0064-lineage-evidence-and-tree-assembly.md | 37 + docs/adr/0065-prov-o-provenance-boundary.md | 38 + .../0066-position-preserving-image-content.md | 41 + docs/adr/0067-visual-region-vision-agent.md | 46 + .../0068-row-level-synthetic-seed-cleanup.md | 44 + docs/adr/0069-member-locale-preference.md | 19 + ...xtual-orchestrator-upstream-integration.md | 68 + .../0071-post-scoped-llm-session-metadata.md | 32 + docs/adr/0072-gateway-model-auto-discovery.md | 35 + docs/adr/0073-llm-structure-adjudication.md | 40 + ...4-provider-shaped-multi-agent-synthesis.md | 55 + ...0075-read-time-synthetic-seed-exclusion.md | 24 + docs/adr/0076-paper-grounded-model-policy.md | 86 + ...-single-call-structured-vision-evidence.md | 46 + ...0078-focused-lineage-graph-on-post-open.md | 44 + ...hestrator-owns-default-reasoning-effort.md | 30 + ...tic-backfill-for-missing-project-fields.md | 49 + .../adr/0081-source-author-keyman-evidence.md | 51 + docs/adr/0082-bounded-keyman-backfill.md | 48 + .../0083-orchestrator-runtime-commit-pin.md | 47 + docs/adr/0084-lineage-research-grounding.md | 79 + docs/adr/0085-bounded-post-model-batches.md | 44 + docs/adr/0086-bounded-post-row-loading.md | 28 + .../adr/0087-bounded-vision-image-payloads.md | 31 + ...-orchestrator-auto-protocol-negotiation.md | 37 + ...0089-private-real-data-runtime-boundary.md | 40 + ...0-global-ask-lineage-timeline-expansion.md | 83 + ...091-visual-region-embedding-persistence.md | 37 + docs/adr/0092-internal-relation-evidence.md | 42 + ...98-valkey-backed-post-content-ingestion.md | 102 + .../adr/0099-badge-and-accent-color-tokens.md | 73 + .../0100-major-event-requester-processor.md | 49 + ...richment-timeout-does-not-block-summary.md | 50 + .../0102-semantic-source-unit-boundaries.md | 51 + ...itespace-is-not-authoritative-structure.md | 38 + ...l-visual-regions-retain-parent-evidence.md | 44 + .../adr/0109-oidc-deep-link-state-recovery.md | 35 + .../0110-buyer-image-evidence-rendering.md | 36 + .../0111-project-bound-major-event-actions.md | 48 + docs/adr/0112-project-bound-summary-events.md | 27 + .../0114-stale-summary-buyer-continuity.md | 45 + .../0115-explicit-terminal-content-retry.md | 42 + .../0116-serialized-post-vision-analysis.md | 40 + .../adr/0117-catalog-backed-semantic-hints.md | 39 + ...-uiux-standard-guide-v3-design-overhaul.md | 36 + docs/adr/README.md | 19 + .../ACTUAL_RUNTIME_EVIDENCE_2026-08-18.md | 31 + ...UAL_RUNTIME_EVIDENCE_2026-08-18_GATEWAY.md | 20 + ...ME_EVIDENCE_2026-08-18_SEMANTIC_PROJECT.md | 32 + ...RUNTIME_EVIDENCE_2026-08-18_TEAM_REPORT.md | 18 + ...TUAL_RUNTIME_EVIDENCE_2026-08-19_VISION.md | 23 + .../ANALYSIS_RUN_REGISTRY_REFERENCES.md | 10 +- docs/doctoring/DESIGN_TOKEN_REFERENCES.md | 14 +- docs/image-content-schema.md | 37 +- docs/lineage-bi-research-notes.md | 47 +- docs/ontology/lineageweave-kg.ttl | 37 +- docs/product-technical-gap-baseline.md | 26 + docs/storybook-inventory.md | 11 +- fix_prompts.py | 46 + frontend/index.html | 6 + frontend/package.json | 3 +- frontend/pnpm-lock.yaml | 29 + frontend/src/App.css | 396 ++- frontend/src/App.test.tsx | 1090 ++++--- frontend/src/App.tsx | 2612 ++++++++++++----- frontend/src/LineageDag.tsx | 30 +- frontend/src/PostBody.test.tsx | 321 ++ frontend/src/PostBody.tsx | 283 +- frontend/src/api.ts | 480 ++- frontend/src/components/AdminPanel.tsx | 63 + frontend/src/components/BuyerNav.test.tsx | 23 + frontend/src/components/BuyerNav.tsx | 39 + frontend/src/components/CitationChip.tsx | 3 +- frontend/src/components/CutoffKnownBody.tsx | 8 +- frontend/src/components/FiveW1H.tsx | 69 + .../src/components/LineageEntityPicker.tsx | 5 +- .../src/components/StatusAlert.stories.tsx | 23 - frontend/src/components/StatusAlert.test.tsx | 23 - frontend/src/components/StatusAlert.tsx | 18 - frontend/src/config.ts | 16 +- frontend/src/i18n.test.ts | 91 + frontend/src/i18n.ts | 1437 +++++++++ frontend/src/index.css | 136 +- frontend/src/main.tsx | 14 +- frontend/src/oidcReturnUrl.test.ts | 69 + frontend/src/oidcReturnUrl.ts | 73 + frontend/src/postBodyDisplay.entities.test.ts | 9 + frontend/src/postBodyDisplay.test.ts | 193 +- frontend/src/postBodyDisplay.ts | 382 +-- frontend/src/styles/tokens.css | 173 +- frontend/src/styles/tokens.test.ts | 73 + frontend/src/vite-env.d.ts | 4 + frontend/tsconfig.app.json | 2 +- lineageweave/__init__.py | 2 +- .../abbreviation_tree_corroboration.py | 149 - lineageweave/adjudication_client.py | 22 +- lineageweave/affiliate_tree.py | 1 + lineageweave/caldav_client.py | 57 + lineageweave/chunking.py | 446 ++- lineageweave/commitment_extraction.py | 5 +- lineageweave/corporate_hierarchy_inference.py | 8 +- lineageweave/customer_group_tree.py | 139 - lineageweave/customer_hint_resolution.py | 97 + lineageweave/embedded_image_payload.py | 95 - lineageweave/embedding_client.py | 114 +- .../entity_relationship_classification.py | 5 +- lineageweave/five_w1h.py | 123 + lineageweave/fixtures.py | 1 + lineageweave/http_client.py | 19 +- lineageweave/image_content.py | 251 +- lineageweave/keyman_extraction.py | 31 +- lineageweave/leftover_pairs.py | 2 +- lineageweave/lineage_persistence.py | 11 +- lineageweave/llm_context.py | 59 + lineageweave/ontology.py | 1 + lineageweave/organization_name_resolution.py | 30 +- lineageweave/period_report.py | 4 +- lineageweave/post_chat.py | 136 +- lineageweave/post_content_normalization.py | 190 +- lineageweave/post_content_persistence.py | 359 +++ lineageweave/post_evaluation.py | 14 +- lineageweave/post_structure.py | 162 + lineageweave/post_summary.py | 762 ++++- lineageweave/prov_o.py | 9 + lineageweave/reconstruct.py | 3 + lineageweave/relation_verification.py | 18 +- lineageweave/semantic_hints.py | 188 ++ lineageweave/server.py | 5 + lineageweave/synthetic_seed_cleanup.py | 229 ++ lineageweave/tepp_client.py | 9 +- lineageweave/tepp_result.py | 210 -- lineageweave/vision_image.py | 58 + migrations/0001_initial_schema.sql | 24 +- migrations/0009_shared_metric_bank.sql | 13 + ...pair.sql => 0012_report_leftover_pair.sql} | 2 +- migrations/0016_cross_post_actor_identity.sql | 7 +- .../0020_analysis_run_retention_purge.sql | 47 +- migrations/0023_analysis_run_outbox.sql | 8 +- migrations/0026_post_content_artifacts.sql | 65 + .../0027_abbreviation_tree_corroboration.sql | 20 - migrations/0027_analysis_run_tepp_result.sql | 12 + migrations/0028_analysis_run_tepp_result.sql | 211 -- .../0028_internal_relation_evidence.sql | 16 + .../0029_analysis_run_tepp_accepted.sql | 230 -- migrations/0029_report_team_grouping.sql | 19 + migrations/0030_report_project_grouping.sql | 15 + migrations/0031_semantic_project_mentions.sql | 17 + migrations/0032_semantic_search_trigram.sql | 20 + migrations/0033_source_state_provenance.sql | 19 + migrations/0034_source_context_provenance.sql | 21 + migrations/0035_body_search_prefix.sql | 13 + migrations/0036_normalized_body_search.sql | 28 + migrations/0037_source_record_identity.sql | 18 + migrations/0038_source_named_hints.sql | 12 + migrations/0039_source_org_named_hints.sql | 13 + migrations/0040_post_summary_contract.sql | 15 + .../0041_source_hint_lookup_indexes.sql | 27 + migrations/0042_voc_type_vocabulary.sql | 20 + migrations/0043_bookmark.sql | 10 + migrations/0044_member_locale_preference.sql | 16 + .../0045_post_content_image_regions.sql | 28 + .../0046_post_content_structure_evidence.sql | 13 + ...7_post_content_image_region_embeddings.sql | 19 + migrations/0048_post_summary_five_w1h.sql | 17 + migrations/0049_source_record_key_lookup.sql | 8 + .../0050_post_content_ingestion_queue.sql | 41 + ...> 0060_role_responsibility_agent_type.sql} | 0 migrations/0100_major_event_action.sql | 16 + .../0101_project_bound_major_event_action.sql | 18 + .../0102_project_bound_summary_event.sql | 18 + migrations/0103_tenant_settings.sql | 6 + .../0028_analysis_run_tepp_result.sql | 27 - .../0029_analysis_run_tepp_accepted.sql | 28 - migrations/rollback/0103_tenant_settings.sql | 1 + patch_api.py | 35 + patch_app_fetch.py | 24 + patch_app_order.py | 36 + patch_app_test.py | 15 + patch_main.py | 41 + patch_main_pool.py | 66 + pyproject.toml | 9 +- scripts/backfill_post_content.py | 283 ++ scripts/backfill_post_keymen.py | 233 ++ scripts/backfill_post_summaries.py | 303 ++ scripts/import_postgresql_posts.py | 558 ++++ scripts/queue_post_content_backfill.py | 186 ++ scripts/requeue_failed_post_content.py | 78 + scripts/seed_demo_data.py | 424 +-- scripts/smoke_test_oidc.py | 2 + .../synthetic_invoice_embedded_image.html | 10 - tests/test_abbreviation_tree_corroboration.py | 130 - tests/test_adaptive_orchestration_defaults.py | 16 +- tests/test_adjudication_client.py | 23 + tests/test_analysis_run_create.py | 2 +- ...test_analysis_run_reconstruction_schema.py | 3 - tests/test_analysis_run_registry_schema.py | 145 - tests/test_analysis_run_start.py | 278 +- .../test_analysis_run_tepp_accepted_schema.py | 157 - tests/test_analysis_run_tepp_result_schema.py | 139 - tests/test_analysis_run_worker.py | 86 + tests/test_caldav_client.py | 54 + tests/test_chunking.py | 240 +- tests/test_contextual_orchestrator_start.py | 120 + tests/test_contextual_orchestrator_vision.py | 93 + tests/test_corporate_hierarchy_inference.py | 26 + tests/test_customer_group_tree.py | 69 - tests/test_customer_group_tree_labels.py | 36 - tests/test_customer_hint_ingestion.py | 116 + tests/test_customer_hint_resolution.py | 51 + tests/test_documentation_hygiene.py | 27 +- tests/test_embedded_image_payload.py | 109 - tests/test_embedding_client.py | 34 +- tests/test_embedding_client_edges.py | 102 + tests/test_entity_relationship_ingestion.py | 57 + tests/test_five_w1h.py | 64 + tests/test_global_ask_sources.py | 360 +++ tests/test_http_client_edges.py | 50 + tests/test_image_content.py | 99 +- tests/test_image_content_gateway.py | 65 + tests/test_import_postgresql_posts.py | 223 ++ tests/test_ingestion_transaction_contracts.py | 74 +- tests/test_leftover_pairs.py | 2 +- tests/test_lineage_ingestion.py | 64 +- tests/test_llm_context.py | 41 + tests/test_migration_replay.py | 35 + tests/test_ontology.py | 28 +- tests/test_organization_name_resolution.py | 20 + ..._organization_name_resolution_ingestion.py | 75 + tests/test_person_mention_projection.py | 100 +- tests/test_post_chat.py | 110 + tests/test_post_chat_ingestion.py | 295 ++ tests/test_post_content_normalization.py | 289 +- tests/test_post_content_persistence.py | 83 + tests/test_post_content_persistence_edges.py | 336 +++ tests/test_post_content_queue.py | 427 +++ tests/test_post_content_worker.py | 285 ++ tests/test_post_eligibility.py | 18 + tests/test_post_evaluation_ingestion.py | 76 + tests/test_post_structure.py | 45 + tests/test_post_summary.py | 214 +- tests/test_real_provider_integration.py | 36 +- tests/test_relation_verification_edges.py | 73 + tests/test_relation_verification_internal.py | 70 + tests/test_report_team_grouping.py | 49 + tests/test_schema.py | 47 +- tests/test_seed_late_demo_post.py | 147 - tests/test_seed_report_run.py | 9 +- tests/test_seed_tepp_run.py | 51 - tests/test_semantic_author_provenance.py | 17 + tests/test_semantic_hints.py | 170 ++ tests/test_semantic_keyman_context.py | 20 + tests/test_semantic_project_evidence.py | 19 + tests/test_source_post_revision.py | 3 - tests/test_source_state_serialization.py | 36 + tests/test_stale_summary_continuity.py | 36 + tests/test_static_sql_review_contracts.py | 76 + tests/test_structured_vision.py | 33 + tests/test_synthetic_seed_cleanup.py | 284 ++ tests/test_tepp_client.py | 17 + tests/test_tepp_public_content.py | 42 - tests/test_tepp_result.py | 73 - tests/test_tepp_transport_evidence.py | 106 - tests/test_tied_organization_no_create.py | 25 + tests/test_vision_image.py | 42 + tests/test_vision_image_formats.py | 52 + update_app.py | 28 + uv.lock | 8 +- 390 files changed, 27833 insertions(+), 6909 deletions(-) delete mode 100644 CHANGELOG.d/2.10.0-leftover-pairs-and-rankings.md delete mode 100644 CHANGELOG.d/2.10.1-analysis-run-accessible-next-action.md delete mode 100644 CHANGELOG.d/2.10.2-embedded-image-html-parser.md delete mode 100644 CHANGELOG.d/2.10.3-stale-hidden-run-list.md delete mode 100644 CHANGELOG.d/2.10.4-retention-purge-reconstruction-children.md delete mode 100644 CHANGELOG.d/2.11.0-customer-group-tree-searxng.md delete mode 100644 CHANGELOG.d/2.12.0-persistable-tepp-result.md delete mode 100644 CHANGELOG.d/2.12.1-tepp-accepted-transport-evidence.md delete mode 100644 CHANGELOG.d/2.12.2-tepp-accepted-clocks.md delete mode 100644 CHANGELOG.d/2.12.3-late-demo-cutoff-post.md create mode 100644 CHANGELOG.d/2.12.6-buyer-image-source-safety.md create mode 100644 CHANGELOG.d/2.12.6-buyer-main-port.md create mode 100644 CHANGELOG.d/2.12.6-oidc-deep-link-safety.md create mode 100644 CHANGELOG.d/2.12.6-oidc-resource-audience.md create mode 100644 CHANGELOG.d/2.12.7-safe-sql-audit.md create mode 100644 CHANGELOG.d/2.13.0-buyer-gnb-i18n.md create mode 100644 CHANGELOG.d/2.13.1-badge-color-tokens.md create mode 100644 CHANGELOG.d/2.13.1-mixed-body-indentation.md create mode 100644 CHANGELOG.d/2.13.1-partial-image-regions.md create mode 100644 CHANGELOG.d/2.13.1-separate-source-tables.md create mode 100644 CHANGELOG.d/2.13.1-source-indent-semantics.md create mode 100644 CHANGELOG.d/2.21.0-synthetic-cleanup-scope.md create mode 100644 CHANGELOG.d/2.21.1-orchestrator-pin-doc.md create mode 100644 add_translations.py delete mode 100644 backend/app/abbreviation_tree_corroboration_ingestion.py create mode 100644 backend/app/analysis_run_worker.py delete mode 100644 backend/app/customer_group_tree_ingestion.py create mode 100644 backend/app/customer_hint_ingestion.py create mode 100644 backend/app/demo_scope.py create mode 100644 backend/app/five_w1h_ingestion.py create mode 100644 backend/app/post_content_queue.py create mode 100644 backend/app/post_content_worker.py create mode 100644 backend/app/post_eligibility.py create mode 100644 backend/tests/test_auth_jwks.py create mode 100644 backend/tests/test_oidc_audience.py create mode 100644 docker/contextual-orchestrator/Dockerfile create mode 100644 docker/contextual-orchestrator/agents.json create mode 100644 docker/contextual-orchestrator/start.py create mode 100644 docker/postgres-init/migrate.sh rename docs/adr/{0030-rankweave-fusion-fail-closed.md => 0024-rankweave-fusion-fail-closed.md} (91%) create mode 100644 docs/adr/0028-keyverse-oidc-provider.md create mode 100644 docs/adr/0029-zotero-local-reproducibility.md create mode 100644 docs/adr/0030-external-llm-gateway-environment.md delete mode 100644 docs/adr/0031-embedded-image-html-parser.md create mode 100644 docs/adr/0032-bounded-browser-projections.md delete mode 100644 docs/adr/0032-retention-purge-reconstruction-children.md delete mode 100644 docs/adr/0033-customer-group-tree-abbreviation-corroboration.md create mode 100644 docs/adr/0033-vision-raster-normalization.md delete mode 100644 docs/adr/0034-persistable-tepp-result.md create mode 100644 docs/adr/0034-team-multiple-membership-period-reports.md create mode 100644 docs/adr/0035-project-period-reports.md delete mode 100644 docs/adr/0035-tepp-accepted-transport-evidence.md create mode 100644 docs/adr/0036-semantic-project-and-keyman-evidence.md create mode 100644 docs/adr/0037-buyer-gnb-and-product-surface.md create mode 100644 docs/adr/0038-calendar-source-contract.md create mode 100644 docs/adr/0039-global-ask-agent-source-boundary.md create mode 100644 docs/adr/0040-source-state-provenance.md create mode 100644 docs/adr/0041-source-context-vs-authorization-scope.md create mode 100644 docs/adr/0042-source-hints-before-customer-binding.md create mode 100644 docs/adr/0043-indexed-body-search-boundary.md create mode 100644 docs/adr/0044-source-author-account-keyman-hints.md create mode 100644 docs/adr/0045-orchestrator-credential-source.md create mode 100644 docs/adr/0046-source-record-identity.md create mode 100644 docs/adr/0047-global-ask-semantic-retrieval.md rename docs/adr/{0028-persist-lsirm-leftover-pairs.md => 0048-persist-lsirm-leftover-pairs.md} (94%) rename docs/adr/{0029-leftover-pair-report-ui.md => 0049-leftover-pair-report-ui.md} (87%) rename docs/adr/{0024-seed-period-report-analysis-run.md => 0050-seed-period-report-analysis-run.md} (98%) create mode 100644 docs/adr/0051-source-named-hints.md create mode 100644 docs/adr/0052-plain-orchestrator-semantic-evidence.md create mode 100644 docs/adr/0053-related-post-body-evidence.md create mode 100644 docs/adr/0054-author-linked-keyman-evidence.md create mode 100644 docs/adr/0055-board-semantic-project-evidence.md create mode 100644 docs/adr/0056-source-body-availability-and-reconciliation.md create mode 100644 docs/adr/0057-immutable-source-identity-reconciliation.md create mode 100644 docs/adr/0058-source-publication-state-gate.md create mode 100644 docs/adr/0059-unknown-publication-state-surface.md create mode 100644 docs/adr/0060-source-voc-type-vocabulary.md create mode 100644 docs/adr/0061-post-body-character-reference-decoding.md create mode 100644 docs/adr/0062-semantic-unit-embedding.md create mode 100644 docs/adr/0063-third-normal-form-bookmarks.md create mode 100644 docs/adr/0064-lineage-evidence-and-tree-assembly.md create mode 100644 docs/adr/0065-prov-o-provenance-boundary.md create mode 100644 docs/adr/0066-position-preserving-image-content.md create mode 100644 docs/adr/0067-visual-region-vision-agent.md create mode 100644 docs/adr/0068-row-level-synthetic-seed-cleanup.md create mode 100644 docs/adr/0069-member-locale-preference.md create mode 100644 docs/adr/0070-contextual-orchestrator-upstream-integration.md create mode 100644 docs/adr/0071-post-scoped-llm-session-metadata.md create mode 100644 docs/adr/0072-gateway-model-auto-discovery.md create mode 100644 docs/adr/0073-llm-structure-adjudication.md create mode 100644 docs/adr/0074-provider-shaped-multi-agent-synthesis.md create mode 100644 docs/adr/0075-read-time-synthetic-seed-exclusion.md create mode 100644 docs/adr/0076-paper-grounded-model-policy.md create mode 100644 docs/adr/0077-single-call-structured-vision-evidence.md create mode 100644 docs/adr/0078-focused-lineage-graph-on-post-open.md create mode 100644 docs/adr/0079-orchestrator-owns-default-reasoning-effort.md create mode 100644 docs/adr/0080-semantic-backfill-for-missing-project-fields.md create mode 100644 docs/adr/0081-source-author-keyman-evidence.md create mode 100644 docs/adr/0082-bounded-keyman-backfill.md create mode 100644 docs/adr/0083-orchestrator-runtime-commit-pin.md create mode 100644 docs/adr/0084-lineage-research-grounding.md create mode 100644 docs/adr/0085-bounded-post-model-batches.md create mode 100644 docs/adr/0086-bounded-post-row-loading.md create mode 100644 docs/adr/0087-bounded-vision-image-payloads.md create mode 100644 docs/adr/0088-orchestrator-auto-protocol-negotiation.md create mode 100644 docs/adr/0089-private-real-data-runtime-boundary.md create mode 100644 docs/adr/0090-global-ask-lineage-timeline-expansion.md create mode 100644 docs/adr/0091-visual-region-embedding-persistence.md create mode 100644 docs/adr/0092-internal-relation-evidence.md create mode 100644 docs/adr/0098-valkey-backed-post-content-ingestion.md create mode 100644 docs/adr/0099-badge-and-accent-color-tokens.md create mode 100644 docs/adr/0100-major-event-requester-processor.md create mode 100644 docs/adr/0101-enrichment-timeout-does-not-block-summary.md create mode 100644 docs/adr/0102-semantic-source-unit-boundaries.md create mode 100644 docs/adr/0103-source-whitespace-is-not-authoritative-structure.md create mode 100644 docs/adr/0104-partial-visual-regions-retain-parent-evidence.md create mode 100644 docs/adr/0109-oidc-deep-link-state-recovery.md create mode 100644 docs/adr/0110-buyer-image-evidence-rendering.md create mode 100644 docs/adr/0111-project-bound-major-event-actions.md create mode 100644 docs/adr/0112-project-bound-summary-events.md create mode 100644 docs/adr/0114-stale-summary-buyer-continuity.md create mode 100644 docs/adr/0115-explicit-terminal-content-retry.md create mode 100644 docs/adr/0116-serialized-post-vision-analysis.md create mode 100644 docs/adr/0117-catalog-backed-semantic-hints.md create mode 100644 docs/adr/0118-uiux-standard-guide-v3-design-overhaul.md create mode 100644 docs/adr/README.md create mode 100644 docs/doctoring/ACTUAL_RUNTIME_EVIDENCE_2026-08-18.md create mode 100644 docs/doctoring/ACTUAL_RUNTIME_EVIDENCE_2026-08-18_GATEWAY.md create mode 100644 docs/doctoring/ACTUAL_RUNTIME_EVIDENCE_2026-08-18_SEMANTIC_PROJECT.md create mode 100644 docs/doctoring/ACTUAL_RUNTIME_EVIDENCE_2026-08-18_TEAM_REPORT.md create mode 100644 docs/doctoring/ACTUAL_RUNTIME_EVIDENCE_2026-08-19_VISION.md create mode 100644 docs/product-technical-gap-baseline.md create mode 100644 fix_prompts.py create mode 100644 frontend/src/PostBody.test.tsx create mode 100644 frontend/src/components/AdminPanel.tsx create mode 100644 frontend/src/components/BuyerNav.test.tsx create mode 100644 frontend/src/components/BuyerNav.tsx create mode 100644 frontend/src/components/FiveW1H.tsx delete mode 100644 frontend/src/components/StatusAlert.stories.tsx delete mode 100644 frontend/src/components/StatusAlert.test.tsx delete mode 100644 frontend/src/components/StatusAlert.tsx create mode 100644 frontend/src/i18n.test.ts create mode 100644 frontend/src/i18n.ts create mode 100644 frontend/src/oidcReturnUrl.test.ts create mode 100644 frontend/src/oidcReturnUrl.ts create mode 100644 frontend/src/postBodyDisplay.entities.test.ts create mode 100644 frontend/src/styles/tokens.test.ts delete mode 100644 lineageweave/abbreviation_tree_corroboration.py create mode 100644 lineageweave/caldav_client.py delete mode 100644 lineageweave/customer_group_tree.py create mode 100644 lineageweave/customer_hint_resolution.py delete mode 100644 lineageweave/embedded_image_payload.py create mode 100644 lineageweave/five_w1h.py create mode 100644 lineageweave/llm_context.py create mode 100644 lineageweave/post_content_persistence.py create mode 100644 lineageweave/post_structure.py create mode 100644 lineageweave/semantic_hints.py create mode 100644 lineageweave/synthetic_seed_cleanup.py delete mode 100644 lineageweave/tepp_result.py create mode 100644 lineageweave/vision_image.py rename migrations/{0026_report_leftover_pair.sql => 0012_report_leftover_pair.sql} (97%) create mode 100644 migrations/0026_post_content_artifacts.sql delete mode 100644 migrations/0027_abbreviation_tree_corroboration.sql create mode 100644 migrations/0027_analysis_run_tepp_result.sql delete mode 100644 migrations/0028_analysis_run_tepp_result.sql create mode 100644 migrations/0028_internal_relation_evidence.sql delete mode 100644 migrations/0029_analysis_run_tepp_accepted.sql create mode 100644 migrations/0029_report_team_grouping.sql create mode 100644 migrations/0030_report_project_grouping.sql create mode 100644 migrations/0031_semantic_project_mentions.sql create mode 100644 migrations/0032_semantic_search_trigram.sql create mode 100644 migrations/0033_source_state_provenance.sql create mode 100644 migrations/0034_source_context_provenance.sql create mode 100644 migrations/0035_body_search_prefix.sql create mode 100644 migrations/0036_normalized_body_search.sql create mode 100644 migrations/0037_source_record_identity.sql create mode 100644 migrations/0038_source_named_hints.sql create mode 100644 migrations/0039_source_org_named_hints.sql create mode 100644 migrations/0040_post_summary_contract.sql create mode 100644 migrations/0041_source_hint_lookup_indexes.sql create mode 100644 migrations/0042_voc_type_vocabulary.sql create mode 100644 migrations/0043_bookmark.sql create mode 100644 migrations/0044_member_locale_preference.sql create mode 100644 migrations/0045_post_content_image_regions.sql create mode 100644 migrations/0046_post_content_structure_evidence.sql create mode 100644 migrations/0047_post_content_image_region_embeddings.sql create mode 100644 migrations/0048_post_summary_five_w1h.sql create mode 100644 migrations/0049_source_record_key_lookup.sql create mode 100644 migrations/0050_post_content_ingestion_queue.sql rename migrations/{0012_role_responsibility_agent_type.sql => 0060_role_responsibility_agent_type.sql} (100%) create mode 100644 migrations/0100_major_event_action.sql create mode 100644 migrations/0101_project_bound_major_event_action.sql create mode 100644 migrations/0102_project_bound_summary_event.sql create mode 100644 migrations/0103_tenant_settings.sql delete mode 100644 migrations/rollback/0028_analysis_run_tepp_result.sql delete mode 100644 migrations/rollback/0029_analysis_run_tepp_accepted.sql create mode 100644 migrations/rollback/0103_tenant_settings.sql create mode 100644 patch_api.py create mode 100644 patch_app_fetch.py create mode 100644 patch_app_order.py create mode 100644 patch_app_test.py create mode 100644 patch_main.py create mode 100644 patch_main_pool.py create mode 100644 scripts/backfill_post_content.py create mode 100644 scripts/backfill_post_keymen.py create mode 100644 scripts/backfill_post_summaries.py create mode 100644 scripts/import_postgresql_posts.py create mode 100644 scripts/queue_post_content_backfill.py create mode 100644 scripts/requeue_failed_post_content.py delete mode 100644 tests/fixtures/synthetic_invoice_embedded_image.html delete mode 100644 tests/test_abbreviation_tree_corroboration.py create mode 100644 tests/test_adjudication_client.py delete mode 100644 tests/test_analysis_run_tepp_accepted_schema.py delete mode 100644 tests/test_analysis_run_tepp_result_schema.py create mode 100644 tests/test_analysis_run_worker.py create mode 100644 tests/test_caldav_client.py create mode 100644 tests/test_contextual_orchestrator_start.py create mode 100644 tests/test_contextual_orchestrator_vision.py delete mode 100644 tests/test_customer_group_tree.py delete mode 100644 tests/test_customer_group_tree_labels.py create mode 100644 tests/test_customer_hint_ingestion.py create mode 100644 tests/test_customer_hint_resolution.py delete mode 100644 tests/test_embedded_image_payload.py create mode 100644 tests/test_embedding_client_edges.py create mode 100644 tests/test_entity_relationship_ingestion.py create mode 100644 tests/test_five_w1h.py create mode 100644 tests/test_global_ask_sources.py create mode 100644 tests/test_http_client_edges.py create mode 100644 tests/test_image_content_gateway.py create mode 100644 tests/test_import_postgresql_posts.py create mode 100644 tests/test_llm_context.py create mode 100644 tests/test_migration_replay.py create mode 100644 tests/test_organization_name_resolution_ingestion.py create mode 100644 tests/test_post_chat_ingestion.py create mode 100644 tests/test_post_content_persistence.py create mode 100644 tests/test_post_content_persistence_edges.py create mode 100644 tests/test_post_content_queue.py create mode 100644 tests/test_post_content_worker.py create mode 100644 tests/test_post_eligibility.py create mode 100644 tests/test_post_evaluation_ingestion.py create mode 100644 tests/test_post_structure.py create mode 100644 tests/test_relation_verification_edges.py create mode 100644 tests/test_relation_verification_internal.py create mode 100644 tests/test_report_team_grouping.py delete mode 100644 tests/test_seed_late_demo_post.py create mode 100644 tests/test_semantic_author_provenance.py create mode 100644 tests/test_semantic_hints.py create mode 100644 tests/test_semantic_keyman_context.py create mode 100644 tests/test_semantic_project_evidence.py create mode 100644 tests/test_source_state_serialization.py create mode 100644 tests/test_stale_summary_continuity.py create mode 100644 tests/test_static_sql_review_contracts.py create mode 100644 tests/test_structured_vision.py create mode 100644 tests/test_synthetic_seed_cleanup.py delete mode 100644 tests/test_tepp_public_content.py delete mode 100644 tests/test_tepp_result.py delete mode 100644 tests/test_tepp_transport_evidence.py create mode 100644 tests/test_vision_image.py create mode 100644 tests/test_vision_image_formats.py create mode 100644 update_app.py diff --git a/.env.example b/.env.example index 03b0fb45e..11d0a674b 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,11 @@ VALKEY_PORT=16379 KEYCLOAK_ADMIN=admin KEYCLOAK_ADMIN_PASSWORD=admin_dev_only KEYCLOAK_PORT=18080 +OIDC_CLOCK_SKEW_SECONDS=5 +# Browser/API access tokens must be minted for this backend resource. +# Local Keycloak emits this audience through realm-export.json. Production +# Keyverse/OIDC deployments may instead set KEYVERSE_AUDIENCE/OIDC_AUDIENCE. +OIDC_AUDIENCE=lineageweave-api BACKEND_PORT=18420 @@ -27,4 +32,18 @@ BACKEND_PORT=18420 # running contextual-orchestrator to turn the channels on. ORCHESTRATOR_BASE_URL= ORCHESTRATOR_API_KEY= -VISION_MODEL= + +# GitHub workflows inject the canonical provider names from masked secrets. +# Non-GitHub Compose runs also accept the operator's ~/.env compatibility +# names below; docker-compose maps them to the canonical names without +# exposing them to the frontend or committing them. +# Canonical provider endpoint for contextual-orchestrator. +LLM_GATEWAY_API_URL= +# Compatibility alias; LLM_GATEWAY_API_URL wins when both are set. +LLM_GATEWAY_URL= +LLM_GATEWAY_API_KEY= +LLM_GATEWAY_EMBEDDING_MODEL= +LLM_API_GATEWAY= +LLM_API_KEY= +CALDAV_BASE_URL= +CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS= diff --git a/.gitignore b/.gitignore index 92f76829b..54a94e390 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ __pycache__/ .DS_Store .codegraph/ .env - +.coverage diff --git a/AGENTS.md b/AGENTS.md index e26c146d2..1728f9e61 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,20 +7,23 @@ Cross-agent conventions for `LineageWeave`, readable by any coding agent A demo BI prototype that reconstructs git-branch-style lineage between scattered short records. See [ARCHITECTURE.md](ARCHITECTURE.md) for the -design and [`docs/lineage-bi-research-notes.md`](docs/lineage-bi-research-notes.md) -for the literature it is grounded in. +design, [ADR 0084](docs/adr/0084-lineage-research-grounding.md) for the +normative research-grounding policy, and +[`docs/lineage-bi-research-notes.md`](docs/lineage-bi-research-notes.md) for +supporting literature and aggregate evidence. -## Hard rule: no real data, ever +## Hard rule: no real data in repository artifacts -This repo ships **synthetic data only** (`lineageweave/fixtures.py`) and -must never reference, by name or otherwise identifiably, any real -organization whose data motivated this design. Never add a fixture, test -case, screenshot, or example derived from a real organization's records. If you are extending this repo to validate against -real data, do that validation entirely outside this repository (a private -scratch script against a local database is fine) and only bring back -**aggregate, non-identifying findings** -- see how -`docs/lineage-bi-research-notes.md`'s "2.6%" validation number is phrased: -a statistic, never a title, name, or id. +This repository ships **synthetic fixtures only** (`lineageweave/fixtures.py`) +and must never commit or expose, by name or otherwise identifiably, any real +organization's records. Never add a real record to a fixture, test case, +screenshot, example, log, benchmark artifact, or documentation. + +The private runtime is different: the product is expected to read an +authorized real PostgreSQL source through its configured import/data boundary. +Keep those records outside git and return only authorized, provenance-bearing +product evidence. Validation results brought back into this repository must be +aggregate and non-identifying -- a statistic, never a title, name, or id. ## Reuse before you build @@ -44,6 +47,111 @@ reimplementing them: Before adding a new dependency, check whether an existing org repo already does it (`gh repo list ContextualWisdomLab`). +## Decision records and model boundary + +- Read the applicable `docs/adr/` records before making an architectural, + schema, provider, model, or runtime decision. Record a new decision before + implementing a new policy; do not resolve ADR conflicts by intuition. +- All LLM, VISION, embedding, and structured-output traffic crosses + `contextual-orchestrator`. This repository never calls a provider API + directly and never uses a monkey patch to repair an upstream capability. +- Compose loads provider transport credentials from `~/.env` into the + orchestrator service. Never copy those values into this repository, an + image, a fixture, a log, or a committed agent configuration. +- `LLM_GATEWAY_MODEL`, `VISION_MODEL`, and provider-specific model selectors + are not LineageWeave configuration. Model discovery, capability selection, + reasoning effort, protocol negotiation, and VISION selection belong to + contextual-orchestrator and must follow its paper-grounded ADRs. +- MLX and any other local runtime are not public provider contracts. Use the + provider-neutral gateway boundary; historical benchmark material is not + runtime configuration. + +## ADR-first and paper-grounded model decisions + +ADRs are normative. Before making an architectural, schema, provider, agent, +LLM, VISION, routing, reasoning-effort, or persistence decision, read the +relevant ADRs first. If the ADR does not cover the decision, write or update +the ADR before changing code, tests, Docker configuration, or runtime policy. +Do not use an implementation preference to silently override an ADR. + +Model-related decisions are governed by [ADR +0076](docs/adr/0076-paper-grounded-model-policy.md) and may rely only on the +paper sources cited there and in contextual-orchestrator's literature register: +the Fugu technical report, TRINITY, and Conductor. Provider model ordering, +model-name size guesses, undocumented benchmarks, and local intuition are not +evidence for model quality, routing, reasoning effort, agent count, synthesis, +or VISION selection. If the papers do not support a policy, leave it +undecided or unavailable rather than inventing a heuristic. + +The canonical provider credentials are runtime-only from `~/.env` through the +Compose `env_file` boundary. Never copy `~/.env` into the repository or image, +print its values, or persist them. Do not add `LLM_GATEWAY_MODEL`; the upstream +contextual-orchestrator owns model discovery and selection. + +## LLM and VISION boundary + +- Use `LLM_GATEWAY_API_KEY` and `LLM_GATEWAY_API_URL` from the user's `~/.env` + at runtime. Keep compatibility aliases only at the process boundary; do not + introduce a second credential source or a repository-local secret. +- Every LLM and VISION operation goes through contextual-orchestrator. This + includes adjudication, summaries, Keyman/entity extraction, post chat, + paragraph structure, image region recognition, OCR, image descriptions, and + embeddings. Do not call a provider SDK or raw `/v1/chat/completions` or + `/v1/responses` endpoint from LineageWeave. +- One post shares one orchestrator session id across its LLM and VISION work. + Pass bounded provenance metadata with each request, including post id, + corporate entity code, PU, author id, source system, and visibility when + available. Do not persist an ad hoc `user_account + post_id` session key; + use normalized third-normal-form tables and the ADR-defined foreign keys. +- Do not set `LLM_GATEWAY_MODEL` or select a model by provider order, model + name, parameter count, or local intuition. Blank provider agents must be + expanded and selected by contextual-orchestrator. Reasoning effort defaults + to `auto`; `low`, `medium`, `high`, and `xhigh` are capability/paper-policy + inputs, not a local model ranking heuristic. Never force `none` merely + because a model is not known to be a reasoning model. +- Responses API `developer` and Chat Completions `system` are compatible + instruction roles at the orchestrator boundary. The orchestrator owns the + translation and provider capability handling; do not fork prompts per + transport in this repository. +- Treat `LLM_GATEWAY_API_URL` as an opaque OpenAI-compatible gateway endpoint. + Do not add MLX/local-server URL schemes, port lists, local defaults, + chat-template injection, or vendor-specific bootstrap exceptions in + LineageWeave. Provider-specific capability translation belongs upstream. +- `response_format`, `tools`, Responses API requests, `json_object`, and + `json_schema` must remain multi-agent workflows. A structured response or a + repair attempt must not silently fall back to a single-agent passthrough. + Preserve schema validation, synthesis, repair, session, and cost lineage. +- VISION is an orchestrator capability, not a frontend-only enhancement. For + unsupported image formats, convert at ingestion; for transparent PNGs, + flatten transparent pixels onto white for the derived analysis image while + retaining the original asset and provenance. Recognize image DOM/visual + regions before OCR, descriptions, Keyman extraction, or embeddings. Store + region-level evidence; never show an internal LLM instruction such as + `This post is an image` to a buyer. + +## Source parsing and semantic units + +- Preserve the source representation and provenance, then derive semantic + paragraph/list/table/image-region units for search, ontology, and embeddings. + Do not flatten a post into one opaque body string. +- Paragraph structure may come from HTML DOM and CSS, visible leading spaces or + ` `, and OOXML/MS Word paragraph or run properties. Combine those + signals with contextual-orchestrator adjudication when evidence conflicts; + heuristics are not authoritative and must not be the only fallback for an + unresolved structure decision. +- Source-system codes may be enriched with catalog display names under ADR + 0117. Pass those names to contextual-orchestrator as labeled lookup hints + only; never promote them to an entity binding, customer fact, project fact, + or imported-author affiliation without post evidence. +- Remove presentation-only visual line alignment inside a paragraph (for + example continuation lines manually aligned after `-`, `*`, `1.`, or `.`) + from derived semantic text, while retaining the source body and meaningful + list/heading nesting. A buyer-facing post view must render semantic + paragraphs, not the authoring application's spacing workaround. +- Image descriptions, OCR text, and region evidence are analysis artifacts, + not buyer-facing prompt instructions. Buyer UI shows the source content and + useful captions/evidence only, with provenance where appropriate. + ## Pluggable channels: never fake a missing signal `NullEmbeddingClient`, `NullAdjudicationClient`, @@ -58,13 +166,9 @@ confidently-negative signal are different things. Keyman extraction, entity-relationship classification, post summary, in-popup chat, and commitment derivation go through contextual-orchestrator the same way adjudication does -- never a raw LLM API. Demo TEPP seed goes through -`tepp_client` the same way: a missing transport or an unpublished -envelope is Failed (`tepp_not_available` / `tepp_result_not_persisted`). -A published accepted acknowledgement is Failed / -`tepp_completed_result_unsupported` and may be shown as aggregate -transport evidence (ADR 0035). Never stamp Succeeded from that ack or -from a LineageWeave-local completed envelope, and never invent a theta -or a local psychometric substitute. +`tepp_client` the same way: a missing transport or an unused accepted +envelope is Failed (`tepp_not_available` / `tepp_result_not_persisted`), +never a fabricated theta or a local psychometric substitute. ## Tests @@ -82,7 +186,7 @@ in the same spirit) -- never against real data, per the hard rule above. against a live local stack (`make up`) and self-skip without one -- see [README.md](README.md#local-product-stack-docker-compose). -Period leftover pairs (ADR 0028 / 0029) are computed in +Period leftover pairs (ADR 0017 / 0018) are computed in `lineageweave/leftover_pairs.py` from the residual after a real GRM/GPCM score, never invented. Missing cells stay out of the Gabriel factorization. Closest and farthest post–criterion pairs @@ -101,9 +205,8 @@ pnpm run lint && pnpm run test && pnpm run build A run-bearing analysis-run registry empties only after an unrevoked `analysis_run_retention_grant` and `GRANT analysis_run_retention_admin` -(ADR 0020 / v0.87.0). The documented phrase is not a secret. The same -call empties reconstruction children when those tables exist -(ADR 0032 / v2.10.4). Do not expose purge on a public HTTP route. +(ADR 0020 / v0.87.0). The documented phrase is not a secret. Do not +expose purge on a public HTTP route. `POST /api/analysis-runs` records Pending lineage only (ADR 0017 / v2.7.1). TEPP and period-report kinds 422 before any snapshot write. @@ -113,9 +216,6 @@ v0.88.0). Do not invent a theta. Opening a cutoff-rewritten title shows **Body this run knew** from `source_post_revision` beside the live rewrite (ADR 0025 / v2.1.0). Do not invent the earlier sentence when no revision covers the cutoff. -After `make seed`, the January 12 Demo Corp lineage and TEPP runs list -Demo public post and do not list Late Demo public post (2026-01-13). -The live post list still shows Late Demo (ADR 0016). A corporate-entity similarity result has three outcomes: unique, miss, or tie (ADR 0026). A tie is not a miss. Keep the organization name @@ -130,13 +230,6 @@ R&R chips read the catalog id stored on `post_summary_role` backfill leaves a role unbound when two same-named mentions already exist on the post. -A listed analysis-run that then 404s must stay generic: do not name the thread or the cutoff, -and do not say the run is not visible (ADR 0014 / ADR 0018). After -that 404, re-read the authorized list so the stale row does not stay -clickable. Announce the next action with `role="alert"` without -moving focus. Remaining visible runs stay clickable. Request remains -the named reconstruction control. - ## CI gates `.github/workflows/tests.yml` runs the full suite on every PR to `main`. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 5bdebfab1..d0280ff97 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -64,7 +64,7 @@ flowchart LR | `chunking.py` | Splits a document into meaning-identifiable units (paragraph, sentence, DOM, conversation-turn) plus embedded-image extraction, in document order | | `embedding_client.py` | Pluggable text-embedding channel (`Null` default, `OpenAiCompatible` real impl) + `chunked_max_similarity` | | `adjudication_client.py` | Pluggable LLM-judgment channel (`Null` default, `ContextualOrchestrator` real impl) | -| `image_content.py` | Pluggable vision channel: OCR + object recognition/tagging for embedded images (`Null` default, `OpenAiCompatibleVisionClient` real impl). The product popup (`frontend/src/PostBody.tsx`) and `extract_base64_images` parse with the same HTML rules as `chunk_by_dom` (ADR 0031) so invoice-like `alt` values still show the picture; GET does not call the vision client. | +| `image_content.py` | Pluggable vision channel: OCR + object recognition/tagging for embedded images (`Null` default, `OpenAiCompatibleVisionClient` real impl). The product popup (`frontend/src/PostBody.tsx`) renders each `data:image` payload in document order so the buyer sees the picture, not the base64 string; GET does not call the vision client. | | `tepp_client.py` | TEPP's published `AnalysisRunRequest` wire contract, pluggable transport | | `rankweave_client.py` | Fail-closed RankWeave ranking port (`weighted_reciprocal_rank_fuse` in-process; never invent a fused score or a theta) | | `reconstruct.py` | The pipeline: group → candidate window → score → fuse → thread | @@ -81,8 +81,8 @@ flowchart LR | `ontology.py` | Loads `docs/ontology/lineageweave-kg.ttl`, the formal OWL 2/RDFS/SKOS vocabulary for the Knowledge Graph's node/edge types (ADR 0004) | | `period_report.py` | Fit GRM/GPCM on persisted IRT rows, FIPC-select, EAP-score a period (ADR 0003 slice 3; Bock & Mislevy, 1982) | | `fixtures.py` | Synthetic demo dataset -- no real data ships in this repo | -| `server.py` | Stdlib HTTP server: `GET /api/lineage` (JSON graph) + static viewer | -| `web/index.html` | Self-contained SVG DAG viewer, no build step, no external script dependency | +| `server.py` | Legacy stdlib HTTP server for the library-level synthetic fixture demo; production uses FastAPI/PostgreSQL | +| `web/index.html` | Legacy self-contained SVG DAG viewer; production UI is the React/Vite frontend | > **Known local-test-environment limitation:** `adjudication_client.py`'s > `mode="verify"` call depends on contextual-orchestrator's @@ -122,12 +122,13 @@ flowchart LR `rankweave_client.py`'s default transport raises `RankWeaveNotAvailable`. `GET /api/rankings` then returns `rankweave_not_available` and an empty ranking list. Hidden posts - are omitted from every channel. See ADR 0030. + are omitted from every channel. See ADR 0024. ## Standards and citations -See [`docs/lineage-bi-research-notes.md`](docs/lineage-bi-research-notes.md) -for the full APA 7th reference list this design is grounded in. +See [ADR 0084](docs/adr/0084-lineage-research-grounding.md) for the normative +research-grounding policy and [`docs/lineage-bi-research-notes.md`](docs/lineage-bi-research-notes.md) +for the full APA 7th reference list and supporting aggregate evidence. ## Product schema (Phase 1 of a larger roadmap) @@ -496,9 +497,6 @@ Event Lineage panel as that run's tree. `make seed` also records a TEPP measurement run through `tepp_client` on that same snapshot; the default transport is unavailable, so that run is Failed rather than a fabricated score. -A second Demo Corp TEPP run uses an in-process published accepted -acknowledgement and stays Failed / `tepp_completed_result_unsupported` -with aggregate transport evidence (ADR 0035). The home list is clickable: `GET /api/analysis-runs/{id}` fills a labeled detail (cutoff, requested date, 12-character digest prefixes with full digests on hover, counts, status history) @@ -514,9 +512,7 @@ measurement service) so `tepp_not_available` is not mistaken for a calibrated negative result. A failed lineage row tells the operator to retry reconstruction, not to connect TEPP. A failed period-report row tells the operator to rebuild the report. A pending TEPP row -does not claim a calibrated measurement and does not say -reconstruction. The list button accessible name includes the -next-action sentence; detail repeats it (ADR 0014). A pending lineage row +does not claim a calibrated measurement. A pending lineage row says reconstruction has not started yet; open it and start reconstruction. The payload is lookup labels plus non-negative aggregate counts -- never @@ -524,11 +520,8 @@ source SQL, a DSN, a raw record, or a provider body. After `make seed`, Demo Analyst and Demo Admin see "Lineage reconstruction · Succeeded · Demo Corp" with "3 documents" and Pending / Running / Succeeded times, the designed A-100 fork as clickable reconstructed edges, Claimed -then Delivered outbox times, "TEPP measurement · Failed · Demo -Corp" whose detail history ends in Failed / `tepp_not_available`, -and a second "TEPP measurement · Failed · Demo Corp" whose detail -shows Measurement evidence for the published accepted acknowledgement -(ADR 0035). +then Delivered outbox times, and "TEPP measurement · Failed · Demo +Corp" whose detail history ends in Failed / `tepp_not_available`. Seed also records "Period report · Succeeded · Demo Corp" on that same snapshot after the calibrated report tables are written (ADR 0024). Open that row to confirm the cutoff posts; mean θ stays @@ -537,10 +530,7 @@ A run-bearing registry is emptied only after an unrevoked `analysis_run_retention_grant` and `GRANT analysis_run_retention_admin`, then `purge_analysis_run_registry('approved-retention-purge')` (ADR 0020); a raw `DELETE` and a runtime role that only knows the -public phrase stay rejected. When start reconstruction has persisted -run-scoped edges, that same call empties those children instead of -stopping on an immutable-trigger or foreign-key error (ADR 0032). -Repeated chip and close controls use +public phrase stay rejected. Repeated chip and close controls use `frontend/src/styles/tokens.css` and the Storybook inventory. ## Phase 6a: fast-mlsirm dependency + Rust toolchain (infra only) @@ -598,7 +588,7 @@ on those same fixed parameters (Kim, 2006 FIPC). After scoring, `information_polytomous` ranks the shared-bank items by Fisher information at the group's mean θ (Lord, 1980 max-info CAT). Rankings persist to `report_item_information`. After those IRT main effects, -residual SVD leftover pairs (Jeon et al., 2021; ADR 0028) persist to +residual SVD leftover pairs (Jeon et al., 2021; ADR 0017) persist to `report_leftover_pair`. Results persist to `report_period_score` / `report_member_score`. `GET /api/reports/{grouping}` lists the trend; @@ -706,15 +696,13 @@ when absent) alongside its text -- `_BlockTextExtractor` tracks it through the existing start/end-tag stack rather than adding a second pass over the document. -Wiring: `backend/app/config.py` gained `Settings.vision_model` (env -`VISION_MODEL`) -- empty means the vision channel is unavailable, the -same "no fake channel" discipline as every other pluggable client, not a -guessed default model. `backend/app/main.py`'s `_vision_client()` factory +Wiring: `backend/app/main.py`'s `_vision_client()` factory returns a real `OpenAiCompatibleVisionClient` (via `orchestrator_vision_client`, which appends `/v1` so the same `ORCHESTRATOR_BASE_URL` other channels use lands on -`/v1/chat/completions`) only when base URL, API key, and model are all -set, else `NullImageContentClient()`; it is +`/v1/chat/completions`) when base URL and API key are set, else +`NullImageContentClient()`. The request omits `model`; contextual-orchestrator +selects the registered vision-capable agent. It is called at all three raw-`post_body`-reading endpoints (`extract-keymen`, post summary, commitment derivation) and threaded through `post_chat_ingestion.gather_chat_sources()` so every RAG source document @@ -800,7 +788,7 @@ subclasses of the real external PROV-O classes (imported via the `prov:` namespace), kept distinct from the ontology's existing `:Person` (node_type's cataloged Keyman with a stable `person_id`) since an R&R actor is a free-text name with no cataloged identity of its own. -`migrations/0012_role_responsibility_agent_type.sql` renames the +`migrations/0060_role_responsibility_agent_type.sql` renames the `post_summary_role` column via `RENAME COLUMN` (preserves existing rows) rather than a drop/recreate. The popup's R&R list shows a Person/Organization badge and the inferred affiliation; only a person @@ -968,15 +956,3 @@ so it also covers the multi-entity opposite-order case a per-name lock would still deadlock on. Every already-cataloged entity still resolves through the unchanged, lock-free similarity-matching fast path; only the rare creation branch serializes. - -## Phase 14: customer-group tree plus Searxng abbreviation cross-check - -Operators navigate the authorized Group / Company / Plant catalog -(`GET /api/customer-group-tree`), not only the post-scoped affiliate -tree or the flat `/api/me` corp list. Abbreviations on a post are -cross-checked against that tree through the existing Searxng client -(`abbreviation_tree_corroboration`). A unique corroborated node binds; -a down, empty, or tied search stays unbound and does not invent a -parent or AUTO row. See -[ADR 0033](docs/adr/0033-customer-group-tree-abbreviation-corroboration.md). -This path does not reimplement ADR 0008 or ADR 0010. diff --git a/CHANGELOG.d/0.71.2-leftover-pairs.md b/CHANGELOG.d/0.71.2-leftover-pairs.md index 30a56188f..0c6b1e1f7 100644 --- a/CHANGELOG.d/0.71.2-leftover-pairs.md +++ b/CHANGELOG.d/0.71.2-leftover-pairs.md @@ -3,7 +3,7 @@ ## Added - Persist closest and farthest leftover pairs from the residual - interaction map after GRM/GPCM scoring (ADR 0028). + interaction map after GRM/GPCM scoring (ADR 0017). - After `make seed`, period reports show the closest and farthest leftover pairs above the member list; clicking a pair opens that post - (ADR 0029). A leftover pair for a hidden post is omitted. + (ADR 0018). A leftover pair for a hidden post is omitted. diff --git a/CHANGELOG.d/2.10.0-leftover-pairs-and-rankings.md b/CHANGELOG.d/2.10.0-leftover-pairs-and-rankings.md deleted file mode 100644 index 0b712589e..000000000 --- a/CHANGELOG.d/2.10.0-leftover-pairs-and-rankings.md +++ /dev/null @@ -1,13 +0,0 @@ -# 2.10.0 — Leftover pairs and fail-closed Rankings - -## Added - -- Home Rankings panel fuses visible posts through `RankWeaveClient` - (ADR 0030). After login with the port disabled or the library - missing, Demo Analyst sees **Rankings · RankWeave not available**. - An accepted hit lists the title; click opens that post. A hidden - post is omitted. Never invent a fused score or a theta. -- Period reports persist closest and farthest leftover post–criterion - pairs after IRT main effects (ADR 0028 / 0029). After `make seed`, - leftover pairs sit above the member list; clicking a pair opens that - post. A leftover pair for a hidden post is omitted. diff --git a/CHANGELOG.d/2.10.1-analysis-run-accessible-next-action.md b/CHANGELOG.d/2.10.1-analysis-run-accessible-next-action.md deleted file mode 100644 index 3a0e27a02..000000000 --- a/CHANGELOG.d/2.10.1-analysis-run-accessible-next-action.md +++ /dev/null @@ -1,5 +0,0 @@ -# 2.10.1 Include next-action in analysis-run accessible names - -List button names include the kind-specific next-action sentence. -Open a Failed TEPP row and hear connect the measurement service -(ADR 0014). diff --git a/CHANGELOG.d/2.10.2-embedded-image-html-parser.md b/CHANGELOG.d/2.10.2-embedded-image-html-parser.md deleted file mode 100644 index cb3b3f820..000000000 --- a/CHANGELOG.d/2.10.2-embedded-image-html-parser.md +++ /dev/null @@ -1,5 +0,0 @@ -# 2.10.2 Parse invoice HTML images with an HTML parser - -Opening a post whose embedded picture uses invoice-like HTML -(`alt="Invoice > 1000"`) shows the picture between the surrounding -sentences. The raw base64 string is gone (ADR 0031). diff --git a/CHANGELOG.d/2.10.3-stale-hidden-run-list.md b/CHANGELOG.d/2.10.3-stale-hidden-run-list.md deleted file mode 100644 index b1d8cda09..000000000 --- a/CHANGELOG.d/2.10.3-stale-hidden-run-list.md +++ /dev/null @@ -1,7 +0,0 @@ -# 2.10.3 Drop a stale analysis-run row after its detail 404s - -Opening a listed analysis-run that then 404s drops that stale row -from the home list after an authorized re-read, announces the next -action with a status alert, and leaves Request as the named -reconstruction control. The message still does not name the thread -or the cutoff (ADR 0014 / ADR 0018). diff --git a/CHANGELOG.d/2.10.4-retention-purge-reconstruction-children.md b/CHANGELOG.d/2.10.4-retention-purge-reconstruction-children.md deleted file mode 100644 index 1343d7588..000000000 --- a/CHANGELOG.d/2.10.4-retention-purge-reconstruction-children.md +++ /dev/null @@ -1,6 +0,0 @@ -# 2.10.4 Empty reconstruction children during granted retention purge - -After a Demo Corp lineage reconstruction has started, the same -granted retention purge empties reconstruction edges, the -reconstruction digest, and frozen snapshot members (ADR 0032). -Do not `DISABLE TRIGGER` as superuser. diff --git a/CHANGELOG.d/2.11.0-customer-group-tree-searxng.md b/CHANGELOG.d/2.11.0-customer-group-tree-searxng.md deleted file mode 100644 index 6ffd0f554..000000000 --- a/CHANGELOG.d/2.11.0-customer-group-tree-searxng.md +++ /dev/null @@ -1,7 +0,0 @@ -# 2.11.0 Customer-group tree and Searxng abbreviation cross-check - -Home shows the authorized Group / Company / Plant forest. A click opens -that Demo Corp node as the corporate-entity report grouping. Post -abbreviations are cross-checked against that tree through the existing -Searxng client. When Searxng is down, empty, or tied, the mention stays -unbound — no invented parent and no AUTO row (ADR 0033). diff --git a/CHANGELOG.d/2.12.0-persistable-tepp-result.md b/CHANGELOG.d/2.12.0-persistable-tepp-result.md deleted file mode 100644 index cba49b6f1..000000000 --- a/CHANGELOG.d/2.12.0-persistable-tepp-result.md +++ /dev/null @@ -1,8 +0,0 @@ -# 2.12.0 Persistable TEPP result is Succeeded - -A live TEPP transport that returns a time / multilevel / -multi-affiliation envelope is stored on the analysis-run and marked -Succeeded. Home list and detail show clocks and affiliation counts. -A screen reader on that Succeeded Demo Corp row hears the next action. -An accepted ack or a missing transport stays Failed. No invented theta -(ADR 0034). diff --git a/CHANGELOG.d/2.12.1-tepp-accepted-transport-evidence.md b/CHANGELOG.d/2.12.1-tepp-accepted-transport-evidence.md deleted file mode 100644 index 6619d9285..000000000 --- a/CHANGELOG.d/2.12.1-tepp-accepted-transport-evidence.md +++ /dev/null @@ -1,9 +0,0 @@ -# 2.12.1 TEPP accepted acknowledgements are transport evidence - -A published TEPP `AnalysisRunAccepted` envelope is stored as -aggregate transport evidence. The run stays Failed / -`tepp_completed_result_unsupported`. A LineageWeave-local -`time_multilevel_multi_affiliation` envelope is not a completed TEPP -measurement and must not stamp Succeeded. Authorized detail shows -**Measurement evidence** with a copyable SHA-256. No invented theta -(ADR 0035). diff --git a/CHANGELOG.d/2.12.2-tepp-accepted-clocks.md b/CHANGELOG.d/2.12.2-tepp-accepted-clocks.md deleted file mode 100644 index 56af89e3a..000000000 --- a/CHANGELOG.d/2.12.2-tepp-accepted-clocks.md +++ /dev/null @@ -1,7 +0,0 @@ -# 2.12.2 TEPP accepted evidence stores distinct receipt and row-write clocks - -Accepted transport evidence persists `received_at` as the -transport-response receipt and `recorded_at` as the row-write -instant. Measurement evidence shows the second clock only when those -instants differ. Digest recomputation is unchanged. No invented -theta (ADR 0035 follow-up). diff --git a/CHANGELOG.d/2.12.3-late-demo-cutoff-post.md b/CHANGELOG.d/2.12.3-late-demo-cutoff-post.md deleted file mode 100644 index d442279e0..000000000 --- a/CHANGELOG.d/2.12.3-late-demo-cutoff-post.md +++ /dev/null @@ -1,6 +0,0 @@ -# 2.12.3 Late Demo public post - -Seed inserts Late Demo public post after the January 12 knowledge -cutoff so the ADR 0016 list filter has a falsifiable own-corp -counter-example. No second cutoff implementation. TEPP honesty -unchanged. diff --git a/CHANGELOG.d/2.12.6-buyer-image-source-safety.md b/CHANGELOG.d/2.12.6-buyer-image-source-safety.md new file mode 100644 index 000000000..f4aeb5293 --- /dev/null +++ b/CHANGELOG.d/2.12.6-buyer-image-source-safety.md @@ -0,0 +1,6 @@ +# 2.12.6 — Validate buyer image sources + +## Fixed + +- Buyer image rendering now rejects script, SVG, external, and malformed + source URLs before they reach an image element. diff --git a/CHANGELOG.d/2.12.6-buyer-main-port.md b/CHANGELOG.d/2.12.6-buyer-main-port.md new file mode 100644 index 000000000..5bb247a44 --- /dev/null +++ b/CHANGELOG.d/2.12.6-buyer-main-port.md @@ -0,0 +1,11 @@ +# 2.12.6 — Buyer evidence surface main port + +## Changed + +- Ports the accumulated Buyer Board, Customer master, Calendar, Global Ask, provenance, source-context, and runtime-hardening tree onto protected `main` after #74 was squash-merged independently. +- Preserves the current main release lineage instead of regressing package metadata to the older semantic-branch versions. + +## Fixed + +- Preserves the resolved embedding protocol/static-analysis corrections and source-import/content-normalization hardening accumulated after the #74 branch point. +- Removes stale-parent CI evidence from the merge decision; exact-head checks must run again on the repaired ancestry. diff --git a/CHANGELOG.d/2.12.6-oidc-deep-link-safety.md b/CHANGELOG.d/2.12.6-oidc-deep-link-safety.md new file mode 100644 index 000000000..48599f982 --- /dev/null +++ b/CHANGELOG.d/2.12.6-oidc-deep-link-safety.md @@ -0,0 +1,7 @@ +# 2.12.6 — Bound OIDC deep-link state parsing + +## Fixed + +- OIDC callback state is parsed at most once and length-bounded before JSON + handling, preventing recursive encoded state from exhausting the browser + stack while preserving safe same-origin post deep links. diff --git a/CHANGELOG.d/2.12.6-oidc-resource-audience.md b/CHANGELOG.d/2.12.6-oidc-resource-audience.md new file mode 100644 index 000000000..65f88adc9 --- /dev/null +++ b/CHANGELOG.d/2.12.6-oidc-resource-audience.md @@ -0,0 +1,8 @@ +# 2.12.6 — Resource-bound OIDC verification + +## Security + +- Browser/API bearer verification now requires a non-empty JWT `kid` and an exact acceptable RSA/RS256 JWKS key; a missing `kid` no longer falls back to the first provider key. +- Access tokens must include the configured LineageWeave API audience. A valid token minted by the same issuer for another resource is rejected. +- External Keyverse/OIDC deployments must set `KEYVERSE_AUDIENCE` or `OIDC_AUDIENCE`; the backend does not infer a resource-server audience from the browser OAuth client id. The local Keycloak realm remains self-contained and emits `lineageweave-api` in access-token `aud`. +- Token-side corporate/PU attributes remain non-authoritative; account permissions and affiliations continue to resolve from LineageWeave PostgreSQL. diff --git a/CHANGELOG.d/2.12.7-safe-sql-audit.md b/CHANGELOG.d/2.12.7-safe-sql-audit.md new file mode 100644 index 000000000..dedb645f8 --- /dev/null +++ b/CHANGELOG.d/2.12.7-safe-sql-audit.md @@ -0,0 +1,6 @@ +## 2.12.7 — Audited safe SQL composition + +- Require every accepted Semgrep SQL-composition suppression to use the exact rule identifier and an adjacent written audit reason. +- Verify that schema-fixed eligibility/source-context fragments keep request values in asyncpg parameters. +- Verify that synthetic-cleanup catalog identifiers are quoted before execution while UUID values remain parameters. +- Replace the document-structure protocol no-op body with an explicit `NotImplementedError` contract. diff --git a/CHANGELOG.d/2.13.0-buyer-gnb-i18n.md b/CHANGELOG.d/2.13.0-buyer-gnb-i18n.md new file mode 100644 index 000000000..0209834a7 --- /dev/null +++ b/CHANGELOG.d/2.13.0-buyer-gnb-i18n.md @@ -0,0 +1,7 @@ +# Buyer GNB and multilingual product surface + +- Added the buyer GNB: Board, Customer master, Calendar, and Ask Agent. +- Added direct related-post controls from lineage and Keyman/R&R evidence. +- Added Korean, English, Chinese, Japanese, and Vietnamese locale switching. +- Moved analysis-run and period-report controls into collapsed advanced review + tools instead of the default buyer surface. diff --git a/CHANGELOG.d/2.13.1-badge-color-tokens.md b/CHANGELOG.d/2.13.1-badge-color-tokens.md new file mode 100644 index 000000000..e5b591ed9 --- /dev/null +++ b/CHANGELOG.d/2.13.1-badge-color-tokens.md @@ -0,0 +1,7 @@ +# Badge and accent colors are design tokens + +- Extracted ~13 repeated inline hex colors in `App.css` (lineage-link + accents, R&R actor-type badges, relation-verification status badges) into + named tokens in `styles/tokens.css`, each with a `prefers-color-scheme: + dark` variant. Dark mode previously left every pastel badge exactly as + light as it is in light mode (ADR 0099). diff --git a/CHANGELOG.d/2.13.1-mixed-body-indentation.md b/CHANGELOG.d/2.13.1-mixed-body-indentation.md new file mode 100644 index 000000000..a287b3058 --- /dev/null +++ b/CHANGELOG.d/2.13.1-mixed-body-indentation.md @@ -0,0 +1,7 @@ +# Mixed table and paragraph indentation + +## Fixed + +- Match persisted post units to their source text instead of using ordinal + position, so a table or embedded image cannot shift the fallback indentation + of a later unresolved paragraph. diff --git a/CHANGELOG.d/2.13.1-partial-image-regions.md b/CHANGELOG.d/2.13.1-partial-image-regions.md new file mode 100644 index 000000000..b3da62568 --- /dev/null +++ b/CHANGELOG.d/2.13.1-partial-image-regions.md @@ -0,0 +1,5 @@ +## Preserve partial visual regions + +- Retain valid salient image regions for panel-level OCR and search. +- Also analyze the parent image when locator coverage is partial so text outside + the returned panels remains searchable. diff --git a/CHANGELOG.d/2.13.1-separate-source-tables.md b/CHANGELOG.d/2.13.1-separate-source-tables.md new file mode 100644 index 000000000..0896de45c --- /dev/null +++ b/CHANGELOG.d/2.13.1-separate-source-tables.md @@ -0,0 +1,7 @@ +# Preserve adjacent source tables + +## Fixed + +- Keep consecutive persisted rows in separate buyer-facing tables when the + source post contains more than one HTML table, preserving the authored table + boundary without changing source text or semantic row content. diff --git a/CHANGELOG.d/2.13.1-source-indent-semantics.md b/CHANGELOG.d/2.13.1-source-indent-semantics.md new file mode 100644 index 000000000..bf67e7931 --- /dev/null +++ b/CHANGELOG.d/2.13.1-source-indent-semantics.md @@ -0,0 +1,7 @@ +## Fix source-only indentation depth + +- Keep leading spaces and ` ` available as diagnostics without persisting + them as authoritative structure; only declared HTML/CSS/OOXML or list + nesting is explicit. +- Keep expected structure and embedding channel failures retryable while + propagating unexpected defects to the durable ingestion ledger. diff --git a/CHANGELOG.d/2.21.0-synthetic-cleanup-scope.md b/CHANGELOG.d/2.21.0-synthetic-cleanup-scope.md new file mode 100644 index 000000000..28564ee2d --- /dev/null +++ b/CHANGELOG.d/2.21.0-synthetic-cleanup-scope.md @@ -0,0 +1,4 @@ +### Fixed + +- Restrict synthetic-seed cleanup to `DEMO-*` corporate entities so a real + PostgreSQL import cannot delete blank-context posts from a production scope. diff --git a/CHANGELOG.d/2.21.1-orchestrator-pin-doc.md b/CHANGELOG.d/2.21.1-orchestrator-pin-doc.md new file mode 100644 index 000000000..7aae0ae69 --- /dev/null +++ b/CHANGELOG.d/2.21.1-orchestrator-pin-doc.md @@ -0,0 +1,6 @@ +# 2.21.1 — Align the orchestrator runtime pin record + +## Fixed + +- Align ADR 0083 with the immutable contextual-orchestrator commit used by the + Compose image and add a regression check so the image and ADR cannot drift. diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d82c581e..c8ed1a099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,20 @@ All notable changes to this project are documented here. Format follows environment, so local OIDC and synthetic-data workflows resolve the same pinned dependencies as CI. +## [2.12.6] - 2026-08-20 + +### Added + +- Production OIDC can now use a real Keyverse issuer through + `KEYVERSE_ISSUER` and `KEYVERSE_CLIENT_ID`. The backend discovers the + provider's JWKS and verifies the issuer; Compose keeps local Keycloak only + as an explicit development fallback and does not emulate Keyverse. +- Relation verification now preserves a separately authorized internal source + post containing normalized organization and relationship context. Open that + evidence from the counterparty popup without treating it as an external URL. +- Large corpora now use bounded post and Event Lineage landing projections so + buyers can open complete post-specific detail from a responsive first view. + ## [2.12.5] - 2026-08-18 ### Fixed diff --git a/CLAUDE.md b/CLAUDE.md index 08e3682a6..1bcf50763 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,54 +10,40 @@ To empty a run-bearing registry, insert an unrevoked `GRANT analysis_run_retention_admin` (ADR 0020). Then `select purge_analysis_run_registry('approved-retention-purge')`, export `analysis_run_retention_event`, delete those rows, and roll -back 0020 then 0018. The same call empties reconstruction children -when ADR 0021 tables exist (ADR 0032). The published phrase is not a -secret. Do not `DISABLE TRIGGER` as superuser. Do not grant the admin -role or a retention grant to the application `DATABASE_URL` login. -ADR 0019 is the R&R catalog-id bind, not this purge. Person catalog -identity on that role row is ADR 0027 (`cataloged_person_id`). +back 0020 then 0018. The published phrase is not a secret. Do not +`DISABLE TRIGGER` as superuser. Do not grant the admin role or a +retention grant to the application `DATABASE_URL` login. ADR 0019 +is the R&R catalog-id bind, not this purge. Person catalog identity +on that role row is ADR 0027 (`cataloged_person_id`). ## Analysis-run seed (v0.96.0) -`make seed` writes a Demo Corp lineage run, a Failed missing-transport -TEPP run, a Failed accepted-evidence TEPP run, and a Succeeded -period-report run on the same snapshot (ADR 0013 / ADR 0024 / ADR 0035). -The TEPP path goes through `tepp_client`. A missing transport or an -unpublished envelope is Failed (`tepp_not_available` / -`tepp_result_not_persisted`). A published accepted acknowledgement is -Failed (`tepp_completed_result_unsupported`) and is shown as aggregate -transport evidence. Do not stamp Succeeded from that ack. Do not invent -a theta or a local psychometric substitute. Measurement evidence shows -Received, and recorded only when that row-write instant differs. -The home list caption stays `kind · status · entity`; the machine -failure code is detail-only (ADR 0014). Open a Failed TEPP row, then -connect a live TEPP transport or read aggregate transport evidence. -Do not treat that row as a validated multilevel estimate. A failed lineage row retries reconstruction -- it does not +`make seed` writes a Demo Corp lineage run, a TEPP run, and a Succeeded +period-report run on the same snapshot (ADR 0013 / ADR 0024). The TEPP path goes through `tepp_client`. A missing +transport or an unused accepted envelope is Failed +(`tepp_not_available` / `tepp_result_not_persisted`). Do not invent a +theta or a local psychometric substitute. The home list caption stays +`kind · status · entity`; the machine failure code is detail-only +(ADR 0014). Open a Failed TEPP row, then connect a live TEPP +transport. A failed lineage row retries reconstruction -- it does not mention TEPP. A failed period-report row rebuilds the report. A -pending TEPP row does not claim a calibrated measurement and does -not say reconstruction. The list button name includes the -next-action sentence. A pending lineage row says reconstruction has -not started yet. +pending TEPP row does not claim a calibrated measurement. A pending +lineage row says reconstruction has not started yet. Digest prefixes stay audible; hover a prefix to read the full digest. Opening a cutoff title shows the live post. Titles marked updated after cutoff were rewritten after the run; the opened body names both clocks and shows **Body this run knew** beside the live rewrite. Compare those two texts before treating the live body as reconstructed evidence (ADR 0016 / 0025). -The January 12 Demo Corp lineage and TEPP runs list Demo public post -and do not list Late Demo public post (2026-01-13). The live post -list still shows Late Demo. `POST /api/analysis-runs` records Pending lineage only on an authorized cutoff capture (ADR 0017). TEPP and period-report kinds are 422. The Request button waits until affiliated corps load; choose a corp if the token walks more than one. `POST /api/analysis-runs/{id}/start` commits Running plus a durable outbox row, then reconstructs that frozen cutoff bag (ADR 0021 / ADR 0023) or submits TEPP through -`tepp_client` (ADR 0022 / ADR 0035). A missing transport or unpublished -envelope is Failed. A published accepted acknowledgement is Failed -transport evidence, not a completed measurement. Failed TEPP is -terminal — connect a TEPP transport from that Failed row or read the -stored evidence. Create does not invent a Pending +`tepp_client` (ADR 0022). A missing transport or unused accepted +envelope is Failed. Failed TEPP is terminal — connect a TEPP +transport from that Failed row. Create does not invent a Pending TEPP row. Do not invent a theta. Hover the Result prefix to read the parent-choice digest. After `make seed`, open **Period report · Succeeded · Demo Corp**, @@ -84,7 +70,3 @@ cited source. After that next action, the popup lands the first cited evidence. Changing the week first still focuses the report period field. Mean θ stays on the period-report panel. -A listed analysis-run that then 404s stays generic: do not name the -thread or the cutoff. After that 404, re-read the authorized list so -the stale row does not stay clickable. Announce the next action with -`role="alert"` without moving focus. diff --git a/Makefile b/Makefile index 7f9de4bcf..6468b8857 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,20 @@ .PHONY: up down logs smoke seed ps +# Keep provider credentials outside the repository. Compose interpolation must +# read the same home env file as the orchestrator container's env_file. +COMPOSE := docker compose --env-file "$$HOME/.env" + up: - docker compose up -d + $(COMPOSE) up -d down: - docker compose down + $(COMPOSE) down logs: - docker compose logs -f + $(COMPOSE) logs -f ps: - docker compose ps + $(COMPOSE) ps # Real OIDC round-trip against the running Keycloak container: logs in as # the synthetic demo user, verifies the returned JWT's signature against diff --git a/README.md b/README.md index c143f84fc..632f9b5a3 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,9 @@ signal reliably tells you which record continues which -- see the validation numbers and the literature this design follows. LineageWeave fuses several independent, individually-weak signals (temporal proximity, a shared grouping key, text similarity, and an optional LLM judgment) instead -of trusting any one of them alone. +of trusting any one of them alone. The normative research-grounding policy is +[ADR 0084](docs/adr/0084-lineage-research-grounding.md); the linked notes +retain the supporting bibliography and aggregate evidence. ## How it fits with the rest of the ecosystem @@ -109,6 +111,14 @@ make smoke # real login as the synthetic demo user + JWT signature make down ``` +Outside GitHub, `make up` reads `~/.env` through Compose's `--env-file`. +Configure the contextual-orchestrator provider there with +`LLM_GATEWAY_API_URL` and `LLM_GATEWAY_API_KEY`; the key is never committed or +printed. `LLM_GATEWAY_URL`, `LLM_API_GATEWAY`, and `LLM_API_KEY` remain +compatibility aliases only. `ORCHESTRATOR_BASE_URL` and +`ORCHESTRATOR_API_KEY` are separate, internal +LineageWeave-to-orchestrator settings. + Postgres and Keycloak are built (`docker/postgres-init/`, `docker/keycloak/`) rather than bind-mounted, so the keycloak database's init script and the realm seed ship inside the images themselves -- portable to any Docker host @@ -172,9 +182,7 @@ cd frontend && cp .env.example .env.local && pnpm install && pnpm run dev # Empty a run-bearing registry: insert analysis_run_retention_grant # for session_user, GRANT analysis_run_retention_admin, then # select purge_analysis_run_registry('approved-retention-purge'). -# The published token is not a grant (ADR 0020). After a Succeeded -# start, that same call also empties reconstruction children -# (ADR 0032). +# The published token is not a grant (ADR 0020). # -> http://localhost:5173, click "Log in", redirects through the real # Keycloak login page for demo.analyst / lineageweave-demo-only ``` diff --git a/add_translations.py b/add_translations.py new file mode 100644 index 000000000..8448d75ee --- /dev/null +++ b/add_translations.py @@ -0,0 +1,48 @@ +import re + +with open("frontend/src/i18n.ts", "r") as f: + content = f.read() + +translations = { + "Admin": { + "ko": "관리자", + "zh": "管理员", + "ja": "管理者", + "vi": "Quản trị viên" + }, + "Admin settings": { + "ko": "관리자 설정", + "zh": "管理员设置", + "ja": "管理者設定", + "vi": "Cài đặt quản trị viên" + }, + "Tenant brand name": { + "ko": "테넌트 브랜드명", + "zh": "租户品牌名称", + "ja": "テナントブランド名", + "vi": "Tên thương hiệu khách thuê" + }, + "Save settings": { + "ko": "설정 저장", + "zh": "保存设置", + "ja": "設定を保存", + "vi": "Lưu cài đặt" + }, + "Settings saved!": { + "ko": "설정이 저장되었습니다!", + "zh": "设置已保存！", + "ja": "設定が保存されました！", + "vi": "Đã lưu cài đặt!" + } +} + +for eng, trans in translations.items(): + content = content.replace(f' Refresh: "새로 고침",', f' Refresh: "새로 고침",\n "{eng}": "{trans["ko"]}",') + content = content.replace(f' Refresh: "조회",', f' Refresh: "조회",\n "{eng}": "{trans["ko"]}",') + + content = content.replace(f' Refresh: "刷新",', f' Refresh: "刷新",\n "{eng}": "{trans["zh"]}",') + content = content.replace(f' Refresh: "更新",', f' Refresh: "更新",\n "{eng}": "{trans["ja"]}",') + content = content.replace(f' Refresh: "Làm mới",', f' Refresh: "Làm mới",\n "{eng}": "{trans["vi"]}",') + +with open("frontend/src/i18n.ts", "w") as f: + f.write(content) diff --git a/backend/app/abbreviation_tree_corroboration_ingestion.py b/backend/app/abbreviation_tree_corroboration_ingestion.py deleted file mode 100644 index e8e7d6e92..000000000 --- a/backend/app/abbreviation_tree_corroboration_ingestion.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Persist Searxng abbreviation matches against the authorized tree.""" - -from __future__ import annotations - -import asyncio -from typing import Any - -import asyncpg - -from lineageweave.abbreviation_tree_corroboration import ( - AbbreviationTreeMatch, - TreeEntityCandidate, - abbreviation_candidates, - corroborate_abbreviation_against_tree, -) -from lineageweave.customer_group_tree import CatalogEntityRow, authorized_catalog_ids -from lineageweave.relation_verification import RelationVerificationClient - - -async def collect_post_organization_names(conn: asyncpg.Connection, post_id: str) -> tuple[str, ...]: - """Organization strings already extracted onto this post. - - Keyman affiliations and classified counterparties are the mentions - operators can see. This path does not invent a name from post text. - """ - rows = await conn.fetch( - """ - select distinct name from ( - select pa.affiliated_organization_name as name - from post_person_mention ppm - join person_affiliation pa on pa.person_id = ppm.person_id - where ppm.post_id = $1 - union - select c.counterparty_entity_name as name - from post_counterparty_entity c - where c.post_id = $1 - ) mentioned - where name is not null and btrim(name) <> '' - order by name - """, - post_id, - ) - return tuple(row["name"] for row in rows) - - -async def load_authorized_tree_candidates( - conn: asyncpg.Connection, - affiliated_entity_ids: list[str], -) -> tuple[TreeEntityCandidate, ...]: - """Catalog nodes the account may corroborate an abbreviation against.""" - entity_rows = await conn.fetch( - """ - select corporate_entity_id, parent_entity_id, entity_name, entity_level_code - from corporate_entity - """ - ) - entities = tuple( - CatalogEntityRow( - entity_id=str(row["corporate_entity_id"]), - parent_entity_id=str(row["parent_entity_id"]) if row["parent_entity_id"] is not None else None, - entity_name=row["entity_name"], - entity_level_code=row["entity_level_code"], - ) - for row in entity_rows - ) - needed = authorized_catalog_ids(entities, affiliated_entity_ids) - return tuple( - TreeEntityCandidate(entity_id=row.entity_id, entity_name=row.entity_name) - for row in entities - if row.entity_id in needed - ) - - -async def persist_abbreviation_tree_match( - conn: asyncpg.Connection, - match: AbbreviationTreeMatch, -) -> None: - """Upsert one raw mention's tree-constrained Searxng outcome.""" - await conn.execute( - """ - insert into abbreviation_tree_corroboration - (raw_organization_name, corporate_entity_id, - verification_status_code, verification_evidence_url) - values ($1, $2, $3, $4) - on conflict (raw_organization_name) do update set - corporate_entity_id = excluded.corporate_entity_id, - verification_status_code = excluded.verification_status_code, - verification_evidence_url = excluded.verification_evidence_url, - corroborated_at = now() - """, - match.raw_organization_name, - match.corporate_entity_id, - match.verification_status_code, - match.verification_evidence_url, - ) - - -async def fetch_post_abbreviation_matches( - conn: asyncpg.Connection, - post_id: str, -) -> list[dict[str, Any]]: - """Cached tree matches for organization names already on this post.""" - names = await collect_post_organization_names(conn, post_id) - if not names: - return [] - rows = await conn.fetch( - """ - select raw_organization_name, corporate_entity_id, - verification_status_code, verification_evidence_url - from abbreviation_tree_corroboration - where raw_organization_name = any($1::text[]) - order by raw_organization_name - """, - list(names), - ) - return [ - { - "raw_organization_name": row["raw_organization_name"], - "corporate_entity_id": ( - str(row["corporate_entity_id"]) if row["corporate_entity_id"] is not None else None - ), - "verification_status_code": row["verification_status_code"], - "verification_evidence_url": row["verification_evidence_url"], - } - for row in rows - ] - - -async def corroborate_post_abbreviations( - conn: asyncpg.Connection, - verification_client: RelationVerificationClient, - post_id: str, - affiliated_entity_ids: list[str], -) -> list[AbbreviationTreeMatch]: - """Run Searxng against the authorized tree for this post's mentions.""" - names = await collect_post_organization_names(conn, post_id) - candidates = await load_authorized_tree_candidates(conn, affiliated_entity_ids) - to_check = abbreviation_candidates(names, candidates) - matches: list[AbbreviationTreeMatch] = [] - for raw_name in to_check: - match = await asyncio.to_thread( - corroborate_abbreviation_against_tree, - raw_name, - candidates, - verification_client, - ) - await persist_abbreviation_tree_match(conn, match) - matches.append(match) - return matches diff --git a/backend/app/analysis_run_ingestion.py b/backend/app/analysis_run_ingestion.py index 9b903ea87..f7da2969b 100644 --- a/backend/app/analysis_run_ingestion.py +++ b/backend/app/analysis_run_ingestion.py @@ -11,10 +11,8 @@ records lineage only. It does not reconstruct lineage, accept a TEPP kind, or invent a score. ``enqueue_pending_analysis_run`` then ``deliver_queued_analysis_run`` later reconstruct lineage (ADR 0021 / -ADR 0023) or submit TEPP through ``tepp_client`` (ADR 0022 / ADR 0035). -A published accepted acknowledgement is stored as aggregate transport -evidence; neither path invents a TEPP score or stamps Succeeded from -that ack. +ADR 0023) or submit TEPP through ``tepp_client`` (ADR 0022). Neither +path invents a TEPP score. """ from __future__ import annotations @@ -28,9 +26,10 @@ import asyncpg +from backend.app.demo_scope import has_real_source_context, is_demo_scope from backend.app.knowledge_graph import labels_for_codes +from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL from lineageweave import __version__ as PACKAGE_VERSION -from lineageweave.tepp_result import tepp_accepted_evidence_sha256 _LINEAGE_RUN_KIND = "analysis_run_lineage" _TEPP_RUN_KIND = "analysis_run_tepp" @@ -42,7 +41,7 @@ "analysis_run_tepp": "tepp-run-v1", } -_RUN_LIST_SQL = """ +_RUN_LIST_SQL = f""" select run.analysis_run_id, run.run_kind_code, @@ -56,6 +55,7 @@ scope.process_unit_id, scope.scope_key, corp.entity_name as scope_entity_name, + corp.corporate_entity_code as scope_entity_code, status.status_code, status.failure_code from analysis_run run @@ -84,6 +84,7 @@ select 1 from source_post p where p.thread_group_key = scope.scope_key and p.created_at <= run.knowledge_cutoff + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias="p")} and ( p.visibility_code = 'public' or p.corporate_entity_id = any($2::uuid[]) @@ -93,7 +94,7 @@ order by run.requested_at desc """ -_RUN_DETAIL_SQL = """ +_RUN_DETAIL_SQL = f""" select run.analysis_run_id, run.run_kind_code, @@ -134,9 +135,10 @@ scope.scope_kind_code = 'analysis_scope_thread_group' and exists ( select 1 from source_post p - where p.thread_group_key = scope.scope_key - and p.created_at <= run.knowledge_cutoff - and ( + where p.thread_group_key = scope.scope_key + and p.created_at <= run.knowledge_cutoff + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias="p")} + and ( p.visibility_code = 'public' or p.corporate_entity_id = any($2::uuid[]) ) @@ -186,62 +188,6 @@ def live_write_after_cutoff(updated_at: datetime, knowledge_cutoff: datetime) -> return _as_utc(updated_at) > _as_utc(knowledge_cutoff) -async def _tepp_accepted_by_run( - conn: asyncpg.Connection, - run_ids: list[str], -) -> dict[str, asyncpg.Record]: - """Load published TEPP accepted evidence for the given authorized runs. - - Missing ``analysis_run_tepp_accepted`` means migration 0029 is not - applied. Treat that as no stored transport evidence rather than 500. - Hidden runs never appear in ``run_ids``. - """ - if not run_ids: - return {} - try: - rows = await conn.fetch( - """ - select analysis_run_id, contract_version, accepted_run_id, - run_state, idempotency_key, evidence_sha256, - received_at, recorded_at - from analysis_run_tepp_accepted - where analysis_run_id = any($1::uuid[]) - """, - run_ids, - ) - except asyncpg.UndefinedTableError: - return {} - return {str(row["analysis_run_id"]): row for row in rows} - - -def project_tepp_transport_evidence(row: Any) -> dict[str, Any] | None: - """Project accepted evidence only when the stored digest recomputes. - - Counts, theta, topics, and completed-artifact identity stay omitted. - A digest mismatch fails closed so a substituted row is not shown. - """ - expected = tepp_accepted_evidence_sha256( - contract_version=int(row["contract_version"]), - accepted_run_id=str(row["accepted_run_id"]), - run_state=str(row["run_state"]), - idempotency_key=str(row["idempotency_key"]), - ) - stored = str(row["evidence_sha256"]) - if stored != expected: - return None - return { - "tepp_evidence_kind": "aggregate transport evidence", - "tepp_contract_version": int(row["contract_version"]), - "tepp_accepted_run_id": str(row["accepted_run_id"]), - "tepp_run_state": str(row["run_state"]), - "tepp_idempotency_key": str(row["idempotency_key"]), - "tepp_evidence_sha256": stored, - "tepp_received_at": _iso(row["received_at"]), - "tepp_recorded_at": _iso(row["recorded_at"]), - "tepp_completed_artifact_available": False, - } - - async def _counts_by_run( conn: asyncpg.Connection, run_ids: list[str], @@ -249,8 +195,9 @@ async def _counts_by_run( """Load aggregate snapshot counts for the given runs.""" if not run_ids: return {} - rows = await conn.fetch( - """ + # Safe SQL: this immutable aggregate query has closed schema text; run ids are bound below. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select run.analysis_run_id, counts.count_type_code, counts.count_value from analysis_run run join analysis_source_count counts @@ -341,9 +288,7 @@ async def _serialize_runs( """Project registry rows into the authorized buyer-facing payload.""" if not rows: return [] - run_ids = [str(row["analysis_run_id"]) for row in rows] - count_rows = await _counts_by_run(conn, run_ids) - tepp_rows = await _tepp_accepted_by_run(conn, run_ids) + count_rows = await _counts_by_run(conn, [str(row["analysis_run_id"]) for row in rows]) labels = await labels_for_codes( conn, [row["run_kind_code"] for row in rows] @@ -389,11 +334,6 @@ async def _serialize_runs( grouping_key = scope_grouping_key(row) if grouping_key: item["scope_grouping_key"] = grouping_key - tepp = tepp_rows.get(run_id) - if tepp is not None: - projected = project_tepp_transport_evidence(tepp) - if projected is not None: - item.update(projected) payload.append(item) return payload @@ -403,12 +343,20 @@ async def fetch_visible_analysis_runs( account_id: str, affiliated_entity_ids: list[str], ) -> list[dict[str, Any]]: - """Runs the account requested or whose scope they may already walk.""" - rows = await conn.fetch( + """Runs the account requested or whose scope they may already walk. + + Once real source-import evidence is visible, the synthetic `make seed` + Demo Corp runs stop appearing here -- a buyer must not mistake that + fabricated narrative for real evidence (ADR 0001 / ADR 0042). + """ + # Safe SQL: this immutable module query contains only closed schema SQL; request values remain bound below. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli _RUN_LIST_SQL, account_id, affiliated_entity_ids, ) + if rows and await has_real_source_context(conn, affiliated_entity_ids): + rows = [row for row in rows if not is_demo_scope(row["scope_entity_code"])] return await _serialize_runs(conn, rows) @@ -419,7 +367,8 @@ async def fetch_visible_analysis_run( affiliated_entity_ids: list[str], ) -> dict[str, Any] | None: """One visible run, or None when it is missing or hidden.""" - rows = await conn.fetch( + # Safe SQL: this immutable module query contains only closed schema SQL; request values remain bound below. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli _RUN_DETAIL_SQL, account_id, affiliated_entity_ids, @@ -502,8 +451,9 @@ async def fetch_reconstructed_edges( return None, [] if header is None: return None, [] - rows = await conn.fetch( - """ + # Safe SQL: eligibility fragments are immutable schema predicates; the run id remains a bound parameter. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select edge.parent_post_id, parent_post.post_title as parent_post_title, @@ -515,8 +465,12 @@ async def fetch_reconstructed_edges( child_post.corporate_entity_id as child_corporate_entity_id, edge.fused_score from analysis_run_lineage_edge edge - join source_post parent_post on parent_post.post_id = edge.parent_post_id - join source_post child_post on child_post.post_id = edge.child_post_id + join source_post parent_post + on parent_post.post_id = edge.parent_post_id + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias="parent_post")} + join source_post child_post + on child_post.post_id = edge.child_post_id + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias="child_post")} where edge.analysis_run_id = $1 order by parent_post.post_title, child_post.post_title """, @@ -584,36 +538,44 @@ async def fetch_visible_scope_posts( "post_id, post_title, visibility_code, corporate_entity_id, updated_at" ) if scope_kind_code == "analysis_scope_corporate_entity" and corporate_entity_id: - rows = await conn.fetch( + # Safe SQL: the selected columns and eligibility predicate are closed constants; ids are bound. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli f"select {columns} " "from source_post where corporate_entity_id = $1 " "and created_at <= $2 " + f"and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')} " "order by created_at, post_title", corporate_entity_id, knowledge_cutoff, ) elif scope_kind_code == "analysis_scope_process_unit" and process_unit_id: - rows = await conn.fetch( + # Safe SQL: the selected columns and eligibility predicate are closed constants; ids are bound. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli f"select {columns} " "from source_post where process_unit_id = $1 " "and created_at <= $2 " + f"and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')} " "order by created_at, post_title", process_unit_id, knowledge_cutoff, ) elif scope_kind_code == "analysis_scope_thread_group" and scope_key: - rows = await conn.fetch( + # Safe SQL: the selected columns and eligibility predicate are closed constants; keys are bound. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli f"select {columns} " "from source_post where thread_group_key = $1 " "and created_at <= $2 " + f"and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')} " "order by created_at, post_title", scope_key, knowledge_cutoff, ) elif scope_kind_code == "analysis_scope_all_visible": - rows = await conn.fetch( + # Safe SQL: the selected columns and eligibility predicate are closed constants; cutoff is bound. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli f"select {columns} " "from source_post where created_at <= $1 " + f"and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')} " "order by created_at, post_title", knowledge_cutoff, ) @@ -808,12 +770,12 @@ async def create_pending_analysis_run( "Request a corporate-entity run. Other scopes are not available yet.", ) cutoff_explicit = knowledge_cutoff is not None + database_now = await conn.fetchval("select clock_timestamp()") if knowledge_cutoff is None: - knowledge_cutoff = datetime.now(timezone.utc) + knowledge_cutoff = database_now elif knowledge_cutoff.tzinfo is None: knowledge_cutoff = knowledge_cutoff.replace(tzinfo=timezone.utc) - now = datetime.now(timezone.utc) - if knowledge_cutoff > now: + if knowledge_cutoff > database_now: raise AnalysisRunCreateError( 422, "Choose a knowledge cutoff at or before now, then request the run again.", @@ -831,12 +793,15 @@ async def create_pending_analysis_run( key, ) - rows = await conn.fetch( - """ + # Safe SQL: the eligibility predicate is an immutable schema fragment; corporate id and cutoff are bound. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select post_id, post_title, thread_group_key, created_at, visibility_code, corporate_entity_id from source_post - where corporate_entity_id = $1 and created_at <= $2 + where corporate_entity_id = $1 + and created_at <= $2 + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias="source_post")} order by created_at, post_title """, corp_id, @@ -884,14 +849,13 @@ async def create_pending_analysis_run( insert into analysis_source_snapshot (snapshot_sha256, source_contract_version, maximum_available_time, captured_at, created_at) - values ($1, $2, $3, $4, $4) + values ($1, $2, $3, clock_timestamp(), clock_timestamp()) on conflict (snapshot_sha256) do nothing returning analysis_source_snapshot_id """, capture.snapshot_sha256, _CAPTURE_CONTRACT_VERSION, capture.maximum_available_time, - now, ) if snapshot_id is None: snapshot_id = await conn.fetchval( @@ -933,7 +897,7 @@ async def create_pending_analysis_run( requested_by_account_id, knowledge_cutoff, configuration_schema_version, configuration_sha256, code_revision_sha, requested_at) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9) + values ($1, $2, $3, $4, $5, $6, $7, $8, clock_timestamp()) returning analysis_run_id """, snapshot_id, @@ -944,7 +908,6 @@ async def create_pending_analysis_run( capture.configuration_schema_version, capture.configuration_sha256, capture.code_revision_sha, - now, ) except asyncpg.UniqueViolationError: raced = await conn.fetchrow( @@ -985,10 +948,9 @@ async def create_pending_analysis_run( """ insert into analysis_run_status_event (analysis_run_id, status_ordinal, status_code, occurred_at) - values ($1, 1, 'analysis_status_pending', $2) + values ($1, 1, 'analysis_status_pending', clock_timestamp()) """, run_id, - now, ) created = await fetch_visible_analysis_run( conn, diff --git a/backend/app/analysis_run_outbox.py b/backend/app/analysis_run_outbox.py index 7abeb2b31..487948ba8 100644 --- a/backend/app/analysis_run_outbox.py +++ b/backend/app/analysis_run_outbox.py @@ -10,7 +10,6 @@ import hashlib import json from datetime import datetime, timezone -from typing import Any import redis.asyncio as redis diff --git a/backend/app/analysis_run_start.py b/backend/app/analysis_run_start.py index 6cf3b657e..2387d940b 100644 --- a/backend/app/analysis_run_start.py +++ b/backend/app/analysis_run_start.py @@ -2,13 +2,9 @@ ADR 0021 reconstructs lineage. ADR 0022 starts TEPP through ``tepp_client`` only. ADR 0023 enqueues that work on a durable outbox -so a crash after Running does not lose the item. ADR 0035 stores a -published TEPP accepted acknowledgement as aggregate transport -evidence and never stamps Succeeded from that ack or from a -LineageWeave-local completed envelope. Accepted evidence stores -transport-response receipt and row-write time as distinct clocks -when those instants differ. Period-report stays another path. -Neither start invents a theta or a calibrated report score. +so a crash after Running does not lose the item. Period-report stays +another path. Neither start invents a theta or a calibrated report +score. """ from __future__ import annotations @@ -25,17 +21,18 @@ AnalysisRunCreateError, fetch_visible_analysis_run, ) +from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL from backend.app.analysis_run_outbox import ( latest_outbox_delivery_is_claimed, latest_outbox_delivery_is_delivered, outbox_request_digest, ) from backend.app.lineage_ingestion import records_from_source_posts +from lineageweave.adjudication_client import AdjudicationClient from lineageweave.http_client import HttpClientError, post_json from lineageweave.lineage_persistence import lineage_edge_specs from lineageweave.models import Edge from lineageweave.tepp_client import AnalysisRunRequest, TeppClient, TeppNotAvailable -from lineageweave.tepp_result import TeppAcceptedEvidence, parse_tepp_accepted_evidence _LINEAGE_KIND = "analysis_run_lineage" _TEPP_KIND = "analysis_run_tepp" @@ -91,7 +88,7 @@ def start_kind_rejection(run_kind_code: str) -> AnalysisRunStartError | None: ) -def configured_tepp_client(transport_url: str = "") -> TeppClient: +def configured_tepp_client(transport_url: str = "", api_key: str = "") -> TeppClient: """Build a TEPP client from an optional HTTP transport URL. An empty URL keeps the default unavailable transport. A set URL @@ -104,8 +101,9 @@ def configured_tepp_client(transport_url: str = "") -> TeppClient: def transport(payload: dict[str, Any]) -> dict[str, Any]: try: - return post_json(url, payload, headers={}, timeout=30.0) - except (HttpClientError, ValueError, TypeError) as exc: + headers = {"authorization": f"Bearer {api_key}"} if api_key.strip() else {} + return post_json(url, payload, headers=headers, timeout=30.0) + except (HttpClientError, OSError, ValueError, TypeError) as exc: raise TeppNotAvailable(str(exc)) from exc return TeppClient(transport=transport) @@ -132,49 +130,70 @@ def tepp_run_request( ) -def tepp_accepted_clocks( - *, - started_at: datetime, - received_at: datetime, - recorded_at: datetime, -) -> tuple[datetime, datetime]: - """Return receipt then row-write clocks, monotonic versus start. - - ``received_at`` is the transport-response receipt. ``recorded_at`` - is the later row-write instant. A clock that runs backward is - clamped forward so ``started_at <= received_at <= recorded_at``. - Equal instants stay equal; this helper does not invent a later - recorded clock. - """ - receipt = received_at if received_at >= started_at else started_at - recorded = recorded_at if recorded_at >= receipt else receipt - return receipt, recorded - - -def tepp_submit_outcome( +def _tepp_submission( client: TeppClient, request: AnalysisRunRequest, -) -> tuple[str, str | None, TeppAcceptedEvidence | None]: - """Submit through ``tepp_client``. Never invent or persist a theta. - - A missing transport is ``tepp_not_available``. A published - ``AnalysisRunAccepted`` envelope is Failed / - ``tepp_completed_result_unsupported`` and returned as aggregate - transport evidence. A LineageWeave-local completed envelope or any - other unpublished shape is Failed / ``tepp_result_not_persisted``. - Succeeded is never stamped from an accepted ack. +) -> tuple[str, str, dict[str, Any] | None]: + """Submit through ``tepp_client`` and require a completed result envelope. + + TEPP's target HTTP contract is asynchronous. An ``accepted`` response is + therefore not a measurement and remains ``tepp_result_not_persisted``. + Only a provider-authoritative completed envelope can enter the database. """ try: - envelope = client.submit_analysis_run(request) + response = client.submit_analysis_run(request) except TeppNotAvailable: return _FAILED, "tepp_not_available", None - parsed = parse_tepp_accepted_evidence( - envelope, - expected_idempotency_key=request.idempotency_key, - ) - if parsed is None: + if not isinstance(response, dict): + return _FAILED, "tepp_result_not_persisted", None + if response.get("status") not in {"completed", "succeeded"}: return _FAILED, "tepp_result_not_persisted", None - return _FAILED, "tepp_completed_result_unsupported", parsed + if not isinstance(response.get("result"), dict): + return _FAILED, "tepp_result_not_persisted", None + remote_run_id = response.get("analysis_run_id") or response.get("run_id") + if not isinstance(remote_run_id, str) or not remote_run_id.strip(): + return _FAILED, "tepp_result_not_persisted", None + return _SUCCEEDED, "", response + + +def tepp_submit_outcome( + client: TeppClient, + request: AnalysisRunRequest, +) -> tuple[str, str]: + """Compatibility projection of the TEPP submission outcome.""" + status_code, failure_code, _ = _tepp_submission(client, request) + return status_code, failure_code + + +async def _persist_tepp_result( + conn: asyncpg.Connection, + *, + analysis_run_id: str, + envelope: dict[str, Any], +) -> bool: + """Persist only a validated, remote-completed TEPP envelope.""" + remote_run_id = envelope.get("analysis_run_id") or envelope.get("run_id") + if not isinstance(remote_run_id, str) or not remote_run_id.strip(): + return False + result_json = json.dumps(envelope, separators=(",", ":"), sort_keys=True) + result_sha256 = hashlib.sha256(result_json.encode("utf-8")).hexdigest() + try: + async with conn.transaction(): + await conn.execute( + """ + insert into analysis_run_tepp_result + (analysis_run_id, remote_run_id, result_json, result_sha256) + values ($1, $2, $3::jsonb, $4) + on conflict (analysis_run_id) do nothing + """, + analysis_run_id, + remote_run_id, + result_json, + result_sha256, + ) + except (asyncpg.PostgresError, TypeError, ValueError): + return False + return True def start_write_conflict_error() -> AnalysisRunStartError: @@ -208,13 +227,16 @@ async def _cutoff_source_posts( affiliated_entity_ids: list[str], ) -> list[asyncpg.Record]: """ABAC-visible cutoff rows with the grouping keys reconstruct needs.""" - rows = await conn.fetch( - """ + # Safe SQL: the eligibility predicate is an immutable schema fragment; both request values are bound. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select post_id, post_title, created_at, visibility_code, corporate_entity_id, process_unit_id, thread_group_key, secondary_grouping_key from source_post - where corporate_entity_id = $1 and created_at <= $2 + where corporate_entity_id = $1 + and created_at <= $2 + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias="source_post")} order by created_at, post_title """, corporate_entity_id, @@ -236,14 +258,17 @@ async def _snapshot_member_posts( """Load frozen capture rows, or empty when the member table is absent.""" try: return list( - await conn.fetch( - """ + # Safe SQL: the eligibility predicate is an immutable schema fragment; snapshot id is bound. + await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select post.post_id, post.post_title, post.created_at, post.visibility_code, post.corporate_entity_id, post.process_unit_id, post.thread_group_key, post.secondary_grouping_key from analysis_source_snapshot_member member - join source_post post on post.post_id = member.source_post_id + join source_post post + on post.post_id = member.source_post_id + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias="post")} where member.analysis_source_snapshot_id = $1 order by post.created_at, post.post_title """, @@ -267,12 +292,11 @@ async def _append_status( """ insert into analysis_run_status_event (analysis_run_id, status_ordinal, status_code, occurred_at, failure_code) - values ($1, $2, $3, $4, $5) + values ($1, $2, $3, clock_timestamp(), $4) """, analysis_run_id, status_ordinal, status_code, - occurred_at, failure_code, ) @@ -491,7 +515,7 @@ async def enqueue_pending_analysis_run( "or TEPP measurement.", ) - now = datetime.now(timezone.utc) + now = await conn.fetchval("select clock_timestamp()") digest = outbox_request_digest( analysis_run_id=str(locked["analysis_run_id"]), work_kind_code=str(locked["run_kind_code"]), @@ -534,16 +558,14 @@ async def deliver_queued_analysis_run( account_id: str, affiliated_entity_ids: list[str], tepp_client: TeppClient | None = None, + adjudication_client: AdjudicationClient | None = None, valkey_stream_entry_id: str | None = None, ) -> dict[str, Any]: """Claim the outbox row and finish ThreadWeave or TEPP. A delivered row replays the stored result. Missing work is 409. - TEPP stays Failed when the transport is missing, the envelope is - not the published accepted acknowledgement, or TEPP has not - published a completed-result contract. A published accepted - envelope is stored as aggregate transport evidence. No theta is - invented and Succeeded is never stamped. + TEPP stays Failed when the transport is missing or the envelope is + not persistable. No theta is invented. """ try: UUID(analysis_run_id) @@ -612,6 +634,7 @@ async def deliver_queued_analysis_run( analysis_run_id=analysis_run_id, locked=outbox, affiliated_entity_ids=affiliated_entity_ids, + adjudication_client=adjudication_client, ) finished = datetime.now(timezone.utc) if finished < now: @@ -638,6 +661,7 @@ async def start_pending_analysis_run( account_id: str, affiliated_entity_ids: list[str], tepp_client: TeppClient | None = None, + adjudication_client: AdjudicationClient | None = None, valkey_stream_entry_id: str | None = None, ) -> dict[str, Any]: """Enqueue then deliver on one connection. @@ -661,6 +685,7 @@ async def start_pending_analysis_run( account_id=account_id, affiliated_entity_ids=affiliated_entity_ids, tepp_client=tepp_client, + adjudication_client=adjudication_client, valkey_stream_entry_id=valkey_stream_entry_id, ) @@ -671,6 +696,7 @@ async def _deliver_lineage_reconstruction( analysis_run_id: str, locked: asyncpg.Record, affiliated_entity_ids: list[str], + adjudication_client: AdjudicationClient | None = None, ) -> None: """Persist ThreadWeave parent choices for the frozen bag.""" now = datetime.now(timezone.utc) @@ -687,7 +713,7 @@ async def _deliver_lineage_reconstruction( knowledge_cutoff=locked["knowledge_cutoff"], affiliated_entity_ids=affiliated_entity_ids, ) - edges = lineage_edge_specs(records_from_source_posts(rows)) + edges = lineage_edge_specs(records_from_source_posts(rows), llm=adjudication_client) digest = reconstruction_result_digest(edges) finished = datetime.now(timezone.utc) if finished < now: @@ -695,13 +721,12 @@ async def _deliver_lineage_reconstruction( await conn.execute( """ insert into analysis_run_reconstruction - (analysis_run_id, result_sha256, edge_count, reconstructed_at) - values ($1, $2, $3, $4) + (analysis_run_id, result_sha256, edge_count, reconstructed_at, recorded_at) + values ($1, $2, $3, clock_timestamp(), clock_timestamp()) """, analysis_run_id, digest, len(edges), - finished, ) for edge in edges: await conn.execute( @@ -726,42 +751,6 @@ async def _deliver_lineage_reconstruction( ) -async def _persist_tepp_accepted( - conn: asyncpg.Connection, - analysis_run_id: str, - evidence: TeppAcceptedEvidence, - received_at: datetime, - recorded_at: datetime, -) -> bool: - """Store published accepted evidence with receipt and row-write clocks. - - Missing table is not success. Callers pass transport-response - receipt as ``received_at`` and the row-write instant as - ``recorded_at``. This function binds those two values as given and - does not invent a later recorded clock when they are equal. - """ - try: - await conn.execute( - """ - insert into analysis_run_tepp_accepted - (analysis_run_id, contract_version, accepted_run_id, run_state, - idempotency_key, evidence_sha256, received_at, recorded_at) - values ($1, $2, $3, $4, $5, $6, $7, $8) - """, - analysis_run_id, - evidence.contract_version, - evidence.accepted_run_id, - evidence.run_state, - evidence.idempotency_key, - evidence.evidence_sha256(), - received_at, - recorded_at, - ) - except asyncpg.UndefinedTableError: - return False - return True - - async def _deliver_tepp_measurement( conn: asyncpg.Connection, *, @@ -770,32 +759,30 @@ async def _deliver_tepp_measurement( tepp_client: TeppClient, ) -> None: """Submit the frozen snapshot through ``tepp_client``. Never persist a theta.""" - started_at = datetime.now(timezone.utc) + now = datetime.now(timezone.utc) request = tepp_run_request( idempotency_key=str(locked["idempotency_key"]), snapshot_sha256=str(locked["snapshot_sha256"]), knowledge_cutoff=locked["knowledge_cutoff"], corporate_entity_id=str(locked["corporate_entity_id"]), ) - status_code, failure_code, accepted = tepp_submit_outcome(tepp_client, request) - received_at = datetime.now(timezone.utc) - recorded_at = datetime.now(timezone.utc) - receipt, recorded = tepp_accepted_clocks( - started_at=started_at, - received_at=received_at, - recorded_at=recorded_at, - ) - if accepted is not None: - stored = await _persist_tepp_accepted( - conn, analysis_run_id, accepted, receipt, recorded - ) - if not stored: - status_code, failure_code = _FAILED, "tepp_result_not_persisted" + status_code, failure_code, envelope = _tepp_submission(tepp_client, request) + if status_code == _SUCCEEDED and envelope is not None: + if not await _persist_tepp_result( + conn, + analysis_run_id=analysis_run_id, + envelope=envelope, + ): + status_code = _FAILED + failure_code = "tepp_result_not_persisted" + finished = datetime.now(timezone.utc) + if finished < now: + finished = now await _append_status( conn, analysis_run_id, await _next_status_ordinal(conn, analysis_run_id), status_code, - recorded, + finished, failure_code, ) diff --git a/backend/app/analysis_run_worker.py b/backend/app/analysis_run_worker.py new file mode 100644 index 000000000..43b8d17b7 --- /dev/null +++ b/backend/app/analysis_run_worker.py @@ -0,0 +1,83 @@ +"""Consume durable analysis-run wake-ups from the Valkey stream. + +PostgreSQL remains the source of truth. The worker only uses Valkey to wake +the existing idempotent delivery function; the account that created the run +supplies the internal visibility scope, and no event body is trusted. +""" + +from __future__ import annotations + +import asyncpg +import redis.asyncio as redis +from uuid import UUID + +from lineageweave.adjudication_client import AdjudicationClient +from lineageweave.tepp_client import TeppClient + +from backend.app.analysis_run_outbox import OUTBOX_STREAM_KEY +from backend.app.analysis_run_start import deliver_queued_analysis_run + + +async def consume_analysis_run_stream_once( + client: redis.Redis, + pool: asyncpg.Pool, + *, + last_id: str, + tepp_client: TeppClient, + adjudication_client: AdjudicationClient, +) -> str: + """Consume one batch and return the last inspected Valkey entry id. + + Invalid or stale entries are acknowledged by advancing the cursor; the + durable PostgreSQL outbox remains available for a later explicit retry. + """ + batches = await client.xread({OUTBOX_STREAM_KEY: last_id}, count=10, block=1000) + for _stream_name, entries in batches: + for entry_id, fields in entries: + analysis_run_id = str(fields.get("analysis_run_id", "")).strip() + try: + UUID(analysis_run_id) + except ValueError: + analysis_run_id = "" + if analysis_run_id: + async with pool.acquire() as conn: + async with conn.transaction(): + owner = await conn.fetchrow( + """ + select requested_by_account_id + from analysis_run + where analysis_run_id = $1::uuid + """, + analysis_run_id, + ) + if owner is not None: + await deliver_queued_analysis_run( + conn, + analysis_run_id=analysis_run_id, + account_id=str(owner["requested_by_account_id"]), + affiliated_entity_ids=[], + tepp_client=tepp_client, + adjudication_client=adjudication_client, + valkey_stream_entry_id=str(entry_id), + ) + last_id = str(entry_id) + return last_id + + +async def run_analysis_run_worker( + client: redis.Redis, + pool: asyncpg.Pool, + *, + tepp_client: TeppClient, + adjudication_client: AdjudicationClient, +) -> None: + """Run the single-process wake-up consumer until task cancellation.""" + last_id = "0-0" + while True: + last_id = await consume_analysis_run_stream_once( + client, + pool, + last_id=last_id, + tepp_client=tepp_client, + adjudication_client=adjudication_client, + ) diff --git a/backend/app/auth.py b/backend/app/auth.py index e5f07fff2..155974d52 100644 --- a/backend/app/auth.py +++ b/backend/app/auth.py @@ -1,12 +1,14 @@ -"""OIDC-validated login. A bearer access token is verified against -Keycloak's own live JWKS (real signature verification, not a shared-secret -shortcut) and resolved to a user_account row via external_subject_id -- +"""OIDC-validated login. A bearer access token is verified against the +configured provider's live JWKS (Keyverse in production, local Keycloak in +Compose development; never a shared-secret shortcut) and resolved to a +user_account row via external_subject_id -- corp_code/pu_code are attributes read from the DB's account_affiliation, never trusted directly off the token, per the schema's design (see migrations/0001_initial_schema.sql). -JWKS is fetched through ``lineageweave.http_client.get_json`` (http(s) -allowlist) so a mis-set KEYCLOAK_BASE_URL cannot become a file-scheme read. +JWKS is fetched through OIDC discovery or an explicit JWKS URI using +``lineageweave.http_client.get_json``. The HTTP client rejects non-http(s) +schemes, so provider configuration cannot become a file-scheme read. """ from __future__ import annotations @@ -25,34 +27,83 @@ from lineageweave.http_client import HttpClientError, get_json _bearer_scheme = HTTPBearer(auto_error=True) -_jwks_cache: dict[str, dict] = {} +_jwks_cache: dict[tuple[str, str, str], dict] = {} -def _jwks(settings: Settings) -> dict: - """Return the realm JWKS, cached per URI for the process lifetime.""" - cached = _jwks_cache.get(settings.keycloak_jwks_uri) +def _jwks_cache_key(settings: Settings) -> tuple[str, str, str]: + """Bind cached keys to the exact issuer and key-discovery configuration.""" + return ( + settings.oidc_issuer, + settings.oidc_discovery_uri, + settings.oidc_jwks_uri_override, + ) + + +def _jwks(settings: Settings, *, force_refresh: bool = False) -> dict: + """Return provider JWKS, refreshing explicitly when signing keys rotate.""" + cache_key = _jwks_cache_key(settings) + cached = None if force_refresh else _jwks_cache.get(cache_key) if cached is None: try: - cached = get_json(settings.keycloak_jwks_uri, timeout=10) + if settings.oidc_jwks_uri_override: + jwks_uri = settings.oidc_jwks_uri_override + else: + metadata = get_json(settings.oidc_discovery_uri, timeout=10) + jwks_uri = metadata.get("jwks_uri") + if not isinstance(jwks_uri, str) or not jwks_uri.strip(): + raise ValueError("OIDC discovery document has no jwks_uri") + cached = get_json(jwks_uri, timeout=10) except (HttpClientError, OSError, ValueError) as exc: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, - f"could not fetch JWKS from {settings.keycloak_jwks_uri}: {exc}", + f"could not fetch OIDC JWKS for {settings.oidc_issuer}: {exc}", ) from exc - _jwks_cache[settings.keycloak_jwks_uri] = cached + _jwks_cache[cache_key] = cached return cached def _signing_key_from_jwks(jwks: dict, token: str): - """Pick the JWKS RSA key that matches the JWT kid, without urllib.""" - header = jwt.get_unverified_header(token) + """Require a non-empty JWT ``kid`` and an exact acceptable RSA key match.""" + try: + header = jwt.get_unverified_header(token) + except jwt.PyJWTError as exc: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "invalid access-token header") from exc + if header.get("alg") != "RS256": + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "access token must use RS256") kid = header.get("kid") + if not isinstance(kid, str) or not kid.strip(): + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "access token must include a non-empty kid") for key in jwks.get("keys", []): - if kid is None or key.get("kid") == kid: + if not isinstance(key, dict) or key.get("kid") != kid: + continue + if key.get("kty") != "RSA": + continue + if key.get("alg") not in (None, "RS256"): + continue + if key.get("use") not in (None, "sig"): + continue + key_ops = key.get("key_ops") + if key_ops is not None and ( + not isinstance(key_ops, list) or "verify" not in key_ops + ): + continue + try: return RSAAlgorithm.from_jwk(json.dumps(key)) + except (KeyError, TypeError, ValueError) as exc: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "matching JWKS key is invalid") from exc raise HTTPException(status.HTTP_401_UNAUTHORIZED, f"no JWKS key matched kid={kid!r}") +def _signing_key(settings: Settings, token: str): + """Resolve a signing key and refresh JWKS once when a new ``kid`` appears.""" + try: + return _signing_key_from_jwks(_jwks(settings), token) + except HTTPException as exc: + if not str(exc.detail).startswith("no JWKS key matched kid="): + raise + return _signing_key_from_jwks(_jwks(settings, force_refresh=True), token) + + @dataclass(frozen=True) class CurrentAccount: """The provisioned account that a verified access token resolved to.""" @@ -60,6 +111,7 @@ class CurrentAccount: user_account_id: str external_subject_id: str display_name: str + preferred_locale: str | None corporate_entity_ids: frozenset[str] permission_codes: frozenset[str] @@ -69,17 +121,24 @@ def has_permission(self, permission_code: str) -> bool: def _decode_access_token(token: str, settings: Settings) -> dict: + """Validate signature, issuer, resource audience, time claims, and subject.""" try: - signing_key = _signing_key_from_jwks(_jwks(settings), token) - return jwt.decode( + claims = jwt.decode( token, - key=signing_key, + key=_signing_key(settings, token), algorithms=["RS256"], - issuer=settings.keycloak_issuer, - options={"verify_aud": False}, + issuer=settings.oidc_issuer, + audience=settings.oidc_audience, + leeway=settings.oidc_clock_skew_seconds, ) + except HTTPException: + raise except jwt.PyJWTError as exc: raise HTTPException(status.HTTP_401_UNAUTHORIZED, f"invalid token: {exc}") from exc + subject = claims.get("sub") + if not isinstance(subject, str) or not subject.strip(): + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "access token has no subject") + return claims async def get_current_account( @@ -93,7 +152,7 @@ async def get_current_account( async with pool.acquire() as conn: account_row = await conn.fetchrow( - "select user_account_id, display_name from user_account where external_subject_id = $1", + "select user_account_id, display_name, preferred_locale from user_account where external_subject_id = $1", subject, ) if account_row is None: @@ -121,6 +180,7 @@ async def get_current_account( user_account_id=str(account_row["user_account_id"]), external_subject_id=subject, display_name=account_row["display_name"], + preferred_locale=account_row["preferred_locale"], corporate_entity_ids=frozenset(str(row["corporate_entity_id"]) for row in entity_rows), permission_codes=frozenset(row["permission_code"] for row in permission_rows), ) diff --git a/backend/app/config.py b/backend/app/config.py index 082f8756a..02dc8dc34 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -25,35 +25,29 @@ class Settings: # docker-compose the two differ (internal DNS name vs. the # host-published port a browser actually hits). keycloak_issuer: str + # Production may use the organization's Keyverse OIDC issuer. The + # keycloak fields above remain the explicit local-development fallback. + oidc_issuer: str + oidc_client_id: str + # Resource audience the backend accepts. This is deliberately separate + # from the browser OAuth client id: an access token issued for another + # resource at the same trusted issuer must not become a LineageWeave API + # credential merely because its signature is valid. + oidc_audience: str + oidc_discovery_uri: str + oidc_jwks_uri_override: str + oidc_clock_skew_seconds: int # Exact browser origins allowed by CORS. Comma-separated FRONTEND_ORIGINS; # never a wildcard -- the backend only serves the product UI. frontend_origins: list[str] - # Keyman extraction is a hard dependency of POST /api/posts/{id}/extract-keymen - # only -- every other endpoint works with these unset. Empty string, not - # a fabricated default, when unconfigured (see keyman_ingestion.py). orchestrator_base_url: str orchestrator_api_key: str - # A vision-capable model name on the same contextual-orchestrator gateway - # (orchestrator_base_url/_api_key) -- describes embedded post images - # before an LLM call or embedding sees the post body (ADR: see - # lineageweave/post_content_normalization.py). Empty means the image - # channel is unavailable, same "no fake channel" discipline as every - # other pluggable client. - vision_model: str - # Event queue for post/ticket activity (XADD/XRANGE), per the brief's - # "Event Queue, not MQ" requirement -- see backend/app/activity_stream.py. + embedding_model: str valkey_url: str - # Self-hosted Searxng instance relation_verification.py's real client - # checks Knowledge Graph relation inferences against (ADR 0005). Empty - # means the verification channel is unavailable, same "no fake - # channel" discipline as every other pluggable client. searxng_base_url: str - # Optional TEPP HTTP transport. Empty keeps TeppClient's default - # unavailable transport. Never a local psychometric substitute. tepp_transport_url: str - # RankWeave ranking port (ADR 0030). True = fail-closed - # RankWeaveNotAvailable -- never invent a fused score. Default false - # uses the in-process library already required by reconstruct.py. + tepp_api_key: str + caldav_base_url: str rankweave_disabled: bool @property @@ -66,6 +60,45 @@ def load_settings() -> Settings: """Read Settings from the environment, with local-dev defaults only.""" keycloak_base_url = os.environ.get("KEYCLOAK_BASE_URL", "http://localhost:18080") keycloak_realm = os.environ.get("KEYCLOAK_REALM", "lineageweave-demo") + keycloak_client_id = os.environ.get("KEYCLOAK_CLIENT_ID", "lineageweave-frontend") + keycloak_issuer = os.environ.get( + "KEYCLOAK_ISSUER", f"{keycloak_base_url}/realms/{keycloak_realm}" + ) + keyverse_issuer = os.environ.get("KEYVERSE_ISSUER", "").strip() + generic_oidc_issuer = os.environ.get("OIDC_ISSUER", "").strip() + external_oidc = bool(keyverse_issuer or generic_oidc_issuer) + oidc_issuer = (keyverse_issuer or generic_oidc_issuer or keycloak_issuer).rstrip("/") + oidc_client_id = ( + os.environ.get("KEYVERSE_CLIENT_ID", "").strip() + or os.environ.get("OIDC_CLIENT_ID", "").strip() + or keycloak_client_id + ) + configured_audience = ( + os.environ.get("KEYVERSE_AUDIENCE", "").strip() + or os.environ.get("OIDC_AUDIENCE", "").strip() + ) + if external_oidc and not configured_audience: + raise ValueError( + "external OIDC requires KEYVERSE_AUDIENCE or OIDC_AUDIENCE; " + "do not infer a resource-server audience from the browser client id" + ) + oidc_audience = configured_audience or "lineageweave-api" + oidc_discovery_uri = os.environ.get("KEYVERSE_DISCOVERY_URI", "").strip() or os.environ.get( + "OIDC_DISCOVERY_URI", "" + ).strip() + if not oidc_discovery_uri: + discovery_base = oidc_issuer if external_oidc else keycloak_base_url + oidc_discovery_uri = ( + f"{discovery_base.rstrip('/')}/realms/{keycloak_realm}/.well-known/openid-configuration" + if not external_oidc + else f"{discovery_base.rstrip('/')}/.well-known/openid-configuration" + ) + try: + oidc_clock_skew_seconds = int(os.environ.get("OIDC_CLOCK_SKEW_SECONDS", "5")) + except ValueError as exc: + raise ValueError("OIDC_CLOCK_SKEW_SECONDS must be an integer") from exc + if not 0 <= oidc_clock_skew_seconds <= 60: + raise ValueError("OIDC_CLOCK_SKEW_SECONDS must be between 0 and 60") return Settings( database_url=os.environ.get( "DATABASE_URL", @@ -73,10 +106,22 @@ def load_settings() -> Settings: ), keycloak_base_url=keycloak_base_url, keycloak_realm=keycloak_realm, - keycloak_client_id=os.environ.get("KEYCLOAK_CLIENT_ID", "lineageweave-frontend"), - keycloak_issuer=os.environ.get( - "KEYCLOAK_ISSUER", f"{keycloak_base_url}/realms/{keycloak_realm}" + keycloak_client_id=keycloak_client_id, + keycloak_issuer=keycloak_issuer, + oidc_issuer=oidc_issuer, + oidc_client_id=oidc_client_id, + oidc_audience=oidc_audience, + oidc_discovery_uri=oidc_discovery_uri, + oidc_jwks_uri_override=( + os.environ.get("KEYVERSE_JWKS_URI", "").strip() + or os.environ.get("OIDC_JWKS_URI", "").strip() + or ( + f"{keycloak_base_url}/realms/{keycloak_realm}/protocol/openid-connect/certs" + if not external_oidc + else "" + ) ), + oidc_clock_skew_seconds=oidc_clock_skew_seconds, frontend_origins=[ origin.strip() for origin in os.environ.get("FRONTEND_ORIGINS", "http://localhost:5173").split(",") @@ -84,10 +129,12 @@ def load_settings() -> Settings: ], orchestrator_base_url=os.environ.get("ORCHESTRATOR_BASE_URL", ""), orchestrator_api_key=os.environ.get("ORCHESTRATOR_API_KEY", ""), - vision_model=os.environ.get("VISION_MODEL", ""), + embedding_model=os.environ.get("LLM_GATEWAY_EMBEDDING_MODEL", "").strip(), valkey_url=os.environ.get("VALKEY_URL", "redis://localhost:16379/0"), searxng_base_url=os.environ.get("SEARXNG_BASE_URL", ""), tepp_transport_url=os.environ.get("TEPP_TRANSPORT_URL", ""), + tepp_api_key=os.environ.get("TEPP_API_KEY", ""), + caldav_base_url=os.environ.get("CALDAV_BASE_URL", "").strip(), rankweave_disabled=os.environ.get("RANKWEAVE_DISABLED", "") .strip() .lower() diff --git a/backend/app/corporate_entity_ingestion.py b/backend/app/corporate_entity_ingestion.py index 5c42c0b52..bfc0e740d 100644 --- a/backend/app/corporate_entity_ingestion.py +++ b/backend/app/corporate_entity_ingestion.py @@ -29,6 +29,7 @@ CorporateEntityCandidate, score_corporate_entity, ) +from lineageweave.http_client import HttpClientError from lineageweave.relation_verification import ( STATUS_CORROBORATED, RelationVerificationClient, @@ -38,19 +39,6 @@ _MAX_HIERARCHY_DEPTH = 4 _CREATION_LOCK_KEY = "lineageweave:corporate_entity_creation" -# The post-lock re-check exists only to catch a genuine concurrent -# duplicate CREATE of this exact name (see get_or_create_corporate_entity's -# docstring) -- never a fuzzy resolution against an unrelated-but-similar -# sibling or parent. score_corporate_entity's default 0.6 threshold is -# deliberately loose for real mention resolution (an abbreviation, a -# trailing legal suffix), but that same looseness is wrong here: a child -# whose name contains its own just-created parent's name as a prefix -# ("Acme" -> "Acme Gwangju Plant") scores ~0.7 against that parent alone, -# so a loose re-check would silently bind the child TO the parent instead -# of creating its own row. 1.0 (post-normalization exact match) is the -# only threshold that means "this really is the same entity." -_EXACT_MATCH_SIMILARITY = 1.0 - def _auto_entity_code(organization_name: str) -> str: """Return a deterministic, namespace-separated code.""" @@ -151,19 +139,33 @@ async def get_or_create_corporate_entity( if _depth >= _MAX_HIERARCHY_DEPTH or not inference_client.available: return None - proposal = await asyncio.to_thread( - inference_client.infer, - normalized_name, - context_text, - ) + try: + proposal = await asyncio.to_thread( + inference_client.infer, + normalized_name, + context_text, + ) + except (HttpClientError, OSError, TimeoutError): + # A provider timeout is an unavailable enrichment channel, not a + # reason to discard the source-grounded summary. Keep the actor + # unbound and let an explicit retry attempt catalog enrichment later. + return None if proposal is None or not verification_client.available: return None - placement_result = await asyncio.to_thread( - verification_client.verify, - normalized_name, - _hierarchy_verification_label(proposal), - ) + try: + placement_result = await asyncio.to_thread( + verification_client.verify, + normalized_name, + _hierarchy_verification_label(proposal), + ) + except (HttpClientError, OSError): + # A transient search-provider failure (DNS, timeout, non-2xx) here + # must not crash the caller (extract-keymen / post summary + # ingestion) -- treat it the same as "not corroborated this run": + # the entity simply isn't auto-created, same conservative outcome + # as a real search that found nothing. + return None if placement_result.status_code != STATUS_CORROBORATED: return None @@ -173,11 +175,16 @@ async def get_or_create_corporate_entity( normalized_parent = proposal.parent_name.strip() if not normalized_parent or normalized_parent.casefold() in visited_names: return None - parent_result = await asyncio.to_thread( - verification_client.verify, - normalized_parent, - f"immediate parent of {normalized_name}", - ) + try: + parent_result = await asyncio.to_thread( + verification_client.verify, + normalized_parent, + f"immediate parent of {normalized_name}", + ) + except (HttpClientError, OSError): + # Same fail-closed-without-crashing behavior as the placement + # verification above. + return None if parent_result.status_code != STATUS_CORROBORATED: return None parent_entity_id = await get_or_create_corporate_entity( @@ -198,15 +205,14 @@ async def get_or_create_corporate_entity( "select pg_advisory_xact_lock(hashtext($1))", _CREATION_LOCK_KEY, ) - # Exact match only (see _EXACT_MATCH_SIMILARITY): this re-check's - # sole purpose is catching a genuine concurrent duplicate CREATE of - # THIS name, not re-resolving against a merely-similar candidate -- - # the parent this call may have just created above is now in the - # reloaded pool and must not be mistaken for this (distinct) entity. + # ponytail: the lock recheck is exact-only; fuzzy matching here can + # mistake an inferred child for the parent just created above. The + # initial lookup remains fuzzy, while this check only prevents a + # concurrent insert of the same normalized name. fresh = score_corporate_entity( normalized_name, await _reload_candidates(conn), - min_similarity=_EXACT_MATCH_SIMILARITY, + min_similarity=1.0, ) if fresh.kind == RESOLUTION_UNIQUE and fresh.catalog_id is not None: _remember_candidate(candidates, fresh.catalog_id, normalized_name) diff --git a/backend/app/customer_group_tree_ingestion.py b/backend/app/customer_group_tree_ingestion.py deleted file mode 100644 index e4b79c76d..000000000 --- a/backend/app/customer_group_tree_ingestion.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Load the authorized customer-group forest from PostgreSQL.""" - -from __future__ import annotations - -from typing import Any - -import asyncpg - -from lineageweave.customer_group_tree import ( - CatalogEntityRow, - TreeAbbreviation, - build_customer_group_forest, -) -from lineageweave.relation_verification import STATUS_CORROBORATED - -from .knowledge_graph import labels_for_codes - - -async def fetch_customer_group_forest( - conn: asyncpg.Connection, - affiliated_entity_ids: list[str], -) -> list[dict[str, Any]]: - """Authorized Group / Company / Plant forest for one account.""" - entity_rows = await conn.fetch( - """ - select corporate_entity_id, parent_entity_id, entity_name, entity_level_code - from corporate_entity - """ - ) - entities = tuple( - CatalogEntityRow( - entity_id=str(row["corporate_entity_id"]), - parent_entity_id=str(row["parent_entity_id"]) if row["parent_entity_id"] is not None else None, - entity_name=row["entity_name"], - entity_level_code=row["entity_level_code"], - ) - for row in entity_rows - ) - alias_rows = await conn.fetch( - """ - select raw_organization_name, corporate_entity_id, - verification_status_code, verification_evidence_url - from abbreviation_tree_corroboration - where verification_status_code = $1 - and corporate_entity_id is not null - """, - STATUS_CORROBORATED, - ) - abbreviations = tuple( - ( - str(row["corporate_entity_id"]), - TreeAbbreviation( - raw_organization_name=row["raw_organization_name"], - verification_status_code=row["verification_status_code"], - verification_evidence_url=row["verification_evidence_url"], - ), - ) - for row in alias_rows - ) - forest = [ - node.to_dict() - for node in build_customer_group_forest(entities, affiliated_entity_ids, abbreviations) - ] - await _attach_lookup_labels(conn, forest) - return forest - - -def _collect_level_codes(nodes: list[dict[str, Any]]) -> list[str]: - """Every entity-level code in the forest.""" - codes: list[str] = [] - for node in nodes: - if node.get("entity_level_code"): - codes.append(node["entity_level_code"]) - codes.extend(_collect_level_codes(node.get("children", []))) - return codes - - -def _apply_lookup_labels(nodes: list[dict[str, Any]], labels: dict[str, str]) -> None: - """Write display labels onto the JSON forest, falling back to the code.""" - for node in nodes: - level = node.get("entity_level_code") - node["entity_level_label"] = labels.get(level, level) if level else None - _apply_lookup_labels(node.get("children", []), labels) - - -async def _attach_lookup_labels(conn: asyncpg.Connection, forest: list[dict[str, Any]]) -> None: - """Hydrate ``entity_level_label`` from lookup rows.""" - _apply_lookup_labels(forest, await labels_for_codes(conn, _collect_level_codes(forest))) diff --git a/backend/app/customer_hint_ingestion.py b/backend/app/customer_hint_ingestion.py new file mode 100644 index 000000000..497c66728 --- /dev/null +++ b/backend/app/customer_hint_ingestion.py @@ -0,0 +1,167 @@ +"""Resolves one observed customer-hint code (`source_post.source_customer_code`) +to a real-world `corporate_entity`, using the text of posts that share the +code as evidence -- and, per the same corroboration discipline +`organization_name_resolution_ingestion.py` already applies to in-text +abbreviations (ADR 0008), never binding a new customer name that external +search did not corroborate. An uncorroborated or unresolved guess leaves +the hint exactly as unresolved as it started; it never invents a Customer +Master entity from a single ungrounded LLM answer. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import asyncpg + +from lineageweave.customer_hint_resolution import CustomerHintResolutionClient +from lineageweave.image_content import NullImageContentClient +from lineageweave.organization_name_resolution import resolve_and_verify_organization_name +from lineageweave.post_content_normalization import normalize_post_body +from lineageweave.relation_verification import STATUS_CORROBORATED, RelationVerificationClient + + +_EXCERPT_LENGTH = 1500 + + +async def resolve_customer_hint( + conn: asyncpg.Connection, + resolution_client: CustomerHintResolutionClient, + verification_client: RelationVerificationClient, + hint_code: str, +) -> dict[str, Any] | None: + """Resolve one `source_customer_code` hint to a real `corporate_entity`. + + Returns ``None`` when the resolver is unavailable, no eligible posts + carry this hint, or the proposed name was not externally corroborated. + Otherwise creates (or reuses, by case-insensitive exact name) the + entity and reclaims every post sharing this hint that still sits at + its account's default placeholder entity, returning the entity + id/name plus how many posts were reclaimed. + """ + if not resolution_client.available: + return None + # Five rows and 20,000 raw body characters per row bound both transfer and + # parsing before deterministic normalization. The SQL remains literal; + # only the observed hint code is a bound value. + rows = await conn.fetch( + """ + select post_title, left(post_body, 20000) as post_body + from source_post + where source_customer_code = $1 + and nullif(btrim(source_post.source_draft_code), '') is null + and nullif(btrim(source_post.source_deleted_flag), '') is null + and not ( + ( + nullif(btrim(source_post.source_author_code), '') is null + and nullif(btrim(source_post.source_author_name), '') is null + and nullif(btrim(source_post.source_company_code), '') is null + and nullif(btrim(source_post.source_company_name), '') is null + and nullif(btrim(source_post.source_process_unit_code), '') is null + and nullif(btrim(source_post.source_process_unit_name), '') is null + and nullif(btrim(source_post.source_sales_pool_code), '') is null + and nullif(btrim(source_post.source_sales_pool_name), '') is null + and nullif(btrim(source_post.source_customer_code), '') is null + and nullif(btrim(source_post.source_customer_name), '') is null + and nullif(btrim(source_post.source_project_code), '') is null + and nullif(btrim(source_post.source_project_name), '') is null + ) + and exists ( + select 1 + from source_post real_post + where ( + nullif(btrim(real_post.source_author_code), '') is not null + or nullif(btrim(real_post.source_author_name), '') is not null + or nullif(btrim(real_post.source_company_code), '') is not null + or nullif(btrim(real_post.source_company_name), '') is not null + or nullif(btrim(real_post.source_process_unit_code), '') is not null + or nullif(btrim(real_post.source_process_unit_name), '') is not null + or nullif(btrim(real_post.source_sales_pool_code), '') is not null + or nullif(btrim(real_post.source_sales_pool_name), '') is not null + or nullif(btrim(real_post.source_customer_code), '') is not null + or nullif(btrim(real_post.source_customer_name), '') is not null + or nullif(btrim(real_post.source_project_code), '') is not null + or nullif(btrim(real_post.source_project_name), '') is not null + ) + ) + ) + order by created_at desc + limit 5 + """, + hint_code, + ) + if not rows: + return None + + vision_client = NullImageContentClient() + excerpts = "\n---\n".join( + f"{row['post_title']}\n" + f"{normalize_post_body(row['post_body'], vision_client=vision_client).text[:_EXCERPT_LENGTH]}" + for row in rows + ) + resolution = await asyncio.to_thread( + resolve_and_verify_organization_name, + hint_code, + excerpts, + resolution_client, + verification_client, + ) + if resolution is None or resolution.verification_status_code != STATUS_CORROBORATED: + return None + + entity_name = resolution.resolved_organization_name + existing = await conn.fetchrow( + "select corporate_entity_id from corporate_entity where lower(entity_name) = lower($1)", + entity_name, + ) + if existing is not None: + entity_id = existing["corporate_entity_id"] + else: + # ON CONFLICT, not a plain INSERT: re-resolving the same hint_code + # is not guaranteed to get byte-identical LLM phrasing back, so the + # name-based lookup above can miss an entity this same hint already + # created -- corporate_entity_code (deterministic from hint_code) + # is the stable identity key a retry must key off instead. + entity_code = f"HINT-{hint_code}" + # Safe SQL: the statement is a literal migration-shaped query; both observed values are bound. + created = await conn.fetchrow( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + """ + insert into corporate_entity (corporate_entity_code, entity_name, entity_level_code) + values ($1, $2, 'company') + on conflict (corporate_entity_code) + do update set entity_name = excluded.entity_name + returning corporate_entity_id + """, + entity_code, + entity_name, + ) + entity_id = created["corporate_entity_id"] + + # `corporate_entity_id` is NOT NULL, so a bulk-imported real record + # never sits at NULL waiting to be resolved -- it defaults to whatever + # entity its shared placeholder `author_account_id` happens to be + # affiliated with (the same shared-placeholder shape as the + # `author_affiliations` hint leak fixed in semantic_hints.py). Only + # reclaim a post still sitting at that default, never one some other + # resolution already bound to a specific entity. + linked = await conn.fetch( + """ + update source_post + set corporate_entity_id = $1 + where source_customer_code = $2 + and corporate_entity_id in ( + select corporate_entity_id from account_affiliation + where user_account_id = source_post.author_account_id + ) + returning post_id + """, + entity_id, + hint_code, + ) + return { + "corporate_entity_id": str(entity_id), + "entity_name": entity_name, + "linked_post_count": len(linked), + "verification_evidence_url": resolution.verification_evidence_url, + } diff --git a/backend/app/demo_scope.py b/backend/app/demo_scope.py new file mode 100644 index 000000000..6d25bee24 --- /dev/null +++ b/backend/app/demo_scope.py @@ -0,0 +1,83 @@ +"""Shared demo-vs-real scope helpers (ADR 0001 / ADR 0042). + +`make seed`'s Demo Corp narrative exists so a fresh, dataless install has +something to show. Once an account can see at least one post carrying real +source-import evidence, the synthetic Demo Corp tree is no longer needed to +fill an empty screen and must stop appearing next to real evidence -- a +buyer must never mistake a fabricated contact (e.g. Ada West, Priya Nair) +for a real one. +""" + +from __future__ import annotations + +import asyncpg + + +def is_demo_scope(corporate_entity_code: str | None) -> bool: + """True for `make seed`'s synthetic Demo Corp tree (``DEMO-*`` codes).""" + return bool(corporate_entity_code) and corporate_entity_code.startswith("DEMO-") + + +async def has_real_source_context( + conn: asyncpg.Connection, corporate_entity_ids: list[str] +) -> bool: + """Return whether the account can see imported source evidence.""" + return bool( + # Safe SQL: this is immutable schema text; authorized entity ids are bound through $1. + await conn.fetchval( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + """ + select exists ( + select 1 + from source_post + where (visibility_code = 'public' + or corporate_entity_id = any($1::uuid[])) + and ( + nullif(btrim(source_post.source_author_code), '') is not null + or nullif(btrim(source_post.source_author_name), '') is not null + or nullif(btrim(source_post.source_company_code), '') is not null + or nullif(btrim(source_post.source_company_name), '') is not null + or nullif(btrim(source_post.source_process_unit_code), '') is not null + or nullif(btrim(source_post.source_process_unit_name), '') is not null + or nullif(btrim(source_post.source_sales_pool_code), '') is not null + or nullif(btrim(source_post.source_sales_pool_name), '') is not null + or nullif(btrim(source_post.source_customer_code), '') is not null + or nullif(btrim(source_post.source_customer_name), '') is not null + or nullif(btrim(source_post.source_project_code), '') is not null + or nullif(btrim(source_post.source_project_name), '') is not null + ) + ) + """, + list(corporate_entity_ids), + ) + ) + + +async def fetch_demo_corporate_entity_ids(conn: asyncpg.Connection) -> set[str]: + """Return synthetic-only Demo entity IDs, excluding imported entities.""" + rows = await conn.fetch( + """ + select entity.corporate_entity_id + from corporate_entity entity + where entity.corporate_entity_code like 'DEMO-%' + and not exists ( + select 1 + from source_post real_post + where real_post.corporate_entity_id = entity.corporate_entity_id + and ( + nullif(btrim(real_post.source_author_code), '') is not null + or nullif(btrim(real_post.source_author_name), '') is not null + or nullif(btrim(real_post.source_company_code), '') is not null + or nullif(btrim(real_post.source_company_name), '') is not null + or nullif(btrim(real_post.source_process_unit_code), '') is not null + or nullif(btrim(real_post.source_process_unit_name), '') is not null + or nullif(btrim(real_post.source_sales_pool_code), '') is not null + or nullif(btrim(real_post.source_sales_pool_name), '') is not null + or nullif(btrim(real_post.source_customer_code), '') is not null + or nullif(btrim(real_post.source_customer_name), '') is not null + or nullif(btrim(real_post.source_project_code), '') is not null + or nullif(btrim(real_post.source_project_name), '') is not null + ) + ) + """ + ) + return {str(row["corporate_entity_id"]) for row in rows} diff --git a/backend/app/entity_relationship_ingestion.py b/backend/app/entity_relationship_ingestion.py index 091e58f40..da22a9136 100644 --- a/backend/app/entity_relationship_ingestion.py +++ b/backend/app/entity_relationship_ingestion.py @@ -7,6 +7,7 @@ from __future__ import annotations import asyncio +import json from collections.abc import Mapping, Sequence from typing import Any @@ -36,32 +37,37 @@ async def ingest_post_entity_relationships( should check `client.available` first, same discipline as every other pluggable channel in this repo. """ - if not organization_names: - return [] - - relationships = await asyncio.to_thread( - client.classify, post_title, post_body, organization_names - ) + if organization_names: + relationships = await asyncio.to_thread( + client.classify, post_title, post_body, organization_names + ) + else: + relationships = [] - for relationship in relationships: + requested_names = set(organization_names) + relationships = [ + relationship + for relationship in relationships + if relationship.organization_name in requested_names + ] + # This is a replacement projection, not an append-only cache. A later + # extraction can remove an organization (including an our-side-only + # affiliation), so stale relationship rows must disappear atomically. + async with conn.transaction(): await conn.execute( - """ - insert into post_counterparty_entity (post_id, counterparty_entity_name, relationship_type_code) - values ($1, $2, $3) - on conflict (post_id, counterparty_entity_name) - do update set - relationship_type_code = excluded.relationship_type_code, - -- A re-classification invalidates any prior verification -- - -- that search was run against the OLD relationship_label, - -- see relation_verification.py. - verification_status_code = 'verify_pending', - verification_evidence_url = null, - verification_checked_at = null - """, + "delete from post_counterparty_entity where post_id = $1", post_id, - relationship.organization_name, - relationship.relationship_type_code, ) + for relationship in relationships: + await conn.execute( + """ + insert into post_counterparty_entity (post_id, counterparty_entity_name, relationship_type_code) + values ($1, $2, $3) + """, + post_id, + relationship.organization_name, + relationship.relationship_type_code, + ) return relationships @@ -93,7 +99,8 @@ async def fetch_post_counterparties(conn: asyncpg.Connection, post_id: str) -> l rows = await conn.fetch( """ select c.counterparty_entity_name, c.relationship_type_code, v.lookup_label as relationship_label, - c.verification_status_code, c.verification_evidence_url + c.verification_status_code, c.verification_evidence_url, + c.verification_evidence_post_id from post_counterparty_entity c join common_lookup_value v on v.lookup_code = c.relationship_type_code where c.post_id = $1 @@ -107,3 +114,145 @@ async def fetch_post_counterparties(conn: asyncpg.Connection, post_id: str) -> l for row in candidate_rows ] return attach_resolved_entity_ids(rows, candidates) + + +async def fetch_relationship_network( + conn: asyncpg.Connection, corporate_entity_ids: Sequence[str] +) -> list[dict[str, Any]]: + """Every counterparty's full observed relationship network, entity-level. + + ``post_counterparty_entity`` classifies one counterparty name's + relationship to us per post (e.g. this specific post is + ``rel_voc`` -- Voice of Customer). A real counterparty is not + limited to one such role over its lifetime: the same organization + can be a customer in one post, a competitor in another (their own + product line competes with ours elsewhere), the customer of our + customer in a third, or a supplier -- Customer Master's per-post + reads never rolled these up, so buyers could only see one role at + a time and never the entity's whole network. This groups every + visible, eligible post's classifications by counterparty name, + keeping every distinct relationship type observed (not just the + most frequent), so a buyer can see a name marked both Customer and + Competitor and know that reflects the real, mixed relationship + rather than a classification error. + + Unresolved names keep ``corporate_entity_id`` null, same + missing-vs-guessed discipline as :func:`attach_resolved_entity_ids`. + Capped at the 100 entities with the most total observed posts; ties + break on name for a stable order. + """ + if not corporate_entity_ids: + return [] + # Safe SQL: this is immutable schema text; authorized entity ids are bound through $1. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + """ + with scoped as ( + select counterparty.counterparty_entity_name, + counterparty.relationship_type_code, + lookup.lookup_label as relationship_label + from post_counterparty_entity counterparty + join source_post post on post.post_id = counterparty.post_id + join common_lookup_value lookup + on lookup.lookup_code = counterparty.relationship_type_code + where (post.visibility_code = 'public' + or post.corporate_entity_id = any($1::uuid[])) + and nullif(btrim(post.source_draft_code), '') is null + and nullif(btrim(post.source_deleted_flag), '') is null + and not ( + ( + nullif(btrim(post.source_author_code), '') is null + and nullif(btrim(post.source_author_name), '') is null + and nullif(btrim(post.source_company_code), '') is null + and nullif(btrim(post.source_company_name), '') is null + and nullif(btrim(post.source_process_unit_code), '') is null + and nullif(btrim(post.source_process_unit_name), '') is null + and nullif(btrim(post.source_sales_pool_code), '') is null + and nullif(btrim(post.source_sales_pool_name), '') is null + and nullif(btrim(post.source_customer_code), '') is null + and nullif(btrim(post.source_customer_name), '') is null + and nullif(btrim(post.source_project_code), '') is null + and nullif(btrim(post.source_project_name), '') is null + ) + and exists ( + select 1 + from source_post real_post + where ( + nullif(btrim(real_post.source_author_code), '') is not null + or nullif(btrim(real_post.source_author_name), '') is not null + or nullif(btrim(real_post.source_company_code), '') is not null + or nullif(btrim(real_post.source_company_name), '') is not null + or nullif(btrim(real_post.source_process_unit_code), '') is not null + or nullif(btrim(real_post.source_process_unit_name), '') is not null + or nullif(btrim(real_post.source_sales_pool_code), '') is not null + or nullif(btrim(real_post.source_sales_pool_name), '') is not null + or nullif(btrim(real_post.source_customer_code), '') is not null + or nullif(btrim(real_post.source_customer_name), '') is not null + or nullif(btrim(real_post.source_project_code), '') is not null + or nullif(btrim(real_post.source_project_name), '') is not null + ) + ) + ) + ), grouped as ( + select counterparty_entity_name, + relationship_type_code, + relationship_label, + count(*) as post_count + from scoped + group by counterparty_entity_name, + relationship_type_code, + relationship_label + ), entity_totals as ( + select counterparty_entity_name, sum(post_count) as total_post_count + from grouped + group by counterparty_entity_name + ), top_entities as materialized ( + select counterparty_entity_name, total_post_count + from entity_totals + order by total_post_count desc, counterparty_entity_name + limit 100 + ) + select top_entities.counterparty_entity_name, + top_entities.total_post_count, + json_agg( + json_build_object( + 'relationship_type_code', grouped.relationship_type_code, + 'relationship_label', grouped.relationship_label, + 'post_count', grouped.post_count + ) + order by grouped.post_count desc, + grouped.relationship_type_code + ) as relationships + from top_entities + join grouped + on grouped.counterparty_entity_name = top_entities.counterparty_entity_name + group by top_entities.counterparty_entity_name, + top_entities.total_post_count + order by top_entities.total_post_count desc, + top_entities.counterparty_entity_name + """, + list(corporate_entity_ids), + ) + candidate_rows = await conn.fetch("select corporate_entity_id, entity_name from corporate_entity") + candidates = [ + CorporateEntityCandidate(str(row["corporate_entity_id"]), row["entity_name"]) + for row in candidate_rows + ] + network: list[dict[str, Any]] = [] + for row in rows: + relationships = ( + json.loads(row["relationships"]) + if isinstance(row["relationships"], str) + else row["relationships"] + ) + network.append( + { + "counterparty_entity_name": row["counterparty_entity_name"], + "corporate_entity_id": resolve_corporate_entity( + row["counterparty_entity_name"], candidates + ), + "total_post_count": row["total_post_count"], + "relationships": relationships, + "multi_role": len(relationships) > 1, + } + ) + return network diff --git a/backend/app/five_w1h_ingestion.py b/backend/app/five_w1h_ingestion.py new file mode 100644 index 000000000..736a8ecd6 --- /dev/null +++ b/backend/app/five_w1h_ingestion.py @@ -0,0 +1,52 @@ +"""Authorized read projection for the post detail 5W1H panel.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import asyncpg + +from lineageweave.five_w1h import assemble_five_w1h_slots, slots_payload + +from .entity_relationship_ingestion import fetch_post_counterparties +from .post_chat_ingestion import find_linked_post_ids +from .post_summary_ingestion import fetch_persisted_summary + + +async def load_five_w1h_slots( + conn: asyncpg.Connection, + post_id: str, + can_see_post: Callable[[asyncpg.Record], bool], +) -> dict[str, Any]: + """Build 5W1H from stored projections and visible lineage only.""" + summary = await fetch_persisted_summary(conn, post_id) or {} + evidence_claims = await conn.fetch( + """ + select slot_code, value_text, evidence_text + from post_summary_five_w1h + where post_id = $1 + order by slot_code, value_ordinal + """, + post_id, + ) + linked = await find_linked_post_ids(conn, post_id) + candidate_ids = sorted(linked.direct | linked.indirect) + linked_titles: list[str] = [] + if candidate_ids: + rows = await conn.fetch( + "select post_id, post_title, visibility_code, corporate_entity_id " + "from source_post where post_id = any($1::uuid[])", + candidate_ids, + ) + linked_titles = [row["post_title"] for row in rows if can_see_post(row)] + + counterparties = await fetch_post_counterparties(conn, post_id) + slots = assemble_five_w1h_slots( + roles=summary.get("roles_and_responsibilities", []), + key_events=summary.get("key_events", []), + counterparties=[row["counterparty_entity_name"] for row in counterparties], + lineage_node_labels=linked_titles, + evidence_claims=[dict(row) for row in evidence_claims], + ) + return {"post_id": post_id, "slots": slots_payload(slots)} diff --git a/backend/app/issue_ticket_ingestion.py b/backend/app/issue_ticket_ingestion.py index c6ad8d454..2cb3fe01d 100644 --- a/backend/app/issue_ticket_ingestion.py +++ b/backend/app/issue_ticket_ingestion.py @@ -18,6 +18,9 @@ import asyncpg from .knowledge_graph import labels_for_codes +from .post_eligibility import source_context_present_sql + +_SOURCE_CONTEXT_PRESENT_SQL = source_context_present_sql("p") def _parse_due_date(due_date: str | None) -> date | None: @@ -135,7 +138,8 @@ async def fetch_upcoming_commitments(conn: asyncpg.Connection) -> list[dict[str, "issue_ticket.assigned_account_id, issue_ticket.due_date, " "issue_ticket.commitment_summary, issue_ticket.created_at, " "issue_ticket.updated_at, " - "p.post_title, p.visibility_code, p.corporate_entity_id " + "p.post_title, p.visibility_code, p.corporate_entity_id, " + f"({_SOURCE_CONTEXT_PRESENT_SQL}) as has_real_source_context " "from issue_ticket " "join source_post p on p.post_id = issue_ticket.post_id " "where issue_ticket.due_date is not null " @@ -150,6 +154,7 @@ async def fetch_upcoming_commitments(conn: asyncpg.Connection) -> list[dict[str, "post_title": row["post_title"], "visibility_code": row["visibility_code"], "corporate_entity_id": str(row["corporate_entity_id"]), + "has_real_source_context": bool(row["has_real_source_context"]), } for row in rows ], diff --git a/backend/app/keyman_ingestion.py b/backend/app/keyman_ingestion.py index 97c81525f..ade1044a1 100644 --- a/backend/app/keyman_ingestion.py +++ b/backend/app/keyman_ingestion.py @@ -216,6 +216,7 @@ async def ingest_post_keymen( resolution_client: OrganizationNameResolutionClient | None = None, verification_client: RelationVerificationClient | None = None, hierarchy_inference_client: CorporateHierarchyInferenceClient | None = None, + context_hints: str = "", persist_graph: bool = True, ) -> list[PersonMention]: """Extracts, persists, and returns the `PersonMention`s found in one post. @@ -240,7 +241,13 @@ async def ingest_post_keymen( resolution_client = resolution_client or NullOrganizationNameResolutionClient() verification_client = verification_client or NullRelationVerificationClient() hierarchy_inference_client = hierarchy_inference_client or NullCorporateHierarchyInferenceClient() - mentions = await asyncio.to_thread(client.extract, post_title, post_body) + extract_with_hints = getattr(client, "extract_with_hints", None) + if callable(extract_with_hints): + mentions = await asyncio.to_thread( + extract_with_hints, post_title, post_body, context_hints + ) + else: + mentions = await asyncio.to_thread(client.extract, post_title, post_body) candidates = await _load_corporate_entity_candidates(conn) resolved_by_mention: list[tuple[PersonMention, list[tuple[str, str, str | None]]]] = [] for mention in mentions: diff --git a/backend/app/knowledge_graph.py b/backend/app/knowledge_graph.py index ce7289bb5..71304ce92 100644 --- a/backend/app/knowledge_graph.py +++ b/backend/app/knowledge_graph.py @@ -13,6 +13,7 @@ import asyncpg +from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL from lineageweave.ontology import ontology_annotations from lineageweave.knowledge_graph import ( EDGE_AFFILIATION, @@ -255,12 +256,14 @@ async def visible_mention_post_ids( can_see_post, ) -> list[str]: """Visible post ids supported by Keyman or R&R person evidence.""" - rows = await conn.fetch( - """ + # Safe SQL: the eligibility predicate is an immutable schema fragment; person id is bound. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select post.post_id, post.visibility_code, post.corporate_entity_id from combined_post_person_mention mention join source_post post on post.post_id = mention.post_id where mention.person_id = $1 + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} order by post.created_at, post.post_id """, person_id, @@ -273,12 +276,14 @@ async def visible_affiliation_post_ids( can_see_post, ) -> list[str]: """Visible posts that mention an entity via a person or a direct org mention.""" - rows = await conn.fetch( - """ + # Safe SQL: the eligibility predicate is an immutable schema fragment; entity id is bound. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select distinct post.post_id, post.visibility_code, post.corporate_entity_id, post.created_at from source_post post - where post.post_id in ( + where {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} + and post.post_id in ( select mention.post_id from person_affiliation affiliation join combined_post_person_mention mention @@ -302,12 +307,14 @@ async def visible_team_mention_post_ids( can_see_post, ) -> list[str]: """Visible post ids supported by a cataloged team mention.""" - rows = await conn.fetch( - """ + # Safe SQL: the eligibility predicate is an immutable schema fragment; team id is bound. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select post.post_id, post.visibility_code, post.corporate_entity_id from post_team_mention mention join source_post post on post.post_id = mention.post_id where mention.team_id = $1 + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} order by post.created_at, post.post_id """, team_id, @@ -477,8 +484,12 @@ async def hydrate_related_nodes( } if person_ids else {} posts = { str(row["post_id"]): row - for row in await conn.fetch( - "select post_id, post_title from source_post where post_id = any($1::uuid[])", + # Safe SQL: the eligibility predicate is an immutable schema fragment; post ids are bound. + for row in await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f"select post_id, post_title, " + "btrim(left(source_post_search_text(post_body), 420)) as post_body_excerpt, " + "char_length(coalesce(post_body, '')) > 420 as post_body_truncated " + f"from source_post where post_id = any($1::uuid[]) and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}", post_ids, ) } if post_ids else {} @@ -516,6 +527,8 @@ async def hydrate_related_nodes( item["person_side_label"] = side_labels.get(side, side) elif node_type_code == NODE_POST and node_id in posts: item["label"] = posts[node_id]["post_title"] + item["post_body_excerpt"] = posts[node_id]["post_body_excerpt"] + item["post_body_truncated"] = posts[node_id]["post_body_truncated"] elif node_type_code == NODE_CORPORATE_ENTITY and node_id in corps: item["label"] = corps[node_id]["entity_name"] elif node_type_code == NODE_TEAM and node_id in teams: @@ -549,6 +562,56 @@ async def related_for_person( return await related_for_start(conn, NODE_PERSON, person_id, visible_post_ids) +async def fetch_person_role_history( + conn: asyncpg.Connection, + person_id: str, + visible_post_ids: list[str], +) -> list[dict[str, Any]]: + """This Keyman's responsibility and affiliated organization across time. + + RWR's related-nodes view answers "what else connects to this + person"; it does not answer "how has this specific person's role + changed" -- the same cataloged_person can be affiliated with + different organizations, or described with a different + responsibility, in posts at different times (a job change, a title + change, a move between projects). ``post_summary_role`` already + carries this per post; this simply orders it chronologically for + one person instead of leaving a buyer to open every post that + mentions them and compare manually. + + ``visible_post_ids`` must already be ABAC-filtered by the caller + (see ``visible_mention_post_ids``); this function does not itself + check visibility. An empty result means no role classification + exists for this person on any post the account can see, not that + the person is unknown. + """ + if not visible_post_ids: + return [] + rows = await conn.fetch( + """ + select role.post_id, post.post_title, post.created_at, + role.responsibility, role.affiliated_organization_name + from post_summary_role role + join source_post post on post.post_id = role.post_id + where role.cataloged_person_id = $1 + and role.post_id = any($2::uuid[]) + order by post.created_at asc, role.post_id + """, + person_id, + visible_post_ids, + ) + return [ + { + "post_id": str(row["post_id"]), + "post_title": row["post_title"], + "created_at": row["created_at"].isoformat(), + "responsibility": row["responsibility"], + "affiliated_organization_name": row["affiliated_organization_name"], + } + for row in rows + ] + + async def related_for_entity( conn: asyncpg.Connection, entity_id: str, diff --git a/backend/app/lineage_ingestion.py b/backend/app/lineage_ingestion.py index e81ee630d..f1e76d495 100644 --- a/backend/app/lineage_ingestion.py +++ b/backend/app/lineage_ingestion.py @@ -15,6 +15,7 @@ import asyncpg +from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL from lineageweave.lineage_persistence import lineage_edge_specs from lineageweave.models import Edge, Record @@ -76,7 +77,7 @@ async def rebuild_lineage(conn: asyncpg.Connection) -> list[Edge]: rows = await conn.fetch( "select post_id, post_title, voc_type_code, created_at, corporate_entity_id, " "process_unit_id, thread_group_key, secondary_grouping_key " - "from source_post" + f"from source_post where {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}" ) edges = lineage_edge_specs(records_from_source_posts(rows)) await persist_lineage_edges(conn, edges) @@ -86,18 +87,62 @@ async def rebuild_lineage(conn: asyncpg.Connection) -> list[Edge]: async def visible_lineage_graph( conn: asyncpg.Connection, can_see_post, + limit: int = 500, + focus_post_id: str | None = None, ) -> dict[str, Any]: - """ABAC-filtered ``{nodes, edges}`` matching the stdlib demo graph shape.""" + """ABAC-filtered graph bounded for the browser's initial viewport. + + The persisted graph can contain tens of thousands of posts. The UI opens + individual posts for complete lineage, while this landing projection keeps + only the newest ``limit`` visible nodes and edges between them. + """ posts = await conn.fetch( "select post_id, post_title, voc_type_code, visibility_code, " "corporate_entity_id, process_unit_id, thread_group_key, created_at " - "from source_post" + f"from source_post where {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}" ) - visible = [row for row in posts if can_see_post(row)] - visible_ids = {str(row["post_id"]) for row in visible} + visible_all = [row for row in posts if can_see_post(row)] edge_rows = await conn.fetch( "select parent_post_id, child_post_id, fused_score from post_lineage_edge" ) + + if focus_post_id is None: + visible = sorted( + visible_all, + key=lambda row: (row["created_at"], str(row["post_id"])), + reverse=True, + )[:limit] + truncated = len(visible_all) > len(visible) + else: + focus_id = str(focus_post_id) + focus_visible = any(str(row["post_id"]) == focus_id for row in visible_all) + neighbors: dict[str, set[str]] = {} + for edge in edge_rows: + parent_id = str(edge["parent_post_id"]) + child_id = str(edge["child_post_id"]) + neighbors.setdefault(parent_id, set()).add(child_id) + neighbors.setdefault(child_id, set()).add(parent_id) + + component_ids: set[str] = set() + frontier = [focus_id] if focus_visible else [] + while frontier: + current_id = frontier.pop() + if current_id in component_ids: + continue + component_ids.add(current_id) + frontier.extend(neighbors.get(current_id, set()) - component_ids) + + # An isolated post has no DAG to render; the post-lineage endpoint + # still reports its empty direct/indirect lists. + if len(component_ids) <= 1: + visible = [] + else: + visible = [ + row for row in visible_all if str(row["post_id"]) in component_ids + ] + truncated = False + + visible_ids = {str(row["post_id"]) for row in visible} visible_edges = [ row for row in edge_rows @@ -128,4 +173,4 @@ async def visible_lineage_graph( } for row in visible_edges ] - return {"nodes": nodes, "edges": edges} + return {"nodes": nodes, "edges": edges, "truncated": truncated} diff --git a/backend/app/main.py b/backend/app/main.py index f944f3c81..fb943315f 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -20,34 +20,51 @@ from __future__ import annotations import asyncio +import json from contextlib import asynccontextmanager +from dataclasses import asdict from datetime import datetime -from typing import Any +from typing import Any, Literal from uuid import UUID import asyncpg import redis.asyncio as redis -from fastapi import Depends, FastAPI, HTTPException, status +from fastapi import Depends, FastAPI, HTTPException, Query, status from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel +from lineageweave.adjudication_client import ( + ContextualOrchestratorAdjudicationClient, + NullAdjudicationClient, +) from lineageweave.commitment_extraction import ( ContextualOrchestratorCommitmentExtractionClient, NullCommitmentExtractionClient, ) +from lineageweave.caldav_client import ( + CALDAV_UNAVAILABLE_NEXT_ACTION, + build_caldav_client, +) from lineageweave.entity_relationship_classification import ( ContextualOrchestratorEntityRelationshipClient, NullEntityRelationshipClient, ) from lineageweave.image_content import orchestrator_vision_client +from lineageweave.embedding_client import orchestrator_embedding_client +from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata from lineageweave.corporate_hierarchy_inference import ( ContextualOrchestratorHierarchyInferenceClient, NullCorporateHierarchyInferenceClient, ) from lineageweave.keyman_extraction import ( + COUNTERPARTY, ContextualOrchestratorKeymanExtractionClient, NullKeymanExtractionClient, ) +from lineageweave.customer_hint_resolution import ( + ContextualOrchestratorCustomerHintResolutionClient, + NullCustomerHintResolutionClient, +) from lineageweave.organization_name_resolution import ( ContextualOrchestratorOrganizationNameResolutionClient, NullOrganizationNameResolutionClient, @@ -55,6 +72,7 @@ from lineageweave.post_chat import ( ContextualOrchestratorPostChatClient, NullPostChatClient, + cited_post_evidence, cited_post_summaries, ) from lineageweave.post_content_normalization import normalize_post_body @@ -63,8 +81,11 @@ NullPostEvaluationClient, RUBRIC_VERSION, ) +from lineageweave.post_structure import ContextualOrchestratorPostStructureClient, NullPostStructureClient from lineageweave.post_summary import ContextualOrchestratorPostSummaryClient, NullPostSummaryClient from lineageweave.relation_verification import NullRelationVerificationClient, SearxngRelationVerificationClient +from lineageweave.semantic_hints import customer_hint_trust, format_semantic_hints +from lineageweave.ontology import LW from lineageweave.rankweave_client import build_rankweave_client from backend.app.analysis_run_ingestion import ( @@ -80,6 +101,14 @@ deliver_queued_analysis_run, enqueue_pending_analysis_run, ) +from backend.app.analysis_run_worker import run_analysis_run_worker +from backend.app.post_content_queue import ( + ensure_post_content_job, + post_content_api_status, + post_content_is_complete, + publish_post_content_event, +) +from backend.app.post_content_worker import run_post_content_worker from backend.app.source_post_revision import fetch_known_at_revision, parse_as_of_clock from backend.app.activity_stream import ( create_valkey_client, @@ -89,19 +118,17 @@ ticket_created_summary, ticket_status_changed_summary, ) -from backend.app.abbreviation_tree_corroboration_ingestion import ( - corroborate_post_abbreviations, - fetch_post_abbreviation_matches, -) from backend.app.affiliate_tree_ingestion import fetch_affiliate_forest, fetch_voc_evidence -from backend.app.customer_group_tree_ingestion import fetch_customer_group_forest from backend.app.auth import CurrentAccount, get_current_account from backend.app.config import load_settings +from backend.app.customer_hint_ingestion import resolve_customer_hint from backend.app.db import create_pool, get_pool from backend.app.entity_relationship_ingestion import ( fetch_post_counterparties, + fetch_relationship_network, ingest_post_entity_relationships, ) +from backend.app.five_w1h_ingestion import load_five_w1h_slots from backend.app.post_evaluation_ingestion import fetch_post_evaluation, ingest_post_evaluation from backend.app.ranking_ingestion import load_visible_ranking_posts from backend.app.report_ingestion import ( @@ -125,6 +152,7 @@ from backend.app.keyman_ingestion import ingest_post_keymen from backend.app.knowledge_graph import ( corporate_entity_exists, + fetch_person_role_history, fetch_post_keymen, labels_for_codes, person_exists, @@ -143,9 +171,21 @@ fetch_persisted_chats, find_linked_post_ids, gather_chat_sources, + gather_global_chat_sources, persist_post_chat, ) -from backend.app.post_summary_ingestion import fetch_persisted_summary, persist_post_summary +from backend.app.post_summary_ingestion import ( + fetch_persisted_summary, + persist_post_summary, + require_summary_source_body, +) +from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL +from backend.app.demo_scope import ( + fetch_demo_corporate_entity_ids, + has_real_source_context, + is_demo_scope, +) +from lineageweave.http_client import HttpClientError _POST_READ = "post_read" _POST_ADMIN = "post_admin" @@ -158,9 +198,36 @@ async def lifespan(app: FastAPI): settings = load_settings() app.state.pool = await create_pool(settings.database_url) app.state.valkey = create_valkey_client(settings.valkey_url) + app.state.analysis_run_worker = asyncio.create_task( + run_analysis_run_worker( + app.state.valkey, + app.state.pool, + tepp_client=configured_tepp_client( + settings.tepp_transport_url, + settings.tepp_api_key, + ), + adjudication_client=_adjudication_client(), + ) + ) + app.state.post_content_worker = asyncio.create_task( + run_post_content_worker( + app.state.valkey, + app.state.pool, + vision_factory=_vision_client, + embedding_factory=_embedding_client, + structure_factory=_post_structure_client, + ) + ) try: yield finally: + app.state.analysis_run_worker.cancel() + app.state.post_content_worker.cancel() + await asyncio.gather( + app.state.analysis_run_worker, + app.state.post_content_worker, + return_exceptions=True, + ) await app.state.pool.close() await app.state.valkey.aclose() @@ -224,6 +291,22 @@ def _organization_name_resolution_client(): ) +def _customer_hint_resolution_client(): + """Live orchestrator client when configured; otherwise the unavailable null. + + A longer timeout than the other channels' default 30s: this prompt + carries up to five posts' excerpts, not one short mention -- a real + resolve call measured 137.6s live end-to-end (90s was not enough + margin and made every real hint 503; 200s gives real headroom). + """ + settings = load_settings() + if not (settings.orchestrator_base_url and settings.orchestrator_api_key): + return NullCustomerHintResolutionClient() + return ContextualOrchestratorCustomerHintResolutionClient( + base_url=settings.orchestrator_base_url, api_key=settings.orchestrator_api_key, timeout=200.0 + ) + + def _corporate_hierarchy_inference_client(): """Live orchestrator client when configured; otherwise the unavailable null.""" settings = load_settings() @@ -244,6 +327,33 @@ def _post_summary_client(): ) +def _adjudication_client(): + """Live orchestrator client when configured; otherwise the unavailable null. + + reconstruct.py's DEFAULT_CHANNEL_WEIGHTS gives this channel the most + weight (0.40) of the four -- it is the only one that reasons about + content instead of approximating it (ADR 0064) -- but nothing ever + passed a real client through lineage_edge_specs() to reconstruct(), + so every lineage reconstruction had silently run on the weaker + 3-channel fallback since the feature was built. + """ + settings = load_settings() + if not (settings.orchestrator_base_url and settings.orchestrator_api_key): + return NullAdjudicationClient() + return ContextualOrchestratorAdjudicationClient( + base_url=settings.orchestrator_base_url, api_key=settings.orchestrator_api_key + ) + + +def _post_structure_client(): + settings = load_settings() + if not (settings.orchestrator_base_url and settings.orchestrator_api_key): + return NullPostStructureClient() + return ContextualOrchestratorPostStructureClient( + base_url=settings.orchestrator_base_url, api_key=settings.orchestrator_api_key + ) + + def _post_chat_client(): """Live orchestrator client when configured; otherwise the unavailable null.""" settings = load_settings() @@ -267,17 +377,24 @@ def _commitment_extraction_client(): def _vision_client(): """Live vision client when configured; otherwise the unavailable null. - Same contextual-orchestrator gateway as every other channel, plus a - vision-capable model name (``VISION_MODEL``) -- unlike the other - channels, a missing model name alone (base_url/api_key present but no - model) also means unavailable, since there is no sane default model - to guess. + Same contextual-orchestrator gateway as every other channel. The model is + intentionally omitted so contextual-orchestrator selects the registered + vision-capable provider agent; LineageWeave never selects ``VISION_MODEL``. """ settings = load_settings() return orchestrator_vision_client( settings.orchestrator_base_url, settings.orchestrator_api_key, - settings.vision_model, + ) + + +def _embedding_client(): + """Build the orchestrator embedding client, or an unavailable channel.""" + settings = load_settings() + return orchestrator_embedding_client( + settings.orchestrator_base_url, + settings.orchestrator_api_key, + settings.embedding_model, ) @@ -292,7 +409,7 @@ def _post_evaluation_client(): def _rankweave_client(): - """In-process RankWeave unless RANKWEAVE_DISABLED=1 (ADR 0030).""" + """In-process RankWeave unless RANKWEAVE_DISABLED=1 (ADR 0024).""" return build_rankweave_client(disabled=load_settings().rankweave_disabled) @@ -303,11 +420,21 @@ def _can_see_post(account: CurrentAccount, post: asyncpg.Record) -> bool: return str(post["corporate_entity_id"]) in account.corporate_entity_ids +def _is_synthetic_demo_member(member: dict[str, Any], demo_entity_ids: set[str]) -> bool: + """Identify one pure seed row without hiding real rows sharing its entity.""" + return bool(demo_entity_ids) and member["corporate_entity_id"] in demo_entity_ids and not bool( + member.get("has_real_source_context", False) + ) + + def _serialize_post(post: asyncpg.Record, labels: dict[str, str] | None = None) -> dict[str, Any]: """Turn a ``source_post`` row into the public JSON shape.""" resolved = labels or {} voc = post["voc_type_code"] visibility = post["visibility_code"] + project_evidence = post.get("project_evidence") or [] + if isinstance(project_evidence, str): + project_evidence = json.loads(project_evidence) return { "post_id": str(post["post_id"]), "post_title": post["post_title"], @@ -315,17 +442,178 @@ def _serialize_post(post: asyncpg.Record, labels: dict[str, str] | None = None) "voc_type_label": resolved.get(voc, voc), "visibility_code": visibility, "visibility_label": resolved.get(visibility, visibility), + "source_stage_code": post.get("source_stage_code"), + "source_detail_state_code": post.get("source_detail_state_code"), + "source_draft_code": post.get("source_draft_code"), + "source_deleted_flag": post.get("source_deleted_flag"), + "publication_state_code": _publication_state_code(post), + "source_author_code": post.get("source_author_code"), + "source_author_name": post.get("source_author_name"), + "source_company_code": post.get("source_company_code"), + "source_company_name": post.get("source_company_name"), + "source_process_unit_code": post.get("source_process_unit_code"), + "source_process_unit_name": post.get("source_process_unit_name"), + "source_sales_pool_code": post.get("source_sales_pool_code"), + "source_sales_pool_name": post.get("source_sales_pool_name"), + "source_customer_code": post.get("source_customer_code"), + "source_customer_name": post.get("source_customer_name"), + "source_project_code": post.get("source_project_code"), + "source_project_name": post.get("source_project_name"), + "source_system_code": post.get("source_system_code"), + "source_record_key": post.get("source_record_key"), + "post_body_excerpt": post.get("post_body_excerpt"), + "post_body_truncated": post.get("post_body_truncated", False), + "project_evidence": project_evidence, "created_at": post["created_at"].isoformat(), } +def _publication_state_code(post: asyncpg.Record) -> str: + """Expose raw lifecycle evidence without guessing its source semantics.""" + if str(post.get("source_deleted_flag") or "").strip(): + return "source_deletion_marker" + if str(post.get("source_draft_code") or "").strip(): + return "source_draft_marker" + return "publication_state_unknown" + + +async def _load_project_evidence( + conn: asyncpg.Connection, + post_id: str, + source_project_code: str | None, + source_project_name: str | None, +) -> list[dict[str, Any]]: + """Merge explicit source hints and stored semantic project candidates.""" + evidence: list[dict[str, Any]] = [] + source_code = source_project_code.strip() if source_project_code else "" + source_name = source_project_name.strip() if source_project_name else "" + if source_code or source_name: + source_field = ( + "source_post.source_project_name" + if source_name + else "source_post.source_project_code" + ) + evidence.append( + { + "project_key": source_code or source_name, + "project_name": source_name or source_code, + "evidence": source_field, + "confidence": None, + "ontology_iri": str(LW.Project), + "ontology_label": "Project", + "extraction_method": "source_field_hint", + "resolution_status": "hint_only", + "provenance": source_field, + } + ) + rows = await conn.fetch( + """ + select project_key, project_name, evidence_text, confidence, + ontology_iri, extraction_method + from post_project_mention + where post_id = $1 + order by confidence desc, project_name, project_key + """, + post_id, + ) + evidence.extend( + { + "project_key": row["project_key"], + "project_name": row["project_name"], + "evidence": row["evidence_text"], + "confidence": float(row["confidence"]), + "ontology_iri": row["ontology_iri"], + "ontology_label": "Project", + "extraction_method": row["extraction_method"], + "resolution_status": "semantic_candidate", + "provenance": "post_project_mention.evidence_text", + } + for row in rows + ) + return evidence + + async def _lookup_post_labels(conn: asyncpg.Connection, rows: list[asyncpg.Record]) -> dict[str, str]: """Resolve voc_type / visibility codes against common_lookup_value.""" codes = [row["voc_type_code"] for row in rows] + [row["visibility_code"] for row in rows] return await labels_for_codes(conn, codes) +async def _post_filter_options( + conn: asyncpg.Connection, corporate_entity_ids: frozenset[str] +) -> tuple[list[dict[str, str]], list[dict[str, str]]]: + """Return every authorized filter value, not only values on the current page.""" + visibility_sql = f""" + select distinct post.visibility_code as code, + coalesce(lookup.lookup_label, post.visibility_code) as label, + coalesce(lookup.display_order, 2147483647) as display_order + from source_post post + left join common_lookup_value lookup + on lookup.lookup_category = 'post_visibility' + and lookup.lookup_code = post.visibility_code + where (post.visibility_code = 'public' + or post.corporate_entity_id::text = any($1::text[])) + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} + order by display_order, code + """ + type_sql = f""" + select distinct post.voc_type_code as code, + coalesce(lookup.lookup_label, post.voc_type_code) as label, + coalesce(lookup.display_order, 2147483647) as display_order + from source_post post + left join common_lookup_value lookup + on lookup.lookup_category = 'voc_type' + and lookup.lookup_code = post.voc_type_code + where (post.visibility_code = 'public' + or post.corporate_entity_id::text = any($1::text[])) + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} + order by display_order, code + """ + # Safe SQL: both query strings are closed lookup statements; entity ids remain asyncpg parameters. + visibility_rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + visibility_sql, list(corporate_entity_ids) + ) + # Safe SQL: both query strings are closed lookup statements; entity ids remain asyncpg parameters. + type_rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + type_sql, list(corporate_entity_ids) + ) + return ( + [{"code": row["code"], "label": row["label"]} for row in type_rows], + [{"code": row["code"], "label": row["label"]} for row in visibility_rows], + ) + + @app.get("/healthz") + +@app.get("/api/settings", response_model=dict) +async def read_tenant_settings( + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +): + async with pool.acquire() as conn: + row = await conn.fetchrow("SELECT brand_name FROM tenant_settings WHERE id = 1") + if not row: + return {"brandName": "LineageWeave"} + return {"brandName": row["brand_name"]} + +@app.patch("/api/settings", response_model=dict) +async def update_tenant_settings( + payload: dict, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +): + # Only admins can change settings + _require_post_admin(account) + brand_name = payload.get("brandName", "LineageWeave") + async with pool.acquire() as conn: + await conn.execute( + "INSERT INTO tenant_settings (id, brand_name) VALUES (1, $1) " + "ON CONFLICT (id) DO UPDATE SET brand_name = $1", + brand_name + ) + return {"brandName": brand_name} + + async def healthz() -> dict[str, str]: """Liveness probe: the process is up. Does not touch Postgres.""" return {"status": "ok"} @@ -363,38 +651,453 @@ async def read_me( return { "user_account_id": account.user_account_id, "display_name": account.display_name, + "preferred_locale": account.preferred_locale, "permission_codes": sorted(account.permission_codes), "corporate_entities": entities, } -@app.get("/api/customer-group-tree") -async def read_customer_group_tree( +class LocalePreferenceRequest(BaseModel): + preferred_locale: Literal["en", "ko", "zh", "ja", "vi"] + + +class CustomerHintResolveRequest(BaseModel): + hint_code: str + + +@app.patch("/api/me/preferences") +async def update_me_preferences( + preference: LocalePreferenceRequest, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, str]: + """Persist member preferences without putting them in browser-only state.""" + async with pool.acquire() as conn: + await conn.execute( + "update user_account set preferred_locale = $1 where user_account_id = $2", + preference.preferred_locale, + account.user_account_id, + ) + return {"preferred_locale": preference.preferred_locale} + + +@app.get("/api/customer-master") +async def read_customer_master( + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """Return the authorized customer catalog and its cataloged Keymen.""" + _require_post_read(account) + if not account.corporate_entity_ids: + return { + "corporate_entities": [], + "keymen": [], + "source_customer_hints": [], + "source_author_hints": [], + "relationship_network": [], + } + + async with pool.acquire() as conn: + # Safe SQL: the evidence query uses only closed schema fragments; authorized entity ids are bound. + source_customer_rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" + with scoped as ( + select post_id, post_title, created_at, + nullif(btrim(source_customer_code), '') as customer_code, + nullif(btrim(source_customer_name), '') as customer_name, + case when nullif(btrim(source_customer_code), '') is null + then nullif(btrim(source_customer_name), '') + else null end as customer_name_group + from source_post + where (nullif(btrim(source_customer_code), '') is not null + or nullif(btrim(source_customer_name), '') is not null) + and (visibility_code = 'public' or corporate_entity_id = any($1::uuid[])) + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')} + ), ranked as ( + select scoped.*, + row_number() over ( + partition by customer_code, customer_name_group + order by created_at desc, post_id desc + ) as related_rank + from scoped + ), groups as ( + select customer_code, customer_name_group, + max(customer_name) as customer_name, + count(*) as post_count + from ranked + group by customer_code, customer_name_group + ), top_groups as materialized ( + select * + from groups + order by post_count desc, customer_code, customer_name + limit 100 + ), related as ( + select ranked.customer_code, ranked.customer_name_group, + json_agg( + json_build_object( + 'post_id', post.post_id::text, + 'post_title', post.post_title + ) + order by ranked.created_at desc, ranked.post_id desc + ) as related_posts + from ranked + join top_groups + on top_groups.customer_code is not distinct from ranked.customer_code + and top_groups.customer_name_group is not distinct from ranked.customer_name_group + join source_post post on post.post_id = ranked.post_id + where ranked.related_rank <= 20 + group by ranked.customer_code, ranked.customer_name_group + ) + select top_groups.customer_code, top_groups.customer_name, top_groups.post_count, + coalesce(related.related_posts, '[]'::json) as related_posts + from top_groups + left join related + on related.customer_code is not distinct from top_groups.customer_code + and related.customer_name_group is not distinct from top_groups.customer_name_group + order by top_groups.post_count desc, top_groups.customer_code, top_groups.customer_name + """, + list(account.corporate_entity_ids), + ) + # Safe SQL: the evidence query uses only closed schema fragments; authorized entity ids are bound. + source_author_rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" + with scoped as ( + select post.post_id, post.post_title, post.created_at, + btrim(post.source_author_code) as author_code, + case + when post.source_author_name is null + or btrim(post.source_author_name) = '' + or lower(btrim(post.source_author_name)) = lower(btrim(post.source_author_code)) + then null + else btrim(post.source_author_name) + end as source_author_name, + post.author_account_id, + author.display_name as account_display_name + from source_post post + join user_account author on author.user_account_id = post.author_account_id + where post.source_author_code is not null + and btrim(post.source_author_code) <> '' + and (post.visibility_code = 'public' or post.corporate_entity_id = any($1::uuid[])) + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} + ), ranked as ( + select scoped.*, + row_number() over ( + partition by author_code, author_account_id, account_display_name + order by created_at desc, post_id desc + ) as related_rank + from scoped + ), groups as ( + select author_code, author_account_id, account_display_name, + max(source_author_name) as author_name, + count(*) as post_count + from ranked + group by author_code, author_account_id, account_display_name + ), keyman_mentions as ( + select ranked.author_code, ranked.author_account_id, + ranked.account_display_name, ranked.post_id, + person.person_id, person.person_name, + person.person_side_code, person.last_known_job_title + from ranked + join post_summary_role role + on role.post_id = ranked.post_id + and role.actor_type_code = 'prov_person' + join cataloged_person person + on person.person_id = role.cataloged_person_id + and person.person_side_code = 'our_side' + where role.cataloged_person_id is not null + union + select ranked.author_code, ranked.author_account_id, + ranked.account_display_name, ranked.post_id, + person.person_id, person.person_name, + person.person_side_code, person.last_known_job_title + from ranked + join post_person_mention mention + on mention.post_id = ranked.post_id + join cataloged_person person + on person.person_id = mention.person_id + and person.person_side_code = 'our_side' + ), keyman_authors as ( + select distinct author_code, author_account_id, account_display_name + from keyman_mentions + ), top_groups as materialized ( + select groups.* + from groups + left join keyman_authors + on keyman_authors.author_code = groups.author_code + and keyman_authors.author_account_id = groups.author_account_id + and keyman_authors.account_display_name = groups.account_display_name + order by (keyman_authors.author_code is not null) desc, + groups.post_count desc, groups.author_code + limit 100 + ), keyman_groups as ( + select mentions.author_code, mentions.author_account_id, + mentions.account_display_name, + mentions.person_id, mentions.person_name, + mentions.person_side_code, mentions.last_known_job_title, + count(distinct mentions.post_id) as mention_count + from keyman_mentions mentions + join top_groups + on top_groups.author_code = mentions.author_code + and top_groups.author_account_id = mentions.author_account_id + and top_groups.account_display_name = mentions.account_display_name + group by mentions.author_code, mentions.author_account_id, + mentions.account_display_name, mentions.person_id, + mentions.person_name, mentions.person_side_code, + mentions.last_known_job_title + ), keyman_related as ( + select author_code, author_account_id, account_display_name, + json_agg( + json_build_object( + 'person_id', person_id::text, + 'person_name', person_name, + 'person_side_code', person_side_code, + 'last_known_job_title', last_known_job_title, + 'mention_count', mention_count, + 'provenance', + 'post_person_mention.person_id|post_summary_role.cataloged_person_id/source_post.author_account_id' + ) + order by mention_count desc, person_name, person_id + ) as keyman_hints + from keyman_groups + group by author_code, author_account_id, account_display_name + ), related as ( + select ranked.author_code, ranked.author_account_id, ranked.account_display_name, + json_agg( + json_build_object( + 'post_id', post.post_id::text, + 'post_title', post.post_title + ) + order by ranked.created_at desc, ranked.post_id desc + ) as related_posts + from ranked + join top_groups + on top_groups.author_code = ranked.author_code + and top_groups.author_account_id = ranked.author_account_id + and top_groups.account_display_name = ranked.account_display_name + join source_post post on post.post_id = ranked.post_id + where ranked.related_rank <= 20 + group by ranked.author_code, ranked.author_account_id, ranked.account_display_name + ) + select top_groups.author_code, top_groups.author_name, top_groups.author_account_id, + top_groups.account_display_name, top_groups.post_count, + coalesce(keyman_related.keyman_hints, '[]'::json) as keyman_hints, + coalesce(related.related_posts, '[]'::json) as related_posts + from top_groups + left join keyman_related + on keyman_related.author_code = top_groups.author_code + and keyman_related.author_account_id = top_groups.author_account_id + and keyman_related.account_display_name = top_groups.account_display_name + left join related + on related.author_code = top_groups.author_code + and related.author_account_id = top_groups.author_account_id + and related.account_display_name = top_groups.account_display_name + order by top_groups.post_count desc, top_groups.author_code + """, + list(account.corporate_entity_ids), + ) + entity_rows = await conn.fetch( + """ + select corporate_entity_id, corporate_entity_code, entity_name, + entity_level_code, parent_entity_id + from corporate_entity + where corporate_entity_id = any($1::uuid[]) + order by entity_name + """, + list(account.corporate_entity_ids), + ) + has_source_context = bool(source_customer_rows or source_author_rows) + if not has_source_context: + has_source_context = await has_real_source_context( + conn, list(account.corporate_entity_ids) + ) + if has_source_context: + synthetic_only_entity_ids = await fetch_demo_corporate_entity_ids(conn) + entity_rows = [ + row + for row in entity_rows + if str(row["corporate_entity_id"]) not in synthetic_only_entity_ids + ] + entity_ids = [row["corporate_entity_id"] for row in entity_rows] + source_author_affiliations = await _load_account_affiliation_hints( + conn, + [str(row["author_account_id"]) for row in source_author_rows], + [str(entity_id) for entity_id in entity_ids], + ) + keyman_rows = await conn.fetch( + """ + select person.person_id, person.person_name, person.person_side_code, + person.last_known_job_title, + affiliation.affiliated_organization_name, + affiliation.affiliated_corporate_entity_id, + affiliation.role_title, + entity.entity_name + from cataloged_person person + join person_affiliation affiliation on affiliation.person_id = person.person_id + left join corporate_entity entity + on entity.corporate_entity_id = affiliation.affiliated_corporate_entity_id + where affiliation.affiliated_corporate_entity_id = any($1::uuid[]) + order by person.person_name, affiliation.affiliated_organization_name + """, + entity_ids, + ) + side_labels = await labels_for_codes(conn, [row["person_side_code"] for row in keyman_rows]) + entity_level_labels = await labels_for_codes(conn, [row["entity_level_code"] for row in entity_rows]) + relationship_network = await fetch_relationship_network(conn, entity_ids) + + keymen_by_id: dict[str, dict[str, Any]] = {} + for row in keyman_rows: + person_id = str(row["person_id"]) + keyman = keymen_by_id.setdefault( + person_id, + { + "person_id": person_id, + "person_name": row["person_name"], + "person_side_code": row["person_side_code"], + "person_side_label": side_labels.get(row["person_side_code"], row["person_side_code"]), + "last_known_job_title": row["last_known_job_title"], + "affiliations": [], + }, + ) + keyman["affiliations"].append( + { + "organization_name": row["affiliated_organization_name"], + "corporate_entity_id": ( + str(row["affiliated_corporate_entity_id"]) + if row["affiliated_corporate_entity_id"] is not None + else None + ), + "entity_name": row["entity_name"], + "role_title": row["role_title"], + } + ) + + return { + "corporate_entities": [ + { + "corporate_entity_id": str(row["corporate_entity_id"]), + "corporate_entity_code": row["corporate_entity_code"], + "entity_name": row["entity_name"], + "entity_level_code": row["entity_level_code"], + "entity_level_label": entity_level_labels.get( + row["entity_level_code"], row["entity_level_code"] + ), + "parent_entity_id": ( + str(row["parent_entity_id"]) if row["parent_entity_id"] is not None else None + ), + } + for row in entity_rows + ], + "keymen": list(keymen_by_id.values()), + "source_customer_hints": [ + { + "customer_code": row["customer_code"], + "customer_name": row["customer_name"], + "post_count": row["post_count"], + "related_posts": ( + json.loads(row["related_posts"]) + if isinstance(row["related_posts"], str) + else row["related_posts"] or [] + ), + "resolution_status": "hint_only", + "hint_trust": customer_hint_trust(row["customer_name"], row["customer_code"]), + "provenance": "source_post.source_customer_code/source_post.source_customer_name", + } + for row in source_customer_rows + ], + "source_author_hints": [ + { + "author_code": row["author_code"], + "author_name": row["author_name"], + "author_account_id": str(row["author_account_id"]), + "account_display_name": row["account_display_name"], + "account_affiliations": source_author_affiliations.get( + str(row["author_account_id"]), [] + ), + "post_count": row["post_count"], + "keyman_hints": ( + json.loads(row["keyman_hints"]) + if isinstance(row["keyman_hints"], str) + else row["keyman_hints"] or [] + ), + "related_posts": ( + json.loads(row["related_posts"]) + if isinstance(row["related_posts"], str) + else row["related_posts"] or [] + ), + "resolution_status": ( + "our_side_context_only" + if source_author_affiliations.get(str(row["author_account_id"]), []) + else "source_author_hint_only" + ), + "provenance": ( + "source_post.author_account_id/user_account.display_name/" + "account_affiliation.corporate_entity_id/source_post.source_author_code/source_post.source_author_name" + ), + } + for row in source_author_rows + ], + "relationship_network": relationship_network, + } + + +@app.post("/api/customer-master/resolve-hint") +async def resolve_customer_master_hint( + request: CustomerHintResolveRequest, account: CurrentAccount = Depends(get_current_account), pool: asyncpg.Pool = Depends(get_pool), ) -> dict[str, Any]: - """Authorized Group / Company / Plant forest this token may navigate. + """Resolve one observed customer-hint code to a real corporate_entity. - Affiliated corps pull in ancestors and descendants. A catalog row - the account does not touch is omitted -- a missing affiliation is - not a guessed parent. Corroborated abbreviations attach as - alternative labels; Searxng is not called on this read. + Gated by post_admin, not post_read: this is a write action with a + real LLM-call cost, same discipline as extract-keymen/verify-relations. + Only an externally-corroborated proposed name ever creates or binds an + entity (`backend.app.customer_hint_ingestion`) -- an unresolved or + uncorroborated hint is returned as such, never guessed into the + catalog. """ - _require_post_read(account) + _require_post_admin(account) async with pool.acquire() as conn: - trees = await fetch_customer_group_forest(conn, list(account.corporate_entity_ids)) - return {"trees": trees} + try: + resolution = await resolve_customer_hint( + conn, + _customer_hint_resolution_client(), + _relation_verification_client(), + request.hint_code, + ) + except (HttpClientError, OSError) as exc: + # resolve_and_verify_organization_name's resolution/verification + # calls raise on a failed request rather than silently returning + # "unresolved" -- a failed call is not the same claim as "the + # model looked and found nothing" (same discipline as + # verify-relations' identical try/except). + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Hint resolution is unavailable: the orchestrator or search provider did not respond", + ) from exc + if resolution is None: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "this hint could not be resolved to a corroborated organization name", + ) + return resolution @app.get("/api/lineage") async def read_lineage_graph( + limit: int = Query(500, ge=1, le=2000), + post_id: str | None = Query(None, min_length=1), account: CurrentAccount = Depends(get_current_account), pool: asyncpg.Pool = Depends(get_pool), ) -> dict[str, Any]: - """ABAC-filtered reconstruct graph for the product UI (same shape as the demo server).""" + """ABAC-filtered reconstruct graph bounded for browser rendering.""" _require_post_read(account) async with pool.acquire() as conn: - return await visible_lineage_graph(conn, lambda row: _can_see_post(account, row)) + return await visible_lineage_graph( + conn, + lambda row: _can_see_post(account, row), + limit=limit, + focus_post_id=post_id, + ) @app.post("/api/lineage/rebuild") @@ -415,19 +1118,281 @@ async def rebuild_lineage_graph( @app.get("/api/posts") async def list_posts( + limit: int = Query(50, ge=1, le=200), + offset: int = Query(0, ge=0), + search: str | None = Query(None, max_length=200), + voc_type: list[str] | None = Query(None, max_length=80), + visibility: str | None = Query(None, max_length=80), + sort: Literal["newest", "oldest", "title"] = Query("newest"), account: CurrentAccount = Depends(get_current_account), pool: asyncpg.Pool = Depends(get_pool), -) -> list[dict[str, Any]]: - """List source_post rows the account is allowed to see (RBAC then ABAC).""" +) -> dict[str, Any]: + """List authorized posts, with semantic evidence search when requested.""" _require_post_read(account) + search_term = search.strip() if search and search.strip() else None async with pool.acquire() as conn: - rows = await conn.fetch( - "select post_id, post_title, voc_type_code, visibility_code, corporate_entity_id, created_at " - "from source_post order by created_at desc" + voc_type_options, visibility_options = await _post_filter_options( + conn, account.corporate_entity_ids + ) + body_search_ids: list[str] = [] + if search_term: + # Safe SQL: search SQL is a closed schema query; search_term is bound through $1. + body_rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" + select post_id + from source_post + where {SOURCE_POST_ELIGIBILITY_SQL.format(alias="source_post")} + and (lower(left(source_post_search_text(post_body), 16384)) + like '%' || lower($1) || '%' + or to_tsvector('simple', source_post_search_text(post_body)) + @@ plainto_tsquery('simple', $1)) + order by + case when lower(left(source_post_search_text(post_body), 16384)) + like '%' || lower($1) || '%' then 0 else 1 end, + ts_rank( + to_tsvector('simple', source_post_search_text(post_body)), + plainto_tsquery('simple', $1) + ) desc, + post_id + """, + search_term, + ) + body_search_ids = [str(row["post_id"]) for row in body_rows] + # Safe SQL: page SQL is a closed schema query; every request value is an asyncpg parameter. + rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" + with page as ( + select post.post_id, post.post_title, post.voc_type_code, post.visibility_code, + post.source_stage_code, post.source_detail_state_code, + post.source_draft_code, post.source_deleted_flag, + post.source_author_code, post.source_author_name, + post.source_company_code, post.source_company_name, + post.source_process_unit_code, post.source_process_unit_name, + post.source_sales_pool_code, post.source_sales_pool_name, + post.source_customer_code, post.source_customer_name, + post.source_project_code, post.source_project_name, + post.source_system_code, + post.source_record_key, + post.corporate_entity_id, post.created_at, + case + when $1::text is null then 0 + when lower(coalesce(post.post_title, '')) like '%' || lower($1) || '%' then 0 + when post.post_id = any($5::uuid[]) then 1 + else 2 + end as search_priority, + count(*) over() as total_count + from source_post post + where (post.visibility_code = 'public' + or post.corporate_entity_id::text = any($2::text[])) + and {SOURCE_POST_ELIGIBILITY_SQL.format(alias="post")} + and ( + $1::text is null + or post.post_title ilike '%' || $1 || '%' + or post.thread_group_key ilike '%' || $1 || '%' + or post.secondary_grouping_key ilike '%' || $1 || '%' + or concat_ws(' ', + post.source_stage_code, + post.source_detail_state_code, + post.source_draft_code, + post.source_deleted_flag, + post.source_author_code, + post.source_author_name, + post.source_company_code, + post.source_company_name, + post.source_process_unit_code, + post.source_process_unit_name, + post.source_sales_pool_code, + post.source_sales_pool_name, + post.source_customer_code, + post.source_customer_name, + post.source_project_code, + post.source_project_name, + post.source_system_code, + post.source_record_key + ) ilike '%' || $1 || '%' + or replace(post.post_id::text, '-', '') ilike '%' || lower($1) || '%' + or ( + char_length($1) >= 3 + and ( + similarity(replace(post.post_id::text, '-', ''), lower($1)) >= 0.78 + or similarity(lower(coalesce(post.source_record_key, '')), lower($1)) >= 0.78 + or word_similarity(lower($1), lower(post.post_title)) >= 0.45 + or word_similarity(lower($1), lower(post.secondary_grouping_key)) >= 0.45 + or word_similarity( + lower($1), + lower(concat_ws(' ', + post.source_stage_code, + post.source_detail_state_code, + post.source_draft_code, + post.source_deleted_flag, + post.source_author_code, + post.source_author_name, + post.source_company_code, + post.source_company_name, + post.source_process_unit_code, + post.source_process_unit_name, + post.source_sales_pool_code, + post.source_sales_pool_name, + post.source_customer_code, + post.source_customer_name, + post.source_project_code, + post.source_project_name + )) + ) >= 0.45 + ) + ) + or post.post_id = any($5::uuid[]) + or exists ( + select 1 from post_project_mention project + where project.post_id = post.post_id + and (project.project_name ilike '%' || $1 || '%' + or project.evidence_text ilike '%' || $1 || '%' + or project.ontology_iri ilike '%' || $1 || '%' + or (char_length($1) >= 3 and word_similarity(lower($1), lower(project.project_name)) >= 0.45)) + ) + or exists ( + select 1 from post_summary_role role + where role.post_id = post.post_id + and (role.actor_name ilike '%' || $1 || '%' + or role.responsibility ilike '%' || $1 || '%' + or coalesce(role.affiliated_organization_name, '') ilike '%' || $1 || '%' + or (char_length($1) >= 3 and word_similarity(lower($1), lower(role.actor_name)) >= 0.45)) + ) + or exists ( + select 1 + from post_person_mention mention + join cataloged_person person on person.person_id = mention.person_id + where mention.post_id = post.post_id + and ( + person.person_name ilike '%' || $1 || '%' + or (char_length($1) >= 3 and word_similarity(lower($1), lower(person.person_name)) >= 0.45) + ) + ) + or exists ( + select 1 from post_summary_result summary + where summary.post_id = post.post_id + and summary.korean_summary ilike '%' || $1 || '%' + ) + or exists ( + select 1 from post_summary_event event + where event.post_id = post.post_id + and event.event_text ilike '%' || $1 || '%' + ) + or exists ( + select 1 from corporate_entity customer + where customer.corporate_entity_id = post.corporate_entity_id + and (customer.entity_name ilike '%' || $1 || '%' + or customer.corporate_entity_code ilike '%' || $1 || '%') + ) + or exists ( + select 1 from process_unit process + where process.process_unit_id = post.process_unit_id + and (process.process_unit_name ilike '%' || $1 || '%' + or process.process_unit_code ilike '%' || $1 || '%') + ) + or exists ( + select 1 from user_account author + where author.user_account_id = post.author_account_id + and (author.display_name ilike '%' || $1 || '%' + or author.email_address ilike '%' || $1 || '%') + ) + or exists ( + select 1 + from account_affiliation affiliation + join corporate_entity affiliated + on affiliated.corporate_entity_id = affiliation.corporate_entity_id + where affiliation.user_account_id = post.author_account_id + and (affiliated.entity_name ilike '%' || $1 || '%' + or affiliated.corporate_entity_code ilike '%' || $1 || '%') + ) + ) + and ($3::text[] is null or post.voc_type_code = any($3::text[])) + and ($4::text is null or post.visibility_code = $4) + order by + search_priority asc, + case + when $1::text is not null and post.post_id = any($5::uuid[]) + then array_position($5::uuid[], post.post_id) + end asc, + case when $8::text = 'title' then lower(coalesce(post.post_title, '')) end asc, + case when $8::text = 'oldest' then post.created_at end asc, + case when $8::text in ('newest', 'title') then post.created_at end desc, + post.post_id desc + offset $6 + limit $7 + ) + select page.*, + case + when $1::text is not null + and strpos(lower(source_post_search_text(post.post_body)), lower($1)) > 0 + then btrim(substring( + source_post_search_text(post.post_body) + from greatest( + 1, + strpos(lower(source_post_search_text(post.post_body)), lower($1)) - 140 + ) for 420 + )) + else btrim(left(source_post_search_text(post.post_body), 420)) + end as post_body_excerpt, + char_length(coalesce(post.post_body, '')) > 420 as post_body_truncated, + coalesce(projects.project_evidence, '[]'::json) as project_evidence + from page + join source_post post on post.post_id = page.post_id + left join lateral ( + select json_agg( + json_build_object( + 'project_key', project.project_key, + 'project_name', project.project_name, + 'evidence', project.evidence_text, + 'confidence', project.confidence, + 'ontology_iri', project.ontology_iri, + 'ontology_label', 'Project', + 'extraction_method', project.extraction_method, + 'resolution_status', 'semantic_candidate', + 'provenance', 'post_project_mention.evidence_text' + ) + order by project.confidence desc, project.project_name, project.project_key + ) as project_evidence + from ( + select project_key, project_name, evidence_text, confidence, + ontology_iri, extraction_method + from post_project_mention + where post_id = page.post_id + order by confidence desc, project_name, project_key + limit 5 + ) project + ) projects on true + order by + case when $1::text is not null then page.search_priority end asc, + case + when $1::text is not null and page.search_priority = 1 + then array_position($5::uuid[], page.post_id) + end asc, + case when $8::text = 'title' then lower(coalesce(page.post_title, '')) end asc, + case when $8::text = 'oldest' then page.created_at end asc, + case when $8::text in ('newest', 'title') then page.created_at end desc, + page.post_id desc + """, + search_term, + list(account.corporate_entity_ids), + [code.strip() for code in voc_type if code.strip()] if voc_type else None, + visibility.strip() if visibility and visibility.strip() else None, + body_search_ids, + offset, + limit, + sort, ) visible = [row for row in rows if _can_see_post(account, row)] labels = await _lookup_post_labels(conn, visible) - return [_serialize_post(row, labels) for row in visible] + total_count = int(rows[0]["total_count"]) if rows else 0 + return { + "posts": [_serialize_post(row, labels) for row in visible], + "total_count": total_count, + "limit": limit, + "offset": offset, + "voc_type_options": voc_type_options, + "visibility_options": visibility_options, + } @app.get("/api/posts/{post_id}") @@ -457,9 +1422,17 @@ async def read_post( "then compare the known body with the live body.", ) from exc async with pool.acquire() as conn: - row = await conn.fetchrow( - "select post_id, post_title, post_body, voc_type_code, visibility_code, corporate_entity_id, created_at " - "from source_post where post_id = $1", + # Safe SQL: the eligibility predicate is an immutable schema fragment; post id is bound. + row = await conn.fetchrow( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + "select post_id, post_title, post_body, voc_type_code, visibility_code, " + "source_stage_code, source_detail_state_code, source_draft_code, source_deleted_flag, " + "source_author_code, source_author_name, source_company_code, source_company_name, " + "source_process_unit_code, source_process_unit_name, " + "source_sales_pool_code, source_sales_pool_name, " + "source_customer_code, source_customer_name, source_project_code, source_project_name, " + "source_system_code, source_record_key, " + "corporate_entity_id, created_at " + f"from source_post where post_id = $1 and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}", post_id, ) if row is None: @@ -467,15 +1440,174 @@ async def read_post( if not _can_see_post(account, row): raise HTTPException(status.HTTP_403_FORBIDDEN, "not authorized to view this post") labels = await _lookup_post_labels(conn, [row]) + project_evidence = await _load_project_evidence( + conn, post_id, row["source_project_code"], row["source_project_name"] + ) known_at = None if as_of_clock is not None: known_at = await fetch_known_at_revision(conn, post_id, as_of_clock) - payload = {**_serialize_post(row, labels), "post_body": row["post_body"]} + payload = { + **_serialize_post(row, labels), + "post_body": row["post_body"], + "project_evidence": project_evidence, + } if known_at is not None: payload["known_at"] = known_at return payload +@app.get("/api/posts/{post_id}/content") +async def read_post_content( + post_id: str, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), + valkey: redis.Redis = Depends(get_valkey), +) -> dict[str, Any]: + """Return persisted content evidence; never derive or invent buyer copy.""" + await _load_visible_post(post_id, account, pool) + queue_event: tuple[str, str] | None = None + async with pool.acquire() as conn: + unit_rows = await conn.fetch( + """ + select unit.unit_index, unit.unit_kind_code, unit.unit_label, unit.unit_text, + coalesce(structure.indent_level, 0) as indent_level, + structure.decision_source_code, structure.confidence, + structure.evidence_text + from post_content_unit unit + left join post_content_unit_structure structure + on structure.post_content_unit_id = unit.post_content_unit_id + where unit.post_id = $1 + order by unit.unit_index + """, + post_id, + ) + content_status = post_content_api_status( + None, + content_present=bool(unit_rows), + ) + body_row = await conn.fetchrow( + "select post_body from source_post where post_id = $1", post_id + ) + raw_body = None if body_row is None else body_row["post_body"] + if isinstance(raw_body, str) and raw_body.strip(): + content_present = bool(unit_rows) + content_complete = await post_content_is_complete( + conn, + post_id, + embedding_model_code=load_settings().embedding_model, + require_structure=bool( + load_settings().orchestrator_base_url + and load_settings().orchestrator_api_key + ), + ) + async with conn.transaction(): + job = await ensure_post_content_job( + conn, + post_id, + raw_body, + content_complete=content_complete, + ) + content_status = post_content_api_status( + job.status_code, + content_present=content_present, + ) + if job.should_publish: + queue_event = (job.post_id, job.source_body_sha256) + rows = await conn.fetch( + """ + select image.post_content_image_id, unit.unit_index, image.mime_type, image.description_status_code, + image.extracted_text, image.caption, + coalesce( + array_agg(tag.tag_text order by tag.tag_text) + filter (where tag.tag_text is not null), + '{}'::text[] + ) as tags + from post_content_unit unit + join post_content_image image + on image.post_content_unit_id = unit.post_content_unit_id + left join post_content_image_tag tag + on tag.post_content_image_id = image.post_content_image_id + where unit.post_id = $1 + group by image.post_content_image_id, unit.unit_index, image.mime_type, image.description_status_code, + image.extracted_text, image.caption + order by unit.unit_index + """, + post_id, + ) + region_rows = await conn.fetch( + """ + select image.post_content_image_id, region.region_index, + region.x_ratio, region.y_ratio, region.width_ratio, region.height_ratio, + region.description_status_code, region.extracted_text, region.caption, + coalesce( + array_agg(tag.tag_text order by tag.tag_text) + filter (where tag.tag_text is not null), + '{}'::text[] + ) as tags + from post_content_image image + join post_content_image_region region + on region.post_content_image_id = image.post_content_image_id + left join post_content_image_region_tag tag + on tag.post_content_image_region_id = region.post_content_image_region_id + where image.post_content_image_id = any($1::uuid[]) + group by image.post_content_image_id, region.region_index, + region.x_ratio, region.y_ratio, region.width_ratio, region.height_ratio, + region.description_status_code, region.extracted_text, region.caption + order by image.post_content_image_id, region.region_index + """, + [row["post_content_image_id"] for row in rows], + ) if rows else [] + if queue_event is not None: + await publish_post_content_event( + valkey, + post_id=queue_event[0], + source_body_digest=queue_event[1], + ) + regions_by_image: dict[str, list[dict[str, Any]]] = {} + for row in region_rows: + regions_by_image.setdefault(str(row["post_content_image_id"]), []).append( + { + "region_index": row["region_index"], + "x_ratio": row["x_ratio"], + "y_ratio": row["y_ratio"], + "width_ratio": row["width_ratio"], + "height_ratio": row["height_ratio"], + "status_code": row["description_status_code"], + "extracted_text": row["extracted_text"], + "caption": row["caption"], + "tags": list(row["tags"] or []), + } + ) + return { + "status": content_status, + "units": [ + { + "unit_index": row["unit_index"], + "unit_kind_code": row["unit_kind_code"], + "unit_label": row["unit_label"], + "unit_text": row["unit_text"], + "indent_level": row["indent_level"], + "indent_source_code": row["decision_source_code"] or "unresolved", + "indent_confidence": float(row["confidence"] or 0), + "indent_evidence": row["evidence_text"] or "", + } + for row in unit_rows + ], + "images": [ + { + "unit_index": row["unit_index"], + "mime_type": row["mime_type"], + "status_code": row["description_status_code"], + "extracted_text": row["extracted_text"], + "caption": row["caption"], + "tags": list(row["tags"] or []), + "regions": regions_by_image.get(str(row["post_content_image_id"]), []), + } + for row in rows + ] + } + + async def _load_visible_post( post_id: str, account: CurrentAccount, @@ -484,9 +1616,22 @@ async def _load_visible_post( """Load one post the account may see, or raise 404 / 403.""" _require_post_read(account) async with pool.acquire() as conn: - row = await conn.fetchrow( - "select post_id, post_title, voc_type_code, visibility_code, corporate_entity_id, created_at " - "from source_post where post_id = $1", + # Safe SQL: the eligibility predicate is an immutable schema fragment; post id is bound. + row = await conn.fetchrow( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + """ + select source_post.post_id, source_post.post_title, source_post.voc_type_code, + source_post.visibility_code, source_post.corporate_entity_id, + source_post.created_at, source_post.author_account_id, + source_post.source_process_unit_code, source_post.source_author_code, + source_post.source_company_code, source_post.source_customer_code, + source_post.source_project_code, source_post.source_sales_pool_code, + customer.corporate_entity_code + from source_post + left join corporate_entity customer + on customer.corporate_entity_id = source_post.corporate_entity_id + where source_post.post_id = $1 + and """ + f"{SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}", post_id, ) if row is None: @@ -496,6 +1641,186 @@ async def _load_visible_post( return row +async def _load_post_semantic_hints(conn: asyncpg.Connection, post_id: str) -> str: + """Render author, business-unit, sales-pool, and customer hints without treating them as proof.""" + rows = await conn.fetch( + """ + select author.user_account_id as author_account_id, + author.display_name as author_name, + post.source_author_code, + post.source_author_name, + post.source_company_code, + post.source_company_name, + source_company.entity_name as source_company_catalog_name, + post.source_process_unit_code, + post.source_process_unit_name, + source_process_unit.process_unit_name as source_process_unit_catalog_name, + post.source_sales_pool_code, + post.source_sales_pool_name, + post.source_customer_code, + post.source_customer_name, + source_customer.entity_name as source_customer_catalog_name, + post.source_project_code, + post.source_project_name, + post.secondary_grouping_key as project_field, + customer.entity_name as customer_name, + affiliated.entity_name as author_affiliation_name + from source_post post + join user_account author on author.user_account_id = post.author_account_id + left join corporate_entity customer on customer.corporate_entity_id = post.corporate_entity_id + left join corporate_entity source_company + on source_company.corporate_entity_code = nullif(btrim(post.source_company_code), '') + left join process_unit source_process_unit + on source_process_unit.process_unit_code = nullif(btrim(post.source_process_unit_code), '') + left join corporate_entity source_customer + on source_customer.corporate_entity_code = nullif(btrim(post.source_customer_code), '') + left join account_affiliation account_aff + on account_aff.user_account_id = post.author_account_id + left join corporate_entity affiliated + on affiliated.corporate_entity_id = account_aff.corporate_entity_id + where post.post_id = $1 + """, + post_id, + ) + if not rows: + return "no structured hints available" + first = rows[0] + source_context_present = any( + first[field] is not None + for field in ( + "source_author_code", + "source_author_name", + "source_company_code", + "source_company_name", + "source_process_unit_code", + "source_process_unit_name", + "source_sales_pool_code", + "source_sales_pool_name", + "source_customer_code", + "source_customer_name", + "source_project_code", + "source_project_name", + ) + ) + source_author_name = first["source_author_name"] + if source_author_name and source_author_name == first["source_author_code"]: + source_author_name = None + return format_semantic_hints( + author_name=source_author_name or first["author_name"], + author_account_id=str(first["author_account_id"]), + author_account_name=first["author_name"], + author_affiliations=( + str(row["author_affiliation_name"]) + for row in rows + if row["author_affiliation_name"] + ), + order_pool_code=first["source_sales_pool_code"], + order_pool_name=first["source_sales_pool_name"], + project_field=first["project_field"], + customer_name=first["customer_name"], + source_author_code=first["source_author_code"], + source_author_name=source_author_name, + source_company_code=first["source_company_code"], + source_company_name=first["source_company_name"], + source_company_catalog_name=first["source_company_catalog_name"], + source_business_unit_code=first["source_process_unit_code"], + source_process_unit_name=first["source_process_unit_name"], + source_process_unit_catalog_name=first["source_process_unit_catalog_name"], + source_sales_pool_code=first["source_sales_pool_code"], + source_sales_pool_name=first["source_sales_pool_name"], + source_customer_code=first["source_customer_code"], + source_customer_name=first["source_customer_name"], + source_customer_catalog_name=first["source_customer_catalog_name"], + source_project_code=first["source_project_code"], + source_project_name=first["source_project_name"], + source_context_present=source_context_present, + ) + + +async def _load_account_affiliation_hints( + conn: asyncpg.Connection, + account_ids: list[str], + corporate_entity_ids: list[str], +) -> dict[str, list[dict[str, Any]]]: + """Load authorized account affiliations as non-binding Keyman context.""" + if not account_ids or not corporate_entity_ids: + return {} + rows = await conn.fetch( + """ + select affiliation.user_account_id, + entity.corporate_entity_id, + entity.entity_name, + process.process_unit_code, + process.process_unit_name + from account_affiliation affiliation + join corporate_entity entity + on entity.corporate_entity_id = affiliation.corporate_entity_id + left join process_unit process + on process.process_unit_id = affiliation.process_unit_id + where affiliation.user_account_id = any($1::uuid[]) + and affiliation.corporate_entity_id = any($2::uuid[]) + order by entity.entity_name, process.process_unit_code + """, + account_ids, + corporate_entity_ids, + ) + affiliations: dict[str, list[dict[str, Any]]] = {} + for row in rows: + account_id = str(row["user_account_id"]) + affiliations.setdefault(account_id, []).append( + { + "corporate_entity_id": str(row["corporate_entity_id"]), + "entity_name": row["entity_name"], + "process_unit_code": row["process_unit_code"], + "process_unit_name": row["process_unit_name"], + } + ) + return affiliations + + +async def _load_source_author_context( + conn: asyncpg.Connection, + post_id: str, + corporate_entity_ids: list[str], +) -> dict[str, Any] | None: + """Return source-author/account context without binding a cataloged person.""" + row = await conn.fetchrow( + """ + select post.author_account_id, + author.display_name as account_display_name, + nullif(btrim(post.source_author_code), '') as source_author_code, + nullif(btrim(post.source_author_name), '') as source_author_name + from source_post post + join user_account author on author.user_account_id = post.author_account_id + where post.post_id = $1 + """, + post_id, + ) + if row is None: + return None + account_id = str(row["author_account_id"]) + affiliations = ( + await _load_account_affiliation_hints(conn, [account_id], corporate_entity_ids) + ).get(account_id, []) + source_author_name = row["source_author_name"] + if source_author_name and source_author_name.casefold() == str(row["source_author_code"] or '').casefold(): + source_author_name = None + return { + "author_account_id": account_id, + "account_display_name": row["account_display_name"], + "source_author_code": row["source_author_code"], + "source_author_name": source_author_name, + "account_affiliations": affiliations, + "resolution_status": ( + "our_side_context_only" if affiliations else "source_author_hint_only" + ), + "provenance": ( + "source_post.author_account_id/user_account.display_name/" + "account_affiliation.corporate_entity_id/source_post.source_author_code/source_post.source_author_name" + ), + } + + @app.get("/api/posts/{post_id}/keymen") async def read_post_keymen( post_id: str, @@ -506,7 +1831,14 @@ async def read_post_keymen( post = await _load_visible_post(post_id, account, pool) async with pool.acquire() as conn: keymen = await fetch_post_keymen(conn, post_id) - return {"post_id": str(post["post_id"]), "keymen": keymen} + source_author_context = await _load_source_author_context( + conn, post_id, list(account.corporate_entity_ids) + ) + return { + "post_id": str(post["post_id"]), + "keymen": keymen, + "source_author_context": source_author_context, + } @app.get("/api/keymen/{person_id}/related") @@ -528,11 +1860,13 @@ async def read_related_keymen( person_id, ) related = await related_for_person(conn, person_id, visible_post_ids) + role_history = await fetch_person_role_history(conn, person_id, visible_post_ids) return { "person_id": str(person["person_id"]), "person_name": person["person_name"], "person_side_code": person["person_side_code"], "related": related, + "role_history": role_history, } @@ -632,64 +1966,6 @@ async def read_post_affiliate_tree( return {"post_id": str(post["post_id"]), "trees": trees} -@app.get("/api/posts/{post_id}/abbreviation-tree-matches") -async def read_post_abbreviation_tree_matches( - post_id: str, - account: CurrentAccount = Depends(get_current_account), - pool: asyncpg.Pool = Depends(get_pool), -) -> dict[str, Any]: - """Cached Searxng tree matches for organization names on this post. - - Does not call Searxng. A missing cache row means the mention has - not been cross-checked yet, not that a parent was invented. - """ - post = await _load_visible_post(post_id, account, pool) - async with pool.acquire() as conn: - matches = await fetch_post_abbreviation_matches(conn, post_id) - return {"post_id": str(post["post_id"]), "matches": matches} - - -@app.post("/api/posts/{post_id}/corroborate-abbreviations") -async def corroborate_post_abbreviation_tree( - post_id: str, - account: CurrentAccount = Depends(get_current_account), - pool: asyncpg.Pool = Depends(get_pool), -) -> dict[str, Any]: - """Cross-check this post's abbreviations against the customer-group tree. - - Reuses the existing Searxng client. Fail-closed: unavailable search - is 503, not an invented parent or AUTO row. A tied or empty result - stays unbound. post_admin only -- a real external-search write. - """ - _require_post_admin(account) - post = await _load_visible_post(post_id, account, pool) - client = _relation_verification_client() - if not client.available: - raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Abbreviation tree corroboration is unavailable: set SEARXNG_BASE_URL", - ) - async with pool.acquire() as conn: - matches = await corroborate_post_abbreviations( - conn, - client, - post_id, - list(account.corporate_entity_ids), - ) - return { - "post_id": str(post["post_id"]), - "matches": [ - { - "raw_organization_name": match.raw_organization_name, - "corporate_entity_id": match.corporate_entity_id, - "verification_status_code": match.verification_status_code, - "verification_evidence_url": match.verification_evidence_url, - } - for match in matches - ], - } - - @app.get("/api/posts/{post_id}/voc-evidence") async def read_post_voc_evidence( post_id: str, @@ -707,6 +1983,7 @@ async def verify_post_entity_relationships( post_id: str, account: CurrentAccount = Depends(get_current_account), pool: asyncpg.Pool = Depends(get_pool), + valkey: redis.Redis = Depends(get_valkey), ) -> dict[str, Any]: """Checks this post's `verify_pending` counterparty relationships (entity_relationship_classification's LLM output) against external @@ -725,7 +2002,29 @@ async def verify_post_entity_relationships( "Relation verification is unavailable: set SEARXNG_BASE_URL", ) async with pool.acquire() as conn: - verified = await verify_post_relations(conn, client, post_id) + try: + verified = await verify_post_relations( + conn, + client, + post_id, + visible_corporate_entity_ids=account.corporate_entity_ids, + ) + except (HttpClientError, OSError) as exc: + # verify_post_relations() deliberately raises on a failed search + # (a failed search is not "searched and found nothing" -- see + # its docstring); this is the one caller, so it is the right + # place to turn that into a clean 503 instead of a raw 500. + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Relation verification is unavailable: the search provider did not respond", + ) from exc + await publish_activity_event( + valkey, + post_id, + "relations_verified", + account.user_account_id, + f"Relations verified: {len(verified)} counterparty relationship(s) checked", + ) return { "post_id": str(post["post_id"]), "verified": [ @@ -733,6 +2032,7 @@ async def verify_post_entity_relationships( "counterparty_entity_name": row.counterparty_entity_name, "verification_status_code": row.verification_status_code, "verification_evidence_url": row.verification_evidence_url, + "verification_evidence_post_id": row.verification_evidence_post_id, } for row in verified ], @@ -744,6 +2044,7 @@ async def extract_post_keymen( post_id: str, account: CurrentAccount = Depends(get_current_account), pool: asyncpg.Pool = Depends(get_pool), + valkey: redis.Redis = Depends(get_valkey), ) -> dict[str, Any]: """Runs Keyman extraction over a post's own title+body and persists the result (cataloged_person / person_affiliation / post_person_mention / @@ -754,43 +2055,65 @@ async def extract_post_keymen( """ _require_post_admin(account) post = await _load_visible_post(post_id, account, pool) - keyman_client = _keyman_extraction_client() - if not keyman_client.available: - raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Keyman extraction is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", - ) - relationship_client = _entity_relationship_client() - async with pool.acquire() as conn: - body_row = await conn.fetchrow("select post_body from source_post where post_id = $1", post_id) - raw_body = "" if body_row is None else body_row["post_body"] - # HTML/base64-image content must never reach an LLM prompt raw -- - # tags dilute the model's attention and a base64 payload sent as - # literal text either blows the token budget or is silently - # ignored (see lineageweave/post_content_normalization.py). - post_body = normalize_post_body(raw_body, vision_client=_vision_client()).text - mentions = await ingest_post_keymen( - conn, - keyman_client, - post_id, - post["post_title"], - post_body, - resolution_client=_organization_name_resolution_client(), - verification_client=_relation_verification_client(), - hierarchy_inference_client=_corporate_hierarchy_inference_client(), - persist_graph=False, - ) - organization_names = sorted( - {name for mention in mentions for name in mention.affiliated_organization_names} - ) - # relationship_client is gated by the same settings check as - # keyman_client above (both read ORCHESTRATOR_BASE_URL/_API_KEY), - # so reaching here means it is available too. - relationships = await ingest_post_entity_relationships( - conn, relationship_client, post_id, post["post_title"], post_body, organization_names - ) - async with conn.transaction(): - await persist_edges_for_post(conn, post_id) + post_metadata = build_post_llm_metadata(post_id, post) + with use_llm_metadata(post_metadata): + keyman_client = _keyman_extraction_client() + if not keyman_client.available: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Keymen extraction is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", + ) + relationship_client = _entity_relationship_client() + async with pool.acquire() as conn: + body_row = await conn.fetchrow("select post_body from source_post where post_id = $1", post_id) + raw_body = "" if body_row is None else body_row["post_body"] + # HTML/base64-image content must never reach an LLM prompt raw -- + # tags dilute the model's attention and a base64 payload sent as + # literal text either blows the token budget or is silently + # ignored (see lineageweave/post_content_normalization.py). + post_body = ( + await asyncio.to_thread(normalize_post_body, raw_body, _vision_client()) + ).text + context_hints = await _load_post_semantic_hints(conn, post_id) + mentions = await ingest_post_keymen( + conn, + keyman_client, + post_id, + post["post_title"], + post_body, + resolution_client=_organization_name_resolution_client(), + verification_client=_relation_verification_client(), + hierarchy_inference_client=_corporate_hierarchy_inference_client(), + context_hints=context_hints, + persist_graph=False, + ) + # Live bug (2026-08-19): an organization affiliated ONLY with an + # our_side person (our own factory, our own affiliate) got fed + # into the counterparty-relationship classifier the same as any + # external org -- forced to pick from six codes that all assume + # an external counterparty, it had no correct answer and landed + # on the closest wrong one (typically "Partner"). Only classify + # organizations a counterparty-side mention actually names. + organization_names = sorted( + { + name + for mention in mentions + if mention.person_side_code == COUNTERPARTY + for name in mention.affiliated_organization_names + } + ) + relationships = await ingest_post_entity_relationships( + conn, relationship_client, post_id, post["post_title"], post_body, organization_names + ) + async with conn.transaction(): + await persist_edges_for_post(conn, post_id) + await publish_activity_event( + valkey, + post_id, + "keymen_extracted", + account.user_account_id, + f"Keymen extracted: {len(mentions)} mention(s) found", + ) return { "post_id": str(post["post_id"]), "extracted_count": len(mentions), @@ -799,6 +2122,7 @@ async def extract_post_keymen( "person_name": mention.person_name, "person_side_code": mention.person_side_code, "affiliated_organization_names": list(mention.affiliated_organization_names), + "job_title": mention.job_title, } for mention in mentions ], @@ -830,16 +2154,24 @@ async def read_post_lineage( candidate_ids = linked.direct | linked.indirect rows = {} if candidate_ids: - fetched = await conn.fetch( - "select post_id, post_title, visibility_code, corporate_entity_id " - "from source_post where post_id = any($1::uuid[])", + # Safe SQL: the eligibility predicate is an immutable schema fragment; candidate ids are bound. + fetched = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + "select post_id, post_title, visibility_code, corporate_entity_id, " + "btrim(left(source_post_search_text(post_body), 420)) as post_body_excerpt, " + "char_length(coalesce(post_body, '')) > 420 as post_body_truncated " + f"from source_post where post_id = any($1::uuid[]) and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}", list(candidate_ids), ) rows = {str(row["post_id"]): row for row in fetched} def _visible_summaries(ids: frozenset[str]) -> list[dict[str, Any]]: return [ - {"post_id": post_id_, "post_title": rows[post_id_]["post_title"]} + { + "post_id": post_id_, + "post_title": rows[post_id_]["post_title"], + "post_body_excerpt": rows[post_id_].get("post_body_excerpt"), + "post_body_truncated": rows[post_id_].get("post_body_truncated", False), + } for post_id_ in ids if post_id_ in rows and _can_see_post(account, rows[post_id_]) ] @@ -881,6 +2213,7 @@ async def evaluate_post( post_id: str, account: CurrentAccount = Depends(get_current_account), pool: asyncpg.Pool = Depends(get_pool), + valkey: redis.Redis = Depends(get_valkey), ) -> dict[str, Any]: """LLM-as-a-Judge a post through fast-mlsirm and persist the IRT row. @@ -889,21 +2222,34 @@ async def evaluate_post( """ _require_post_admin(account) post = await _load_visible_post(post_id, account, pool) - client = _post_evaluation_client() - if not client.available: - raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Post evaluation is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", - ) - async with pool.acquire() as conn: - body_row = await conn.fetchrow("select post_body from source_post where post_id = $1", post_id) - normalized_body = normalize_post_body( - "" if body_row is None else body_row["post_body"], vision_client=_vision_client() - ).text - async with pool.acquire() as conn: - rows = await ingest_post_evaluation( - conn, client, post_id, post["post_title"], normalized_body - ) + post_metadata = build_post_llm_metadata(post_id, post) + with use_llm_metadata(post_metadata): + client = _post_evaluation_client() + if not client.available: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Post evaluation is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", + ) + async with pool.acquire() as conn: + body_row = await conn.fetchrow("select post_body from source_post where post_id = $1", post_id) + normalized_body = ( + await asyncio.to_thread( + normalize_post_body, + "" if body_row is None else body_row["post_body"], + _vision_client(), + ) + ).text + async with pool.acquire() as conn: + rows = await ingest_post_evaluation( + conn, client, post_id, post["post_title"], normalized_body + ) + await publish_activity_event( + valkey, + post_id, + "post_evaluated", + account.user_account_id, + f"Post evaluated: {len(rows)} rubric criterion response(s)", + ) return { "post_id": str(post["post_id"]), "rubric_version": RUBRIC_VERSION, @@ -933,9 +2279,17 @@ async def compare_period_groupings( raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, str(exc)) from exc async with pool.acquire() as conn: rows = await fetch_period_comparison(conn, period_code) + demo_entity_ids: set[str] = set() + if rows and await has_real_source_context(conn, list(account.corporate_entity_ids)): + demo_entity_ids = await fetch_demo_corporate_entity_ids(conn) visible: list[dict[str, Any]] = [] for row in rows: - members = [member for member in row["members"] if _can_see_post(account, member)] + members = [ + member + for member in row["members"] + if _can_see_post(account, member) + and not _is_synthetic_demo_member(member, demo_entity_ids) + ] if not members: continue visible.append({**row, "members": [], "post_count": len(members)}) @@ -954,9 +2308,17 @@ async def list_period_reports( raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "unknown grouping_kind") async with pool.acquire() as conn: summaries = await list_period_report_summaries(conn, grouping_kind) + demo_entity_ids: set[str] = set() + if summaries and await has_real_source_context(conn, list(account.corporate_entity_ids)): + demo_entity_ids = await fetch_demo_corporate_entity_ids(conn) visible: list[dict[str, Any]] = [] for summary in summaries: - members = [member for member in summary["members"] if _can_see_post(account, member)] + members = [ + member + for member in summary["members"] + if _can_see_post(account, member) + and not _is_synthetic_demo_member(member, demo_entity_ids) + ] if not members: continue visible.append({**summary, "members": [], "post_count": len(members)}) @@ -980,15 +2342,32 @@ async def read_period_reports( raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, str(exc)) from exc async with pool.acquire() as conn: reports = await fetch_period_reports(conn, grouping_kind, period_code) + demo_entity_ids: set[str] = set() + if reports and await has_real_source_context(conn, list(account.corporate_entity_ids)): + demo_entity_ids = await fetch_demo_corporate_entity_ids(conn) visible: list[dict[str, Any]] = [] for report in reports: - members = [member for member in report["members"] if _can_see_post(account, member)] + members = [ + member + for member in report["members"] + if _can_see_post(account, member) + and not _is_synthetic_demo_member(member, demo_entity_ids) + ] if not members: continue leftover_pairs = [ pair for pair in report.get("leftover_pairs", []) if _can_see_post(account, pair) + and not _is_synthetic_demo_member(pair, demo_entity_ids) + ] + members = [ + {key: value for key, value in member.items() if key != "has_real_source_context"} + for member in members + ] + leftover_pairs = [ + {key: value for key, value in pair.items() if key != "has_real_source_context"} + for pair in leftover_pairs ] visible.append( {**report, "members": members, "leftover_pairs": leftover_pairs, "post_count": len(members)} @@ -1027,6 +2406,7 @@ async def read_post_summary( post_id: str, account: CurrentAccount = Depends(get_current_account), pool: asyncpg.Pool = Depends(get_pool), + valkey: redis.Redis = Depends(get_valkey), ) -> dict[str, Any]: """A Korean summary, key events, and R&R for the popup. @@ -1036,28 +2416,97 @@ async def read_post_summary( fabricated summary. """ post = await _load_visible_post(post_id, account, pool) + post_metadata = build_post_llm_metadata(post_id, post) + queue_event: tuple[str, str] | None = None async with pool.acquire() as conn: + body_row = await conn.fetchrow( + "select post_body from source_post where post_id = $1", post_id + ) + try: + raw_body = require_summary_source_body( + None if body_row is None else body_row["post_body"] + ) + except ValueError as exc: + raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, str(exc)) from exc stored = await fetch_persisted_summary(conn, post_id) if stored is not None: return stored - client = _post_summary_client() - if not client.available: - raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Post summary is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", + stale = await fetch_persisted_summary(conn, post_id, allow_stale=True) + with use_llm_metadata(post_metadata): + client = _post_summary_client() + if not client.available: + if stale is not None: + return stale + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Post summary is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", + ) + normalized = await asyncio.to_thread(normalize_post_body, raw_body) + normalized_body = normalized.text + context_hints = await _load_post_semantic_hints(conn, post_id) + summarize_with_hints = getattr(client, "summarize_with_hints", None) + try: + if callable(summarize_with_hints): + summary = await asyncio.to_thread( + summarize_with_hints, post["post_title"], normalized_body, context_hints + ) + else: + summary = await asyncio.to_thread(client.summarize, post["post_title"], normalized_body) + except (HttpClientError, KeyError, OSError, TypeError, ValueError) as exc: + if stale is not None: + return stale + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Post summary is unavailable: contextual-orchestrator returned no complete evidence object", + ) from exc + payload = await persist_post_summary( + conn, + post_id, + summary, + post_body=normalized_body, + hierarchy_inference_client=_corporate_hierarchy_inference_client(), + verification_client=_relation_verification_client(), + ) + content_complete = await post_content_is_complete( + conn, + post_id, + embedding_model_code=load_settings().embedding_model, + require_structure=bool( + load_settings().orchestrator_base_url + and load_settings().orchestrator_api_key + ), + ) + async with conn.transaction(): + job = await ensure_post_content_job( + conn, + post_id, + raw_body, + content_complete=content_complete, ) - body_row = await conn.fetchrow("select post_body from source_post where post_id = $1", post_id) - normalized_body = normalize_post_body(body_row["post_body"], vision_client=_vision_client()).text - summary = await asyncio.to_thread( - client.summarize, post["post_title"], normalized_body + if job.should_publish: + queue_event = (job.post_id, job.source_body_sha256) + if queue_event is not None: + await publish_post_content_event( + valkey, + post_id=queue_event[0], + source_body_digest=queue_event[1], ) - return await persist_post_summary( + return payload + + +@app.get("/api/posts/{post_id}/five-w1h") +async def read_post_five_w1h( + post_id: str, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """Return an evidence-only 5W1H projection for an authorized post.""" + await _load_visible_post(post_id, account, pool) + async with pool.acquire() as conn: + return await load_five_w1h_slots( conn, post_id, - summary, - post_body=normalized_body, - hierarchy_inference_client=_corporate_hierarchy_inference_client(), - verification_client=_relation_verification_client(), + lambda row: _can_see_post(account, row), ) @@ -1067,6 +2516,12 @@ class ChatRequest(BaseModel): question: str +class GlobalAskRequest(BaseModel): + """JSON body for the buyer's source-grounded Global Ask Agent.""" + + question: str + + @app.get("/api/posts/{post_id}/chat") async def read_post_chat( post_id: str, @@ -1091,6 +2546,7 @@ async def chat_about_post( request: ChatRequest, account: CurrentAccount = Depends(get_current_account), pool: asyncpg.Pool = Depends(get_pool), + valkey: redis.Redis = Depends(get_valkey), ) -> dict[str, Any]: """In-popup chat: answers `request.question` using this post's own content plus its Event-Lineage-linked posts (direct and Knowledge- @@ -1105,7 +2561,8 @@ async def chat_about_post( question = request.question.strip() if not question: raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "question is required") - await _load_visible_post(post_id, account, pool) + post = await _load_visible_post(post_id, account, pool) + post_metadata = build_post_llm_metadata(post_id, post) async with pool.acquire() as conn: stored = await fetch_persisted_chat(conn, post_id, question) if stored is not None: @@ -1118,19 +2575,34 @@ async def chat_about_post( "cited_posts": stored["cited_posts"], "source_post_ids": source_ids, } - client = _post_chat_client() - if not client.available: - raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Post chat is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", + with use_llm_metadata(post_metadata): + client = _post_chat_client() + if not client.available: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Post chat is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", + ) + sources = await gather_chat_sources( + conn, post_id, lambda row: _can_see_post(account, row), vision_client=_vision_client() ) - sources = await gather_chat_sources( - conn, post_id, lambda row: _can_see_post(account, row), vision_client=_vision_client() - ) - answer = client.answer(question, sources) + try: + with use_llm_metadata(post_metadata): + answer = await asyncio.to_thread(client.answer, question, sources) + except (HttpClientError, KeyError, OSError, ValueError) as exc: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Post chat is unavailable: contextual-orchestrator returned no complete evidence object", + ) from exc cited_ids = list(answer.cited_post_ids) async with pool.acquire() as conn: await persist_post_chat(conn, post_id, question, answer.answer_text, cited_ids) + await publish_activity_event( + valkey, + post_id, + "chat_answered", + account.user_account_id, + f"Chat answered: {question}", + ) return { "post_id": post_id, "answer_text": answer.answer_text, @@ -1140,6 +2612,104 @@ async def chat_about_post( } +@app.post("/api/ask") +async def ask_agent( + request: GlobalAskRequest, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """Answer a buyer question from authorized post and graph evidence.""" + question = request.question.strip() + if not question: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "question is required") + _require_post_read(account) + client = _post_chat_client() + if not client.available: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Ask Agent is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", + ) + async with pool.acquire() as conn: + sources = await gather_global_chat_sources( + conn, + lambda row: _can_see_post(account, row), + account.corporate_entity_ids, + question=question, + ) + if not sources: + return { + "answer_text": "", + "cited_post_ids": [], + "cited_posts": [], + "source_post_ids": [], + "cited_post_evidence": [], + "next_action": "No authorized source posts are available for this question.", + } + try: + answer = await asyncio.to_thread(client.answer, question, sources) + except (HttpClientError, KeyError, OSError, ValueError) as exc: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + f"Ask Agent is unavailable: {exc}", + ) from exc + cited_ids = list(answer.cited_post_ids) + return { + "answer_text": answer.answer_text, + "cited_post_ids": cited_ids, + "cited_posts": cited_post_summaries(sources, cited_ids), + "cited_post_evidence": cited_post_evidence(sources, cited_ids), + "source_post_ids": [source.post_id for source in sources], + } + + +class PostBookmarkRequest(BaseModel): + bookmarked: bool + + +@app.get("/api/posts/{post_id}/bookmark") +async def read_post_bookmark( + post_id: str, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + await _load_visible_post(post_id, account, pool) + async with pool.acquire() as conn: + row = await conn.fetchrow( + "select 1 from bookmark where user_account_id = $1 and post_id = $2", + account.user_account_id, + post_id, + ) + return {"post_id": post_id, "bookmarked": row is not None} + + +@app.post("/api/posts/{post_id}/bookmark") +async def write_post_bookmark( + post_id: str, + request: PostBookmarkRequest, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + await _load_visible_post(post_id, account, pool) + async with pool.acquire() as conn: + if request.bookmarked: + await conn.execute( + """ + insert into bookmark (user_account_id, post_id) + values ($1, $2) + on conflict (user_account_id, post_id) do nothing + """, + account.user_account_id, + post_id, + ) + else: + await conn.execute( + "delete from bookmark where user_account_id = $1 and post_id = $2", + account.user_account_id, + post_id, + ) + return {"post_id": post_id, "bookmarked": request.bookmarked} + + @app.get("/api/posts/{post_id}/tickets") async def read_post_tickets( post_id: str, @@ -1301,20 +2871,24 @@ async def derive_post_commitment( """ _require_post_admin(account) post = await _load_visible_post(post_id, account, pool) - client = _commitment_extraction_client() - if not client.available: - raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Commitment derivation is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", - ) - async with pool.acquire() as conn: - body_row = await conn.fetchrow("select post_body from source_post where post_id = $1", post_id) - normalized_body = normalize_post_body(body_row["post_body"], vision_client=_vision_client()).text - # TimeML/TempEval document creation time, not wall-clock now: "by next - # Friday" in a January post must resolve to that January, not to the - # Friday after the operator clicked Derive. - reference_date = post["created_at"].date().isoformat() - commitment = client.extract(post["post_title"], normalized_body, reference_date) + post_metadata = build_post_llm_metadata(post_id, post) + with use_llm_metadata(post_metadata): + client = _commitment_extraction_client() + if not client.available: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Commitment derivation is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", + ) + async with pool.acquire() as conn: + body_row = await conn.fetchrow("select post_body from source_post where post_id = $1", post_id) + normalized_body = ( + await asyncio.to_thread(normalize_post_body, body_row["post_body"], _vision_client()) + ).text + # TimeML/TempEval document creation time, not wall-clock now: "by next + # Friday" in a January post must resolve to that January, not to the + # Friday after the operator clicked Derive. + reference_date = post["created_at"].date().isoformat() + commitment = client.extract(post["post_title"], normalized_body, reference_date) if not commitment.has_commitment: return {"post_id": str(post["post_id"]), "has_commitment": False, "ticket": None} async with pool.acquire() as conn: @@ -1419,11 +2993,8 @@ async def start_analysis_run( post_read is enough. Hidden runs 404. Period-report is 422 so this path cannot invent a calibrated score. TEPP goes through - ``tepp_client`` and stays Failed when the transport is missing, the - envelope is unpublished, or TEPP has not published a completed-result - contract. A published accepted acknowledgement is stored as - aggregate transport evidence. Succeeded is never stamped from that - ack. A Succeeded lineage retry returns + ``tepp_client`` and stays Failed when the transport is missing or + the envelope is not persistable. A Succeeded lineage retry returns the stored tree. A Running restart with an undelivered outbox finishes that work. A Running restart without pending work is 409. The outbox commits before reconstruct/TEPP so a crash leaves a @@ -1461,7 +3032,11 @@ async def start_analysis_run( analysis_run_id=analysis_run_id, account_id=account.user_account_id, affiliated_entity_ids=list(account.corporate_entity_ids), - tepp_client=configured_tepp_client(settings.tepp_transport_url), + tepp_client=configured_tepp_client( + settings.tepp_transport_url, + settings.tepp_api_key, + ), + adjudication_client=_adjudication_client(), valkey_stream_entry_id=stream_id, ) except AnalysisRunStartError as exc: @@ -1501,18 +3076,46 @@ async def read_calendar( account: CurrentAccount = Depends(get_current_account), pool: asyncpg.Pool = Depends(get_pool), ) -> dict[str, Any]: - """Every dated, not-closed commitment/ticket the account may see, - soonest first -- the to-do/calendar surface (no Outlook sync yet; - this is the internal data model that a future Outlook connector - would read from). + """Return independent CalDAV events alongside authorized commitments. + + An unavailable optional CalDAV source never hides the internal to-do + projection and never creates a synthetic event. """ _require_post_read(account) + caldav = build_caldav_client(load_settings().caldav_base_url) + events = [] + caldav_available = caldav.available + caldav_next_action = None + if caldav.available: + try: + events = [asdict(event) for event in caldav.list_events()] + except (HttpClientError, OSError, ValueError): + caldav_available = False + caldav_next_action = CALDAV_UNAVAILABLE_NEXT_ACTION + else: + caldav_next_action = CALDAV_UNAVAILABLE_NEXT_ACTION async with pool.acquire() as conn: commitments = await fetch_upcoming_commitments(conn) + demo_entity_ids: set[str] = set() + if commitments and await has_real_source_context(conn, list(account.corporate_entity_ids)): + demo_entity_ids = await fetch_demo_corporate_entity_ids(conn) visible = [c for c in commitments if _can_see_post(account, c)] + # Once real evidence is visible, the synthetic Demo Corp commitments + # (ADR 0001 / ADR 0042) stop appearing beside it. + if demo_entity_ids: + visible = [ + c for c in visible if not _is_synthetic_demo_member(c, demo_entity_ids) + ] for c in visible: - del c["visibility_code"], c["corporate_entity_id"] - return {"commitments": visible} + del c["visibility_code"], c["corporate_entity_id"], c["has_real_source_context"] + return { + "events": events, + "commitments": visible, + "calendar_sources": { + "caldav_available": caldav_available, + "caldav_next_action": caldav_next_action, + }, + } @app.get("/api/rankings") @@ -1520,7 +3123,7 @@ async def read_rankings( account: CurrentAccount = Depends(get_current_account), pool: asyncpg.Pool = Depends(get_pool), ) -> dict[str, Any]: - """RankWeave fusion of ABAC-visible posts (ADR 0030). + """RankWeave fusion of ABAC-visible posts (ADR 0024). Hidden posts are omitted from every channel. Never invents a fused score or a theta. Fail-closed when RankWeave is disabled or the diff --git a/backend/app/post_chat_ingestion.py b/backend/app/post_chat_ingestion.py index 794faa10d..71c0f2053 100644 --- a/backend/app/post_chat_ingestion.py +++ b/backend/app/post_chat_ingestion.py @@ -6,12 +6,21 @@ ABAC is re-checked per candidate post here, never trusted from the Knowledge Graph traversal alone -- a KG edge says two posts are related, not that the requesting account may see both. + +`gather_global_chat_sources` (Global Ask, no starting post) also expands +its single best keyword match through the same `post_lineage_edge` +neighbors, so an answer speaks to a connected timeline rather than one +isolated snapshot -- it does not have a starting post to run the +Knowledge Graph's indirect random-walk expansion from, only the lineage +chain of its own top match. """ from __future__ import annotations +import asyncio +import re from dataclasses import dataclass -from typing import Any, Callable +from typing import Any, Callable, Iterable import asyncpg @@ -33,7 +42,8 @@ ) from lineageweave.post_content_normalization import normalize_post_body -from .knowledge_graph import load_visible_subgraph +from .knowledge_graph import hydrate_related_nodes, load_visible_subgraph +from lineageweave.ontology import ontology_annotations @dataclass(frozen=True) @@ -47,6 +57,170 @@ class LinkedPostIds: indirect: frozenset[str] +async def _normalize_post_body_text( + body: str, + vision_client: ImageContentClient, +) -> str: + """Normalize one source body without blocking the request event loop.""" + normalized = await asyncio.to_thread( + normalize_post_body, + body, + vision_client=vision_client, + ) + return normalized.text + + +async def _graph_facts_for_posts( + conn: asyncpg.Connection, + visible_post_ids: list[str], +) -> tuple[str, ...]: + """Render persisted, ontology-annotated graph facts for visible posts. + + The evidence join is deliberate: a graph edge without a visible evidence + post must never enter an LLM prompt. This is the chat-side trust boundary + in addition to the post-level ABAC check. + """ + if not visible_post_ids: + return () + edge_rows = await conn.fetch( + """ + select edge.source_node_type_code, edge.source_node_id, + edge.target_node_type_code, edge.target_node_id, + edge.edge_type_code, edge.edge_weight, + array_agg(distinct evidence.evidence_post_id::text) as evidence_post_ids + from knowledge_graph_edge edge + join knowledge_graph_edge_evidence evidence + on evidence.knowledge_graph_edge_id = edge.knowledge_graph_edge_id + where evidence.evidence_post_id = any($1::uuid[]) + group by edge.source_node_type_code, edge.source_node_id, + edge.target_node_type_code, edge.target_node_id, + edge.edge_type_code, edge.edge_weight + order by min(edge.edge_type_code), min(edge.source_node_id::text), + min(edge.target_node_id::text) + limit 64 + """, + visible_post_ids, + ) + if not edge_rows: + return () + + endpoint_keys = { + node_key(row["source_node_type_code"], str(row["source_node_id"])) + for row in edge_rows + } + endpoint_keys.update( + node_key(row["target_node_type_code"], str(row["target_node_id"])) + for row in edge_rows + ) + hydrated = await hydrate_related_nodes( + conn, [(key, 1.0) for key in sorted(endpoint_keys)] + ) + labels = { + (item["node_type_code"], item["node_id"]): item + for item in hydrated + } + + facts: list[str] = [] + for row in edge_rows: + source_type = row["source_node_type_code"] + source_id = str(row["source_node_id"]) + target_type = row["target_node_type_code"] + target_id = str(row["target_node_id"]) + source = labels.get((source_type, source_id)) + target = labels.get((target_type, target_id)) + if source is None or target is None: + continue + edge_annotation = ontology_annotations(row["edge_type_code"]) + ontology_iri = edge_annotation.get("ontology_iri") + edge_name = row["edge_type_code"] + if ontology_iri: + edge_name = f"{edge_name} ({ontology_iri})" + evidence_ids = ",".join(sorted(str(value) for value in row["evidence_post_ids"])) + facts.append( + f'{source_type} "{source["label"]}" ' + f'--{edge_name}--> {target_type} "{target["label"]}" ' + f"[evidence_post_id={evidence_ids}]" + ) + return tuple(dict.fromkeys(facts)) + + +_SOURCE_HINT_FIELDS = ( + ("source_system_code", "source system"), + ("source_record_key", "source record key"), + ("source_author_code", "source author code"), + ("source_author_name", "source author name"), + ("source_company_code", "source company code"), + ("source_company_name", "source company name"), + ("source_process_unit_code", "source business unit (PU)"), + ("source_process_unit_name", "source business unit name (PU)"), + ("source_sales_pool_code", "source sales pool"), + ("source_sales_pool_name", "source sales pool name"), + ("source_customer_code", "source customer code"), + ("source_customer_name", "source customer name"), + ("source_project_code", "source project code"), + ("source_project_name", "source project name"), +) + +_GLOBAL_ASK_TERM_PATTERN = re.compile(r"[^\W_]+(?:-[^\W_]+)*", re.UNICODE) +_POST_CHAT_SOURCE_LIMIT = 8 +_POST_CHAT_CANDIDATE_LIMIT = 32 + + +def _source_hint_facts(row: Any) -> tuple[str, ...]: + """Render raw source fields as explicitly weak, column-level evidence.""" + facts: list[str] = [] + for field_name, label in _SOURCE_HINT_FIELDS: + value = row.get(field_name) + if value is not None and str(value).strip(): + facts.append( + f"{label}={str(value).strip()} [provenance=source_post.{field_name}; hint_only]" + ) + return tuple(facts) + + +async def _semantic_facts_for_posts( + conn: asyncpg.Connection, post_ids: list[str] +) -> dict[str, tuple[str, ...]]: + """Load persisted project/role/Keyman facts for already-visible posts.""" + if not post_ids: + return {} + rows = await conn.fetch( + """ + select post_id::text as post_id, + 'project: ' || left(project_name, 200) + || ' | evidence: ' || left(evidence_text, 500) + || ' | ontology_iri: ' || ontology_iri + || ' | extraction_method: ' || extraction_method + || ' | confidence: ' || confidence::text + || ' [provenance=post_project_mention]' as fact + from post_project_mention + where post_id = any($1::uuid[]) + union all + select post_id::text as post_id, + 'actor: ' || left(actor_name, 200) + || ' | responsibility: ' || left(responsibility, 500) + || coalesce(' | affiliation: ' || left(affiliated_organization_name, 200), '') + || ' [provenance=post_summary_role]' as fact + from post_summary_role + where post_id = any($1::uuid[]) + union all + select mention.post_id::text as post_id, + 'Keyman mention: ' || left(person.person_name, 200) + || coalesce(' | context: ' || left(mention.mention_context, 500), '') + || ' [provenance=post_person_mention]' as fact + from post_person_mention mention + join cataloged_person person on person.person_id = mention.person_id + where mention.post_id = any($1::uuid[]) + order by post_id, fact + """, + post_ids, + ) + facts: dict[str, list[str]] = {} + for row in rows: + facts.setdefault(str(row["post_id"]), []).append(row["fact"]) + return {post_id: tuple(dict.fromkeys(values)) for post_id, values in facts.items()} + + async def find_linked_post_ids(conn: asyncpg.Connection, post_id: str) -> LinkedPostIds: """Both link kinds for `post_id`, NOT yet ABAC-filtered -- callers must check `can_see_post` on each id before showing or using it as chat @@ -97,10 +271,13 @@ async def gather_chat_sources( can_see_post: Callable[[asyncpg.Record], bool], vision_client: ImageContentClient | None = None, ) -> list[ChatSourceDocument]: - """Post `post_id` itself, plus every linked post the requesting account - can actually see -- numbered in the order returned, which is the - order `post_chat`'s citations refer back to. Every source's body is - normalized (HTML tags/base64 images never reach the reason-and-cite + """Post `post_id` plus a bounded, deterministic linked-source window. + + Direct Event Lineage neighbors precede indirect Knowledge Graph + neighbors; both groups are identifier-sorted before ABAC filtering. The + current post plus at most seven visible linked posts become the numbered + source set that `post_chat` citations refer back to. Every source's body + is normalized (HTML tags/base64 images never reach the reason-and-cite LLM call raw) before becoming a `ChatSourceDocument` -- see `lineageweave.post_content_normalization`. `vision_client` defaults to unavailable (embedded images become an explicit placeholder, not @@ -111,38 +288,268 @@ async def gather_chat_sources( vision_client = NullImageContentClient() this_post = await conn.fetchrow( - "select post_id, post_title, post_body from source_post where post_id = $1", post_id + "select post_id, post_title, post_body, source_system_code, source_record_key, " + "source_author_code, source_author_name, source_company_code, source_company_name, " + "source_process_unit_code, source_process_unit_name, " + "source_sales_pool_code, source_sales_pool_name, " + "source_customer_code, source_customer_name, source_project_code, " + "source_project_name from source_post where post_id = $1", + post_id, ) if this_post is None: return [] + source_id = str(this_post["post_id"]) + semantic_facts = await _semantic_facts_for_posts(conn, [source_id]) + normalized_body = await _normalize_post_body_text( + this_post["post_body"], + vision_client, + ) sources = [ ChatSourceDocument( - str(this_post["post_id"]), + source_id, this_post["post_title"], - normalize_post_body(this_post["post_body"], vision_client=vision_client).text, + normalized_body, + evidence_facts=_source_hint_facts(this_post) + semantic_facts.get(source_id, ()), ) ] linked = await find_linked_post_ids(conn, post_id) - candidate_ids = linked.direct | linked.indirect + candidate_ids = [ + *sorted(linked.direct), + *sorted(linked.indirect), + ][:_POST_CHAT_CANDIDATE_LIMIT] if not candidate_ids: return sources rows = await conn.fetch( - "select post_id, post_title, post_body, visibility_code, corporate_entity_id " - "from source_post where post_id = any($1::uuid[])", - list(candidate_ids), + "select post_id, post_title, post_body, visibility_code, corporate_entity_id, " + "source_system_code, source_record_key, source_author_code, source_author_name, " + "source_company_code, source_company_name, source_process_unit_code, " + "source_process_unit_name, source_sales_pool_code, source_sales_pool_name, " + "source_customer_code, source_customer_name, " + "source_project_code, source_project_name " + "from source_post where post_id = any($1::uuid[]) " + "order by array_position($1::uuid[], post_id) limit $2", + candidate_ids, + _POST_CHAT_CANDIDATE_LIMIT, ) + visible_source_ids = [post_id] + visible_rows: list[asyncpg.Record] = [] for row in rows: - if can_see_post(row): - sources.append( - ChatSourceDocument( - str(row["post_id"]), - row["post_title"], - normalize_post_body(row["post_body"], vision_client=vision_client).text, - ) + if not can_see_post(row): + continue + visible_rows.append(row) + visible_source_ids.append(str(row["post_id"])) + if len(visible_rows) >= _POST_CHAT_SOURCE_LIMIT - 1: + break + + semantic_facts = await _semantic_facts_for_posts(conn, visible_source_ids) + graph_facts = await _graph_facts_for_posts(conn, visible_source_ids) + sources[0] = ChatSourceDocument( + sources[0].post_id, + sources[0].post_title, + sources[0].post_body, + graph_facts=graph_facts, + evidence_facts=sources[0].evidence_facts, + ) + for row in visible_rows: + normalized_body = await _normalize_post_body_text(row["post_body"], vision_client) + sources.append( + ChatSourceDocument( + str(row["post_id"]), + row["post_title"], + normalized_body, + evidence_facts=_source_hint_facts(row) + + semantic_facts.get(str(row["post_id"]), ()), ) + ) + + return sources + + +async def gather_global_chat_sources( + conn: asyncpg.Connection, + can_see_post: Callable[[asyncpg.Record], bool], + authorized_corporate_entity_ids: Iterable[str] = (), + vision_client: ImageContentClient | None = None, + *, + question: str | None = None, + limit: int = 4, +) -> list[ChatSourceDocument]: + """Assemble a bounded, ABAC-filtered source set for Global Ask. + + The source set is intentionally bounded until retrieval/reranking is + needed for a much larger corpus; every selected body still uses the same + image normalization and persisted graph evidence as post-scoped chat. + """ + if limit <= 0: + return [] + if vision_client is None: + vision_client = NullImageContentClient() + search_terms = tuple( + dict.fromkeys( + token.casefold() + for token in _GLOBAL_ASK_TERM_PATTERN.findall(question or "") + if len(token) >= 2 + and token.casefold() + not in { + "which", + "what", + "where", + "when", + "who", + "why", + "how", + "the", + "this", + "that", + "posts", + "post", + "글", + "게시글", + "질문", + "관련", + "확인되는", + "핵심", + "사실", + "무엇", + "무엇인가요", + "인가요", + } + ) + )[:8] + # A post whose title names the exact thing asked about is a far more + # specific match than one that only shares a generic term (a common + # word, or a hit buried in a 16KB body prefix); weighting every match + # equally and then falling back on created_at desc as the only + # tiebreak let recency crowd out relevance -- a year-old post whose + # title is an exact company-name match lost to four newer, only + # loosely related posts in a live reproduction of this bug. + _MATCH_WEIGHT = {"title": 3.0, "body": 1.0, "source_field": 1.0} + candidate_scores: dict[str, float] = {} + for term in search_terms: + candidate_rows = await conn.fetch( + """ + select post_id, matched_in + from ( + (select post_id, created_at, 'title' as matched_in + from source_post + where post_title ilike '%' || $1 || '%' + limit 32) + union all + (select post_id, created_at, 'body' as matched_in + from source_post + where lower(left(source_post_search_text(post_body), 16384)) + like '%' || lower($1) || '%' + limit 32) + union all + (select post_id, created_at, 'body' as matched_in + from source_post + where to_tsvector('simple', source_post_search_text(post_body)) + @@ plainto_tsquery('simple', $1) + limit 32) + union all + (select post_id, created_at, 'source_field' as matched_in + from source_post + where concat_ws(' ', source_system_code, source_record_key, + source_author_code, source_author_name, + source_company_code, source_company_name, + source_process_unit_code, source_process_unit_name, + source_sales_pool_code, source_sales_pool_name, + source_customer_code, source_customer_name, + source_project_code, source_project_name) + ilike '%' || $1 || '%' + limit 32) + ) matches + order by created_at desc, post_id desc + limit 32 + """, + term, + ) + for row in candidate_rows: + post_id = str(row["post_id"]) + candidate_scores[post_id] = candidate_scores.get(post_id, 0.0) + _MATCH_WEIGHT[row["matched_in"]] + candidate_ids = sorted(candidate_scores, key=lambda post_id: candidate_scores[post_id], reverse=True) + + # A keyword match only proves one post's text is relevant -- the + # account asking almost always wants to know what happened before and + # after that event too, not just this one snapshot. Expand the single + # best match through its direct Event Lineage neighbors + # (`post_lineage_edge`, `lineageweave.reconstruct`'s output), mirroring + # `find_linked_post_ids`'s `.direct` set used by the post-scoped chat + # flow. Only the top match is expanded -- expanding every keyword hit + # would let a loosely related term drag in an unrelated lineage chain. + lineage_neighbor_ids: list[str] = [] + lineage_anchor_id = candidate_ids[0] if candidate_ids else None + if lineage_anchor_id: + lineage_rows = await conn.fetch( + "select child_post_id as other_id from post_lineage_edge where parent_post_id = $1 " + "union select parent_post_id as other_id from post_lineage_edge where child_post_id = $1", + lineage_anchor_id, + ) + lineage_neighbor_ids = sorted( + { + str(row["other_id"]) + for row in lineage_rows + if str(row["other_id"]) not in candidate_scores + } + ) + candidate_ids = list( + dict.fromkeys([lineage_anchor_id, *lineage_neighbor_ids, *candidate_ids[1:]]) + )[:limit] + else: + candidate_ids = [] + lineage_neighbor_id_set = frozenset(lineage_neighbor_ids) + rows = await conn.fetch( + """ + select post_id, post_title, post_body, visibility_code, corporate_entity_id, + source_system_code, source_record_key, source_author_code, source_author_name, + source_company_code, source_company_name, source_process_unit_code, + source_process_unit_name, source_sales_pool_code, source_sales_pool_name, + source_customer_code, source_customer_name, + source_project_code, source_project_name + from source_post + where visibility_code = 'public' + or corporate_entity_id::text = any($1::text[]) + order by array_position($2::uuid[], post_id) nulls last, + created_at desc, post_id desc + limit $3 + """, + list(authorized_corporate_entity_ids), + candidate_ids, + limit, + ) + visible_rows = [row for row in rows if can_see_post(row)][:limit] + visible_ids = [str(row["post_id"]) for row in visible_rows] + anchor_is_visible = lineage_anchor_id in visible_ids + semantic_facts = await _semantic_facts_for_posts(conn, visible_ids) + graph_facts = (await _graph_facts_for_posts(conn, visible_ids))[:16] + sources: list[ChatSourceDocument] = [] + for index, row in enumerate(visible_rows): + normalized_body = await _normalize_post_body_text(row["post_body"], vision_client) + if len(normalized_body) > 4000: + normalized_body = ( + normalized_body[:4000] + + "\n[Source body truncated for Global Ask; open the cited post for the full body.]" + ) + post_id = str(row["post_id"]) + lineage_fact = ( + (f"Event Lineage: reconstructed timeline neighbor of post_id={lineage_anchor_id}",) + if post_id in lineage_neighbor_id_set and anchor_is_visible + else () + ) + sources.append( + ChatSourceDocument( + post_id, + row["post_title"], + normalized_body, + graph_facts=graph_facts if index == 0 else (), + evidence_facts=_source_hint_facts(row) + + semantic_facts.get(post_id, ()) + + lineage_fact, + ) + ) return sources diff --git a/backend/app/post_content_queue.py b/backend/app/post_content_queue.py new file mode 100644 index 000000000..dae640240 --- /dev/null +++ b/backend/app/post_content_queue.py @@ -0,0 +1,454 @@ +"""Durable post-content ingestion jobs with a Valkey wake-up stream.""" + +from __future__ import annotations + +import hashlib +from datetime import timedelta +from dataclasses import dataclass +from typing import Any + +import asyncpg +import redis.asyncio as redis + +POST_CONTENT_STREAM_KEY = "post-content-ingestion" +QUEUED = "post_content_ingestion_queued" +RUNNING = "post_content_ingestion_running" +SUCCEEDED = "post_content_ingestion_succeeded" +FAILED = "post_content_ingestion_failed" +STALE_RUNNING_INTERVAL = timedelta(minutes=15) +_ACTIVE = {QUEUED, RUNNING} +POST_CONTENT_MAX_ATTEMPTS = 3 +POST_CONTENT_RETRY_INTERVAL = timedelta(minutes=5) + + +@dataclass(frozen=True) +class PostContentJobRequest: + post_id: str + source_body_sha256: str + status_code: str + should_publish: bool + + +def source_body_sha256(body: str) -> str: + """Hash the immutable source representation, never the derived content.""" + return hashlib.sha256(body.encode("utf-8")).hexdigest() + + +def post_content_api_status(status_code: str | None, *, content_present: bool) -> str: + if status_code in _ACTIVE: + return "processing" + if status_code == FAILED: + return "unavailable" + if content_present: + return "ready" + return "unavailable" + + +async def post_content_is_complete( + conn: asyncpg.Connection, + post_id: str, + *, + embedding_model_code: str, + require_structure: bool = False, +) -> bool: + """Require configured semantic, structure, and region evidence before ready.""" + return bool( + await conn.fetchval( + """ + select exists( + select 1 + from post_content_unit unit + where unit.post_id = $1 + ) + and ( + $2 = '' + or ( + not exists( + select 1 + from post_content_unit unit + left join post_content_embedding embedding + on embedding.post_content_unit_id = unit.post_content_unit_id + and embedding.embedding_model_code = $2 + where unit.post_id = $1 + and embedding.post_content_embedding_id is null + ) + and not exists( + select 1 + from post_content_unit unit + join post_content_image image + on image.post_content_unit_id = unit.post_content_unit_id + join post_content_image_region region + on region.post_content_image_id = image.post_content_image_id + left join post_content_image_region_embedding embedding + on embedding.post_content_image_region_id = region.post_content_image_region_id + and embedding.embedding_model_code = $2 + where unit.post_id = $1 + and region.description_status_code = 'described' + and embedding.post_content_image_region_embedding_id is null + ) + ) + ) + and ( + not $3::boolean + or not exists( + select 1 + from post_content_unit unit + left join post_content_unit_structure structure + on structure.post_content_unit_id = unit.post_content_unit_id + where unit.post_id = $1 + and unit.unit_kind_code <> 'image' + and ( + structure.post_content_unit_structure_id is null + or structure.decision_source_code = 'unresolved' + ) + ) + ) + """, + post_id, + embedding_model_code, + require_structure, + ) + ) + + +def post_content_stream_fields(*, post_id: str, source_body_digest: str) -> dict[str, str]: + """Valkey carries only the identity and digest needed to wake a worker.""" + return {"post_id": str(post_id), "source_body_sha256": source_body_digest} + + +async def publish_post_content_event( + client: redis.Redis | None, + *, + post_id: str, + source_body_digest: str, +) -> str | None: + """Wake the worker after the PostgreSQL transaction has committed.""" + if client is None: + return None + try: + entry_id = await client.xadd( + POST_CONTENT_STREAM_KEY, + post_content_stream_fields( + post_id=post_id, + source_body_digest=source_body_digest, + ), + maxlen=1000, + approximate=True, + ) + except redis.RedisError: + return None + return str(entry_id) + + +async def _record_status( + conn: asyncpg.Connection, + post_id: str, + status_code: str, + *, + failure_code: str | None = None, + detail_text: str | None = None, +) -> None: + ordinal = await conn.fetchval( + """ + select coalesce(max(status_ordinal), -1) + 1 + from post_content_ingestion_job_status_event + where post_id = $1 + """, + post_id, + ) + await conn.execute( + """ + insert into post_content_ingestion_job_status_event + (post_id, status_ordinal, status_code, failure_code, detail_text) + values ($1, $2, $3, $4, $5) + """, + post_id, + int(ordinal), + status_code, + failure_code, + detail_text, + ) + + +async def transition_post_content_job( + conn: asyncpg.Connection, + post_id: str, + status_code: str, + *, + failure_code: str | None = None, + detail_text: str | None = None, + expected_attempt_count: int | None = None, +) -> bool: + """Update one job attempt and append its lifecycle event atomically. + + ``expected_attempt_count`` fences stale workers after lease recovery. A + late completion from an older attempt must not overwrite the newer + attempt's status or append a misleading lifecycle event. + """ + updated = await conn.execute( + """ + update post_content_ingestion_job + set status_code = $2, + started_at = case + when $2 = $3 then now() + when $2 = $6 then null + else started_at + end, + completed_at = case when $2 in ($4, $5) then now() else null end, + queued_at = case when $2 = $6 then now() else queued_at end, + updated_at = now(), + last_error_code = $7, + last_error_detail = $8 + where post_id = $1 + and ($9::integer is null or attempt_count = $9) + """, + post_id, + status_code, + RUNNING, + SUCCEEDED, + FAILED, + QUEUED, + failure_code, + detail_text, + expected_attempt_count, + ) + if not updated.endswith(" 1"): + return False + await _record_status( + conn, + post_id, + status_code, + failure_code=failure_code, + detail_text=detail_text, + ) + return True + + +async def ensure_post_content_job( + conn: asyncpg.Connection, + post_id: str, + body: str, + *, + content_complete: bool, +) -> PostContentJobRequest: + """Create or requeue the job for the current source-body digest.""" + digest = source_body_sha256(body) + row = await conn.fetchrow( + """ + select source_body_sha256, status_code + from post_content_ingestion_job + where post_id = $1 + for update + """, + post_id, + ) + if row is None: + initial_status = SUCCEEDED if content_complete else QUEUED + await conn.execute( + """ + insert into post_content_ingestion_job + (post_id, source_body_sha256, status_code) + values ($1, $2, $3) + """, + post_id, + digest, + initial_status, + ) + await _record_status(conn, post_id, initial_status) + return PostContentJobRequest( + post_id, + digest, + initial_status, + initial_status == QUEUED, + ) + + status_code = str(row["status_code"]) + needs_requeue = ( + str(row["source_body_sha256"]) != digest + or (status_code == SUCCEEDED and not content_complete) + ) + if needs_requeue: + await conn.execute( + """ + update post_content_ingestion_job + set source_body_sha256 = $2, + status_code = $3, + attempt_count = 0, + queued_at = now(), + started_at = null, + completed_at = null, + updated_at = now(), + last_error_code = null, + last_error_detail = null + where post_id = $1 + """, + post_id, + digest, + QUEUED, + ) + await _record_status(conn, post_id, QUEUED) + status_code = QUEUED + return PostContentJobRequest( + post_id, + digest, + status_code, + status_code == QUEUED, + ) + + +async def requeue_failed_post_content_job( + conn: asyncpg.Connection, + post_id: str, + body: str, +) -> PostContentJobRequest: + """Explicitly requeue one terminal job without weakening automatic retry limits.""" + digest = source_body_sha256(body) + row = await conn.fetchrow( + """ + select status_code + from post_content_ingestion_job + where post_id = $1 + for update + """, + post_id, + ) + if row is None: + raise ValueError(f"post-content job does not exist: {post_id}") + if str(row["status_code"]) != FAILED: + raise ValueError("only a failed post-content job can be explicitly requeued") + await conn.execute( + """ + update post_content_ingestion_job + set source_body_sha256 = $2, + status_code = $3, + attempt_count = 0, + queued_at = now(), + started_at = null, + completed_at = null, + updated_at = now(), + last_error_code = null, + last_error_detail = null + where post_id = $1 + and status_code = $4 + """, + post_id, + digest, + QUEUED, + FAILED, + ) + await _record_status( + conn, + post_id, + QUEUED, + detail_text="operator requested an explicit post-content retry", + ) + return PostContentJobRequest(post_id, digest, QUEUED, True) + + +async def record_post_content_backfill_success( + conn: asyncpg.Connection, + post_id: str, + body: str, +) -> PostContentJobRequest: + """Synchronize a completed operator backfill with the durable job ledger.""" + digest = source_body_sha256(body) + row = await conn.fetchrow( + """ + select status_code + from post_content_ingestion_job + where post_id = $1 + for update + """, + post_id, + ) + if row is not None and str(row["status_code"]) in {QUEUED, RUNNING}: + raise ValueError("cannot finalize a backfill while the job is active") + if row is None: + await conn.execute( + """ + insert into post_content_ingestion_job + (post_id, source_body_sha256, status_code, completed_at) + values ($1, $2, $3, now()) + """, + post_id, + digest, + SUCCEEDED, + ) + else: + await conn.execute( + """ + update post_content_ingestion_job + set source_body_sha256 = $2, + status_code = $3, + started_at = null, + completed_at = now(), + updated_at = now(), + last_error_code = null, + last_error_detail = null + where post_id = $1 + """, + post_id, + digest, + SUCCEEDED, + ) + await _record_status( + conn, + post_id, + SUCCEEDED, + detail_text="operator backfill persisted post-content evidence", + ) + return PostContentJobRequest(post_id, digest, SUCCEEDED, False) + + +async def republish_queued_post_content_jobs( + client: redis.Redis, + pool: asyncpg.Pool, + *, + limit: int = 100, +) -> int: + """Recover queued rows and stale running leases when Valkey lost wake-ups.""" + async with pool.acquire() as conn: + rows = await conn.fetch( + """ + select post_id, source_body_sha256 + from post_content_ingestion_job + where ( + status_code = $1 + and ( + attempt_count = 0 + or queued_at <= now() - $2::interval + ) + ) + or ( + status_code = $3 + and started_at is not null + and started_at < now() - $4::interval + ) + order by queued_at + limit $5 + """, + QUEUED, + POST_CONTENT_RETRY_INTERVAL, + RUNNING, + STALE_RUNNING_INTERVAL, + limit, + ) + published = 0 + for row in rows: + if await publish_post_content_event( + client, + post_id=str(row["post_id"]), + source_body_digest=str(row["source_body_sha256"]), + ): + published += 1 + return published + + +def serialize_job_row(row: Any) -> dict[str, Any]: + """Small internal projection used by diagnostics and tests.""" + return { + "post_id": str(row["post_id"]), + "source_body_sha256": str(row["source_body_sha256"]), + "status_code": str(row["status_code"]), + "attempt_count": int(row["attempt_count"]), + } diff --git a/backend/app/post_content_worker.py b/backend/app/post_content_worker.py new file mode 100644 index 000000000..458b9021f --- /dev/null +++ b/backend/app/post_content_worker.py @@ -0,0 +1,331 @@ +"""Consume Valkey post-content wake-ups and persist derived evidence.""" + +from __future__ import annotations + +import asyncio +import logging +import time +from collections.abc import Callable +from uuid import UUID + +import asyncpg +import redis.asyncio as redis + +from lineageweave.embedding_client import EmbeddingClient +from lineageweave.image_content import ImageContentClient +from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata +from lineageweave.post_content_normalization import normalize_post_body +from lineageweave.post_content_persistence import persist_post_content +from lineageweave.post_structure import PostStructureClient + +from backend.app.config import load_settings +from backend.app.post_content_queue import ( + FAILED, + POST_CONTENT_MAX_ATTEMPTS, + POST_CONTENT_RETRY_INTERVAL, + POST_CONTENT_STREAM_KEY, + QUEUED, + RUNNING, + STALE_RUNNING_INTERVAL, + SUCCEEDED, + post_content_is_complete, + transition_post_content_job, + republish_queued_post_content_jobs, +) + +_logger = logging.getLogger(__name__) +_RECOVERY_INTERVAL_SECONDS = 30.0 +_INCOMPLETE_FAILURE_CODE = "post_content_ingestion_incomplete" +_ATTEMPT_LIMIT_FAILURE_CODE = "post_content_ingestion_attempt_limit" + + +async def _stream_tail(client: redis.Redis) -> str: + """Start after historical wake-ups; the normalized ledger drives recovery.""" + rows = await client.xrevrange(POST_CONTENT_STREAM_KEY, count=1) + return str(rows[0][0]) if rows else "0-0" + + +async def _claim_job( + pool: asyncpg.Pool, + post_id: str, + source_body_digest: str, + *, + embedding_model_code: str, + require_structure: bool = False, +) -> asyncpg.Record | None: + async with pool.acquire() as conn: + async with conn.transaction(): + row = await conn.fetchrow( + f""" + select p.*, j.source_body_sha256 as job_source_body_sha256, + j.status_code as job_status_code, + j.attempt_count as job_attempt_count, + j.started_at as job_started_at, + j.queued_at as job_queued_at + from post_content_ingestion_job j + join source_post p on p.post_id = j.post_id + where j.post_id = $1::uuid + and j.source_body_sha256 = $2 + for update of j, p + """, + post_id, + source_body_digest, + ) + if row is None: + return None + status_code = str(row["job_status_code"]) + attempt_count = int(row["job_attempt_count"]) + if status_code == FAILED: + return None + if status_code == RUNNING and attempt_count >= POST_CONTENT_MAX_ATTEMPTS: + await transition_post_content_job( + conn, + post_id, + FAILED, + failure_code=_ATTEMPT_LIMIT_FAILURE_CODE, + detail_text="post-content ingestion attempt limit was already reached", + ) + return None + if status_code == QUEUED and attempt_count >= POST_CONTENT_MAX_ATTEMPTS: + await transition_post_content_job( + conn, + post_id, + FAILED, + failure_code=_ATTEMPT_LIMIT_FAILURE_CODE, + detail_text="post-content ingestion attempt limit was already reached", + ) + return None + if status_code == QUEUED and attempt_count > 0: + retry_ready = await conn.fetchval( + "select now() >= $1 + $2::interval", + row["job_queued_at"], + POST_CONTENT_RETRY_INTERVAL, + ) + if not retry_ready: + return None + if status_code == SUCCEEDED: + content_complete = await post_content_is_complete( + conn, + post_id, + embedding_model_code=embedding_model_code, + require_structure=require_structure, + ) + if content_complete: + return None + if status_code == RUNNING and row["job_started_at"] is not None: + stale = await conn.fetchval( + "select now() - $1 > $2::interval", + row["job_started_at"], + STALE_RUNNING_INTERVAL, + ) + if not stale: + return None + await conn.execute( + """ + update post_content_ingestion_job + set attempt_count = attempt_count + 1 + where post_id = $1 + """, + post_id, + ) + await transition_post_content_job(conn, post_id, RUNNING) + return row + + +async def _finish_job( + pool: asyncpg.Pool, + post_id: str, + status_code: str, + *, + expected_attempt_count: int, + failure_code: str | None = None, + detail_text: str | None = None, +) -> None: + """Finish only the attempt that actually owns the running lease.""" + async with pool.acquire() as conn: + async with conn.transaction(): + await transition_post_content_job( + conn, + post_id, + status_code, + expected_attempt_count=expected_attempt_count, + failure_code=failure_code, + detail_text=detail_text, + ) + + +async def _finish_failed_job( + pool: asyncpg.Pool, + post_id: str, + *, + failure_code: str, + detail_text: str, + expected_attempt_count: int, +) -> None: + """Schedule one retry, or persist a terminal failure for this attempt. + + The running status and attempt number are locked before the transition so + a worker whose lease was reclaimed cannot retry or terminally fail a newer + attempt. + """ + async with pool.acquire() as conn: + async with conn.transaction(): + attempt_count = int( + await conn.fetchval( + """ + select attempt_count + from post_content_ingestion_job + where post_id = $1 + and status_code = $2 + for update + """, + post_id, + RUNNING, + ) + or -1 + ) + if attempt_count != expected_attempt_count: + return + terminal = attempt_count >= POST_CONTENT_MAX_ATTEMPTS + await transition_post_content_job( + conn, + post_id, + FAILED if terminal else QUEUED, + failure_code=_ATTEMPT_LIMIT_FAILURE_CODE if terminal else failure_code, + detail_text=( + "post-content ingestion reached its bounded retry limit" + if terminal + else detail_text + ), + expected_attempt_count=expected_attempt_count, + ) + + +async def process_post_content_job( + pool: asyncpg.Pool, + *, + post_id: str, + source_body_digest: str, + vision_factory: Callable[[], ImageContentClient], + embedding_factory: Callable[[], EmbeddingClient], + structure_factory: Callable[[], PostStructureClient], +) -> None: + settings = load_settings() + row = await _claim_job( + pool, + post_id, + source_body_digest, + embedding_model_code=settings.embedding_model, + require_structure=bool(settings.orchestrator_base_url and settings.orchestrator_api_key), + ) + if row is None: + return + attempt_count = int(row["job_attempt_count"]) + 1 + try: + raw_body = row["post_body"] + if not isinstance(raw_body, str) or not raw_body.strip(): + raise ValueError("source post has no body") + metadata = build_post_llm_metadata(post_id, row) + embedding_client = embedding_factory() + structure_client = structure_factory() + with use_llm_metadata(metadata): + vision_client = vision_factory() + normalized = await asyncio.to_thread(normalize_post_body, raw_body, vision_client) + async with pool.acquire() as conn: + await persist_post_content( + conn, + post_id, + raw_body, + vision_client=vision_client, + embedding_client=embedding_client, + embedding_model_code=settings.embedding_model or None, + normalized_result=normalized, + structure_client=structure_client, + post_title=str(row["post_title"]), + ) + async with pool.acquire() as conn: + complete = await post_content_is_complete( + conn, + post_id, + embedding_model_code=settings.embedding_model, + require_structure=bool( + settings.orchestrator_base_url and settings.orchestrator_api_key + ), + ) + if not complete: + await _finish_failed_job( + pool, + post_id, + failure_code=_INCOMPLETE_FAILURE_CODE, + detail_text="post-content providers did not produce complete persisted evidence", + expected_attempt_count=attempt_count, + ) + return + except Exception as exc: # noqa: BLE001 - durable failure is recorded for retry. + _logger.exception("post content ingestion failed for post_id=%s", post_id) + await _finish_failed_job( + pool, + post_id, + failure_code="post_content_ingestion_failed", + detail_text=str(exc)[:1000], + expected_attempt_count=attempt_count, + ) + return + await _finish_job(pool, post_id, SUCCEEDED, expected_attempt_count=attempt_count) + + +async def consume_post_content_stream_once( + client: redis.Redis, + pool: asyncpg.Pool, + *, + last_id: str, + vision_factory: Callable[[], ImageContentClient], + embedding_factory: Callable[[], EmbeddingClient], + structure_factory: Callable[[], PostStructureClient], +) -> str: + batches = await client.xread({POST_CONTENT_STREAM_KEY: last_id}, count=10, block=1000) + for _stream_name, entries in batches: + for entry_id, fields in entries: + post_id = str(fields.get("post_id", "")).strip() + digest = str(fields.get("source_body_sha256", "")).strip() + try: + UUID(post_id) + except ValueError: + post_id = "" + if post_id and len(digest) == 64: + await process_post_content_job( + pool, + post_id=post_id, + source_body_digest=digest, + vision_factory=vision_factory, + embedding_factory=embedding_factory, + structure_factory=structure_factory, + ) + last_id = str(entry_id) + return last_id + + +async def run_post_content_worker( + client: redis.Redis, + pool: asyncpg.Pool, + *, + vision_factory: Callable[[], ImageContentClient], + embedding_factory: Callable[[], EmbeddingClient], + structure_factory: Callable[[], PostStructureClient], +) -> None: + """Run the at-least-once consumer and periodically recover queued rows.""" + last_id = await _stream_tail(client) + last_recovery = 0.0 + while True: + now = time.monotonic() + if now - last_recovery >= _RECOVERY_INTERVAL_SECONDS: + await republish_queued_post_content_jobs(client, pool) + last_recovery = now + last_id = await consume_post_content_stream_once( + client, + pool, + last_id=last_id, + vision_factory=vision_factory, + embedding_factory=embedding_factory, + structure_factory=structure_factory, + ) diff --git a/backend/app/post_eligibility.py b/backend/app/post_eligibility.py new file mode 100644 index 000000000..41473d9da --- /dev/null +++ b/backend/app/post_eligibility.py @@ -0,0 +1,45 @@ +"""Shared source-post eligibility SQL for buyer evidence reads.""" + +SOURCE_CONTEXT_COLUMNS = ( + "source_author_code", + "source_author_name", + "source_company_code", + "source_company_name", + "source_process_unit_code", + "source_process_unit_name", + "source_sales_pool_code", + "source_sales_pool_name", + "source_customer_code", + "source_customer_name", + "source_project_code", + "source_project_name", +) + + +def source_context_present_sql(alias: str) -> str: + return " or ".join( + f"nullif(btrim({alias}.{column}), '') is not null" for column in SOURCE_CONTEXT_COLUMNS + ) + + +def source_context_missing_sql(alias: str) -> str: + return " and ".join( + f"nullif(btrim({alias}.{column}), '') is null" for column in SOURCE_CONTEXT_COLUMNS + ) + + +SOURCE_POST_ELIGIBILITY_SQL = ( + "nullif(btrim({alias}.source_draft_code), '') is null " + "and nullif(btrim({alias}.source_deleted_flag), '') is null " + "and not (" + "({missing_context}) " + "and exists (" + "select 1 from source_post real_post " + "where ({present_context})" + ")" + ")" +).format( + alias="{alias}", + missing_context=source_context_missing_sql("{alias}"), + present_context=source_context_present_sql("real_post"), +) diff --git a/backend/app/post_summary_ingestion.py b/backend/app/post_summary_ingestion.py index 03426b0ef..7403185ed 100644 --- a/backend/app/post_summary_ingestion.py +++ b/backend/app/post_summary_ingestion.py @@ -42,12 +42,15 @@ NODE_PERSON, NODE_TEAM, ) -from lineageweave.ontology import ontology_annotations +from lineageweave.ontology import LW, ontology_annotations from lineageweave.post_summary import ( ACTOR_TYPE_ORGANIZATION, ACTOR_TYPE_PERSON, ACTOR_TYPE_TEAM, + KeyEvent, PostSummary, + POST_SUMMARY_CONTRACT_VERSION, + normalize_project_key, RoleResponsibility, ) from lineageweave.relation_verification import ( @@ -61,23 +64,53 @@ from .team_ingestion import upsert_team +SUMMARY_SOURCE_BODY_MISSING = ( + "Post summary is unavailable: the source post body is empty. " + "Re-import the source record with its body before requesting a summary." +) + + +def require_summary_source_body(body: str | None) -> str: + """Reject summary derivation when the evidence body was not imported.""" + if not isinstance(body, str) or not body.strip(): + raise ValueError(SUMMARY_SOURCE_BODY_MISSING) + return body + + async def fetch_persisted_summary( - conn: asyncpg.Connection, post_id: str + conn: asyncpg.Connection, + post_id: str, + *, + allow_stale: bool = False, ) -> dict[str, Any] | None: - """Return the stored summary payload, or None when none has been written. + """Return the stored summary payload, or None when none is usable. ``catalog_node_id`` comes from the role row's catalog foreign keys (ADR 0019 / 0027). This function does not join ``corporate_entity`` - by ``entity_name``. Person chips read ``cataloged_person_id``. + by ``entity_name``. Person chips read ``cataloged_person_id``. A stale + row is returned only when ``allow_stale`` is explicit so a caller can + preserve buyer continuity without presenting old semantics as current. """ header = await conn.fetchrow( - "select korean_summary from post_summary_result where post_id = $1", + "select korean_summary, summary_contract_version " + "from post_summary_result where post_id = $1", post_id, ) if header is None: return None + summary_contract_version = header["summary_contract_version"] + if summary_contract_version != POST_SUMMARY_CONTRACT_VERSION and not allow_stale: + return None events = await conn.fetch( - "select event_text from post_summary_event where post_id = $1 order by event_ordinal", + """ + select event.event_text, event.project_key, mention.project_name + from post_summary_event event + left join post_project_mention mention + on mention.post_id = event.post_id + and mention.project_key = event.project_key + where event.post_id = $1 + order by event.event_ordinal + """, post_id, ) roles = await conn.fetch( @@ -93,6 +126,30 @@ async def fetch_persisted_summary( """, post_id, ) + projects = await conn.fetch( + """ + select project_key, project_name, evidence_text, confidence, ontology_iri, + extraction_method + from post_project_mention + where post_id = $1 + order by project_name, project_key + """, + post_id, + ) + actions = await conn.fetch( + """ + select action.action_text, action.requester_actor_name, + action.processor_actor_name, action.evidence_text, + mention.project_name + from post_summary_action action + left join post_project_mention mention + on mention.post_id = action.post_id + and mention.project_key = action.project_key + where action.post_id = $1 + order by action.action_ordinal + """, + post_id, + ) payload_roles: list[dict[str, Any]] = [] for row in roles: catalog_node_id = None @@ -120,8 +177,42 @@ async def fetch_persisted_summary( return { "post_id": post_id, "korean_summary": header["korean_summary"], + "summary_status": ( + "current" + if summary_contract_version == POST_SUMMARY_CONTRACT_VERSION + else "stale" + ), + "summary_contract_version": summary_contract_version, "key_events": [row["event_text"] for row in events], + "key_event_details": [ + { + "event_text": row["event_text"], + "project_name": row.get("project_name"), + } + for row in events + ], "roles_and_responsibilities": payload_roles, + "major_event_actions": [ + { + "action_text": row["action_text"], + "requester_actor_name": row["requester_actor_name"], + "processor_actor_name": row["processor_actor_name"], + "evidence_text": row["evidence_text"], + "project_name": row["project_name"], + } + for row in actions + ], + "project_mentions": [ + { + "project_key": row["project_key"], + "project_name": row["project_name"], + "evidence": row["evidence_text"], + "confidence": float(row["confidence"]), + "ontology_iri": row["ontology_iri"], + "extraction_method": row["extraction_method"], + } + for row in projects + ], } @@ -149,6 +240,9 @@ async def persist_post_summary( summary replacement transaction while all post-owned rows still commit or roll back together. """ + if post_body is not None: + require_summary_source_body(post_body) + hierarchy_inference_client = ( hierarchy_inference_client or NullCorporateHierarchyInferenceClient() ) @@ -226,19 +320,76 @@ async def _replace_summary_projection( ) await conn.execute("delete from post_team_mention where post_id = $1", post_id) await conn.execute("delete from post_organization_mention where post_id = $1", post_id) + await conn.execute("delete from post_summary_five_w1h where post_id = $1", post_id) + await conn.execute("delete from post_summary_action where post_id = $1", post_id) await conn.execute("delete from post_summary_result where post_id = $1", post_id) + await conn.execute("delete from post_project_mention where post_id = $1", post_id) await conn.execute( - "insert into post_summary_result (post_id, korean_summary) values ($1, $2)", + "insert into post_summary_result " + "(post_id, korean_summary, summary_contract_version) values ($1, $2, $3)", post_id, summary.korean_summary, + POST_SUMMARY_CONTRACT_VERSION, + ) + for project in summary.project_mentions: + project_key = normalize_project_key(project.canonical_name) + if not project_key: + continue + await conn.execute( + """ + insert into post_project_mention + (post_id, project_key, project_name, evidence_text, confidence, + ontology_iri, extraction_method) + values ($1, $2, $3, $4, $5, $6, 'contextual_orchestrator_semantic') + on conflict (post_id, project_key) do update set + project_name = excluded.project_name, + evidence_text = excluded.evidence_text, + confidence = excluded.confidence, + ontology_iri = excluded.ontology_iri, + extraction_method = excluded.extraction_method + """, + post_id, + project_key, + project.project_name, + project.evidence, + project.confidence, + str(LW.Project), + ) + event_details = summary.key_event_details or tuple( + KeyEvent(event_text=event_text) for event_text in summary.key_events ) - for ordinal, event_text in enumerate(summary.key_events): + project_keys = { + normalize_project_key(project.canonical_name) + for project in summary.project_mentions + if normalize_project_key(project.canonical_name) + } + for ordinal, event in enumerate(event_details): + normalized_event_project_key = ( + normalize_project_key(event.project_key) if event.project_key else None + ) + project_key = ( + normalized_event_project_key + if normalized_event_project_key in project_keys + else None + ) await conn.execute( - "insert into post_summary_event (post_id, event_ordinal, event_text) " - "values ($1, $2, $3)", + "insert into post_summary_event (post_id, event_ordinal, event_text, project_key) " + "values ($1, $2, $3, $4)", post_id, ordinal, - event_text, + event.event_text, + project_key, + ) + for ordinal, claim in enumerate(summary.five_w1h_evidence): + await conn.execute( + "insert into post_summary_five_w1h " + "(post_id, slot_code, value_ordinal, value_text, evidence_text) " + "values ($1, $2, $3, $4, $5)", + post_id, + claim.slot_code, + ordinal, + claim.value_text, + claim.evidence_text, ) # ADR 0009 / 0019 / 0027: resolve catalog identity before writing # the role row so fetch never reconstructs it by a non-unique name. @@ -299,6 +450,39 @@ async def _replace_summary_projection( post_id, cataloged_person_id, ) + role_names = {role.actor_name for role in summary.roles_and_responsibilities} + project_keys = { + normalize_project_key(project.canonical_name) + for project in summary.project_mentions + if normalize_project_key(project.canonical_name) + } + for ordinal, action in enumerate(summary.major_event_actions): + actor_names = (action.requester_actor_name, action.processor_actor_name) + if any(name is not None and name not in role_names for name in actor_names): + continue + normalized_action_project_key = ( + normalize_project_key(action.project_key) if action.project_key else None + ) + project_key = ( + normalized_action_project_key + if normalized_action_project_key in project_keys + else None + ) + await conn.execute( + """ + insert into post_summary_action + (post_id, action_ordinal, action_text, requester_actor_name, + processor_actor_name, evidence_text, project_key) + values ($1, $2, $3, $4, $5, $6, $7) + """, + post_id, + ordinal, + action.action_text, + action.requester_actor_name, + action.processor_actor_name, + action.evidence_text, + project_key, + ) await persist_edges_for_post(conn, post_id) diff --git a/backend/app/relation_verification_ingestion.py b/backend/app/relation_verification_ingestion.py index a72dae627..ad93729a4 100644 --- a/backend/app/relation_verification_ingestion.py +++ b/backend/app/relation_verification_ingestion.py @@ -8,6 +8,7 @@ from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass import asyncpg @@ -22,12 +23,75 @@ class VerifiedRelation: counterparty_entity_name: str verification_status_code: str verification_evidence_url: str | None + verification_evidence_post_id: str | None + + +async def _find_internal_evidence_post( + conn: asyncpg.Connection, + post_id: str, + organization_name: str, + relationship_label: str, + visible_corporate_entity_ids: Sequence[str], +) -> str | None: + """Find one authorized source post supporting the same relation context. + + The query searches normalized DOM/image text when it exists and falls back + to the source title/body. Public posts are always eligible; private posts + are restricted to the caller's affiliated corporate entities. The result + is evidence metadata only and never changes the external verification + status. + + Each term is matched against title, body, and content-unit text as three + separately indexed branches unioned together (ADR 0043's title/body + trigram indexes), rather than one `like` over a per-row concatenation -- + the concatenated form forces a sequential scan with a per-row string + build over the whole real-imported corpus (tens of seconds at + real-corpus scale; see the `report_leftover_pair`-class perf class of + bug), where the unioned form stays on indexed bitmap scans. + """ + row = await conn.fetchrow( + """ + with term1_matches as ( + (select post_id from source_post where lower(post_title) like '%' || lower($2) || '%') + union + (select post_id from source_post + where lower(left(source_post_search_text(post_body), 16384)) like '%' || lower($2) || '%') + union + (select post_id from post_content_unit where lower(unit_text) like '%' || lower($2) || '%') + ), + term2_matches as ( + (select post_id from source_post where lower(post_title) like '%' || lower($3) || '%') + union + (select post_id from source_post + where lower(left(source_post_search_text(post_body), 16384)) like '%' || lower($3) || '%') + union + (select post_id from post_content_unit where lower(unit_text) like '%' || lower($3) || '%') + ) + select candidate.post_id + from source_post candidate + join term1_matches t1 on t1.post_id = candidate.post_id + join term2_matches t2 on t2.post_id = candidate.post_id + where candidate.post_id <> $1 + and ( + candidate.visibility_code = 'public' + or candidate.corporate_entity_id::text = any($4::text[]) + ) + order by candidate.updated_at desc, candidate.post_id + limit 1 + """, + post_id, + organization_name, + relationship_label, + list(visible_corporate_entity_ids), + ) + return None if row is None else str(row["post_id"]) async def verify_post_relations( conn: asyncpg.Connection, client: RelationVerificationClient, post_id: str, + visible_corporate_entity_ids: Sequence[str] = (), ) -> list[VerifiedRelation]: """Verifies every counterparty row still `verify_pending` for this post and persists the result. Already-checked rows are left alone -- @@ -53,12 +117,20 @@ async def verify_post_relations( verified: list[VerifiedRelation] = [] for row in rows: + internal_evidence_post_id = await _find_internal_evidence_post( + conn, + post_id, + row["counterparty_entity_name"], + row["relationship_label"], + visible_corporate_entity_ids, + ) result = client.verify(row["counterparty_entity_name"], row["relationship_label"]) await conn.execute( """ update post_counterparty_entity set verification_status_code = $3, verification_evidence_url = $4, + verification_evidence_post_id = $5, verification_checked_at = now() where post_id = $1 and counterparty_entity_name = $2 """, @@ -66,12 +138,14 @@ async def verify_post_relations( row["counterparty_entity_name"], result.status_code, result.evidence_url, + internal_evidence_post_id, ) verified.append( VerifiedRelation( counterparty_entity_name=row["counterparty_entity_name"], verification_status_code=result.status_code, verification_evidence_url=result.evidence_url, + verification_evidence_post_id=internal_evidence_post_id, ) ) return verified diff --git a/backend/app/report_ingestion.py b/backend/app/report_ingestion.py index 9e204ef1b..50614b0ad 100644 --- a/backend/app/report_ingestion.py +++ b/backend/app/report_ingestion.py @@ -17,12 +17,14 @@ from lineageweave.post_evaluation import CRITERION_CODES, RUBRIC_VERSION from .knowledge_graph import labels_for_codes +from .post_eligibility import source_context_present_sql -GROUPING_KINDS = frozenset({"process_unit", "corporate_entity", "thread_group"}) +GROUPING_KINDS = frozenset({"process_unit", "corporate_entity", "thread_group", "team", "project"}) SHARED_METRIC_KIND = "shared_metric" SHARED_METRIC_KEY = "all" _WEEK_PERIOD = re.compile(r"^(\d{4})-W(\d{2})$") _MONTH_PERIOD = re.compile(r"^(\d{4})-(\d{2})$") +_SOURCE_CONTEXT_PRESENT_SQL = source_context_present_sql("p") def parse_period_code(period_code: str) -> tuple[str, int, int]: @@ -48,8 +50,14 @@ def grouping_value(kind: str, row: asyncpg.Record) -> str | None: value = row["process_unit_id"] elif kind == "corporate_entity": value = row["corporate_entity_id"] - else: + elif kind == "thread_group": value = row["thread_group_key"] + elif kind == "team": + value = row["team_id"] + elif kind == "project": + value = row["secondary_grouping_key"] + else: + return None if value is None: return None text = str(value).strip() @@ -59,6 +67,7 @@ def grouping_value(kind: str, row: asyncpg.Record) -> str | None: _EVAL_ROWS_WEEK = """ select e.post_id, e.criterion_code, e.response_category, p.process_unit_id, p.corporate_entity_id, p.thread_group_key, + p.secondary_grouping_key, p.visibility_code, p.post_title from post_evaluation_response e join source_post p on p.post_id = e.post_id @@ -68,9 +77,84 @@ def grouping_value(kind: str, row: asyncpg.Record) -> str | None: _EVAL_ROWS_MONTH = """ select e.post_id, e.criterion_code, e.response_category, p.process_unit_id, p.corporate_entity_id, p.thread_group_key, + p.secondary_grouping_key, + p.visibility_code, p.post_title + from post_evaluation_response e + join source_post p on p.post_id = e.post_id + where e.rubric_version = $1 + and to_char(p.created_at at time zone 'UTC', 'YYYY-MM') = $2 + """ +_EVAL_ROWS_TEAM_WEEK = """ + select e.post_id, e.criterion_code, e.response_category, + p.process_unit_id, p.corporate_entity_id, p.thread_group_key, + p.secondary_grouping_key, + p.visibility_code, p.post_title, team.team_id + from post_evaluation_response e + join source_post p on p.post_id = e.post_id + join post_team_mention mention on mention.post_id = p.post_id + join cataloged_team team on team.team_id = mention.team_id + where e.rubric_version = $1 + and to_char(p.created_at at time zone 'UTC', 'IYYY-"W"IW') = $2 + """ +_EVAL_ROWS_TEAM_MONTH = """ + select e.post_id, e.criterion_code, e.response_category, + p.process_unit_id, p.corporate_entity_id, p.thread_group_key, + p.secondary_grouping_key, + p.visibility_code, p.post_title, team.team_id + from post_evaluation_response e + join source_post p on p.post_id = e.post_id + join post_team_mention mention on mention.post_id = p.post_id + join cataloged_team team on team.team_id = mention.team_id + where e.rubric_version = $1 + and to_char(p.created_at at time zone 'UTC', 'YYYY-MM') = $2 + """ +_EVAL_ROWS_PROJECT_WEEK = """ + select e.post_id, e.criterion_code, e.response_category, + p.process_unit_id, p.corporate_entity_id, p.thread_group_key, + p.secondary_grouping_key, + p.visibility_code, p.post_title + from post_evaluation_response e + join source_post p on p.post_id = e.post_id + left join corporate_entity customer on customer.corporate_entity_id = p.corporate_entity_id + where e.rubric_version = $1 + and to_char(p.created_at at time zone 'UTC', 'IYYY-"W"IW') = $2 + and nullif(p.secondary_grouping_key, '') is not null + and replace(lower(coalesce(customer.entity_name, '')), ' ', '') not in + ('기타', '기타고객', '미등록', '미등록고객', 'unknown', 'unregistered', 'other') + union all + select e.post_id, e.criterion_code, e.response_category, + p.process_unit_id, p.corporate_entity_id, p.thread_group_key, + mention.project_key as secondary_grouping_key, p.visibility_code, p.post_title from post_evaluation_response e join source_post p on p.post_id = e.post_id + join post_project_mention mention on mention.post_id = p.post_id + and mention.confidence >= 0.7 + where e.rubric_version = $1 + and to_char(p.created_at at time zone 'UTC', 'IYYY-"W"IW') = $2 + """ +_EVAL_ROWS_PROJECT_MONTH = """ + select e.post_id, e.criterion_code, e.response_category, + p.process_unit_id, p.corporate_entity_id, p.thread_group_key, + p.secondary_grouping_key, + p.visibility_code, p.post_title + from post_evaluation_response e + join source_post p on p.post_id = e.post_id + left join corporate_entity customer on customer.corporate_entity_id = p.corporate_entity_id + where e.rubric_version = $1 + and to_char(p.created_at at time zone 'UTC', 'YYYY-MM') = $2 + and nullif(p.secondary_grouping_key, '') is not null + and replace(lower(coalesce(customer.entity_name, '')), ' ', '') not in + ('기타', '기타고객', '미등록', '미등록고객', 'unknown', 'unregistered', 'other') + union all + select e.post_id, e.criterion_code, e.response_category, + p.process_unit_id, p.corporate_entity_id, p.thread_group_key, + mention.project_key as secondary_grouping_key, + p.visibility_code, p.post_title + from post_evaluation_response e + join source_post p on p.post_id = e.post_id + join post_project_mention mention on mention.post_id = p.post_id + and mention.confidence >= 0.7 where e.rubric_version = $1 and to_char(p.created_at at time zone 'UTC', 'YYYY-MM') = $2 """ @@ -139,8 +223,16 @@ async def load_period_evaluation_rows( ) -> list[asyncpg.Record]: """Evaluation cells whose post falls in ``period_code``.""" kind, _, _ = parse_period_code(period_code) - query = _EVAL_ROWS_WEEK if kind == "week" else _EVAL_ROWS_MONTH - return await conn.fetch(query, RUBRIC_VERSION, period_code) + if grouping_kind == "team": + query = _EVAL_ROWS_TEAM_WEEK if kind == "week" else _EVAL_ROWS_TEAM_MONTH + elif grouping_kind == "project": + query = _EVAL_ROWS_PROJECT_WEEK if kind == "week" else _EVAL_ROWS_PROJECT_MONTH + else: + query = _EVAL_ROWS_WEEK if kind == "week" else _EVAL_ROWS_MONTH + # Safe SQL: query is selected only from immutable module constants; period values are bound. + return await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + query, RUBRIC_VERSION, period_code + ) async def load_shared_item_bank( @@ -152,7 +244,8 @@ async def load_shared_item_bank( header_sql = ( _SHARED_BANK_HEADER_WEEK if kind == "week" else _SHARED_BANK_HEADER_MONTH ) - header = await conn.fetchrow( + # Safe SQL: header_sql is selected only from immutable module constants; keys are bound. + header = await conn.fetchrow( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli header_sql, SHARED_METRIC_KIND, SHARED_METRIC_KEY, @@ -193,7 +286,8 @@ async def load_previous_group_mean( ) -> float | None: """Mean θ of the latest earlier period for this grouping key.""" kind, _, _ = parse_period_code(period_code) - header = await conn.fetchrow( + # Safe SQL: the period query is selected only from immutable module constants; keys are bound. + header = await conn.fetchrow( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli _PREVIOUS_MEAN_WEEK if kind == "week" else _PREVIOUS_MEAN_MONTH, grouping_kind, grouping_key, @@ -213,7 +307,8 @@ async def load_anchor_item_bank( ) -> tuple[ItemBank, float] | None: """Latest earlier period's item bank and mean θ, if one exists.""" kind, _, _ = parse_period_code(period_code) - header = await conn.fetchrow( + # Safe SQL: the period query is selected only from immutable module constants; keys are bound. + header = await conn.fetchrow( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli _ANCHOR_HEADER_WEEK if kind == "week" else _ANCHOR_HEADER_MONTH, grouping_kind, grouping_key, @@ -366,7 +461,11 @@ async def persist_period_report( def _groups_from_rows( kind: str, rows: list[asyncpg.Record] ) -> dict[str, tuple[list[str], list[tuple[str, str, int]]]]: - """Partition evaluation rows into FIPC groups for one grouping kind.""" + """Partition evaluation rows into FIPC groups for one grouping kind. + + Team rows come from ``post_team_mention``. A post may therefore occur in + more than one returned group without being duplicated inside one group. + """ by_group: dict[str, list[asyncpg.Record]] = defaultdict(list) for row in rows: key = grouping_value(kind, row) @@ -398,10 +497,10 @@ async def rebuild_period_reports( if grouping_kind not in GROUPING_KINDS: raise ValueError(f"unknown grouping_kind {grouping_kind!r}") parse_period_code(period_code) - rows = await load_period_evaluation_rows(conn, grouping_kind, period_code) item_bank = await load_shared_item_bank(conn, period_code) reports: list[PeriodReport] = [] - for kind in ("process_unit", "corporate_entity", "thread_group"): + for kind in ("process_unit", "corporate_entity", "thread_group", "team", "project"): + rows = await load_period_evaluation_rows(conn, kind, period_code) groups = _groups_from_rows(kind, rows) if not groups: continue @@ -448,10 +547,12 @@ async def fetch_period_reports( period_code, RUBRIC_VERSION, ) - members = await conn.fetch( - """ + # Safe SQL: the source-context expression is an immutable schema fragment; report keys are bound. + members = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select m.grouping_key, m.post_id, m.theta_eap, m.theta_sd, p.post_title, p.visibility_code, p.corporate_entity_id, + ({_SOURCE_CONTEXT_PRESENT_SQL}) as has_real_source_context, t.due_date as ticket_due_date, t.ticket_title, t.ticket_status_code from report_member_score m join source_post p on p.post_id = m.post_id @@ -496,11 +597,13 @@ async def fetch_period_reports( period_code, RUBRIC_VERSION, ) - leftover = await conn.fetch( - """ + # Safe SQL: the source-context expression is an immutable schema fragment; report keys are bound. + leftover = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select lp.grouping_key, lp.pair_kind, lp.post_id, lp.criterion_code, lp.leftover_distance, lp.leftover_residual, p.post_title, - p.visibility_code, p.corporate_entity_id + p.visibility_code, p.corporate_entity_id, + ({_SOURCE_CONTEXT_PRESENT_SQL}) as has_real_source_context from report_leftover_pair lp join source_post p on p.post_id = lp.post_id where lp.grouping_kind = $1 and lp.period_code = $2 and lp.rubric_version = $3 @@ -561,6 +664,7 @@ async def fetch_period_reports( "theta_sd": float(row["theta_sd"]), "visibility_code": row["visibility_code"], "corporate_entity_id": str(row["corporate_entity_id"]), + "has_real_source_context": bool(row["has_real_source_context"]), "ticket_due_date": ( None if row["ticket_due_date"] is None @@ -596,6 +700,7 @@ async def fetch_period_reports( "leftover_residual": float(row["leftover_residual"]), "visibility_code": row["visibility_code"], "corporate_entity_id": str(row["corporate_entity_id"]), + "has_real_source_context": bool(row["has_real_source_context"]), } for row in leftover_by_group.get(header["grouping_key"], []) ], @@ -623,9 +728,11 @@ async def list_period_report_summaries( grouping_kind, RUBRIC_VERSION, ) - members = await conn.fetch( - """ + # Safe SQL: the source-context expression is an immutable schema fragment; report keys are bound. + members = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select m.grouping_key, m.period_code, p.visibility_code, p.corporate_entity_id + , ({_SOURCE_CONTEXT_PRESENT_SQL}) as has_real_source_context from report_member_score m join source_post p on p.post_id = m.post_id where m.grouping_kind = $1 and m.rubric_version = $2 @@ -676,6 +783,7 @@ async def list_period_report_summaries( { "visibility_code": member["visibility_code"], "corporate_entity_id": str(member["corporate_entity_id"]), + "has_real_source_context": bool(member["has_real_source_context"]), } for member in members_by_key.get((row["grouping_key"], row["period_code"]), []) ], @@ -685,7 +793,7 @@ async def list_period_report_summaries( async def resolve_grouping_label(conn: asyncpg.Connection, grouping_kind: str, grouping_key: str) -> str: - """Human-readable name for a grouping key (process unit / corp / thread).""" + """Human-readable name for a process unit, corp, thread, team, or project key.""" if grouping_kind == "process_unit": row = await conn.fetchrow( "select process_unit_name from process_unit where process_unit_id::text = $1", @@ -700,6 +808,21 @@ async def resolve_grouping_label(conn: asyncpg.Connection, grouping_kind: str, g ) if row is not None: return str(row["entity_name"]) + elif grouping_kind == "team": + row = await conn.fetchrow( + "select team_name from cataloged_team where team_id::text = $1", + grouping_key, + ) + if row is not None: + return str(row["team_name"]) + elif grouping_kind == "project": + row = await conn.fetchrow( + "select project_name from post_project_mention " + "where project_key = $1 order by confidence desc, project_name limit 1", + grouping_key, + ) + if row is not None: + return str(row["project_name"]) return grouping_key @@ -707,7 +830,7 @@ async def fetch_period_comparison( conn: asyncpg.Connection, period_code: str, ) -> list[dict[str, Any]]: - """Every PU / corp / thread scored on the shared metric for one period.""" + """Every PU / corp / thread / team / project scored on the shared metric.""" parse_period_code(period_code) rows = await conn.fetch( """ @@ -721,9 +844,11 @@ async def fetch_period_comparison( RUBRIC_VERSION, list(GROUPING_KINDS), ) - members = await conn.fetch( - """ + # Safe SQL: the source-context expression is an immutable schema fragment; grouping filters are bound. + members = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + f""" select m.grouping_kind, m.grouping_key, p.visibility_code, p.corporate_entity_id + , ({_SOURCE_CONTEXT_PRESENT_SQL}) as has_real_source_context from report_member_score m join source_post p on p.post_id = m.post_id where m.period_code = $1 and m.rubric_version = $2 @@ -751,6 +876,7 @@ async def fetch_period_comparison( { "visibility_code": member["visibility_code"], "corporate_entity_id": str(member["corporate_entity_id"]), + "has_real_source_context": bool(member["has_real_source_context"]), } for member in members_by_key.get((row["grouping_kind"], row["grouping_key"]), []) ], diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 910a78480..438b4786a 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -23,11 +23,7 @@ from lineageweave.http_client import HttpClientError, get_json, post_form from lineageweave.knowledge_graph import knowledge_graph_edges_for_post -from lineageweave.tepp_client import TeppClient -from lineageweave.tepp_result import ( - accepted_tepp_seed_envelope, - tepp_accepted_evidence_sha256, -) +from lineageweave.post_summary import POST_SUMMARY_CONTRACT_VERSION _POSTGRES_ADMIN_DSN = os.environ.get( "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://lineageweave:lineageweave_dev_only@localhost:15432/lineageweave" @@ -50,11 +46,72 @@ _REVISION_MIGRATION = ( Path(__file__).resolve().parents[2] / "migrations" / "0024_source_post_revision.sql" ) +_POST_CONTENT_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0026_post_content_artifacts.sql" +) _TEPP_RESULT_MIGRATION = ( - Path(__file__).resolve().parents[2] / "migrations" / "0028_analysis_run_tepp_result.sql" + Path(__file__).resolve().parents[2] / "migrations" / "0027_analysis_run_tepp_result.sql" +) +_INTERNAL_RELATION_EVIDENCE_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0028_internal_relation_evidence.sql" +) +_PROJECT_GROUPING_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0030_report_project_grouping.sql" +) +_SEMANTIC_PROJECT_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0031_semantic_project_mentions.sql" +) +_SEMANTIC_SEARCH_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0032_semantic_search_trigram.sql" +) +_SOURCE_STATE_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0033_source_state_provenance.sql" +) +_SOURCE_CONTEXT_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0034_source_context_provenance.sql" +) +_NORMALIZED_BODY_SEARCH_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0036_normalized_body_search.sql" +) +_SOURCE_RECORD_IDENTITY_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0037_source_record_identity.sql" +) +_SOURCE_NAMED_HINTS_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0038_source_named_hints.sql" +) +_SOURCE_ORG_NAMED_HINTS_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0039_source_org_named_hints.sql" +) +_MEMBER_LOCALE_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0044_member_locale_preference.sql" +) +_IMAGE_REGION_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0045_post_content_image_regions.sql" +) +_POST_CONTENT_STRUCTURE_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0046_post_content_structure_evidence.sql" +) +_IMAGE_REGION_EMBEDDING_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0047_post_content_image_region_embeddings.sql" ) -_TEPP_ACCEPTED_MIGRATION = ( - Path(__file__).resolve().parents[2] / "migrations" / "0029_analysis_run_tepp_accepted.sql" +_SUMMARY_FIVE_W1H_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0048_post_summary_five_w1h.sql" +) +_POST_CONTENT_QUEUE_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0050_post_content_ingestion_queue.sql" +) +_MAJOR_EVENT_ACTION_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0100_major_event_action.sql" +) +_PROJECT_BOUND_ACTION_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0101_project_bound_major_event_action.sql" +) +_PROJECT_BOUND_EVENT_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0102_project_bound_summary_event.sql" ) @@ -144,8 +201,31 @@ def seeded_db(demo_analyst_token): cur.execute(_SNAPSHOT_MEMBER_MIGRATION.read_text()) cur.execute(_OUTBOX_MIGRATION.read_text()) cur.execute(_REVISION_MIGRATION.read_text()) + cur.execute(_POST_CONTENT_MIGRATION.read_text()) cur.execute(_TEPP_RESULT_MIGRATION.read_text()) - cur.execute(_TEPP_ACCEPTED_MIGRATION.read_text()) + cur.execute(_INTERNAL_RELATION_EVIDENCE_MIGRATION.read_text()) + cur.execute(_PROJECT_GROUPING_MIGRATION.read_text()) + cur.execute(_SEMANTIC_PROJECT_MIGRATION.read_text()) + cur.execute(_SEMANTIC_SEARCH_MIGRATION.read_text()) + cur.execute(_SOURCE_STATE_MIGRATION.read_text()) + cur.execute(_SOURCE_CONTEXT_MIGRATION.read_text()) + cur.execute(_NORMALIZED_BODY_SEARCH_MIGRATION.read_text()) + cur.execute(_SOURCE_RECORD_IDENTITY_MIGRATION.read_text()) + cur.execute(_SOURCE_NAMED_HINTS_MIGRATION.read_text()) + cur.execute(_SOURCE_ORG_NAMED_HINTS_MIGRATION.read_text()) + cur.execute( + (Path(__file__).resolve().parents[2] / "migrations" / "0040_post_summary_contract.sql") + .read_text() + ) + cur.execute(_MEMBER_LOCALE_MIGRATION.read_text()) + cur.execute(_IMAGE_REGION_MIGRATION.read_text()) + cur.execute(_POST_CONTENT_STRUCTURE_MIGRATION.read_text()) + cur.execute(_IMAGE_REGION_EMBEDDING_MIGRATION.read_text()) + cur.execute(_SUMMARY_FIVE_W1H_MIGRATION.read_text()) + cur.execute(_POST_CONTENT_QUEUE_MIGRATION.read_text()) + cur.execute(_MAJOR_EVENT_ACTION_MIGRATION.read_text()) + cur.execute(_PROJECT_BOUND_ACTION_MIGRATION.read_text()) + cur.execute(_PROJECT_BOUND_EVENT_MIGRATION.read_text()) cur.execute( "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values " "('corporate_entity_level', 'group', 'Group'), " @@ -197,12 +277,6 @@ def seeded_db(demo_analyst_token): (own_group_id,), ) own_corp_id = cur.fetchone()[0] - cur.execute( - "insert into corporate_entity (parent_entity_id, corporate_entity_code, entity_name, entity_level_code) " - "values (%s, 'TEST-PLANT', 'Test Plant', 'plant') returning corporate_entity_id", - (own_corp_id,), - ) - own_plant_id = cur.fetchone()[0] cur.execute( "insert into corporate_entity (corporate_entity_code, entity_name, entity_level_code) " "values ('OTHER-CORP', 'Other Corp', 'group') returning corporate_entity_id" @@ -473,7 +547,6 @@ def _insert_post( "public_post_id": public_post_id, "own_group_id": str(own_group_id), "own_corp_id": str(own_corp_id), - "own_plant_id": str(own_plant_id), "other_corp_id": str(other_corp_id), "own_private_post_id": own_private_post_id, "late_own_private_post_id": late_own_private_post_id, @@ -573,9 +646,6 @@ def test_analysis_runs_are_labeled_aggregates_and_hide_other_scopes( headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) assert hidden.status_code == 404 - assert "tepp_evidence_sha256" not in hidden.text - assert "accepted_run_id" not in hidden.text - assert "aggregate transport evidence" not in hidden.text unauthenticated = client.get("/api/analysis-runs") assert unauthenticated.status_code == 401 @@ -807,7 +877,7 @@ def test_start_analysis_run_recovers_the_a100_fork( '2026-02-15T00:00:00Z', '2026-02-15T00:05:00Z') returning analysis_source_snapshot_id """, - ("b" * 64,), + ("f" * 64,), ) tepp_snapshot_id = cur.fetchone()[0] cur.execute( @@ -822,7 +892,7 @@ def test_start_analysis_run_recovers_the_a100_fork( '2026-02-15T12:30:00Z') returning analysis_run_id """, - (tepp_snapshot_id, requester_id, "c" * 64, "d" * 40), + (tepp_snapshot_id, requester_id, "a" * 64, "b" * 40), ) tepp_run_id = str(cur.fetchone()[0]) cur.execute( @@ -1057,134 +1127,286 @@ def test_start_analysis_run_recovers_the_a100_fork( assert "Pricing renegotiation: revised quote sent" in children -def test_tepp_start_persists_published_accepted_evidence( - client, demo_analyst_token, seeded_db, monkeypatch -) -> None: - """A published accepted ack is stored as transport evidence, never Succeeded.""" - idempotency_key = "buyer-start-tepp-accepted" - monkeypatch.setattr( - "backend.app.main.configured_tepp_client", - lambda _url="": TeppClient( - transport=lambda _payload: accepted_tepp_seed_envelope( - idempotency_key=idempotency_key - ) - ), +def test_me_reflects_the_authenticated_account(client, demo_analyst_token) -> None: + response = client.get("/api/me", headers={"Authorization": f"Bearer {demo_analyst_token}"}) + assert response.status_code == 200 + body = response.json() + assert body["display_name"] == "Test Analyst" + assert "post_read" in body["permission_codes"] + assert any( + entity["entity_name"] == "Test Corp" for entity in body["corporate_entities"] ) + + +def test_customer_master_returns_authorized_catalog_contract(client, demo_analyst_token, seeded_db) -> None: admin_conn = psycopg2.connect(seeded_db["dsn"]) - admin_conn.autocommit = True try: with admin_conn.cursor() as cur: cur.execute( - "select requested_by_account_id from analysis_run " - "where analysis_run_id = %s", - (seeded_db["visible_run_id"],), + "update source_post set source_customer_code = %s, source_author_code = %s, source_author_name = %s where post_id = %s", + ("TEST-CUSTOMER-001", "TEST-AUTHOR-001", "Test Author", seeded_db["public_post_id"]), ) - requester_id = cur.fetchone()[0] + # SOURCE_POST_ELIGIBILITY_SQL treats a post with no source_* + # context as ineligible once any other post has real context + # (the demo-vs-real-data lifecycle rule). source_project_code + # isn't read by the customer/author hint queries, so setting + # it keeps this post eligible for relationship_network + # without adding a second source_customer_hints/ + # source_author_hints row to the exact-list assertions below. cur.execute( - """ - insert into analysis_source_snapshot - (snapshot_sha256, source_contract_version, - maximum_available_time, captured_at) - values (%s, 'source-contract-v1', - '2026-02-15T00:00:00Z', '2026-02-15T00:05:00Z') - returning analysis_source_snapshot_id - """, - ("8" * 64,), + "update source_post set source_project_code = %s where post_id = %s", + ("TEST-PROJECT-001", seeded_db["own_private_post_id"]), ) - snapshot_id = cur.fetchone()[0] cur.execute( - """ - insert into analysis_run - (analysis_source_snapshot_id, run_kind_code, idempotency_key, - requested_by_account_id, knowledge_cutoff, - configuration_schema_version, configuration_sha256, - code_revision_sha, requested_at) - values (%s, 'analysis_run_tepp', 'buyer-start-tepp-accepted', - %s, '2026-02-15T00:00:00Z', 'tepp-run-v1', %s, %s, - '2026-02-15T12:30:00Z') - returning analysis_run_id - """, - (snapshot_id, requester_id, "7" * 64, "6" * 40), + "insert into post_summary_result (post_id, korean_summary, summary_contract_version) " + "values (%s, %s, %s)", + (seeded_db["public_post_id"], "stored summary", POST_SUMMARY_CONTRACT_VERSION), ) - tepp_run_id = str(cur.fetchone()[0]) cur.execute( - """ - insert into analysis_run_scope - (analysis_run_id, scope_kind_code, corporate_entity_id) - values (%s, 'analysis_scope_corporate_entity', %s) - """, - (tepp_run_id, seeded_db["own_corp_id"]), + "insert into post_summary_role " + "(post_id, actor_name, responsibility, actor_type_code, affiliated_organization_name, cataloged_person_id) " + "values (%s, %s, %s, %s, %s, %s)", + ( + seeded_db["public_post_id"], + "Ada West", + "account lead", + "prov_person", + "Test Corp", + seeded_db["our_person_id"], + ), ) + # A real counterparty can hold more than one role over its + # lifetime -- one post classifies "Northridge Grid" as a + # customer, a different visible post classifies the same + # name as a competitor. relationship_network must surface + # both, not just the most recent/frequent one. cur.execute( - """ - insert into analysis_run_status_event - (analysis_run_id, status_ordinal, status_code, occurred_at) - values (%s, 1, 'analysis_status_pending', '2026-02-15T12:31:00Z') - """, - (tepp_run_id,), + "insert into post_counterparty_entity " + "(post_id, counterparty_entity_name, relationship_type_code, verification_status_code) " + "values (%s, 'Northridge Grid', 'rel_voc', 'verify_pending'), " + " (%s, 'Northridge Grid', 'rel_voco', 'verify_pending'), " + " (%s, 'Solo Role Corp', 'rel_vos', 'verify_pending')", + ( + seeded_db["public_post_id"], + seeded_db["own_private_post_id"], + seeded_db["public_post_id"], + ), ) + admin_conn.commit() finally: admin_conn.close() - - measured = client.post( - f"/api/analysis-runs/{tepp_run_id}/start", - headers={"Authorization": f"Bearer {demo_analyst_token}"}, - ) - assert measured.status_code == 200, measured.text - body = measured.json() - assert body["status_label"] == "Failed" - assert body["failure_code"] == "tepp_completed_result_unsupported" - assert body["tepp_evidence_kind"] == "aggregate transport evidence" - assert body["tepp_run_state"] == "accepted" - assert body["tepp_accepted_run_id"] == "demo-tepp-accepted-opaque" - assert body["tepp_completed_artifact_available"] is False - expected = tepp_accepted_evidence_sha256( - contract_version=1, - accepted_run_id="demo-tepp-accepted-opaque", - run_state="accepted", - idempotency_key=idempotency_key, - ) - assert body["tepp_evidence_sha256"] == expected - assert "tepp_affiliation_count" not in body - assert "theta" not in str(body).lower() - - listed = client.get( - "/api/analysis-runs", + response = client.get( + "/api/customer-master", headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) - assert listed.status_code == 200 - listed_run = next( - run for run in listed.json()["analysis_runs"] if run["analysis_run_id"] == tepp_run_id - ) - assert listed_run["status_label"] == "Failed" - assert listed_run["tepp_evidence_sha256"] == expected - assert "tepp_affiliation_count" not in listed_run - - -def test_me_reflects_the_authenticated_account(client, demo_analyst_token) -> None: - response = client.get("/api/me", headers={"Authorization": f"Bearer {demo_analyst_token}"}) assert response.status_code == 200 body = response.json() - assert body["display_name"] == "Test Analyst" - assert "post_read" in body["permission_codes"] + assert set(body) == { + "corporate_entities", "keymen", "source_customer_hints", "source_author_hints", + "relationship_network", + } + entity = next(item for item in body["corporate_entities"] if item["entity_name"] == "Test Corp") + assert { + "corporate_entity_id", "corporate_entity_code", "entity_name", + "entity_level_code", "entity_level_label", "parent_entity_id", + } <= set(entity) + # Live UI finding (2026-08-19): the corporate entity list rendered + # the raw entity_level_code ("company") instead of a human label -- + # confirm this is a real common_lookup_value label, not the code echoed back. + assert entity["entity_level_code"] == "company" + assert entity["entity_level_label"] not in ("", "company") + assert isinstance(body["keymen"], list) + ada_west = next(item for item in body["keymen"] if item["person_name"] == "Ada West") + assert ada_west["person_side_code"] == "our_side" + # Live UI finding (2026-08-19): the Customer Master Keymen list falls + # back to person_side_label, not the raw code, whenever + # last_known_job_title is null -- confirm the label is actually a + # human label from common_lookup_value, not the bare code repeated. + assert ada_west["person_side_label"] not in ("", "our_side") + + network = {row["counterparty_entity_name"]: row for row in body["relationship_network"]} + northridge = network["Northridge Grid"] + assert northridge["multi_role"] is True + assert {rel["relationship_type_code"] for rel in northridge["relationships"]} == {"rel_voc", "rel_voco"} + for rel in northridge["relationships"]: + assert rel["post_count"] == 1 + assert rel["relationship_label"] not in ("", rel["relationship_type_code"]) + solo = network["Solo Role Corp"] + assert solo["multi_role"] is False + assert [rel["relationship_type_code"] for rel in solo["relationships"]] == ["rel_vos"] + # Neither counterparty name matches a cataloged corporate_entity in + # this fixture -- resolution stays null rather than guessing. + assert northridge["corporate_entity_id"] is None + assert solo["corporate_entity_id"] is None + assert body["source_customer_hints"] == [ + { + "customer_code": "TEST-CUSTOMER-001", + "customer_name": None, + "post_count": 1, + "related_posts": [{ + "post_id": seeded_db["public_post_id"], + "post_title": "Public post", + }], + "resolution_status": "hint_only", + "hint_trust": "normal", + "provenance": "source_post.source_customer_code/source_post.source_customer_name", + } + ] + author_hint = body["source_author_hints"] + assert len(author_hint) == 1 + assert author_hint[0]["author_code"] == "TEST-AUTHOR-001" + assert author_hint[0]["author_name"] == "Test Author" + assert author_hint[0]["author_account_id"] + assert author_hint[0]["account_display_name"] == "Test Analyst" + assert author_hint[0]["keyman_hints"] == [ + { + "person_id": seeded_db["our_person_id"], + "person_name": "Ada West", + "person_side_code": "our_side", + "last_known_job_title": None, + "mention_count": 1, + "provenance": "post_person_mention.person_id|post_summary_role.cataloged_person_id/source_post.author_account_id", + } + ] + assert author_hint[0]["related_posts"] == [ + { + "post_id": seeded_db["public_post_id"], + "post_title": "Public post", + } + ] + assert author_hint[0]["resolution_status"] == "our_side_context_only" assert any( - entity["entity_name"] == "Test Corp" for entity in body["corporate_entities"] + affiliation["entity_name"] == "Test Corp" + for affiliation in author_hint[0]["account_affiliations"] + ) + assert "account_affiliation.corporate_entity_id" in author_hint[0]["provenance"] + + +def test_resolve_customer_hint_creates_and_links_a_corroborated_entity( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """A Customer Master hint (an opaque source_customer_code with no name) + must resolve to a real corporate_entity only once external search + corroborates the proposed name -- deterministic fake resolution/ + verification clients (not a real LLM or Searxng call) so this is + CI-stable; the point under test is the resolve-then-persist wiring. + """ + from lineageweave.relation_verification import STATUS_CORROBORATED, RelationVerificationResult + + _grant_post_admin(seeded_db["dsn"]) + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + # corporate_entity_id is NOT NULL: a bulk-imported real record + # defaults to whatever entity its author account is affiliated + # with, never to a null "unresolved" sentinel. own_private_post_id + # already sits at that exact default (its author's own + # account_affiliation row) -- the case this endpoint reclaims. + cur.execute( + "update source_post set source_customer_code = %s where post_id = %s", + ("HINT-CODE-001", seeded_db["own_private_post_id"]), + ) + + class _FakeResolutionClient: + available = True + + def resolve(self, hint_code: str, context_text: str) -> str | None: + assert hint_code == "HINT-CODE-001" + return "Northridge Grid" + + class _FakeVerificationClient: + available = True + + def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: + assert organization_name == "Northridge Grid" + return RelationVerificationResult( + status_code=STATUS_CORROBORATED, evidence_url="https://example.org/northridge" + ) + + monkeypatch.setattr( + "backend.app.main._customer_hint_resolution_client", lambda: _FakeResolutionClient() + ) + monkeypatch.setattr( + "backend.app.main._relation_verification_client", lambda: _FakeVerificationClient() + ) + + response = client.post( + "/api/customer-master/resolve-hint", + json={"hint_code": "HINT-CODE-001"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["entity_name"] == "Northridge Grid" + assert body["linked_post_count"] == 1 + + with admin_conn.cursor() as cur: + cur.execute( + "select entity_name, corporate_entity_code from corporate_entity where corporate_entity_id = %s", + (body["corporate_entity_id"],), + ) + entity_row = cur.fetchone() + assert entity_row == ("Northridge Grid", "HINT-HINT-CODE-001") + cur.execute( + "select corporate_entity_id from source_post where post_id = %s", + (seeded_db["own_private_post_id"],), + ) + assert str(cur.fetchone()[0]) == body["corporate_entity_id"] + finally: + admin_conn.close() + + +def test_resolve_customer_hint_requires_post_admin(client, demo_analyst_token, seeded_db) -> None: + response = client.post( + "/api/customer-master/resolve-hint", + json={"hint_code": "HINT-CODE-001"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) + assert response.status_code == 403 def test_post_list_includes_public_and_own_corp_but_excludes_other_corp(client, demo_analyst_token, seeded_db) -> None: response = client.get("/api/posts", headers={"Authorization": f"Bearer {demo_analyst_token}"}) assert response.status_code == 200 - titles = {post["post_title"] for post in response.json()} + payload = response.json() + titles = {post["post_title"] for post in payload["posts"]} assert titles == { "Public post", "Own-corp private post", "Late own-corp private post", "Edited own-corp private post", } - public = next(post for post in response.json() if post["post_title"] == "Public post") + public = next(post for post in payload["posts"] if post["post_title"] == "Public post") assert public["voc_type_label"] == "Voice of Customer" assert public["visibility_label"] == "Public" + assert {option["code"] for option in payload["voc_type_options"]} == {"voc"} + assert {option["code"] for option in payload["visibility_options"]} == {"public", "private"} + assert next(option for option in payload["visibility_options"] if option["code"] == "public")["label"] == "Public" + + +def test_post_list_supports_bounded_offset_pages(client, demo_analyst_token, seeded_db) -> None: + response = client.get( + "/api/posts?limit=1&offset=1", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + + assert response.status_code == 200, response.text + assert len(response.json()["posts"]) == 1 + assert response.json()["total_count"] == 4 + + title_sorted = client.get( + "/api/posts?limit=1&offset=0&sort=title", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert title_sorted.status_code == 200, title_sorted.text + assert title_sorted.json()["posts"][0]["post_title"] == "Edited own-corp private post" + + invalid_sort = client.get( + "/api/posts?sort=unsupported", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert invalid_sort.status_code == 422 def test_post_detail_uses_lookup_labels_not_raw_codes(client, demo_analyst_token, seeded_db) -> None: @@ -1200,6 +1422,61 @@ def test_post_detail_uses_lookup_labels_not_raw_codes(client, demo_analyst_token assert body["visibility_label"] == "Public" +def test_post_detail_exposes_explicit_and_semantic_project_evidence( + client, demo_analyst_token, seeded_db +) -> None: + conn = psycopg2.connect(seeded_db["dsn"]) + try: + with conn.cursor() as cur: + cur.execute( + "update source_post set source_project_code = %s where post_id = %s", + ("SOURCE-PROJECT-001", seeded_db["public_post_id"]), + ) + cur.execute( + """ + insert into post_project_mention + (post_id, project_key, project_name, evidence_text, confidence, + ontology_iri, extraction_method) + values (%s, %s, %s, %s, %s, %s, %s) + """, + ( + seeded_db["public_post_id"], + "semantic-project", + "Semantic project", + "project was described in the body", + 0.82, + "https://contextualwisdomlab.github.io/lineageweave/ontology#Project", + "contextual_orchestrator_semantic", + ), + ) + conn.commit() + finally: + conn.close() + response = client.get( + f"/api/posts/{seeded_db['public_post_id']}", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200 + evidence = response.json()["project_evidence"] + source = next(row for row in evidence if row["extraction_method"] == "source_field_hint") + semantic = next(row for row in evidence if row["extraction_method"] == "contextual_orchestrator_semantic") + assert source["resolution_status"] == "hint_only" + assert source["confidence"] is None + assert semantic["resolution_status"] == "semantic_candidate" + assert semantic["ontology_label"] == "Project" + + listed = client.get( + "/api/posts?search=semantic-project", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert listed.status_code == 200, listed.text + listed_post = next( + post for post in listed.json()["posts"] if post["post_id"] == seeded_db["public_post_id"] + ) + assert listed_post["project_evidence"][0]["project_name"] == "Semantic project" + assert listed_post["project_evidence"][0]["provenance"] == "post_project_mention.evidence_text" + + def test_post_detail_as_of_returns_the_cutoff_known_body( client, demo_analyst_token, seeded_db ) -> None: @@ -1249,8 +1526,13 @@ def test_persisted_summary_is_returned_without_an_llm(client, demo_analyst_token try: with admin_conn.cursor() as cur: cur.execute( - "insert into post_summary_result (post_id, korean_summary) values (%s, %s)", - (seeded_db["public_post_id"], "저장된 한국어 요약입니다."), + "insert into post_summary_result " + "(post_id, korean_summary, summary_contract_version) values (%s, %s, %s)", + ( + seeded_db["public_post_id"], + "저장된 한국어 요약입니다.", + POST_SUMMARY_CONTRACT_VERSION, + ), ) cur.execute( "insert into post_summary_event (post_id, event_ordinal, event_text) " @@ -1283,6 +1565,39 @@ def test_persisted_summary_is_returned_without_an_llm(client, demo_analyst_token assert role["ontology_label"] == "Role actor (person)" +def test_stale_summary_is_returned_labeled_when_orchestrator_is_unavailable( + client, demo_analyst_token, seeded_db +) -> None: + """A legacy saved summary preserves buyer continuity with an explicit label.""" + os.environ.pop("ORCHESTRATOR_BASE_URL", None) + os.environ.pop("ORCHESTRATOR_API_KEY", None) + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into post_summary_result " + "(post_id, korean_summary, summary_contract_version) values (%s, %s, %s)", + ( + seeded_db["public_post_id"], + "보관된 이전 계약 요약입니다.", + POST_SUMMARY_CONTRACT_VERSION - 1, + ), + ) + finally: + admin_conn.close() + + response = client.get( + f"/api/posts/{seeded_db['public_post_id']}/summary", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200 + body = response.json() + assert body["summary_status"] == "stale" + assert body["summary_contract_version"] == POST_SUMMARY_CONTRACT_VERSION - 1 + assert body["korean_summary"] == "보관된 이전 계약 요약입니다." + + def test_seed_demo_summary_surfaces_on_get_summary(client, demo_analyst_token, seeded_db) -> None: """The same helper `make seed` calls must produce a row GET summary returns -- even with the orchestrator unset. @@ -1673,6 +1988,10 @@ def test_own_corp_post_keymen_are_readable(client, demo_analyst_token, seeded_db "Northridge Grid", "Northridge Holdings", } + context = response.json()["source_author_context"] + assert context["account_display_name"] == "Test Analyst" + assert context["resolution_status"] == "our_side_context_only" + assert any(affiliation["entity_name"] == "Test Corp" for affiliation in context["account_affiliations"]) def test_other_corp_private_post_keymen_are_forbidden(client, demo_analyst_token, seeded_db) -> None: @@ -1706,122 +2025,6 @@ def test_affiliate_tree_walks_ancestors_and_keeps_unresolved_orgs(client, demo_a assert all(person["person_name"] == "Priya Nair" for node in unresolved for person in node["people"]) -def test_customer_group_tree_walks_authorized_ancestors_and_descendants( - client, demo_analyst_token, seeded_db -) -> None: - """Operators navigate Group → Company → Plant, not a flat corp list.""" - response = client.get( - "/api/customer-group-tree", - headers={"Authorization": f"Bearer {demo_analyst_token}"}, - ) - assert response.status_code == 200, response.text - trees = response.json()["trees"] - assert [node["entity_name"] for node in trees] == ["Test Group"] - assert trees[0]["entity_id"] == seeded_db["own_group_id"] - assert trees[0]["entity_level_label"] == "Group" - children = trees[0]["children"] - assert [child["entity_name"] for child in children] == ["Test Corp"] - assert children[0]["entity_id"] == seeded_db["own_corp_id"] - assert children[0]["entity_level_label"] == "Company" - plants = children[0]["children"] - assert [plant["entity_name"] for plant in plants] == ["Test Plant"] - assert plants[0]["entity_id"] == seeded_db["own_plant_id"] - assert plants[0]["entity_level_label"] == "Plant" - assert "Other Corp" not in str(trees) - - -def test_corroborate_abbreviations_requires_post_admin( - client, demo_analyst_token, seeded_db -) -> None: - response = client.post( - f"/api/posts/{seeded_db['own_private_post_id']}/corroborate-abbreviations", - headers={"Authorization": f"Bearer {demo_analyst_token}"}, - ) - assert response.status_code == 403 - - -def test_corroborate_abbreviations_is_unavailable_without_searxng( - client, demo_analyst_token, seeded_db -) -> None: - _grant_post_admin(seeded_db["dsn"]) - response = client.post( - f"/api/posts/{seeded_db['own_private_post_id']}/corroborate-abbreviations", - headers={"Authorization": f"Bearer {demo_analyst_token}"}, - ) - assert response.status_code == 503 - assert "SEARXNG_BASE_URL" in response.json()["detail"] - - -def test_corroborate_abbreviations_binds_a_unique_tree_node( - client, demo_analyst_token, seeded_db, monkeypatch -) -> None: - """Searxng must corroborate TC against Test Corp; a miss invents nothing.""" - from lineageweave.relation_verification import STATUS_CORROBORATED, RelationVerificationResult - - _grant_post_admin(seeded_db["dsn"]) - admin_conn = psycopg2.connect(seeded_db["dsn"]) - admin_conn.autocommit = True - try: - with admin_conn.cursor() as cur: - cur.execute( - "insert into person_affiliation (person_id, affiliated_organization_name) " - "values (%s, 'TC')", - (seeded_db["our_person_id"],), - ) - finally: - admin_conn.close() - - class _FakeVerificationClient: - available = True - - def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: - if organization_name == "Test Corp" and relationship_label == "TC": - return RelationVerificationResult( - STATUS_CORROBORATED, "https://example.test/test-corp-tc" - ) - return RelationVerificationResult("verify_uncorroborated", None) - - monkeypatch.setattr("backend.app.main._relation_verification_client", lambda: _FakeVerificationClient()) - response = client.post( - f"/api/posts/{seeded_db['own_private_post_id']}/corroborate-abbreviations", - headers={"Authorization": f"Bearer {demo_analyst_token}"}, - ) - assert response.status_code == 200, response.text - matches = {row["raw_organization_name"]: row for row in response.json()["matches"]} - assert matches["TC"]["corporate_entity_id"] == seeded_db["own_corp_id"] - assert matches["TC"]["verification_status_code"] == "verify_corroborated" - assert "Test Corp" not in matches - for unresolved_name in ("Northridge Grid", "Northridge Holdings"): - if unresolved_name in matches: - assert matches[unresolved_name]["corporate_entity_id"] is None - assert matches[unresolved_name]["verification_status_code"] == "verify_uncorroborated" - - cached = client.get( - f"/api/posts/{seeded_db['own_private_post_id']}/abbreviation-tree-matches", - headers={"Authorization": f"Bearer {demo_analyst_token}"}, - ) - assert cached.status_code == 200 - cached_matches = {row["raw_organization_name"]: row for row in cached.json()["matches"]} - assert cached_matches["TC"]["corporate_entity_id"] == seeded_db["own_corp_id"] - - tree = client.get( - "/api/customer-group-tree", - headers={"Authorization": f"Bearer {demo_analyst_token}"}, - ) - company = tree.json()["trees"][0]["children"][0] - assert [alias["raw_organization_name"] for alias in company["abbreviations"]] == ["TC"] - - -def test_other_corp_private_abbreviation_cross_check_is_forbidden( - client, demo_analyst_token, seeded_db -) -> None: - listed = client.get( - f"/api/posts/{seeded_db['other_private_post_id']}/abbreviation-tree-matches", - headers={"Authorization": f"Bearer {demo_analyst_token}"}, - ) - assert listed.status_code == 403 - - def test_other_corp_private_affiliate_tree_is_forbidden(client, demo_analyst_token, seeded_db) -> None: response = client.get( f"/api/posts/{seeded_db['other_private_post_id']}/affiliate-tree", @@ -1908,6 +2111,96 @@ def test_related_keymen_use_rwr_and_hide_invisible_posts(client, demo_analyst_to assert own_post["ontology_label"] == "Post" +def test_related_keymen_includes_chronological_role_history(client, demo_analyst_token, seeded_db) -> None: + """Feature request (2026-08-19): clicking a Keyman should show which + company they were affiliated with and how their responsibility + changed over time, not just the RWR-related node list. + """ + admin_conn = psycopg2.connect(seeded_db["dsn"]) + try: + with admin_conn.cursor() as cur: + # Two visible posts, given a known chronological order, each + # classifying a different role/organization for the same + # cataloged person -- simulating a real job change. + cur.execute( + "update source_post set created_at = %s where post_id = %s", + ("2026-01-01T00:00:00+00:00", seeded_db["own_private_post_id"]), + ) + cur.execute( + "update source_post set created_at = %s where post_id = %s", + ("2026-06-01T00:00:00+00:00", seeded_db["public_post_id"]), + ) + for post_id, summary in ( + (seeded_db["own_private_post_id"], "early summary"), + (seeded_db["public_post_id"], "later summary"), + ): + cur.execute( + "insert into post_summary_result (post_id, korean_summary, summary_contract_version) " + "values (%s, %s, %s)", + (post_id, summary, POST_SUMMARY_CONTRACT_VERSION), + ) + cur.execute( + "insert into post_summary_role " + "(post_id, actor_name, responsibility, actor_type_code, affiliated_organization_name, cataloged_person_id) " + "values (%s, %s, %s, %s, %s, %s)", + ( + seeded_db["own_private_post_id"], + "Ada West", + "junior account rep", + "prov_person", + "Northwind Labs", + seeded_db["our_person_id"], + ), + ) + cur.execute( + "insert into post_summary_role " + "(post_id, actor_name, responsibility, actor_type_code, affiliated_organization_name, cataloged_person_id) " + "values (%s, %s, %s, %s, %s, %s)", + ( + seeded_db["public_post_id"], + "Ada West", + "account lead", + "prov_person", + "Test Corp", + seeded_db["our_person_id"], + ), + ) + admin_conn.commit() + finally: + admin_conn.close() + + response = client.get( + f"/api/keymen/{seeded_db['our_person_id']}/related", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200 + body = response.json() + history = body["role_history"] + assert [row["post_id"] for row in history] == [ + seeded_db["own_private_post_id"], + seeded_db["public_post_id"], + ] + assert history[0]["responsibility"] == "junior account rep" + assert history[0]["affiliated_organization_name"] == "Northwind Labs" + assert history[1]["responsibility"] == "account lead" + assert history[1]["affiliated_organization_name"] == "Test Corp" + assert history[0]["created_at"] < history[1]["created_at"] + + +def test_related_keymen_role_history_is_empty_without_any_role_classification( + client, demo_analyst_token, seeded_db +) -> None: + """No post_summary_role rows for this person -- an empty history is + the correct, non-fabricated answer, not an error. + """ + response = client.get( + f"/api/keymen/{seeded_db['our_person_id']}/related", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200 + assert response.json()["role_history"] == [] + + def test_related_corporate_entity_uses_rwr_and_hides_invisible_posts( client, demo_analyst_token, seeded_db ) -> None: @@ -2005,10 +2298,122 @@ def test_extract_keymen_requires_post_admin(client, demo_analyst_token, seeded_d assert response.status_code == 403 +def test_extract_keymen_and_verify_relations_publish_activity_events( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """Live gap (2026-08-19): extract-keymen and verify-relations are real, + consequential write actions (an LLM call, an external-search call) but + never published anything to the post's activity feed -- only ticket + mutations did. An operator reviewing a post's history had no way to + see that Keymen extraction or relation verification ever ran on it. + """ + from lineageweave.keyman_extraction import OUR_SIDE, PersonMention + from lineageweave.relation_verification import STATUS_UNCORROBORATED, RelationVerificationResult + + _grant_post_admin(seeded_db["dsn"]) + + class _FakeKeymanClient: + available = True + + def extract(self, post_title: str, post_body: str) -> list[PersonMention]: + return [PersonMention(person_name="Kim Cheolsu", person_side_code=OUR_SIDE)] + + class _FakeRelationshipClient: + available = True + + def classify(self, post_title: str, post_body: str, organization_names: list[str]): + return [] + + class _FakeVerificationClient: + available = True + + def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: + return RelationVerificationResult(status_code=STATUS_UNCORROBORATED, evidence_url=None) + + monkeypatch.setattr("backend.app.main._keyman_extraction_client", lambda: _FakeKeymanClient()) + monkeypatch.setattr("backend.app.main._entity_relationship_client", lambda: _FakeRelationshipClient()) + monkeypatch.setattr("backend.app.main._relation_verification_client", lambda: _FakeVerificationClient()) + + post_id = seeded_db["own_private_post_id"] + extract_response = client.post( + f"/api/posts/{post_id}/extract-keymen", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert extract_response.status_code == 200, extract_response.text + + verify_response = client.post( + f"/api/posts/{post_id}/verify-relations", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert verify_response.status_code == 200, verify_response.text + + activity_response = client.get( + f"/api/posts/{post_id}/activity", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + events = activity_response.json()["events"] + event_types = [event["event_type"] for event in events] + # XREVRANGE returns newest first: verify-relations ran second. + assert event_types == ["relations_verified", "keymen_extracted"] + assert "1 mention" in events[1]["summary"] + + _ORCHESTRATOR_BASE_URL = os.environ.get("LINEAGEWEAVE_TEST_ORCHESTRATOR_BASE_URL") _ORCHESTRATOR_API_KEY = os.environ.get("LINEAGEWEAVE_TEST_ORCHESTRATOR_API_KEY") +def test_extract_keymen_never_classifies_an_org_named_only_by_our_side_mentions( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """Live bug (2026-08-19): an organization affiliated ONLY with an + our_side person (our own factory, our own affiliate) got fed into the + counterparty-relationship classifier the same as any external org -- + forced to pick from six codes that all assume an external + counterparty, it had no correct answer and landed on the closest + wrong one (observed live as "Partner"). Only a counterparty-side + mention's affiliated organizations may reach that classifier. + """ + from lineageweave.keyman_extraction import COUNTERPARTY, OUR_SIDE, PersonMention + + _grant_post_admin(seeded_db["dsn"]) + + class _FakeKeymanClient: + available = True + + def extract(self, post_title: str, post_body: str) -> list[PersonMention]: + return [ + PersonMention( + person_name="Kim Cheolsu", + person_side_code=OUR_SIDE, + affiliated_organization_names=("Our Own Factory",), + ), + PersonMention( + person_name="Lee Younghee", + person_side_code=COUNTERPARTY, + affiliated_organization_names=("Acme Corp",), + ), + ] + + classified_names: list[str] = [] + + class _FakeRelationshipClient: + available = True + + def classify(self, post_title: str, post_body: str, organization_names: list[str]): + classified_names.extend(organization_names) + return [] + + monkeypatch.setattr("backend.app.main._keyman_extraction_client", lambda: _FakeKeymanClient()) + monkeypatch.setattr("backend.app.main._entity_relationship_client", lambda: _FakeRelationshipClient()) + + response = client.post( + f"/api/posts/{seeded_db['own_private_post_id']}/extract-keymen", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + assert classified_names == ["Acme Corp"] + + def test_extract_keymen_does_not_merge_same_name_people_with_conflicting_titles( client, demo_analyst_token, seeded_db, monkeypatch ) -> None: @@ -2855,6 +3260,84 @@ def test_evaluation_is_empty_before_a_judge_run(client, demo_analyst_token, seed assert response.json()["responses"] == [] +def test_evaluate_publishes_an_activity_event(client, demo_analyst_token, seeded_db, monkeypatch) -> None: + """Live gap (2026-08-19): evaluate is a real, consequential write + action (an LLM-as-a-Judge call), same discipline as extract-keymen -- + it must publish to the post's activity feed too, not only those two. + """ + from backend.app.post_evaluation_ingestion import PersistedEvaluation + + _grant_post_admin(seeded_db["dsn"]) + + class _FakeEvaluationClient: + available = True + + async def _fake_ingest_post_evaluation(conn, client, post_id, post_title, post_body): + return [ + PersistedEvaluation( + criterion_code="specificity", + criterion_label="Specificity", + response_category=2, + rubric_version="v1", + ) + ] + + monkeypatch.setattr("backend.app.main._post_evaluation_client", lambda: _FakeEvaluationClient()) + monkeypatch.setattr("backend.app.main.ingest_post_evaluation", _fake_ingest_post_evaluation) + + post_id = seeded_db["own_private_post_id"] + response = client.post( + f"/api/posts/{post_id}/evaluate", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + + activity_response = client.get( + f"/api/posts/{post_id}/activity", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + events = activity_response.json()["events"] + assert events[0]["event_type"] == "post_evaluated" + assert "1 rubric criterion response" in events[0]["summary"] + + +def test_live_chat_answer_publishes_an_activity_event( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """Live gap (2026-08-20): a live (non-cached) chat answer is a real, + consequential LLM call, same discipline as extract-keymen/evaluate -- + it must publish to the post's activity feed too. A stored/seeded + answer (no live call made) must not. + """ + from lineageweave.post_chat import ChatAnswer + + _grant_post_admin(seeded_db["dsn"]) + + class _FakeChatClient: + available = True + + def answer(self, question: str, sources) -> ChatAnswer: + return ChatAnswer(answer_text="a live answer", cited_post_ids=()) + + monkeypatch.setattr("backend.app.main._post_chat_client", lambda: _FakeChatClient()) + + post_id = seeded_db["own_private_post_id"] + response = client.post( + f"/api/posts/{post_id}/chat", + json={"question": "What happened here that no seed already answers?"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + + activity_response = client.get( + f"/api/posts/{post_id}/activity", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + events = activity_response.json()["events"] + assert events[0]["event_type"] == "chat_answered" + assert "What happened here that no seed already answers?" in events[0]["summary"] + + def test_evaluate_is_unavailable_without_orchestrator(client, demo_analyst_token, seeded_db) -> None: os.environ.pop("ORCHESTRATOR_BASE_URL", None) os.environ.pop("ORCHESTRATOR_API_KEY", None) @@ -3022,6 +3505,8 @@ def test_post_lineage_surfaces_indirect_link_via_shared_keyman(client, demo_anal assert body["direct"] == [] indirect_ids = {post["post_id"] for post in body["indirect"]} assert indirect_ids == {seeded_db["public_post_id"]} + assert body["indirect"][0]["post_body_excerpt"] + assert "post_body_truncated" in body["indirect"][0] def test_other_corp_private_post_summary_is_forbidden(client, demo_analyst_token, seeded_db) -> None: @@ -3154,6 +3639,15 @@ def _insert_post(title, body): assert post_b in body_json["cited_post_ids"] cited_by_id = {row["post_id"]: row["post_title"] for row in body_json["cited_posts"]} assert cited_by_id[post_b] == "Bid follow-up" + cited_evidence = next(row for row in body_json["cited_post_evidence"] if row["post_id"] == post_b) + assert any( + fact["kind"] == "semantic_keyman" and "Shared Keyman" in fact["text"] + for fact in cited_evidence["facts"] + ) + assert all( + "ontology_iri" not in fact["text"] and "contextual_orchestrator" not in fact["text"] + for fact in cited_evidence["facts"] + ) def test_rebuild_lineage_requires_post_admin(client, demo_analyst_token) -> None: @@ -3530,6 +4024,48 @@ def test_calendar_hides_other_corp_private_commitments_and_sorts_by_due_date( assert "corporate_entity_id" not in commitments[0] +def test_calendar_keeps_real_ticket_when_demo_code_is_shared( + client, demo_analyst_token, seeded_db +) -> None: + """A shared DEMO code must filter pure seed tickets row by row.""" + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "update corporate_entity set corporate_entity_code = 'DEMO-SHARED' where corporate_entity_id = %s", + (seeded_db["own_corp_id"],), + ) + cur.execute( + "update source_post set source_author_code = null, source_company_code = null, " + "source_process_unit_code = null, source_sales_pool_code = null, " + "source_customer_code = null, source_project_code = null where post_id = %s", + (seeded_db["own_private_post_id"],), + ) + cur.execute( + "update source_post set source_author_code = 'REAL-AUTHOR', source_company_code = 'REAL-COMPANY' " + "where post_id = %s", + (seeded_db["public_post_id"],), + ) + cur.execute( + "insert into issue_ticket (post_id, ticket_status_code, ticket_title, due_date, commitment_summary) " + "values (%s, 'open', 'Synthetic commitment', '2026-01-01', 'seed')", + (seeded_db["own_private_post_id"],), + ) + cur.execute( + "insert into issue_ticket (post_id, ticket_status_code, ticket_title, due_date, commitment_summary) " + "values (%s, 'open', 'Real commitment', '2026-02-01', 'source-backed')", + (seeded_db["public_post_id"],), + ) + finally: + admin_conn.close() + + response = client.get("/api/calendar", headers={"Authorization": f"Bearer {demo_analyst_token}"}) + assert response.status_code == 200 + titles = [commitment["ticket_title"] for commitment in response.json()["commitments"]] + assert titles == ["Real commitment"] + + def test_calendar_excludes_closed_tickets_and_includes_manual_due_dates( client, demo_analyst_token, seeded_db ) -> None: @@ -4405,3 +4941,35 @@ def test_shared_metric_ranks_two_process_units(client, demo_analyst_token, seede assert [item["rank"] for item in selected] == [1, 2, 3] assert all(item["information"] > 0.0 for item in selected) assert {item["item_code"] for item in selected} == set(CRITERION_CODES) + + +def test_post_search_matches_source_record_key_and_one_character_typo( + client, demo_analyst_token, seeded_db +) -> None: + """The board searches preserved source identity, not only the internal UUID.""" + source_system = "synthetic-source" + source_key = "SYNTHETIC-SOURCE-REC-001" + conn = psycopg2.connect(seeded_db["dsn"]) + conn.autocommit = True + try: + with conn.cursor() as cur: + cur.execute( + "update source_post set source_system_code = %s, source_record_key = %s where post_id = %s", + (source_system, source_key, seeded_db["own_private_post_id"]), + ) + finally: + conn.close() + + headers = {"Authorization": f"Bearer {demo_analyst_token}"} + exact = client.get("/api/posts", params={"search": source_key}, headers=headers) + assert exact.status_code == 200, exact.text + exact_row = next( + post for post in exact.json()["posts"] if post["post_id"] == seeded_db["own_private_post_id"] + ) + assert exact_row["source_system_code"] == source_system + assert exact_row["source_record_key"] == source_key + + typo = source_key[:-1] + "2" + fuzzy = client.get("/api/posts", params={"search": typo}, headers=headers) + assert fuzzy.status_code == 200, fuzzy.text + assert any(post["post_id"] == seeded_db["own_private_post_id"] for post in fuzzy.json()["posts"]) diff --git a/backend/tests/test_auth_jwks.py b/backend/tests/test_auth_jwks.py new file mode 100644 index 000000000..709d2c16e --- /dev/null +++ b/backend/tests/test_auth_jwks.py @@ -0,0 +1,175 @@ +"""Fail-closed JWT verification regressions that do not need live OIDC.""" + +from __future__ import annotations + +import base64 +import json +from types import SimpleNamespace + +import pytest +from fastapi import HTTPException + +import backend.app.auth as auth + + +def _segment(value: dict) -> str: + raw = json.dumps(value, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + +def _unsigned_token(header: dict) -> str: + return f"{_segment(header)}.{_segment({'sub': 'subject'})}.{_segment({'test': 'signature'})}" + + +def test_signing_key_requires_nonempty_exact_kid(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(auth.RSAAlgorithm, "from_jwk", lambda value: ("key", value)) + jwks = { + "keys": [ + {"kid": "first", "kty": "RSA", "alg": "RS256", "n": "x", "e": "AQAB"}, + {"kid": "wanted", "kty": "RSA", "alg": "RS256", "n": "y", "e": "AQAB"}, + ] + } + + key = auth._signing_key_from_jwks( + jwks, _unsigned_token({"alg": "RS256", "kid": "wanted"}) + ) + assert key[0] == "key" + assert '"kid": "wanted"' in key[1] + + with pytest.raises(HTTPException) as missing: + auth._signing_key_from_jwks(jwks, _unsigned_token({"alg": "RS256"})) + assert missing.value.status_code == 401 + + with pytest.raises(HTTPException) as unknown: + auth._signing_key_from_jwks( + jwks, _unsigned_token({"alg": "RS256", "kid": "unknown"}) + ) + assert unknown.value.status_code == 401 + + +def test_signing_key_rejects_non_rs256_header(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(auth.RSAAlgorithm, "from_jwk", lambda value: ("key", value)) + jwks = {"keys": [{"kid": "wanted", "kty": "RSA", "n": "y", "e": "AQAB"}]} + + with pytest.raises(HTTPException) as error: + auth._signing_key_from_jwks( + jwks, _unsigned_token({"alg": "RS512", "kid": "wanted"}) + ) + + assert error.value.status_code == 401 + + +def test_signing_key_requires_rsa_jwk_and_verification_use( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(auth.RSAAlgorithm, "from_jwk", lambda value: ("key", value)) + token = _unsigned_token({"alg": "RS256", "kid": "wanted"}) + + rejected_keys = [ + {"kid": "wanted", "n": "x", "e": "AQAB"}, + {"kid": "wanted", "kty": "EC", "alg": "RS256"}, + {"kid": "wanted", "kty": "RSA", "alg": "RS512", "n": "x", "e": "AQAB"}, + {"kid": "wanted", "kty": "RSA", "use": "enc", "n": "x", "e": "AQAB"}, + {"kid": "wanted", "kty": "RSA", "key_ops": ["encrypt"], "n": "x", "e": "AQAB"}, + {"kid": "wanted", "kty": "RSA", "key_ops": "verify", "n": "x", "e": "AQAB"}, + ] + + for key in rejected_keys: + with pytest.raises(HTTPException) as error: + auth._signing_key_from_jwks({"keys": [key]}, token) + assert error.value.status_code == 401 + + accepted = { + "kid": "wanted", + "kty": "RSA", + "alg": "RS256", + "use": "sig", + "key_ops": ["verify"], + "n": "x", + "e": "AQAB", + } + assert auth._signing_key_from_jwks({"keys": [accepted]}, token)[0] == "key" + + +def test_signing_key_refreshes_jwks_once_for_rotated_kid( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(auth.RSAAlgorithm, "from_jwk", lambda value: ("key", value)) + calls: list[bool] = [] + old_jwks = { + "keys": [{"kid": "old", "kty": "RSA", "alg": "RS256", "n": "x", "e": "AQAB"}] + } + new_jwks = { + "keys": [{"kid": "new", "kty": "RSA", "alg": "RS256", "n": "y", "e": "AQAB"}] + } + + def fake_jwks(settings, *, force_refresh=False): + calls.append(force_refresh) + return new_jwks if force_refresh else old_jwks + + monkeypatch.setattr(auth, "_jwks", fake_jwks) + token = _unsigned_token({"alg": "RS256", "kid": "new"}) + + key = auth._signing_key(SimpleNamespace(), token) + + assert key[0] == "key" + assert calls == [False, True] + + +def test_signing_key_does_not_refresh_for_invalid_header( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[bool] = [] + + def fake_jwks(settings, *, force_refresh=False): + calls.append(force_refresh) + return {"keys": []} + + monkeypatch.setattr(auth, "_jwks", fake_jwks) + + with pytest.raises(HTTPException) as error: + auth._signing_key( + SimpleNamespace(), + _unsigned_token({"alg": "RS512", "kid": "unknown"}), + ) + + assert error.value.status_code == 401 + assert calls == [False] + + +def test_decode_requires_configured_resource_audience(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} + monkeypatch.setattr(auth, "_signing_key", lambda settings, token: "signing-key") + + def fake_decode(token, **kwargs): + captured.update(kwargs) + return {"sub": "subject-1"} + + monkeypatch.setattr(auth.jwt, "decode", fake_decode) + settings = SimpleNamespace( + oidc_issuer="https://id.example", + oidc_audience="https://lineage.example/api", + oidc_clock_skew_seconds=5, + ) + + claims = auth._decode_access_token("token", settings) + + assert claims["sub"] == "subject-1" + assert captured["issuer"] == "https://id.example" + assert captured["audience"] == "https://lineage.example/api" + assert captured["algorithms"] == ["RS256"] + assert "options" not in captured + + +def test_decode_rejects_missing_subject(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(auth, "_signing_key", lambda settings, token: "signing-key") + monkeypatch.setattr(auth.jwt, "decode", lambda *args, **kwargs: {}) + settings = SimpleNamespace( + oidc_issuer="https://id.example", + oidc_audience="lineageweave-api", + oidc_clock_skew_seconds=5, + ) + + with pytest.raises(HTTPException) as error: + auth._decode_access_token("token", settings) + assert error.value.status_code == 401 diff --git a/backend/tests/test_config.py b/backend/tests/test_config.py index ba5dc6882..b5f6e9a12 100644 --- a/backend/tests/test_config.py +++ b/backend/tests/test_config.py @@ -23,6 +23,19 @@ def test_frontend_origins_drop_blank_entries(monkeypatch) -> None: assert load_settings().frontend_origins == ["http://localhost:5173"] +def test_oidc_clock_skew_is_bounded(monkeypatch) -> None: + monkeypatch.setenv("OIDC_CLOCK_SKEW_SECONDS", "12") + assert load_settings().oidc_clock_skew_seconds == 12 + + monkeypatch.setenv("OIDC_CLOCK_SKEW_SECONDS", "61") + try: + load_settings() + except ValueError as exc: + assert "between 0 and 60" in str(exc) + else: + raise AssertionError("clock skew above the bound must be rejected") + + def test_tepp_transport_url_defaults_empty_and_is_not_a_score(monkeypatch) -> None: """Missing TEPP_TRANSPORT_URL keeps the channel dropped.""" monkeypatch.delenv("TEPP_TRANSPORT_URL", raising=False) @@ -31,6 +44,44 @@ def test_tepp_transport_url_defaults_empty_and_is_not_a_score(monkeypatch) -> No assert load_settings().tepp_transport_url == "https://tepp.example/v1/analysis-runs" +def test_keyverse_issuer_overrides_local_keycloak_and_uses_oidc_discovery(monkeypatch) -> None: + """Production Keyverse configuration is standard OIDC, not a local mock.""" + monkeypatch.setenv("KEYVERSE_ISSUER", "https://keyverse.example/tenant/acme") + monkeypatch.setenv("KEYVERSE_CLIENT_ID", "lineageweave-production") + monkeypatch.setenv("KEYVERSE_AUDIENCE", "lineageweave-api") + monkeypatch.delenv("KEYVERSE_DISCOVERY_URI", raising=False) + monkeypatch.delenv("KEYVERSE_JWKS_URI", raising=False) + + settings = load_settings() + + assert settings.oidc_issuer == "https://keyverse.example/tenant/acme" + assert settings.oidc_client_id == "lineageweave-production" + assert settings.oidc_discovery_uri == ( + "https://keyverse.example/tenant/acme/.well-known/openid-configuration" + ) + assert settings.oidc_jwks_uri_override == "" + + +def test_local_keycloak_discovery_uses_backend_reachable_base_url(monkeypatch) -> None: + """Compose discovery uses service DNS, not the browser's localhost issuer.""" + monkeypatch.delenv("KEYVERSE_ISSUER", raising=False) + monkeypatch.delenv("OIDC_ISSUER", raising=False) + monkeypatch.delenv("KEYVERSE_DISCOVERY_URI", raising=False) + monkeypatch.delenv("OIDC_DISCOVERY_URI", raising=False) + monkeypatch.setenv("KEYCLOAK_BASE_URL", "http://keycloak:8080") + monkeypatch.setenv("KEYCLOAK_ISSUER", "http://localhost:18080/realms/lineageweave-demo") + + settings = load_settings() + + assert settings.oidc_issuer == "http://localhost:18080/realms/lineageweave-demo" + assert settings.oidc_discovery_uri == ( + "http://keycloak:8080/realms/lineageweave-demo/.well-known/openid-configuration" + ) + assert settings.oidc_jwks_uri_override == ( + "http://keycloak:8080/realms/lineageweave-demo/protocol/openid-connect/certs" + ) + + def test_rankweave_disabled_defaults_off(monkeypatch) -> None: monkeypatch.delenv("RANKWEAVE_DISABLED", raising=False) assert load_settings().rankweave_disabled is False diff --git a/backend/tests/test_oidc_audience.py b/backend/tests/test_oidc_audience.py new file mode 100644 index 000000000..e78a834a8 --- /dev/null +++ b/backend/tests/test_oidc_audience.py @@ -0,0 +1,43 @@ +"""OIDC resource-audience configuration regressions.""" + +import pytest + +from backend.app.config import load_settings + + +def test_local_oidc_audience_defaults_to_backend_resource(monkeypatch) -> None: + monkeypatch.delenv("KEYVERSE_ISSUER", raising=False) + monkeypatch.delenv("OIDC_ISSUER", raising=False) + monkeypatch.delenv("KEYVERSE_AUDIENCE", raising=False) + monkeypatch.delenv("OIDC_AUDIENCE", raising=False) + assert load_settings().oidc_audience == "lineageweave-api" + + +def test_keyverse_audience_is_explicitly_configurable(monkeypatch) -> None: + monkeypatch.setenv("KEYVERSE_ISSUER", "https://keyverse.example/tenant/acme") + monkeypatch.setenv("KEYVERSE_CLIENT_ID", "lineageweave-browser") + monkeypatch.setenv("KEYVERSE_AUDIENCE", "https://lineage.example/api") + assert load_settings().oidc_audience == "https://lineage.example/api" + + +def test_external_oidc_requires_explicit_resource_audience(monkeypatch) -> None: + monkeypatch.delenv("KEYVERSE_ISSUER", raising=False) + monkeypatch.setenv("OIDC_ISSUER", "https://id.example") + monkeypatch.setenv("OIDC_CLIENT_ID", "lineageweave-browser") + monkeypatch.delenv("KEYVERSE_AUDIENCE", raising=False) + monkeypatch.delenv("OIDC_AUDIENCE", raising=False) + + with pytest.raises(ValueError, match="external OIDC requires"): + load_settings() + + +def test_generic_external_oidc_accepts_explicit_resource_audience(monkeypatch) -> None: + monkeypatch.delenv("KEYVERSE_ISSUER", raising=False) + monkeypatch.setenv("OIDC_ISSUER", "https://id.example") + monkeypatch.setenv("OIDC_CLIENT_ID", "lineageweave-browser") + monkeypatch.setenv("OIDC_AUDIENCE", "https://lineage.example/api") + + settings = load_settings() + + assert settings.oidc_client_id == "lineageweave-browser" + assert settings.oidc_audience == "https://lineage.example/api" diff --git a/docker-compose.yml b/docker-compose.yml index 5087366b4..96ec0b89a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,21 @@ services: timeout: 5s retries: 10 + database_migration: + build: + context: . + dockerfile: docker/postgres-init/Dockerfile + entrypoint: ["/usr/local/bin/lineageweave-migrate"] + environment: + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + POSTGRES_USER: ${POSTGRES_USER:-lineageweave} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-lineageweave_dev_only} + POSTGRES_DB: ${POSTGRES_DB:-lineageweave} + depends_on: + postgres: + condition: service_healthy + valkey: image: valkey/valkey:8-alpine@sha256:a038175878d66b9d274fbf8be73c0305e93798b83917647f167e18cef3c71eec ports: @@ -82,6 +97,35 @@ services: postgres: condition: service_healthy + orchestrator: + # Consume the paper-grounded orchestration service from main; inference + # remains behind its authenticated OpenAI-compatible boundary. + build: + context: ./docker/contextual-orchestrator + dockerfile: Dockerfile + env_file: + - ${HOME}/.env + environment: + AGENTS_FILE: /app/agents.json + PORT: 8000 + CONTEXTUAL_ORCHESTRATOR_TOKEN: ${CONTEXTUAL_ORCHESTRATOR_TOKEN:-${ORCHESTRATOR_API_KEY:-lineageweave-orchestrator-dev-only}} + # Gateway credentials and URL are supplied only by env_file (${HOME}/.env). + # Do not repeat them under environment:, where Compose interpolation can + # overwrite env_file values with an empty host-shell value. + # The upstream default remains 64 KiB for ordinary text APIs. Buyer + # image blocks are base64 data URIs, so the multimodal boundary gets an + # explicit bounded 8 MiB limit rather than an unbounded request size. + CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES: ${CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES:-8388608} + CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS: ${CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS:-host.docker.internal} + command: ["python", "/app/start.py"] + ports: + - "${ORCHESTRATOR_PORT:-18000}:8000" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=2)"] + interval: 5s + timeout: 3s + retries: 20 + backend: build: context: . @@ -95,21 +139,45 @@ services: KEYCLOAK_ISSUER: http://localhost:${KEYCLOAK_PORT:-18080}/realms/lineageweave-demo KEYCLOAK_REALM: lineageweave-demo KEYCLOAK_CLIENT_ID: lineageweave-frontend + # Production may set these to the real Keyverse OIDC provider. Empty + # values keep this stack on its explicit local Keycloak development mode; + # no Keyverse-shaped identity service is created by Compose. + KEYVERSE_ISSUER: ${KEYVERSE_ISSUER:-} + KEYVERSE_CLIENT_ID: ${KEYVERSE_CLIENT_ID:-} + KEYVERSE_AUDIENCE: ${KEYVERSE_AUDIENCE:-} + KEYVERSE_DISCOVERY_URI: ${KEYVERSE_DISCOVERY_URI:-} + KEYVERSE_JWKS_URI: ${KEYVERSE_JWKS_URI:-} + OIDC_ISSUER: ${OIDC_ISSUER:-} + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-} + OIDC_AUDIENCE: ${OIDC_AUDIENCE:-lineageweave-api} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-} + OIDC_JWKS_URI: ${OIDC_JWKS_URI:-} + OIDC_CLOCK_SKEW_SECONDS: ${OIDC_CLOCK_SKEW_SECONDS:-5} FRONTEND_ORIGINS: http://localhost:${FRONTEND_PORT:-15173} VALKEY_URL: redis://valkey:6379/0 # Empty by default: every LLM/vision channel stays the Null client # (dropped, not faked). Set these to a running contextual-orchestrator - # to turn the channels on -- NVIDIA_NIM_API_KEY lives on that - # orchestrator, never in this compose file. - ORCHESTRATOR_BASE_URL: ${ORCHESTRATOR_BASE_URL:-} - ORCHESTRATOR_API_KEY: ${ORCHESTRATOR_API_KEY:-} - VISION_MODEL: ${VISION_MODEL:-} + # to turn the channels on. Provider credentials use LLM_GATEWAY_API_URL / + # LLM_GATEWAY_API_KEY in the orchestrator's private env file. + ORCHESTRATOR_BASE_URL: ${ORCHESTRATOR_BASE_URL:-http://orchestrator:8000} + ORCHESTRATOR_API_KEY: ${ORCHESTRATOR_API_KEY:-${CONTEXTUAL_ORCHESTRATOR_TOKEN:-lineageweave-orchestrator-dev-only}} + # This is a non-secret embedding contract. Provider credentials remain + # private to contextual-orchestrator and are never injected into backend. + LLM_GATEWAY_EMBEDDING_MODEL: ${LLM_GATEWAY_EMBEDDING_MODEL:-text-embedding-3-large} SEARXNG_BASE_URL: http://searxng:8080 + TEPP_TRANSPORT_URL: ${TEPP_TRANSPORT_URL:-} + TEPP_API_KEY: ${TEPP_API_KEY:-} + CALDAV_BASE_URL: ${CALDAV_BASE_URL:-} + RANKWEAVE_DISABLED: ${RANKWEAVE_DISABLED:-} ports: - "${BACKEND_PORT:-18420}:8000" depends_on: postgres: condition: service_healthy + database_migration: + condition: service_completed_successfully + orchestrator: + condition: service_healthy keycloak: condition: service_started valkey: @@ -121,8 +189,8 @@ services: build: context: ./frontend args: - VITE_KEYCLOAK_ISSUER: http://localhost:${KEYCLOAK_PORT:-18080}/realms/lineageweave-demo - VITE_KEYCLOAK_CLIENT_ID: lineageweave-frontend + VITE_KEYVERSE_ISSUER: ${KEYVERSE_ISSUER:-http://localhost:${KEYCLOAK_PORT:-18080}/realms/lineageweave-demo} + VITE_KEYVERSE_CLIENT_ID: ${KEYVERSE_CLIENT_ID:-lineageweave-frontend} VITE_BACKEND_BASE_URL: http://localhost:${BACKEND_PORT:-18420} ports: - "${FRONTEND_PORT:-15173}:8080" diff --git a/docker/contextual-orchestrator/Dockerfile b/docker/contextual-orchestrator/Dockerfile new file mode 100644 index 000000000..f4ef8eab7 --- /dev/null +++ b/docker/contextual-orchestrator/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf + +WORKDIR /app + +# Reuse the upstream implementation without copying it into LineageWeave. +# Pin the runtime to a reviewed immutable upstream commit; model selection, +# structured synthesis, and reasoning policy stay in contextual-orchestrator. +ADD https://github.com/ContextualWisdomLab/contextual-orchestrator/archive/7df051ac2b929e5910071ac1848d0447c5d6744e.tar.gz /tmp/contextual-orchestrator.tar.gz +RUN mkdir /tmp/contextual-orchestrator \ + && tar -xzf /tmp/contextual-orchestrator.tar.gz --strip-components=1 -C /tmp/contextual-orchestrator \ + && cp -R /tmp/contextual-orchestrator/contextual_orchestrator /app/contextual_orchestrator \ + && cp -R /tmp/contextual-orchestrator/examples /app/examples \ + && rm -rf /tmp/contextual-orchestrator /tmp/contextual-orchestrator.tar.gz \ + && useradd --uid 10001 --no-create-home orchestrator + +COPY agents.json /app/agents.json +COPY start.py /app/start.py + +ENV AGENTS_FILE=/app/agents.json \ + PORT=8000 + +USER orchestrator +EXPOSE 8000 +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s \ + CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=2)"] +CMD ["python", "/app/start.py"] diff --git a/docker/contextual-orchestrator/agents.json b/docker/contextual-orchestrator/agents.json new file mode 100644 index 000000000..6eed030a5 --- /dev/null +++ b/docker/contextual-orchestrator/agents.json @@ -0,0 +1,20 @@ +{ + "agents": [ + { + "id": "multimodal_reasoning_agent", + "model": "", + "provider_protocol": "auto", + "base_url": "https://integrate.api.nvidia.com/v1", + "credential_key": "LLM_GATEWAY_API_KEY", + "tags": [ + "reasoning", + "writing", + "planning", + "verification", + "extraction", + "vision" + ], + "priority": 1 + } + ] +} diff --git a/docker/contextual-orchestrator/start.py b/docker/contextual-orchestrator/start.py new file mode 100644 index 000000000..e35c9a436 --- /dev/null +++ b/docker/contextual-orchestrator/start.py @@ -0,0 +1,115 @@ +"""Bootstrap the local NIM credential, then start contextual-orchestrator. + +The provider key is transport-only: it is registered in the orchestrator's +process-local credential store before the server starts and removed from the +process environment before request handling begins. +""" + +from __future__ import annotations + +import os +import sys +import json +from pathlib import Path + + +def _pop_first_env(*names: str) -> str: + """Read the first configured alias without leaving credentials in the environment.""" + for name in names: + value = os.environ.pop(name, "").strip() + if value: + return value + return "" + + +def main() -> None: + """Register the provider credential and delegate to the upstream server.""" + provider_key = _pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY", "NVIDIA_NIM_API_KEY") + if not provider_key: + raise SystemExit("LLM_GATEWAY_API_KEY or LLM_API_KEY is required to start the real LLM service") + auth_token = os.environ.get("CONTEXTUAL_ORCHESTRATOR_TOKEN", "").strip() + if not auth_token: + raise SystemExit("CONTEXTUAL_ORCHESTRATOR_TOKEN is required to start the authenticated LLM service") + + provider_url = _pop_first_env("LLM_GATEWAY_API_URL", "LLM_GATEWAY_URL", "LLM_API_GATEWAY") + if not provider_url: + raise SystemExit("LLM_GATEWAY_API_URL or LLM_GATEWAY_URL is required to start the gateway") + if not provider_url.rstrip("/").endswith("/v1"): + provider_url = provider_url.rstrip("/") + "/v1" + raw_limit = os.environ.pop("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "4096").strip() + try: + max_output_tokens = int(raw_limit) + except ValueError as exc: + raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be an integer") from exc + if not 64 <= max_output_tokens <= 4096: + raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be between 64 and 4096") + raw_body_limit = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES", str(8 * 1024 * 1024)).strip() + try: + max_body_bytes = int(raw_body_limit) + except ValueError as exc: + raise SystemExit("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES must be an integer") from exc + if not 64 * 1024 <= max_body_bytes <= 64 * 1024 * 1024: + raise SystemExit("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES must be between 65536 and 67108864") + agents_path = Path("/tmp/lineageweave-agents.json") + agents = json.loads(Path("/app/agents.json").read_text(encoding="utf-8")) + for agent in agents["agents"]: + agent["base_url"] = provider_url + agent["credential_key"] = "LLM_GATEWAY_API_KEY" + agent.setdefault("provider_protocol", "auto") + embedding_model = os.environ.get("LLM_GATEWAY_EMBEDDING_MODEL", "").strip() + if embedding_model: + embedding_agents = [ + agent + for agent in agents["agents"] + if "embedding" in agent.get("tags", []) + ] + if embedding_agents: + for agent in embedding_agents: + agent["model"] = embedding_model + else: + agents["agents"].append( + { + "id": "llm_gateway_embedding_agent", + "model": embedding_model, + "base_url": provider_url, + "credential_key": "LLM_GATEWAY_API_KEY", + "provider_protocol": "auto", + "tags": ["embedding"], + "priority": 0, + } + ) + agents_path.write_text(json.dumps(agents), encoding="utf-8") + + from contextual_orchestrator.credentials import register_credential + + register_credential("NVIDIA_NIM_API_KEY", provider_key) + register_credential("LLM_GATEWAY_API_KEY", provider_key) + del provider_key + sys.argv = [ + "contextual_orchestrator", + "--serve", + "--agents", + str(agents_path), + "--auto-discover-model-agents", + "--allow-discovery-failures", + "--host", + "0.0.0.0", + "--port", + "8000", + "--allow-public-bind", + "--auth-token", + auth_token, + "--max-output-tokens", + str(max_output_tokens), + "--max-body-bytes", + str(max_body_bytes), + ] + del provider_url + del auth_token + from contextual_orchestrator.__main__ import main as serve + + serve() + + +if __name__ == "__main__": + main() diff --git a/docker/keycloak/realm-export.json b/docker/keycloak/realm-export.json index 5e12e3fd3..be9826ea4 100644 --- a/docker/keycloak/realm-export.json +++ b/docker/keycloak/realm-export.json @@ -23,6 +23,16 @@ "redirectUris": ["http://localhost:5173/*", "http://localhost:15173/*"], "webOrigins": ["http://localhost:5173", "http://localhost:15173"], "protocolMappers": [ + { + "name": "lineageweave-api-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "config": { + "included.custom.audience": "lineageweave-api", + "id.token.claim": "false", + "access.token.claim": "true" + } + }, { "name": "corp-code", "protocol": "openid-connect", diff --git a/docker/postgres-init/Dockerfile b/docker/postgres-init/Dockerfile index ce2cf84df..7088497da 100644 --- a/docker/postgres-init/Dockerfile +++ b/docker/postgres-init/Dockerfile @@ -17,24 +17,42 @@ COPY migrations/0008_post_summary_result.sql /docker-entrypoint-initdb.d/09-post COPY migrations/0009_shared_metric_bank.sql /docker-entrypoint-initdb.d/10-shared-metric-bank.sql COPY migrations/0010_report_item_information.sql /docker-entrypoint-initdb.d/11-report-item-information.sql COPY migrations/0011_post_chat_result.sql /docker-entrypoint-initdb.d/12-post-chat-result.sql -COPY migrations/0012_role_responsibility_agent_type.sql /docker-entrypoint-initdb.d/13-role-responsibility-agent-type.sql -COPY migrations/0013_person_job_title.sql /docker-entrypoint-initdb.d/14-person-job-title.sql -COPY migrations/0014_role_responsibility_team_actor_type.sql /docker-entrypoint-initdb.d/15-role-responsibility-team-actor-type.sql -COPY migrations/0015_organization_name_resolution.sql /docker-entrypoint-initdb.d/16-organization-name-resolution.sql -COPY migrations/0016_cross_post_actor_identity.sql /docker-entrypoint-initdb.d/17-cross-post-actor-identity.sql -COPY migrations/0017_prov_o_standard_relations.sql /docker-entrypoint-initdb.d/18-prov-o-standard-relations.sql -COPY migrations/0018_analysis_run_registry.sql /docker-entrypoint-initdb.d/19-analysis-run-registry.sql -COPY migrations/0019_role_catalog_identity.sql /docker-entrypoint-initdb.d/20-role-catalog-identity.sql -COPY migrations/0020_analysis_run_retention_purge.sql /docker-entrypoint-initdb.d/21-analysis-run-retention-purge.sql -COPY migrations/0021_analysis_run_reconstruction.sql /docker-entrypoint-initdb.d/22-analysis-run-reconstruction.sql -COPY migrations/0022_analysis_source_snapshot_member.sql /docker-entrypoint-initdb.d/23-analysis-source-snapshot-member.sql -COPY migrations/0023_analysis_run_outbox.sql /docker-entrypoint-initdb.d/24-analysis-run-outbox.sql -COPY migrations/0024_source_post_revision.sql /docker-entrypoint-initdb.d/25-source-post-revision.sql -COPY migrations/0025_role_person_catalog_identity.sql /docker-entrypoint-initdb.d/26-role-person-catalog-identity.sql -COPY migrations/0026_report_leftover_pair.sql /docker-entrypoint-initdb.d/27-report-leftover-pair.sql -COPY migrations/0027_abbreviation_tree_corroboration.sql /docker-entrypoint-initdb.d/28-abbreviation-tree-corroboration.sql -COPY migrations/0028_analysis_run_tepp_result.sql /docker-entrypoint-initdb.d/29-analysis-run-tepp-result.sql -COPY migrations/0029_analysis_run_tepp_accepted.sql /docker-entrypoint-initdb.d/30-analysis-run-tepp-accepted.sql +COPY migrations/0012_report_leftover_pair.sql /docker-entrypoint-initdb.d/13-report-leftover-pair.sql +COPY migrations/0060_role_responsibility_agent_type.sql /docker-entrypoint-initdb.d/14-role-responsibility-agent-type.sql +COPY migrations/0013_person_job_title.sql /docker-entrypoint-initdb.d/15-person-job-title.sql +COPY migrations/0014_role_responsibility_team_actor_type.sql /docker-entrypoint-initdb.d/16-role-responsibility-team-actor-type.sql +COPY migrations/0015_organization_name_resolution.sql /docker-entrypoint-initdb.d/17-organization-name-resolution.sql +COPY migrations/0016_cross_post_actor_identity.sql /docker-entrypoint-initdb.d/18-cross-post-actor-identity.sql +COPY migrations/0017_prov_o_standard_relations.sql /docker-entrypoint-initdb.d/19-prov-o-standard-relations.sql +COPY migrations/0018_analysis_run_registry.sql /docker-entrypoint-initdb.d/20-analysis-run-registry.sql +COPY migrations/0019_role_catalog_identity.sql /docker-entrypoint-initdb.d/21-role-catalog-identity.sql +COPY migrations/0020_analysis_run_retention_purge.sql /docker-entrypoint-initdb.d/22-analysis-run-retention-purge.sql +COPY migrations/0021_analysis_run_reconstruction.sql /docker-entrypoint-initdb.d/23-analysis-run-reconstruction.sql +COPY migrations/0022_analysis_source_snapshot_member.sql /docker-entrypoint-initdb.d/24-analysis-source-snapshot-member.sql +COPY migrations/0023_analysis_run_outbox.sql /docker-entrypoint-initdb.d/25-analysis-run-outbox.sql +COPY migrations/0024_source_post_revision.sql /docker-entrypoint-initdb.d/26-source-post-revision.sql +COPY migrations/0025_role_person_catalog_identity.sql /docker-entrypoint-initdb.d/27-role-person-catalog-identity.sql +COPY migrations/0026_post_content_artifacts.sql /docker-entrypoint-initdb.d/28-post-content-artifacts.sql +COPY migrations/0027_analysis_run_tepp_result.sql /docker-entrypoint-initdb.d/29-analysis-run-tepp-result.sql +COPY migrations/0028_internal_relation_evidence.sql /docker-entrypoint-initdb.d/30-internal-relation-evidence.sql +COPY migrations/0029_report_team_grouping.sql /docker-entrypoint-initdb.d/31-report-team-grouping.sql +COPY migrations/0030_report_project_grouping.sql /docker-entrypoint-initdb.d/32-report-project-grouping.sql +COPY migrations/0031_semantic_project_mentions.sql /docker-entrypoint-initdb.d/33-semantic-project-mentions.sql +COPY migrations/0032_semantic_search_trigram.sql /docker-entrypoint-initdb.d/34-semantic-search-trigram.sql +COPY migrations/0033_source_state_provenance.sql /docker-entrypoint-initdb.d/35-source-state-provenance.sql +COPY migrations/0034_source_context_provenance.sql /docker-entrypoint-initdb.d/36-source-context-provenance.sql +COPY migrations/0035_body_search_prefix.sql /docker-entrypoint-initdb.d/37-body-search-prefix.sql +COPY migrations/0036_normalized_body_search.sql /docker-entrypoint-initdb.d/38-normalized-body-search.sql +COPY migrations/0037_source_record_identity.sql /docker-entrypoint-initdb.d/39-source-record-identity.sql +COPY migrations/0038_source_named_hints.sql /docker-entrypoint-initdb.d/40-source-named-hints.sql +COPY migrations/0039_source_org_named_hints.sql /docker-entrypoint-initdb.d/41-source-org-named-hints.sql +COPY migrations/0040_post_summary_contract.sql /docker-entrypoint-initdb.d/42-post-summary-contract.sql + +COPY migrations/ /opt/lineageweave/migrations/ +COPY docker/postgres-init/migrate.sh /usr/local/bin/lineageweave-migrate +USER root +RUN chmod 0755 /usr/local/bin/lineageweave-migrate + # Official image already drops to this account at runtime; declare it so # the Dockerfile itself satisfies DS-0002 (explicit non-root USER). USER postgres diff --git a/docker/postgres-init/migrate.sh b/docker/postgres-init/migrate.sh new file mode 100644 index 000000000..f329117d6 --- /dev/null +++ b/docker/postgres-init/migrate.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu + +: "${POSTGRES_HOST:=postgres}" +: "${POSTGRES_PORT:=5432}" +: "${POSTGRES_USER:?POSTGRES_USER is required}" +: "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}" +: "${POSTGRES_DB:?POSTGRES_DB is required}" +export PGPASSWORD="$POSTGRES_PASSWORD" + +until pg_isready -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; do + sleep 1 +done + +# ponytail: gate at the existing 0012 boundary; replace with a migration ledger +# when a new non-idempotent migration family is introduced. +for migration in /opt/lineageweave/migrations/*.sql; do + migration_name=${migration##*/} + case "$migration_name" in + 0012_*|0013_*|0014_*|0015_*|0016_*|0017_*|0018_*|0019_*|0020_*|0021_*|0022_*|0023_*|0024_*|0025_*|0026_*|0027_*|0028_*|0029_*|0030_*|0031_*|0032_*|0033_*|0034_*|0035_*|0036_*|0037_*|0038_*|0039_*|0040_*|0041_*|0042_*|0043_*|0044_*|0045_*|0046_*|0047_*|0048_*|0049_*|0050_*) ;; + 0060_*|0100_*|0101_*|0102_*) ;; + *) continue ;; + esac + printf 'Applying %s\n' "$migration_name" + psql -X -v ON_ERROR_STOP=1 \ + -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" \ + -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ + -f "$migration" +done diff --git a/docs/PROV_O_IMPLEMENTATION.md b/docs/PROV_O_IMPLEMENTATION.md index 96b471e96..c10e9ebb9 100644 --- a/docs/PROV_O_IMPLEMENTATION.md +++ b/docs/PROV_O_IMPLEMENTATION.md @@ -1,5 +1,8 @@ # W3C PROV-O implementation +> Normative decision: [ADR 0065](adr/0065-prov-o-provenance-boundary.md). +> This document is the implementation contract and acceptance evidence. + ## Requirement LineageWeave must accept, validate, persist, infer, and serialize every normative relation in *PROV-O: The PROV Ontology* without flattening qualified influences or literal-valued properties into the existing navigation graph. diff --git a/docs/PROV_O_IMPLEMENTATION_MATRIX.md b/docs/PROV_O_IMPLEMENTATION_MATRIX.md index 8a3c28617..1dd1d75af 100644 --- a/docs/PROV_O_IMPLEMENTATION_MATRIX.md +++ b/docs/PROV_O_IMPLEMENTATION_MATRIX.md @@ -1,5 +1,8 @@ # PROV-O implementation matrix +> Normative decision: [ADR 0065](adr/0065-prov-o-provenance-boundary.md). +> This file remains the machine-reviewable coverage matrix. + LineageWeave implements the W3C PROV-O Recommendation as a separate standards-complete provenance layer. The product-specific `knowledge_graph_edge` remains a compact navigation projection; it is not used to flatten literal-valued or qualified PROV-O assertions. ## Coverage contract diff --git a/docs/adr/0002-figma-access-boundary.md b/docs/adr/0002-figma-access-boundary.md index e95707453..90c213390 100644 --- a/docs/adr/0002-figma-access-boundary.md +++ b/docs/adr/0002-figma-access-boundary.md @@ -2,6 +2,8 @@ **Decision status:** Accepted **Date:** 2026-08-13 +**Figma File ID:** `1Su3lDRmiZdcUs47t1QwIX` +**Figma File URL:** https://www.figma.com/design/1Su3lDRmiZdcUs47t1QwIX ## Context @@ -38,6 +40,10 @@ statistic, or internal identifier observed while checking the file's metadata is repeated anywhere in this repository, in code, in docs, or in commit history. +The newly created file identified above is the safe design-system boundary +for LineageWeave's buyer surface. It currently contains no copied source +organization content; future token or component work must keep that boundary. + ## Rationale - This repository's oldest and most consistently enforced rule (see diff --git a/docs/adr/0003-fast-mlsirm-report-integration.md b/docs/adr/0003-fast-mlsirm-report-integration.md index e31436b8b..bdf234b65 100644 --- a/docs/adr/0003-fast-mlsirm-report-integration.md +++ b/docs/adr/0003-fast-mlsirm-report-integration.md @@ -100,7 +100,7 @@ than one large PR: `information_polytomous` (Lord, 1980 max-info). Persist the ranking (`report_item_information`) and show the rank-1 item on the Period reports panel. Do not reimplement an information function here. -7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0028 / 0029): after +7. **Leftover-pair slice** (shipped in 0.71.2; ADR 0017 / 0018): after IRT main effects, persist closest and farthest post–criterion pairs from the residual leftover map. Do not fork LSIRM; do not invent a leftover-pair API inside `fast-mlsirm` in this slice. diff --git a/docs/adr/0004-knowledge-graph-ontology.md b/docs/adr/0004-knowledge-graph-ontology.md index 3c7dcfef2..5d0e987d8 100644 --- a/docs/adr/0004-knowledge-graph-ontology.md +++ b/docs/adr/0004-knowledge-graph-ontology.md @@ -57,8 +57,9 @@ prose: `person_side_code`. Issue tickets stay a separate table (`issue_ticket`), not a knowledge-graph node type. - **Object properties** (`owl:ObjectProperty`, each with - `rdfs:domain`/`rdfs:range`): `mentions` (Post -> Person, from - `edge_mention`), `affiliatedWith` (Person -> CorporateEntity, from + `rdfs:domain`/`rdfs:range`): `mentionedIn` (Person -> Post, the + canonical direction stored by `edge_mention`; `mentions` is its + declared RDF inverse), `affiliatedWith` (Person -> CorporateEntity, from `edge_affiliation`), `coMentionedWith` (symmetric, Person <-> Person, from `edge_co_mention`), and one object property per entity- relationship-type code (`hasVocRelationship`, `hasVomRelationship`, diff --git a/docs/adr/0006-role-responsibility-agent-ontology.md b/docs/adr/0006-role-responsibility-agent-ontology.md index 8ded02b81..55b16a8b1 100644 --- a/docs/adr/0006-role-responsibility-agent-ontology.md +++ b/docs/adr/0006-role-responsibility-agent-ontology.md @@ -62,7 +62,7 @@ resolve to a Keyman row. Persistence: `post_summary_role` gains `actor_type_code` (FK to `common_lookup_value`, default `prov_person`) and `affiliated_organization_name`; `person_name` is renamed to -`actor_name` via `migrations/0012_role_responsibility_agent_type.sql`'s +`actor_name` via `migrations/0060_role_responsibility_agent_type.sql`'s `ALTER TABLE ... RENAME COLUMN` (preserves every existing row's data, unlike a drop/recreate) plus the two new `ADD COLUMN IF NOT EXISTS` statements, with `migrations/0001_initial_schema.sql` updated directly diff --git a/docs/adr/0009-cross-post-actor-identity.md b/docs/adr/0009-cross-post-actor-identity.md index 1a970c0fb..d989824f5 100644 --- a/docs/adr/0009-cross-post-actor-identity.md +++ b/docs/adr/0009-cross-post-actor-identity.md @@ -83,10 +83,11 @@ ADR 0007's `:RoleActorTeam`, but a distinct term -- `:Team` is a per-row `actor_type_code` classification, the same `:Person`/`:RoleActorPerson` split ADR 0006 already established). `:mentionsTeam` / `:teamAffiliatedWith` / `:mentionsOrganization` are -new, distinct object properties rather than widening `:mentions`'s -domain/range -- stating `rdfs:domain :mentions` twice (once `:Person`, -once `:Team`) would let RDFS entail every `:mentions` subject is BOTH, -which is false. +new, distinct object properties rather than widening `:mentionedIn`'s +domain/range -- stating `rdfs:domain :mentionedIn` twice (once `:Person`, +once `:Team`) would let RDFS entail every `:mentionedIn` subject is BOTH, +which is false. The natural-language `:mentions` property is retained as +the declared inverse of canonical `:mentionedIn` for RDF consumers. ## Consequences diff --git a/docs/adr/0013-normalized-analysis-run-registry.md b/docs/adr/0013-normalized-analysis-run-registry.md index 1c3567666..15fd040d6 100644 --- a/docs/adr/0013-normalized-analysis-run-registry.md +++ b/docs/adr/0013-normalized-analysis-run-registry.md @@ -252,13 +252,10 @@ Acceptance requires: 4. Add TEPP and contextual-orchestrator adapters only after their versioned contracts are present on reviewed main branches. Seed and `POST /api/analysis-runs/{id}/start` now record Failed TEPP through - `tepp_client` on the frozen snapshot when the transport is missing - or the envelope is unpublished. A published accepted acknowledgement - is stored as aggregate transport evidence and stays Failed / - `tepp_completed_result_unsupported` (ADR 0035). A missing or - unpublished TEPP envelope must stay Failed (`tepp_not_available` / - `tepp_result_not_persisted`) and must not write a local psychometric - substitute or stamp Succeeded. Seed also records a + `tepp_client` on the frozen snapshot; a persistable measurement + remains a later slice. A missing or unused TEPP envelope must stay + Failed (`tepp_not_available` / `tepp_result_not_persisted`) and must + not write a local psychometric substitute. Seed also records a Succeeded `analysis_run_report` on that snapshot after the period-report tables are written (ADR 0024); the registry row does not copy a theta. diff --git a/docs/adr/0014-authorized-analysis-run-read.md b/docs/adr/0014-authorized-analysis-run-read.md index b8df6eef8..10e99d37d 100644 --- a/docs/adr/0014-authorized-analysis-run-read.md +++ b/docs/adr/0014-authorized-analysis-run-read.md @@ -38,39 +38,24 @@ LineageWeave owns a fail-closed read projection of the #89 registry: ## Consequences -`make seed` writes one synthetic Demo Corp lineage run, one Failed -missing-transport TEPP run, one Failed accepted-evidence TEPP run, and -one Succeeded period-report run on the same snapshot so the existing -React home page can show all three kinds without a second application -(ADR 0024 / ADR 0035). The missing-transport TEPP run is +`make seed` writes one synthetic Demo Corp lineage run, one TEPP +run, and one Succeeded period-report run on the same snapshot so the +existing React home page can show all three kinds without a second +application (ADR 0024). The TEPP run is Failed / `tepp_not_available` when the default transport is missing -- the list keeps that machine code off the caption (this decision) and instead tells the operator to open the TEPP run, then connect the measurement -service. The accepted-evidence TEPP row tells the operator to read -aggregate transport evidence and that completed measurement identity -is unavailable. A failed lineage row tells the operator to retry +service. A failed lineage row tells the operator to retry reconstruction, not to connect TEPP. A failed period-report row tells the operator to rebuild the report from a current snapshot. A pending or running TEPP row must not claim a calibrated -measurement and must not say reconstruction. The list -button accessible name is `Open analysis run: {caption}. {nextAction}` -when a next action exists (WCAG 2.2 SC 4.1.2); otherwise the caption -alone. `aria-label` replaces button contents (W3C Accessible Name and -Description Computation 1.1), so the next-action sentence must live -in that name. Detail repeats that sentence. A pending -lineage row says reconstruction has not started yet. The detail now shows the legal +measurement. A pending lineage row says reconstruction has not +started yet. The detail now shows the legal lifecycle the registry already stored. `POST /api/analysis-runs` now records a Pending lineage run on an authorized cutoff capture -(ADR 0017). TEPP and period-report kinds are 422. Reconstruction and -TEPP accepted transport evidence are ADR 0021 / ADR 0035. A fuller Analysis -Run Console remains a later slice. A 404 on a hidden run (including a thread-group row that -still lacks an in-cutoff visible post, ADR 0018) must stay generic: -do not name the thread or the cutoff, and do not say the run is not -visible. Tell the operator to open a visible run from the home list, -or request a lineage reconstruction for a corporation they already -walk. After that 404, re-read `GET /api/analysis-runs` so the stale -list row does not stay clickable, and announce the status with -`role="alert"` (WCAG 2.2 SC 4.1.3) without moving focus. +(ADR 0017). TEPP and period-report kinds are 422. Reconstruction, a +live TEPP transport, and a fuller Analysis Run Console remain later +slices. ## References @@ -82,11 +67,3 @@ Educational Research Association. Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV ontology* (W3C Recommendation). World Wide Web Consortium. https://www.w3.org/TR/2013/REC-prov-o-20130430/ - -World Wide Web Consortium. (2018). *Accessible name and description -computation 1.1* (W3C Recommendation). -https://www.w3.org/TR/accname-1.1/ - -World Wide Web Consortium. (2023). *Web content accessibility -guidelines (WCAG) 2.2* (W3C Recommendation). -https://www.w3.org/TR/WCAG22/ diff --git a/docs/adr/0016-analysis-run-knowledge-cutoff-posts.md b/docs/adr/0016-analysis-run-knowledge-cutoff-posts.md index 553d549b5..373c783ac 100644 --- a/docs/adr/0016-analysis-run-knowledge-cutoff-posts.md +++ b/docs/adr/0016-analysis-run-knowledge-cutoff-posts.md @@ -45,9 +45,8 @@ run. ## Consequences - After `make seed`, the Demo Corp lineage run lists Demo public post - and other in-cutoff Demo Corp titles. Late Demo public post - (2026-01-13) and the later fixture account-review post (2026-02-10) - do not appear. The live post list still shows Late Demo. + and other in-cutoff Demo Corp titles. The later fixture account-review + post (2026-02-10) does not appear. - Open the run: Demo public post is marked updated after cutoff (`updated_at` 2026-01-13). Demo private post is not. - Open a marked title: the popup shows **Body this run knew** from @@ -63,13 +62,6 @@ run. (ADR 0018). A later public post cannot surface a previously hidden thread-group run. -## Follow-up — v2.12.3 Late Demo own-corp counter-example - -v2.12.3 seeds Late Demo public post on 2026-01-13 so the January 12 -Demo Corp lineage and TEPP runs can prove the existing -`created_at <= knowledge_cutoff` filter. This is not a second cutoff -and does not change TEPP honesty. - ## References International Organization for Standardization. (2019). *ISO 8601-1:2019: diff --git a/docs/adr/0018-related-nodes-team-org-walk.md b/docs/adr/0018-related-nodes-team-org-walk.md index c7f020eb0..ae0a1c331 100644 --- a/docs/adr/0018-related-nodes-team-org-walk.md +++ b/docs/adr/0018-related-nodes-team-org-walk.md @@ -49,9 +49,6 @@ Thread-group run list visibility requires at least one ABAC-visible organization chip. - A later public post in a thread group no longer lists a January run that could not have known that post. -- A 404 on that hidden row stays generic: do not name the thread or - the cutoff. After that 404, re-read the authorized home list so the - stale row does not stay clickable (ADR 0014). ## References diff --git a/docs/adr/0020-analysis-run-retention-purge.md b/docs/adr/0020-analysis-run-retention-purge.md index 5d294c41b..928ba87c6 100644 --- a/docs/adr/0020-analysis-run-retention-purge.md +++ b/docs/adr/0020-analysis-run-retention-purge.md @@ -76,15 +76,6 @@ operators who purge from `psql`. Do not expose purge on a public HTTP route. Split the application login from the migration owner so the product role cannot execute the function even as table owner. -Start reconstruction (ADR 0021) adds `analysis_run_lineage_edge`, -`analysis_run_reconstruction`, and `analysis_source_snapshot_member` -with delete-reject triggers. This procedure already disables those -user triggers when `to_regclass` finds the tables, deletes lineage -edges, then reconstruction, then the 0018 rows, then snapshot -members, then the snapshot, and re-enables the triggers (ADR 0032). -A 0020-only database without those relations still purges. Do not -require a superuser `DISABLE TRIGGER` after a Succeeded start. - ## References — APA 7th American Institute of Certified Public Accountants. (2017). *SOC 2®: SOC diff --git a/docs/adr/0022-authorized-tepp-start.md b/docs/adr/0022-authorized-tepp-start.md index a6517ae53..84fc71635 100644 --- a/docs/adr/0022-authorized-tepp-start.md +++ b/docs/adr/0022-authorized-tepp-start.md @@ -41,8 +41,7 @@ authorized transaction: or refused, or Failed / `tepp_result_not_persisted` when TEPP accepts an envelope this product cannot store yet. -Succeeded TEPP remains unpublished; ADR 0035 stores accepted -transport evidence without stamping Succeeded. This slice does not persist a local +Succeeded TEPP stays later. This slice does not persist a local psychometric substitute, does not call contextual-orchestrator as TEPP, and does not stamp Succeeded from an `accepted` envelope. Failed remains terminal. `POST /api/analysis-runs` is lineage-only (ADR 0017) and does diff --git a/docs/adr/0030-rankweave-fusion-fail-closed.md b/docs/adr/0024-rankweave-fusion-fail-closed.md similarity index 91% rename from docs/adr/0030-rankweave-fusion-fail-closed.md rename to docs/adr/0024-rankweave-fusion-fail-closed.md index 4f1a682bf..af1de902e 100644 --- a/docs/adr/0030-rankweave-fusion-fail-closed.md +++ b/docs/adr/0024-rankweave-fusion-fail-closed.md @@ -1,4 +1,4 @@ -# ADR 0030 — Fail-closed RankWeave ranking port +# ADR 0024 — Fail-closed RankWeave ranking port **Decision status:** Accepted **Date:** 2026-08-17 @@ -40,8 +40,9 @@ tables, and does not bind the demo IdP to production Keyverse. `RANKWEAVE_DISABLED=1` keeps the fail-closed transport. The default seeded stack uses the in-process library already required by -`reconstruct.py`. Leftover pairs stay on ADR 0028 / #211. TEPP stays -on ADR 0022 / #214. Keyverse IdP remains a later slice. +`reconstruct.py`. Mailbox stays on ADR 0020 / #217. Conversations stay +on ADR 0021 / #219. Leftover pairs stay on #211. TEPP stays on #214. +Keyverse IdP remains a later slice. ## References diff --git a/docs/adr/0028-keyverse-oidc-provider.md b/docs/adr/0028-keyverse-oidc-provider.md new file mode 100644 index 000000000..2a2c481ed --- /dev/null +++ b/docs/adr/0028-keyverse-oidc-provider.md @@ -0,0 +1,40 @@ +# ADR 0028: Use Keyverse as a real OIDC provider in production + +## Status + +Accepted for version 2.10.0. + +## Context + +LineageWeave must use real user accounts for login, while corporation and PU +attributes remain authorization data. The local Compose stack needs a portable +development identity provider, but a Keycloak container is not Keyverse and +must not be presented as one. + +## Decision + +1. Production sets `KEYVERSE_ISSUER` and `KEYVERSE_CLIENT_ID` to the actual + Keyverse OIDC client configuration. +2. The backend uses OIDC discovery from that issuer and fetches the returned + `jwks_uri` for RS256 verification. `KEYVERSE_DISCOVERY_URI` and + `KEYVERSE_JWKS_URI` are explicit overrides for deployments where discovery + is proxied. +3. The verified `sub` is still resolved to a provisioned `user_account`; the + database remains authoritative for corporation affiliations and permissions. +4. Compose uses its existing local Keycloak realm only when no Keyverse issuer + is configured. It does not add a Keyverse-shaped identity implementation. + +## Consequences + +- A real Keyverse tenant can be used without changing application code. +- A deployment must provision the Keyverse client, redirect URI, and matching + `user_account` rows before login is usable. +- Local OIDC smoke tests continue to prove cryptographic behavior against the + synthetic Keycloak realm, while production validation must run against the + configured Keyverse discovery document. + +## Security boundary + +Non-HTTP(S) discovery and JWKS URLs are rejected by the shared HTTP client. +No bearer token, client secret, or Keyverse credential belongs in this +repository or in the browser bundle. diff --git a/docs/adr/0029-zotero-local-reproducibility.md b/docs/adr/0029-zotero-local-reproducibility.md new file mode 100644 index 000000000..cba377f0b --- /dev/null +++ b/docs/adr/0029-zotero-local-reproducibility.md @@ -0,0 +1,47 @@ +# ADR 0029: Local Zotero research reproducibility + +Status: accepted + +## Decision + +Zotero is an optional research-workstation dependency, not a LineageWeave +runtime service. Store literature locally through Zotero's Connector HTTP +server and verify it through the local Web API. + +The local Web API at `http://127.0.0.1:23119/api/` is read-only in the current +Zotero release. Do not script writes to `/api/users/0/items`; use +`/connector/saveItems` instead. + +The reproducibility seed for the summarization/evidence experiments is: + +- title: `Get to the Point: Summarization with Pointer-Generator Networks` +- DOI: `10.18653/v1/P17-1099` +- open PDF: `https://aclanthology.org/P17-1099.pdf` + +## Save and verify + +With Zotero running and the Connector HTTP server enabled: + +```bash +SESSION_ID="lineageweave-p17-1099-$(date +%s)" +curl -fsS -X POST http://127.0.0.1:23119/connector/saveItems \ + -H 'Content-Type: application/json' \ + -H 'X-Zotero-Connector-API-Version: 3' \ + --data "$(jq -cn --arg session_id \"$SESSION_ID\" '{items:[{itemType:\"journalArticle\",title:\"Get to the Point: Summarization with Pointer-Generator Networks\",DOI:\"10.18653/v1/P17-1099\",url:\"https://aclanthology.org/P17-1099.pdf\",publicationTitle:\"Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics\"}],uri:\"https://aclanthology.org/P17-1099/\",sessionID:$session_id}')" + +curl -fsS 'http://127.0.0.1:23119/api/users/0/items?limit=100' \ + | jq '.[] | select(.data.DOI == "10.18653/v1/P17-1099") | {key, title: .data.title, DOI: .data.DOI, url: .data.url}' +``` + +The first command may return an empty `201` response. Connector `items` is an +array, not the keyed object accepted by older examples; a unique `sessionID` +prevents replay collisions. The second command is +the persisted-library check. A missing Zotero instance does not disable the +product or fabricate literature evidence. + +## Consequences + +- Zotero setup is reproducible without adding a Python or frontend package. +- OA literature remains linked by DOI and URL rather than copied into this + repository. +- Tests and fixtures do not depend on a user's local Zotero library. diff --git a/docs/adr/0030-external-llm-gateway-environment.md b/docs/adr/0030-external-llm-gateway-environment.md new file mode 100644 index 000000000..e518ee114 --- /dev/null +++ b/docs/adr/0030-external-llm-gateway-environment.md @@ -0,0 +1,100 @@ +# ADR-0030: External LLM gateway environment boundary + +- **Status:** Accepted +- **Date:** 2026-08-18 + +## Context + +LineageWeave must send every LLM request through +[contextual-orchestrator](https://github.com/ContextualWisdomLab/contextual-orchestrator). +The provider gateway is an operational secret and must not be copied into the +repository, an image, a fixture, a test, or a GitHub workflow. + +Compose has two distinct authenticated hops: + +1. `LineageWeave -> contextual-orchestrator`, configured by + `ORCHESTRATOR_BASE_URL` and `ORCHESTRATOR_API_KEY`. +2. `contextual-orchestrator -> LLM gateway`, configured by + `LLM_GATEWAY_API_URL` and `LLM_GATEWAY_API_KEY`. + +These credentials are not interchangeable. The first is the local service +boundary; the second is the provider credential. + +The backend must not load `~/.env` wholesale. Compose injects the provider +credential and URL only into contextual-orchestrator; the backend receives +only its internal orchestrator credential and the non-secret embedding model +identifier. This prevents an unrelated application process from holding the +provider secret while preserving the single LLM/Vision boundary. + +## Decision + +For GitHub Actions and any deployment that injects canonical secrets, the +provider variables use these exact names: + +```text +LLM_GATEWAY_API_KEY +LLM_GATEWAY_API_URL +``` + +For the operator's local `~/.env`, Compose also accepts the existing names +`LLM_GATEWAY_URL`, `LLM_API_GATEWAY`, and `LLM_API_KEY` and maps them to the +canonical provider variables. Canonical values win when both names are present. +`make up`, +`make down`, `make logs`, and `make ps` pass that file to Compose with +`--env-file`, so its interpolation and the orchestrator's `env_file` use the +same source. The API key remains process/container environment data and is +never logged, rendered into frontend assets, or checked in. + +GitHub Actions must not depend on a developer's `~/.env`. If a workflow needs +the provider, it must inject the same two names from GitHub-managed secrets at +runtime, with masking enabled; the repository contains no provider secret. + +The contextual-orchestrator service remains the only LLM boundary. LineageWeave +does not call the provider gateway directly and does not create a fallback +local score, summary, extraction, or answer when the gateway is unavailable. + +## Consequences + +- Provider changes are deployment configuration, not source changes. +- A missing or invalid gateway credential fails at the orchestrator boundary; + it must not be replaced by a fabricated channel result. +- `CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS` must explicitly allow the + hostname selected by `LLM_GATEWAY_API_URL`; wildcard allowlists are forbidden. +- Local Compose development permits only the explicitly enumerated + `host.docker.internal:8080` text gateway and `host.docker.internal:18082` + Vision gateway when `LINEAGEWEAVE_ALLOW_LOCAL_LLM_HTTP=1`; arbitrary local + HTTP ports remain rejected. +- `LLM_GATEWAY_MODEL` and `VISION_MODEL` are not selected by LineageWeave. + When they are blank, contextual-orchestrator resolves the registered agent + model, so a local or provider-specific model name cannot leak into this + application or be assumed available on an external gateway. +- `LLM_GATEWAY_EMBEDDING_MODEL` is the explicit allowlisted semantic embedding + model. If it is absent, contextual-orchestrator rejects embedding work + instead of returning its standalone eight-dimensional heuristic vector. +- `LLM_API_KEY`, `LLM_API_GATEWAY`, and `LLM_GATEWAY_URL` are compatibility + aliases only; `LLM_GATEWAY_API_KEY` and `LLM_GATEWAY_API_URL` are the + canonical names for + GitHub and deployment automation. + +Vision follows the same boundary. LineageWeave sends image content blocks only +to contextual-orchestrator's internal OpenAI-compatible endpoint; it never +sends image bytes directly to `LLM_GATEWAY_API_URL`. The contextual-orchestrator +container validates the supported `text` and `image_url` blocks and forwards +the multimodal request to the configured provider. Image data is excluded from +the text used for workflow routing and reasoning classification. + +The upstream HTTP server keeps its ordinary JSON body default at 64 KiB. The +LineageWeave Compose bootstrap passes the explicit bounded +`CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES` value, defaulting to 8 MiB, because +normalized image blocks are base64 data URIs. The limit is bounded at 64 MiB; +unbounded request bodies and per-image provider bypasses are forbidden. + +The selected provider/model must actually support multimodal `image_url` +content. A text-only gateway response such as `Only 'text' content type is +supported` is a provider capability failure, not a successful Vision result; +the channel remains unavailable and no OCR/caption is fabricated. + +Before the image content block is built, the backend decodes supported raster +formats, applies EXIF orientation, composites transparent pixels onto white, +and encodes the payload as PNG. Invalid or undecodable image bytes fail closed; +they are never forwarded as an unvalidated provider request. diff --git a/docs/adr/0031-embedded-image-html-parser.md b/docs/adr/0031-embedded-image-html-parser.md deleted file mode 100644 index 6acdfd71c..000000000 --- a/docs/adr/0031-embedded-image-html-parser.md +++ /dev/null @@ -1,78 +0,0 @@ -# ADR 0031 — Embedded images use an HTML parser and a raster allowlist - -**Decision status:** Accepted -**Date:** 2026-08-17 - -## Context - -The product popup stopped dumping a well-formed -`data:image/png;base64,...` invoice as a base64 wall. The splitter and -`extract_base64_images` still used a `[^>]*` regex. Real invoice HTML -puts `>` inside `alt` or `title` *before* `src`. That shape is legal -HTML (WHATWG, n.d.) and is what `chunk_by_dom` already parses. The regex -missed the picture and put the payload back into the text node. - -The same open MIME class `image/[a-zA-Z0-9.+-]+` accepted -`image/svg+xml`. SVG-as-`` does not run script in current browsers, -but the regex also fed the vision channel. `atob` and -`b64decode(validate=True)` already disagreed on padding. - -ADR 0019 is the R&R catalog-identity decision. This decision is the -viewer/extractor parse contract. Layout clues stay as character offsets -and `chunk_position` rows — never raw HTML in the knowledge graph or in -a persisted post body. - -Persistence of OCR under the figure (Li et al., 2023; Radford et al., -2021) is still the next buyer slice. It must not land on a splitter that -fails the HTML the buyer actually opens. - -## Decision - -The popup (`splitPostBody`), `extract_base64_images`, and `chunk_by_dom` -share one decode helper (`lineageweave.embedded_image_payload`): - -1. Parse with an HTML parser (`DOMParser` in the browser, `html.parser` - in Python). Comments, ` -

Please confirm.

diff --git a/tests/test_abbreviation_tree_corroboration.py b/tests/test_abbreviation_tree_corroboration.py deleted file mode 100644 index e3c4bce97..000000000 --- a/tests/test_abbreviation_tree_corroboration.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Tree-constrained Searxng abbreviation cross-check (ADR 0033).""" - -from __future__ import annotations - -import pytest - -from lineageweave.abbreviation_tree_corroboration import ( - AbbreviationTreeMatch, - TreeEntityCandidate, - abbreviation_candidates, - corroborate_abbreviation_against_tree, - exact_catalog_matches, -) -from lineageweave.relation_verification import ( - STATUS_CORROBORATED, - STATUS_PENDING, - STATUS_UNCORROBORATED, - NullRelationVerificationClient, - RelationVerificationResult, -) - -_TREE = ( - TreeEntityCandidate("group-id", "Demo Group"), - TreeEntityCandidate("corp-id", "Demo Corp"), - TreeEntityCandidate("plant-id", "Demo Plant"), -) - - -class _FakeVerificationClient: - available = True - - def __init__(self, hits: dict[tuple[str, str], RelationVerificationResult]) -> None: - self._hits = hits - self.calls: list[tuple[str, str]] = [] - - def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: - self.calls.append((organization_name, relationship_label)) - return self._hits.get( - (organization_name, relationship_label), - RelationVerificationResult(STATUS_UNCORROBORATED, None), - ) - - -class _RaisingVerificationClient: - available = True - - def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: - raise RuntimeError("searxng timeout") - - -def test_exact_catalog_name_is_not_an_abbreviation_candidate() -> None: - assert abbreviation_candidates(("Demo Corp", "DC", " "), _TREE) == ("DC",) - - -def test_tied_exact_catalog_names_stay_candidates() -> None: - twins = ( - TreeEntityCandidate("a", "Demo Twin"), - TreeEntityCandidate("b", "Demo Twin"), - ) - assert abbreviation_candidates(("Demo Twin",), twins) == ("Demo Twin",) - - -def test_unique_searxng_hit_binds_the_tree_node() -> None: - client = _FakeVerificationClient( - { - ("Demo Corp", "DC"): RelationVerificationResult( - STATUS_CORROBORATED, "https://example.test/demo-corp-dc" - ) - } - ) - match = corroborate_abbreviation_against_tree("DC", _TREE, client) - assert match == AbbreviationTreeMatch( - raw_organization_name="DC", - corporate_entity_id="corp-id", - verification_status_code=STATUS_CORROBORATED, - verification_evidence_url="https://example.test/demo-corp-dc", - ) - assert ("Demo Group", "DC") in client.calls - assert ("Demo Plant", "DC") in client.calls - - -def test_no_searxng_hit_stays_unbound() -> None: - match = corroborate_abbreviation_against_tree("ZZ", _TREE, _FakeVerificationClient({})) - assert match.corporate_entity_id is None - assert match.verification_status_code == STATUS_UNCORROBORATED - assert match.verification_evidence_url is None - - -def test_tied_searxng_hits_stay_unbound() -> None: - client = _FakeVerificationClient( - { - ("Demo Corp", "DX"): RelationVerificationResult( - STATUS_CORROBORATED, "https://example.test/demo-corp" - ), - ("Demo Group", "DX"): RelationVerificationResult( - STATUS_CORROBORATED, "https://example.test/demo-group" - ), - } - ) - match = corroborate_abbreviation_against_tree("DX", _TREE, client) - assert match.corporate_entity_id is None - assert match.verification_status_code == STATUS_UNCORROBORATED - - -def test_unavailable_searxng_is_pending_and_does_not_invent_a_parent() -> None: - match = corroborate_abbreviation_against_tree("DC", _TREE, NullRelationVerificationClient()) - assert match.corporate_entity_id is None - assert match.verification_status_code == STATUS_PENDING - assert match.verification_evidence_url is None - - -def test_empty_mention_is_uncorroborated() -> None: - match = corroborate_abbreviation_against_tree(" ", _TREE, _FakeVerificationClient({})) - assert match.corporate_entity_id is None - assert match.verification_status_code == STATUS_UNCORROBORATED - - -def test_search_failure_is_not_recorded_as_uncorroborated() -> None: - with pytest.raises(RuntimeError, match="searxng timeout"): - corroborate_abbreviation_against_tree("DC", _TREE, _RaisingVerificationClient()) - - -def test_exact_catalog_matches_normalize_legal_suffix() -> None: - matches = exact_catalog_matches("Demo Corp.", _TREE) - assert [row.entity_id for row in matches] == ["corp-id"] - - -def test_exact_catalog_matches_ignore_empty_normalized_names() -> None: - assert exact_catalog_matches(" ", _TREE) == () - assert exact_catalog_matches("Corp.", _TREE) == () diff --git a/tests/test_adaptive_orchestration_defaults.py b/tests/test_adaptive_orchestration_defaults.py index aa84ad30a..911ac0209 100644 --- a/tests/test_adaptive_orchestration_defaults.py +++ b/tests/test_adaptive_orchestration_defaults.py @@ -77,13 +77,23 @@ def test_structured_consumers_request_auto_mode( monkeypatch, module_name, client_factory, invoke, content ) -> None: observed: dict[str, object] = {} + call_count = 0 def fake_post_json(url, payload, *, headers, timeout): + nonlocal call_count + call_count += 1 observed["url"] = url observed["payload"] = payload observed["headers"] = headers observed["timeout"] = timeout - return {"choices": [{"message": {"content": content}}]} + response_content = content + if module_name == "lineageweave.post_summary": + response_content = ( + "요약\nKEY EVENTS: NONE" + if call_count == 1 + else "ROLES:\nNONE\nPROJECTS:\nNONE" + ) + return {"choices": [{"message": {"content": response_content}}]} module = __import__(module_name, fromlist=["post_json"]) monkeypatch.setattr(module, "post_json", fake_post_json) @@ -91,6 +101,9 @@ def fake_post_json(url, payload, *, headers, timeout): invoke(client_factory()) assert observed["payload"]["mode"] == "auto" + assert observed["payload"]["reasoning_effort"] == "auto" + if module_name == "lineageweave.post_summary": + assert call_count == 2 def test_post_evaluation_judge_defaults_to_auto(monkeypatch) -> None: @@ -129,3 +142,4 @@ def fake_post_json(url, payload, *, headers, timeout): client.evaluate("Title", "Body") assert observed["payload"]["mode"] == "auto" + assert observed["payload"]["reasoning_effort"] == "auto" diff --git a/tests/test_adjudication_client.py b/tests/test_adjudication_client.py new file mode 100644 index 000000000..576f5e9c4 --- /dev/null +++ b/tests/test_adjudication_client.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from lineageweave.adjudication_client import ContextualOrchestratorAdjudicationClient + + +def test_adjudication_uses_supported_auto_mode_and_long_local_timeout(monkeypatch) -> None: + captured: dict[str, object] = {} + + def fake_post_json(url, payload, *, headers, timeout): + captured.update(url=url, payload=payload, headers=headers, timeout=timeout) + return {"choices": [{"message": {"content": "0.75"}}]} + + monkeypatch.setattr("lineageweave.adjudication_client.post_json", fake_post_json) + + client = ContextualOrchestratorAdjudicationClient( + base_url="http://orchestrator:8000", api_key="synthetic-token" + ) + + assert client.judge("workshop", "follow-up bid") == 0.75 + assert captured["url"] == "http://orchestrator:8000/v1/chat/completions" + assert captured["payload"]["mode"] == "auto" + assert captured["payload"]["reasoning_effort"] == "auto" + assert captured["timeout"] == 180.0 diff --git a/tests/test_analysis_run_create.py b/tests/test_analysis_run_create.py index 613ddc329..664ecc830 100644 --- a/tests/test_analysis_run_create.py +++ b/tests/test_analysis_run_create.py @@ -161,7 +161,7 @@ def test_create_pending_rejects_tepp_before_touching_the_registry() -> None: class ForbiddenConnection: def __getattr__(self, name: str) -> object: - raise AssertionError(f"TEPP create must not touch the registry ({name})") + raise AttributeError(f"TEPP create must not touch the registry ({name})") async def _run() -> None: with pytest.raises(AnalysisRunCreateError) as err: diff --git a/tests/test_analysis_run_reconstruction_schema.py b/tests/test_analysis_run_reconstruction_schema.py index 4d332f579..30a2b6579 100644 --- a/tests/test_analysis_run_reconstruction_schema.py +++ b/tests/test_analysis_run_reconstruction_schema.py @@ -50,9 +50,6 @@ def test_reconstruction_migration_is_normalized_and_wired() -> None: assert "0023_analysis_run_outbox.sql" in dockerfile assert "0024_source_post_revision.sql" in dockerfile assert "0025_role_person_catalog_identity.sql" in dockerfile - assert "0026_report_leftover_pair.sql" in dockerfile - assert "0028_analysis_run_tepp_result.sql" in dockerfile - assert "0029_analysis_run_tepp_accepted.sql" in dockerfile assert "analysis_run_reconstruction_not_empty" in rollback assert "reject_analysis_run_reconstruction_update" in migration assert "reject_analysis_run_lineage_edge_update" in migration diff --git a/tests/test_analysis_run_registry_schema.py b/tests/test_analysis_run_registry_schema.py index 083eb8fbb..4161f2452 100644 --- a/tests/test_analysis_run_registry_schema.py +++ b/tests/test_analysis_run_registry_schema.py @@ -281,9 +281,6 @@ def test_registry_contract_is_normalized_and_has_one_temporal_authority() -> Non assert "0023_analysis_run_outbox.sql" in dockerfile assert "0024_source_post_revision.sql" in dockerfile assert "0025_role_person_catalog_identity.sql" in dockerfile - assert "0026_report_leftover_pair.sql" in dockerfile - assert "0028_analysis_run_tepp_result.sql" in dockerfile - assert "0029_analysis_run_tepp_accepted.sql" in dockerfile seed = (_ROOT / "scripts" / "seed_demo_data.py").read_text(encoding="utf-8") assert seed.index("0019_role_catalog_identity.sql") < seed.index( "0020_analysis_run_retention_purge.sql" @@ -303,18 +300,6 @@ def test_registry_contract_is_normalized_and_has_one_temporal_authority() -> Non assert seed.index("0024_source_post_revision.sql") < seed.index( "0025_role_person_catalog_identity.sql" ) - assert seed.index("0025_role_person_catalog_identity.sql") < seed.index( - "0026_report_leftover_pair.sql" - ) - assert seed.index("0026_report_leftover_pair.sql") < seed.index( - "0027_abbreviation_tree_corroboration.sql" - ) - assert seed.index("0027_abbreviation_tree_corroboration.sql") < seed.index( - "0028_analysis_run_tepp_result.sql" - ) - assert seed.index("0028_analysis_run_tepp_result.sql") < seed.index( - "0029_analysis_run_tepp_accepted.sql" - ) assert "analysis_run_registry_not_empty" in rollback retention = _RETENTION_MIGRATION.read_text(encoding="utf-8") retention_rollback = _RETENTION_ROLLBACK.read_text(encoding="utf-8") @@ -330,37 +315,6 @@ def test_registry_contract_is_normalized_and_has_one_temporal_authority() -> Non assert "analysis_run_retention_not_approved" in retention assert "analysis_run_retention_not_granted" in retention assert "analysis_run_retention_not_admin" in retention - assert "analysis_run_lineage_edge" in retention - assert "analysis_run_reconstruction" in retention - assert "analysis_source_snapshot_member" in retention - assert "to_regclass" in retention - assert retention.index("delete from analysis_run_lineage_edge") < ( - retention.index("delete from analysis_run_reconstruction") - ) - assert retention.index("delete from analysis_run_reconstruction") < ( - retention.index("delete from analysis_run_status_event") - ) - assert retention.index("delete from analysis_source_count") < ( - retention.index("delete from analysis_source_snapshot_member") - ) - assert retention.index("delete from analysis_source_snapshot_member") < ( - retention.index("delete from analysis_source_snapshot;") - ) - outbox_purge = ( - _ROOT / "migrations" / "0023_analysis_run_outbox.sql" - ).read_text(encoding="utf-8") - assert outbox_purge.index("delete from analysis_run_lineage_edge") < ( - outbox_purge.index("delete from analysis_run_reconstruction") - ) - assert outbox_purge.index("delete from analysis_run_reconstruction") < ( - outbox_purge.index("delete from analysis_run_status_event") - ) - assert outbox_purge.index("delete from analysis_source_count") < ( - outbox_purge.index("delete from analysis_source_snapshot_member") - ) - assert outbox_purge.index("delete from analysis_source_snapshot_member") < ( - outbox_purge.index("delete from analysis_source_snapshot;") - ) assert "analysis_run_retention_event_not_empty" in retention_rollback assert "jsonb" not in retention.casefold() for object_name in re.findall( @@ -967,105 +921,6 @@ def test_approved_retention_purge_empties_a_run_bearing_registry(registry_db) -> assert cursor.fetchone()[0] is None -def test_retention_purge_empties_optional_reconstruction_children( - registry_db, -) -> None: - """Grant plus admin empties ADR 0021 children despite delete-reject triggers.""" - - with registry_db.cursor() as cursor: - _insert_run_bearing_registry( - cursor, - digest="b" * 64, - idempotency_key="retention-purge-children", - ) - cursor.execute( - "select analysis_run_id, analysis_source_snapshot_id " - "from analysis_run" - ) - run_id, snapshot_id = cursor.fetchone() - cursor.execute( - """ - create table analysis_run_reconstruction ( - analysis_run_id uuid primary key - references analysis_run (analysis_run_id), - result_sha256 text not null, - edge_count integer not null, - reconstructed_at timestamptz not null - ); - create table analysis_run_lineage_edge ( - analysis_run_id uuid not null - references analysis_run_reconstruction (analysis_run_id), - child_post_id uuid not null, - parent_post_id uuid not null, - fused_score double precision not null, - primary key (analysis_run_id, child_post_id) - ); - create table analysis_source_snapshot_member ( - analysis_source_snapshot_id uuid not null - references analysis_source_snapshot - (analysis_source_snapshot_id), - source_post_id uuid not null, - primary key ( - analysis_source_snapshot_id, source_post_id - ) - ); - create function reject_reconstruction_child_delete() - returns trigger language plpgsql as $fn$ - begin - raise exception 'analysis_run_reconstruction_is_immutable'; - end - $fn$; - create trigger analysis_run_reconstruction_update_reject - before update or delete on analysis_run_reconstruction - for each row execute function reject_reconstruction_child_delete(); - create trigger analysis_run_lineage_edge_update_reject - before update or delete on analysis_run_lineage_edge - for each row execute function reject_reconstruction_child_delete(); - create trigger analysis_source_snapshot_member_update_reject - before update or delete on analysis_source_snapshot_member - for each row execute function reject_reconstruction_child_delete(); - """ - ) - cursor.execute( - "insert into analysis_run_reconstruction " - "(analysis_run_id, result_sha256, edge_count, reconstructed_at) " - "values (%s, %s, 1, now())", - (run_id, "c" * 64), - ) - cursor.execute( - "insert into analysis_run_lineage_edge " - "(analysis_run_id, child_post_id, parent_post_id, fused_score) " - "values (%s, %s, %s, 0.91)", - (run_id, str(uuid.uuid4()), str(uuid.uuid4())), - ) - cursor.execute( - "insert into analysis_source_snapshot_member " - "(analysis_source_snapshot_id, source_post_id) " - "values (%s, %s)", - (snapshot_id, str(uuid.uuid4())), - ) - with pytest.raises( - psycopg2.errors.RaiseException, - match="analysis_run_reconstruction_is_immutable", - ): - cursor.execute("delete from analysis_run_reconstruction") - _authorize_session_for_purge(cursor) - cursor.execute( - "select purge_analysis_run_registry(%s)", - ("approved-retention-purge",), - ) - cursor.execute("select count(*) from analysis_run") - assert cursor.fetchone()[0] == 0 - cursor.execute("select count(*) from analysis_source_snapshot") - assert cursor.fetchone()[0] == 0 - cursor.execute("select count(*) from analysis_run_reconstruction") - assert cursor.fetchone()[0] == 0 - cursor.execute("select count(*) from analysis_run_lineage_edge") - assert cursor.fetchone()[0] == 0 - cursor.execute("select count(*) from analysis_source_snapshot_member") - assert cursor.fetchone()[0] == 0 - - def test_retention_purge_requires_unrevoked_session_grant(registry_db) -> None: """Admin membership plus the published token cannot purge without a grant.""" diff --git a/tests/test_analysis_run_start.py b/tests/test_analysis_run_start.py index 2fcccd0b1..e46aa4a0c 100644 --- a/tests/test_analysis_run_start.py +++ b/tests/test_analysis_run_start.py @@ -1,22 +1,17 @@ """Start-reconstruction contracts: digest, freeze, 422/409, designed tree.""" -import asyncio from datetime import datetime, timezone -import asyncpg import pytest from backend.app.analysis_run_ingestion import reconstructed_edge_is_visible from backend.app.analysis_run_start import ( AnalysisRunStartError, - _deliver_tepp_measurement, - _persist_tepp_accepted, configured_tepp_client, reconstruction_member_ids, reconstruction_result_digest, start_kind_rejection, start_write_conflict_error, - tepp_accepted_clocks, tepp_run_request, tepp_submit_outcome, ) @@ -24,12 +19,6 @@ from lineageweave.fixtures import sample_records from lineageweave.lineage_persistence import lineage_edge_specs from lineageweave.tepp_client import AnalysisRunRequest, TeppClient, TeppNotAvailable -from lineageweave.tepp_result import ( - TeppAcceptedEvidence, - accepted_tepp_seed_envelope, - parse_tepp_accepted_evidence, - persistable_tepp_seed_envelope, -) def test_reconstruction_digest_is_stable_and_ignores_edge_order() -> None: @@ -139,57 +128,21 @@ def test_tepp_run_request_is_the_published_wire_shape() -> None: def test_tepp_submit_outcome_drops_a_missing_transport() -> None: """A missing TEPP transport is Failed, never a fabricated score.""" - status, failure, result = tepp_submit_outcome(TeppClient(), _tepp_request()) + status, failure = tepp_submit_outcome(TeppClient(), _tepp_request()) assert status == "analysis_status_failed" assert failure == "tepp_not_available" - assert result is None def test_tepp_submit_outcome_does_not_persist_an_empty_envelope() -> None: - """A bare accepted status is not TEPP's published acknowledgement.""" + """An accepted envelope is not a persistable measurement.""" class _Accepting(TeppClient): def __init__(self) -> None: super().__init__(transport=lambda _payload: {"status": "accepted"}) - status, failure, result = tepp_submit_outcome(_Accepting(), _tepp_request()) - assert status == "analysis_status_failed" - assert failure == "tepp_result_not_persisted" - assert result is None - - -def test_tepp_submit_outcome_keeps_a_published_accepted_envelope_failed() -> None: - """A published accepted ack is transport evidence, never Succeeded.""" - request = _tepp_request() - - class _Accepted(TeppClient): - def __init__(self) -> None: - super().__init__( - transport=lambda _payload: accepted_tepp_seed_envelope( - idempotency_key=request.idempotency_key - ) - ) - - status, failure, result = tepp_submit_outcome(_Accepted(), request) - assert status == "analysis_status_failed" - assert failure == "tepp_completed_result_unsupported" - assert result is not None - assert result.run_state == "accepted" - assert result.evidence_kind() == "aggregate transport evidence" - assert "theta" not in result.evidence_sha256() - - -def test_tepp_submit_outcome_rejects_a_local_completed_envelope() -> None: - """A LineageWeave-local completed shape must not become Succeeded.""" - - class _Local(TeppClient): - def __init__(self) -> None: - super().__init__(transport=lambda _payload: persistable_tepp_seed_envelope()) - - status, failure, result = tepp_submit_outcome(_Local(), _tepp_request()) + status, failure = tepp_submit_outcome(_Accepting(), _tepp_request()) assert status == "analysis_status_failed" assert failure == "tepp_result_not_persisted" - assert result is None def test_configured_tepp_client_stays_unavailable_without_http() -> None: @@ -200,231 +153,6 @@ def test_configured_tepp_client_stays_unavailable_without_http() -> None: client.submit_analysis_run(_tepp_request()) -def test_tepp_accepted_clocks_keep_distinct_receipt_and_row_write() -> None: - """Transport receipt and row write stay two values when they differ.""" - started = datetime(2026, 1, 12, 12, 44, tzinfo=timezone.utc) - received = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) - recorded = datetime(2026, 1, 12, 12, 46, tzinfo=timezone.utc) - assert tepp_accepted_clocks( - started_at=started, - received_at=received, - recorded_at=recorded, - ) == (received, recorded) - - -def test_tepp_accepted_clocks_clamp_backward_receipt_to_start() -> None: - """A receipt earlier than start is not stored as a later invention.""" - started = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) - earlier = datetime(2026, 1, 12, 12, 44, tzinfo=timezone.utc) - assert tepp_accepted_clocks( - started_at=started, - received_at=earlier, - recorded_at=earlier, - ) == (started, started) - - -def test_tepp_accepted_clocks_clamp_backward_row_write_to_receipt() -> None: - """A row-write earlier than receipt stays the receipt, not invented later.""" - started = datetime(2026, 1, 12, 12, 44, tzinfo=timezone.utc) - received = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) - earlier = datetime(2026, 1, 12, 12, 44, 30, tzinfo=timezone.utc) - assert tepp_accepted_clocks( - started_at=started, - received_at=received, - recorded_at=earlier, - ) == (received, received) - - -def test_tepp_accepted_clocks_do_not_invent_a_second_instant() -> None: - """Equal receipt and persist stay one stored instant.""" - instant = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) - assert tepp_accepted_clocks( - started_at=instant, - received_at=instant, - recorded_at=instant, - ) == (instant, instant) - - -def _accepted_evidence() -> TeppAcceptedEvidence: - """Published Demo Corp accepted envelope used by persist-path tests.""" - parsed = parse_tepp_accepted_evidence( - accepted_tepp_seed_envelope(idempotency_key="buyer-key"), - expected_idempotency_key="buyer-key", - ) - assert parsed is not None - return parsed - - -def test_persist_tepp_accepted_stores_two_clock_values() -> None: - """The insert binds transport receipt and row-write as distinct values.""" - received = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) - recorded = datetime(2026, 1, 12, 12, 46, tzinfo=timezone.utc) - - class _Conn: - def __init__(self) -> None: - self.bound: tuple[object, ...] | None = None - - async def execute(self, _sql: str, *args: object) -> str: - self.bound = args - return "INSERT 0 1" - - conn = _Conn() - stored = asyncio.run( - _persist_tepp_accepted(conn, "run-id", _accepted_evidence(), received, recorded) - ) - assert stored is True - assert conn.bound is not None - assert conn.bound[6] == received - assert conn.bound[7] == recorded - assert conn.bound[6] != conn.bound[7] - - -def test_persist_tepp_accepted_keeps_equal_clocks_equal() -> None: - """Same-instant receipt and persist are stored once each, not rewritten later.""" - instant = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) - - class _Conn: - def __init__(self) -> None: - self.bound: tuple[object, ...] | None = None - - async def execute(self, _sql: str, *args: object) -> str: - self.bound = args - return "INSERT 0 1" - - conn = _Conn() - stored = asyncio.run( - _persist_tepp_accepted(conn, "run-id", _accepted_evidence(), instant, instant) - ) - assert stored is True - assert conn.bound is not None - assert conn.bound[6] == instant - assert conn.bound[7] == instant - - -def test_persist_tepp_accepted_fails_closed_without_the_table() -> None: - """A missing accepted-evidence table is not success.""" - instant = datetime(2026, 1, 12, 12, 45, tzinfo=timezone.utc) - - class _Conn: - async def execute(self, _sql: str, *_args: object) -> str: - raise asyncpg.UndefinedTableError("undefined_table") - - stored = asyncio.run( - _persist_tepp_accepted(_Conn(), "run-id", _accepted_evidence(), instant, instant) - ) - assert stored is False - - -class _DeliverConn: - """In-memory start connection for TEPP persist and status append.""" - - def __init__(self, *, persist_ok: bool = True) -> None: - self.persist_ok = persist_ok - self.accepted_args: tuple[object, ...] | None = None - self.status_args: tuple[object, ...] | None = None - - async def fetchval(self, _sql: str, *_args: object) -> int: - return 2 - - async def execute(self, sql: str, *args: object) -> str: - if "analysis_run_tepp_accepted" in sql: - if not self.persist_ok: - raise asyncpg.UndefinedTableError("undefined_table") - self.accepted_args = args - return "INSERT 0 1" - if "analysis_run_status_event" in sql: - self.status_args = args - return "INSERT 0 1" - raise AssertionError(sql) - - -def _locked_tepp_row() -> dict[str, object]: - """Frozen Demo Corp TEPP start row. Never invents a theta.""" - return { - "idempotency_key": "buyer-tepp-2026-w07", - "snapshot_sha256": "ab" * 32, - "knowledge_cutoff": datetime(2026, 1, 12, 12, 0, tzinfo=timezone.utc), - "corporate_entity_id": "11111111-1111-1111-1111-111111111111", - } - - -def test_deliver_tepp_measurement_persists_distinct_clocks() -> None: - """Accepted evidence stores receipt then row-write, and stays Failed.""" - request = _tepp_request() - - class _Accepted(TeppClient): - def __init__(self) -> None: - super().__init__( - transport=lambda _payload: accepted_tepp_seed_envelope( - idempotency_key=request.idempotency_key - ) - ) - - conn = _DeliverConn() - asyncio.run( - _deliver_tepp_measurement( - conn, - analysis_run_id="run-id", - locked=_locked_tepp_row(), - tepp_client=_Accepted(), - ) - ) - assert conn.accepted_args is not None - received_at = conn.accepted_args[6] - recorded_at = conn.accepted_args[7] - assert isinstance(received_at, datetime) - assert isinstance(recorded_at, datetime) - assert received_at <= recorded_at - assert conn.status_args is not None - assert conn.status_args[2] == "analysis_status_failed" - assert conn.status_args[4] == "tepp_completed_result_unsupported" - assert conn.status_args[3] == recorded_at - - -def test_deliver_tepp_measurement_fails_closed_when_table_is_missing() -> None: - """A missing accepted table is Failed, never a fabricated measurement.""" - request = _tepp_request() - - class _Accepted(TeppClient): - def __init__(self) -> None: - super().__init__( - transport=lambda _payload: accepted_tepp_seed_envelope( - idempotency_key=request.idempotency_key - ) - ) - - conn = _DeliverConn(persist_ok=False) - asyncio.run( - _deliver_tepp_measurement( - conn, - analysis_run_id="run-id", - locked=_locked_tepp_row(), - tepp_client=_Accepted(), - ) - ) - assert conn.accepted_args is None - assert conn.status_args is not None - assert conn.status_args[2] == "analysis_status_failed" - assert conn.status_args[4] == "tepp_result_not_persisted" - - -def test_deliver_tepp_measurement_does_not_persist_a_missing_transport() -> None: - """A missing TEPP transport writes no accepted evidence row.""" - conn = _DeliverConn() - asyncio.run( - _deliver_tepp_measurement( - conn, - analysis_run_id="run-id", - locked=_locked_tepp_row(), - tepp_client=TeppClient(), - ) - ) - assert conn.accepted_args is None - assert conn.status_args is not None - assert conn.status_args[2] == "analysis_status_failed" - assert conn.status_args[4] == "tepp_not_available" - - def test_hidden_run_start_is_not_found() -> None: """Operators get a 404 next action, not an internal exception name.""" error = AnalysisRunStartError(404, "This analysis run is not visible.") diff --git a/tests/test_analysis_run_tepp_accepted_schema.py b/tests/test_analysis_run_tepp_accepted_schema.py deleted file mode 100644 index 18c380ce6..000000000 --- a/tests/test_analysis_run_tepp_accepted_schema.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Static and optional PostgreSQL contracts for TEPP accepted evidence.""" - -from __future__ import annotations - -import os -import re -import uuid -from pathlib import Path -from urllib.parse import urlsplit, urlunsplit - -import pytest - -_ROOT = Path(__file__).resolve().parents[1] -_INITIAL_MIGRATION = _ROOT / "migrations" / "0001_initial_schema.sql" -_REGISTRY_MIGRATION = _ROOT / "migrations" / "0018_analysis_run_registry.sql" -_TEPP_RESULT_MIGRATION = _ROOT / "migrations" / "0028_analysis_run_tepp_result.sql" -_TEPP_ACCEPTED_MIGRATION = _ROOT / "migrations" / "0029_analysis_run_tepp_accepted.sql" -_TEPP_ACCEPTED_ROLLBACK = ( - _ROOT / "migrations" / "rollback" / "0029_analysis_run_tepp_accepted.sql" -) -_POSTGRES_IMAGE = _ROOT / "docker" / "postgres-init" / "Dockerfile" -_ADMIN_DSN = os.environ.get( - "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" -) -_REQUIRED_TABLES = {"analysis_run_tepp_accepted"} - - -def test_tepp_accepted_migration_is_normalized_and_wired() -> None: - """Static contract: 3NF names, additive to 0028, Dockerfile copy, rollback.""" - migration = _TEPP_ACCEPTED_MIGRATION.read_text(encoding="utf-8") - rollback = _TEPP_ACCEPTED_ROLLBACK.read_text(encoding="utf-8") - dockerfile = _POSTGRES_IMAGE.read_text(encoding="utf-8") - seed = (_ROOT / "scripts" / "seed_demo_data.py").read_text(encoding="utf-8") - created_tables = set( - re.findall(r"create table if not exists\s+([a-z0-9_]+)", migration, re.I) - ) - assert _REQUIRED_TABLES <= created_tables - assert "jsonb" not in migration.casefold() - assert "metadata_payload" not in migration - assert "theta" not in migration.casefold() - assert "affiliation_count" not in migration - assert "interval_count" not in migration - assert "validated multilevel estimate" in migration - assert "0029_analysis_run_tepp_accepted.sql" in dockerfile - assert "0029_analysis_run_tepp_accepted.sql" in seed - assert seed.index("0028_analysis_run_tepp_result.sql") < seed.index( - "0029_analysis_run_tepp_accepted.sql" - ) - assert "analysis_run_tepp_accepted_not_empty" in rollback - assert "drop table if exists analysis_run_tepp_result" not in rollback - assert "reject_analysis_run_tepp_accepted_update" in migration - assert "delete from analysis_run_tepp_accepted" in migration - assert migration.index("delete from analysis_run_tepp_accepted") < ( - migration.index("delete from analysis_run_status_event") - ) - for object_name in re.findall( - r"create table if not exists\s+([a-z0-9_]+)", - migration, - re.I, - ): - assert len(object_name.split("_")) >= 2, object_name - for object_name in re.findall( - r"create or replace function\s+([a-z0-9_]+)", - migration, - ): - assert len(object_name.split("_")) >= 2, object_name - for object_name in re.findall(r"create trigger\s+([a-z0-9_]+)", migration, re.I): - assert len(object_name.split("_")) >= 2, object_name - - -def test_tepp_accepted_migration_is_idempotent_sql() -> None: - """Upgrade-safe: create if not exists and replace, never drop 0028.""" - migration = _TEPP_ACCEPTED_MIGRATION.read_text(encoding="utf-8") - assert "create table if not exists analysis_run_tepp_accepted" in migration - assert "create or replace function purge_analysis_run_registry" in migration - assert "drop table" not in migration.casefold() - assert "analysis_run_tepp_result" in migration - - -def _postgres_available() -> bool: - """Return whether the configured administrator DSN is reachable.""" - try: - import psycopg2 - - psycopg2.connect(_ADMIN_DSN, connect_timeout=2).close() - return True - except Exception: - return False - - -def _database_dsn(database_name: str) -> str: - """Replace only the database path while preserving DSN query options.""" - parsed = urlsplit(_ADMIN_DSN) - return urlunsplit(parsed._replace(path=f"/{database_name}")) - - -@pytest.fixture -def tepp_accepted_db(): - """Yield a throwaway registry plus TEPP-accepted database.""" - if not _postgres_available(): - pytest.skip("a reachable PostgreSQL administrator DSN is required") - import psycopg2 - - database_name = f"lineageweave_tepp_acc_{uuid.uuid4().hex[:12]}" - admin = psycopg2.connect(_ADMIN_DSN) - admin.autocommit = True - try: - with admin.cursor() as cursor: - cursor.execute(f'create database "{database_name}"') - finally: - admin.close() - conn = psycopg2.connect(_database_dsn(database_name)) - conn.autocommit = True - try: - with conn.cursor() as cursor: - cursor.execute(_INITIAL_MIGRATION.read_text(encoding="utf-8")) - cursor.execute(_REGISTRY_MIGRATION.read_text(encoding="utf-8")) - cursor.execute(_TEPP_RESULT_MIGRATION.read_text(encoding="utf-8")) - cursor.execute(_TEPP_ACCEPTED_MIGRATION.read_text(encoding="utf-8")) - cursor.execute(_TEPP_ACCEPTED_MIGRATION.read_text(encoding="utf-8")) - yield conn - finally: - conn.close() - admin = psycopg2.connect(_ADMIN_DSN) - admin.autocommit = True - try: - with admin.cursor() as cursor: - cursor.execute( - "select pg_terminate_backend(pid) from pg_stat_activity " - "where datname = %s and pid <> pg_backend_pid()", - (database_name,), - ) - cursor.execute(f'drop database "{database_name}"') - finally: - admin.close() - - -def test_empty_tepp_accepted_rollback_is_replayable(tepp_accepted_db) -> None: - """An empty accepted-evidence schema can be rolled back twice.""" - with tepp_accepted_db.cursor() as cursor: - cursor.execute( - "select table_name from information_schema.tables " - "where table_schema = 'public' and table_name = any(%s)", - (list(_REQUIRED_TABLES | {"analysis_run_tepp_result"}),), - ) - present = {row[0] for row in cursor.fetchall()} - assert _REQUIRED_TABLES <= present - assert "analysis_run_tepp_result" in present - cursor.execute(_TEPP_ACCEPTED_ROLLBACK.read_text(encoding="utf-8")) - cursor.execute( - "select table_name from information_schema.tables " - "where table_schema = 'public' and table_name = any(%s)", - (list(_REQUIRED_TABLES | {"analysis_run_tepp_result"}),), - ) - remaining = {row[0] for row in cursor.fetchall()} - assert remaining == {"analysis_run_tepp_result"} - cursor.execute(_TEPP_ACCEPTED_ROLLBACK.read_text(encoding="utf-8")) diff --git a/tests/test_analysis_run_tepp_result_schema.py b/tests/test_analysis_run_tepp_result_schema.py deleted file mode 100644 index 46ec05a4f..000000000 --- a/tests/test_analysis_run_tepp_result_schema.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Static and optional PostgreSQL contracts for persistable TEPP results.""" - -from __future__ import annotations - -import os -import re -import uuid -from pathlib import Path -from urllib.parse import urlsplit, urlunsplit - -import pytest - -_ROOT = Path(__file__).resolve().parents[1] -_INITIAL_MIGRATION = _ROOT / "migrations" / "0001_initial_schema.sql" -_REGISTRY_MIGRATION = _ROOT / "migrations" / "0018_analysis_run_registry.sql" -_TEPP_RESULT_MIGRATION = _ROOT / "migrations" / "0028_analysis_run_tepp_result.sql" -_TEPP_RESULT_ROLLBACK = ( - _ROOT / "migrations" / "rollback" / "0028_analysis_run_tepp_result.sql" -) -_POSTGRES_IMAGE = _ROOT / "docker" / "postgres-init" / "Dockerfile" -_ADMIN_DSN = os.environ.get( - "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" -) -_REQUIRED_TABLES = {"analysis_run_tepp_result"} - - -def test_tepp_result_migration_is_normalized_and_wired() -> None: - """Static contract: 3NF names, no payload JSON, Dockerfile copy, rollback.""" - migration = _TEPP_RESULT_MIGRATION.read_text(encoding="utf-8") - rollback = _TEPP_RESULT_ROLLBACK.read_text(encoding="utf-8") - dockerfile = _POSTGRES_IMAGE.read_text(encoding="utf-8") - seed = (_ROOT / "scripts" / "seed_demo_data.py").read_text(encoding="utf-8") - created_tables = set( - re.findall(r"create table if not exists\s+([a-z0-9_]+)", migration, re.I) - ) - assert _REQUIRED_TABLES <= created_tables - assert "jsonb" not in migration.casefold() - assert "metadata_payload" not in migration - assert "theta" not in migration.casefold() - assert "0028_analysis_run_tepp_result.sql" in dockerfile - assert "0028_analysis_run_tepp_result.sql" in seed - assert seed.index("0027_abbreviation_tree_corroboration.sql") < seed.index( - "0028_analysis_run_tepp_result.sql" - ) - assert "analysis_run_tepp_result_not_empty" in rollback - assert "reject_analysis_run_tepp_result_update" in migration - assert "delete from analysis_run_tepp_result" in migration - assert migration.index("delete from analysis_run_tepp_result") < ( - migration.index("delete from analysis_run_status_event") - ) - for object_name in re.findall( - r"create table if not exists\s+([a-z0-9_]+)", - migration, - re.I, - ): - assert len(object_name.split("_")) >= 2, object_name - for object_name in re.findall( - r"create or replace function\s+([a-z0-9_]+)", - migration, - re.I, - ): - assert len(object_name.split("_")) >= 2, object_name - for object_name in re.findall(r"create trigger\s+([a-z0-9_]+)", migration, re.I): - assert len(object_name.split("_")) >= 2, object_name - - -def _postgres_available() -> bool: - """Return whether the configured administrator DSN is reachable.""" - try: - import psycopg2 - - psycopg2.connect(_ADMIN_DSN, connect_timeout=2).close() - return True - except Exception: - return False - - -def _database_dsn(database_name: str) -> str: - """Replace only the database path while preserving DSN query options.""" - parsed = urlsplit(_ADMIN_DSN) - return urlunsplit(parsed._replace(path=f"/{database_name}")) - - -@pytest.fixture -def tepp_result_db(): - """Yield a throwaway registry plus TEPP-result database.""" - if not _postgres_available(): - pytest.skip("a reachable PostgreSQL administrator DSN is required") - import psycopg2 - - database_name = f"lineageweave_tepp_{uuid.uuid4().hex[:12]}" - admin = psycopg2.connect(_ADMIN_DSN) - admin.autocommit = True - try: - with admin.cursor() as cursor: - cursor.execute(f'create database "{database_name}"') - finally: - admin.close() - conn = psycopg2.connect(_database_dsn(database_name)) - conn.autocommit = True - try: - with conn.cursor() as cursor: - cursor.execute(_INITIAL_MIGRATION.read_text(encoding="utf-8")) - cursor.execute(_REGISTRY_MIGRATION.read_text(encoding="utf-8")) - cursor.execute(_TEPP_RESULT_MIGRATION.read_text(encoding="utf-8")) - yield conn - finally: - conn.close() - admin = psycopg2.connect(_ADMIN_DSN) - admin.autocommit = True - try: - with admin.cursor() as cursor: - cursor.execute( - "select pg_terminate_backend(pid) from pg_stat_activity " - "where datname = %s and pid <> pg_backend_pid()", - (database_name,), - ) - cursor.execute(f'drop database "{database_name}"') - finally: - admin.close() - - -def test_empty_tepp_result_rollback_is_replayable(tepp_result_db) -> None: - """An empty TEPP-result schema can be rolled back and removed.""" - with tepp_result_db.cursor() as cursor: - cursor.execute( - "select table_name from information_schema.tables " - "where table_schema = 'public' and table_name = any(%s)", - (list(_REQUIRED_TABLES),), - ) - assert {row[0] for row in cursor.fetchall()} == _REQUIRED_TABLES - cursor.execute(_TEPP_RESULT_ROLLBACK.read_text(encoding="utf-8")) - cursor.execute( - "select table_name from information_schema.tables " - "where table_schema = 'public' and table_name = any(%s)", - (list(_REQUIRED_TABLES),), - ) - assert cursor.fetchall() == [] - cursor.execute(_TEPP_RESULT_ROLLBACK.read_text(encoding="utf-8")) diff --git a/tests/test_analysis_run_worker.py b/tests/test_analysis_run_worker.py new file mode 100644 index 000000000..2a840b081 --- /dev/null +++ b/tests/test_analysis_run_worker.py @@ -0,0 +1,86 @@ +"""Synthetic checks for Valkey analysis-run wake-up consumption.""" + +from __future__ import annotations + +import pytest + +from backend.app import analysis_run_worker +from lineageweave.adjudication_client import NullAdjudicationClient +from lineageweave.tepp_client import TeppClient + + +class _Transaction: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + +class _Connection: + def transaction(self): + return _Transaction() + + async def fetchrow(self, _query, _analysis_run_id): + return {"requested_by_account_id": "synthetic-account"} + + +class _Acquire: + def __init__(self, conn): + self.conn = conn + + async def __aenter__(self): + return self.conn + + async def __aexit__(self, *_args): + return False + + +class _Pool: + def acquire(self): + return _Acquire(_Connection()) + + +class _Valkey: + async def xread(self, _streams, *, count, block): + assert (count, block) == (10, 1000) + return [ + ( + "analysis-run-outbox", + [ + ("1-0", {"analysis_run_id": "00000000-0000-0000-0000-000000000001"}), + ("1-1", {}), + ], + ) + ] + + +@pytest.mark.anyio +async def test_consumer_forwards_valid_event_and_skips_malformed_event(monkeypatch): + calls = [] + + async def fake_deliver(conn, **kwargs): + del conn + calls.append(kwargs) + + monkeypatch.setattr(analysis_run_worker, "deliver_queued_analysis_run", fake_deliver) + + last_id = await analysis_run_worker.consume_analysis_run_stream_once( + _Valkey(), + _Pool(), + last_id="0-0", + tepp_client=TeppClient(), + adjudication_client=NullAdjudicationClient(), + ) + + assert last_id == "1-1" + assert calls == [ + { + "analysis_run_id": "00000000-0000-0000-0000-000000000001", + "account_id": "synthetic-account", + "affiliated_entity_ids": [], + "tepp_client": calls[0]["tepp_client"], + "adjudication_client": calls[0]["adjudication_client"], + "valkey_stream_entry_id": "1-0", + } + ] diff --git a/tests/test_caldav_client.py b/tests/test_caldav_client.py new file mode 100644 index 000000000..2cacb3539 --- /dev/null +++ b/tests/test_caldav_client.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import pytest + +from lineageweave.caldav_client import ( + CalDavEvent, + HttpCalDavClient, + NullCalDavClient, + build_caldav_client, +) + + +def test_missing_base_url_drops_only_the_optional_caldav_channel() -> None: + client = build_caldav_client("") + + assert isinstance(client, NullCalDavClient) + assert not client.available + assert client.list_events() == [] + + +def test_http_client_reads_valid_events_and_ignores_malformed_rows(monkeypatch) -> None: + received = {} + + def fake_get_json(url: str, *, timeout: float) -> dict: + received.update(url=url, timeout=timeout) + return { + "events": [ + { + "event_id": "event-1", + "summary": "Review", + "starts_at": "2026-08-19T09:00:00Z", + }, + { + "event_id": "event-2", + "summary": "", + "starts_at": "2026-08-19T10:00:00Z", + }, + "not-an-event", + ] + } + + monkeypatch.setattr("lineageweave.caldav_client.get_json", fake_get_json) + client = build_caldav_client("https://calendar.example/caldav/") + + assert isinstance(client, HttpCalDavClient) + assert client.list_events() == [ + CalDavEvent("event-1", "Review", "2026-08-19T09:00:00Z") + ] + assert received == {"url": "https://calendar.example/caldav/events", "timeout": 10} + + +def test_invalid_caldav_url_is_rejected() -> None: + with pytest.raises(ValueError, match="CALDAV_BASE_URL"): + build_caldav_client("file:///tmp/events") diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 27e47c238..d37a300cc 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -6,6 +6,8 @@ chunk_by_dom, chunk_by_paragraph, chunk_by_sentence, + chunk_by_source_body, + normalize_semantic_text, ) @@ -70,11 +72,237 @@ def test_chunk_by_dom_nested_blocks_do_not_duplicate_text() -> None: assert chunks[0].label == "p" +def test_chunk_by_dom_groups_table_cells_by_row_instead_of_flattening() -> None: + """Live bug (2026-08-19): each used to push its own independent + chunk with no row grouping, so a real table (headers + N data rows) + degraded into a flat, unattributable list of cell fragments -- e.g. a + 5-column x 13-row table read back as 65 disconnected one-word lines + with no way to tell which cells shared a row. + """ + html = ( + "" + "" + "" + "" + "
No.CompanyResult
1Acme CorpDeclined
2Globex CorpInterested
" + ) + chunks = chunk_by_dom(html) + + texts = [c.text for c in chunks] + assert texts == [ + "No. | Company | Result", + "1 | Acme Corp | Declined", + "2 | Globex Corp | Interested", + ] + assert all(c.label == "tr" for c in chunks) + + +def test_chunk_by_dom_keeps_nested_table_cell_blocks_in_their_row() -> None: + chunks = chunk_by_dom( + "

No.

Company
" + ) + assert [(chunk.label, chunk.text) for chunk in chunks] == [("tr", "No. | Company")] + + +def test_chunk_by_dom_labels_markerless_footnotes() -> None: + chunks = chunk_by_dom("

Body text

*Tier 2: follow-up note

") + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Body text"), + ("footnote", "*Tier 2: follow-up note"), + ] + + +def test_chunk_by_dom_labels_html_and_word_footnote_markup() -> None: + html = ( + "

Body text

" + '

  1. HTML footnote body

' + '

1 Word footnote body

' + ) + + chunks = chunk_by_dom(html) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Body text"), + ("footnote", "HTML footnote body"), + ("footnote", "1 Word footnote body"), + ] + + +def test_chunk_by_dom_does_not_label_body_footnote_citation_as_footnote() -> None: + html = ( + '

Body cites [1].

' + '

[1] Footnote definition.

' + ) + + chunks = chunk_by_dom(html) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Body cites [1]."), + ("footnote", "[1] Footnote definition."), + ] + + +def test_chunk_by_dom_labels_ooxml_footnote_containers() -> None: + chunks = chunk_by_dom( + "OOXML footnote body" + "OOXML endnote body" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("footnote", "OOXML footnote body"), + ("footnote", "OOXML endnote body"), + ] + + +def test_chunk_by_dom_word_table_rows_also_group_cells() -> None: + html = "1Acme Corp" + chunks = chunk_by_dom(html) + + assert [c.text for c in chunks] == ["1 | Acme Corp"] + assert chunks[0].label == "w:tr" + + +def test_chunk_by_dom_keeps_indentation_as_metadata_not_embedding_text() -> None: + html = "

  Level one

    Level two

" + chunks = chunk_by_dom(html) + + assert [chunk.text for chunk in chunks] == ["Level one", "Level two"] + assert [chunk.indent_width for chunk in chunks] == [2, 4] + assert [chunk.declared_indent_width for chunk in chunks] == [0, 0] + + +def test_chunk_by_dom_reads_html_and_word_indentation_declarations() -> None: + html = ( + '

HTML

' + '' + "Word" + ) + chunks = chunk_by_dom(html) + + assert [chunk.text for chunk in chunks] == ["HTML", "Word"] + assert [chunk.indent_width for chunk in chunks] == [4, 4] + assert [chunk.declared_indent_width for chunk in chunks] == [4, 4] + + +def test_chunk_by_dom_reads_the_css_margin_shorthand_not_just_margin_left() -> None: + """Live bug (2026-08-19): a real editor (Word paste, Outlook compose) + declares indentation with the box-model shorthand + ("margin: 0cm 0cm 0cm 56px") far more often than the "margin-left" + longhand -- every nested
  • in a real body used only the shorthand, + so indentation silently read as 0 and every nesting level flattened. + """ + html = ( + '
    • Outer item
    ' + '
    • Nested item
    ' + ) + chunks = chunk_by_dom(html) + + assert [chunk.text for chunk in chunks] == ["Outer item", "Nested item"] + outer, nested = chunks + assert outer.indent_width < nested.indent_width + assert outer.indent_width > 0 + + +def test_chunk_by_dom_uses_list_container_depth_as_explicit_indentation() -> None: + html = "
    1. Outer
      1. Nested
    " + + chunks = chunk_by_dom(html) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [("li", "Outer"), ("li", "Nested")] + assert [chunk.indent_width for chunk in chunks] == [4, 8] + + +def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: + body = """1. Background + continuation stays with the first item. +2. Decision + +| Field | Value | +| --- | --- | +| Owner | Buyer | +""" + + chunks = chunk_by_source_body(body) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("", "1. Background continuation stays with the first item."), + ("", "2. Decision"), + ("tr", "Field | Value"), + ("tr", "Owner | Buyer"), + ] + + +def test_chunk_by_dom_joins_visual_continuation_lines_but_keeps_list_items() -> None: + html = ( + '

    1. 배경
    ' + " 1) 기존 대차는 이전이 필요함
    " + " 콘크리트 양생까지 공장 운영 불가하여 이전 불가
    " + " 2) 신규 대차 제작으로 결정

    " + ) + + chunks = chunk_by_dom(html) + + assert [chunk.text for chunk in chunks] == [ + "1. 배경", + "1) 기존 대차는 이전이 필요함 콘크리트 양생까지 공장 운영 불가하여 이전 불가", + "2) 신규 대차 제작으로 결정", + ] + assert [chunk.indent_width for chunk in chunks] == [0, 4, 4] + + +def test_normalize_semantic_text_removes_visual_hanging_indent_breaks() -> None: + text = ( + "1. 배경\n\n" + " 1) 기존 대차는 이전이 필요함\n" + " 콘크리트 양생까지 공장 운영 불가하여 이전 불가\n" + " 2) 신규 대차 제작" + ) + + assert normalize_semantic_text(text) == ( + "1. 배경\n\n" + "1) 기존 대차는 이전이 필요함 콘크리트 양생까지 공장 운영 불가하여 이전 불가\n" + "2) 신규 대차 제작" + ) + + +def test_normalize_semantic_text_preserves_blank_paragraph_boundaries() -> None: + assert normalize_semantic_text("첫 문단\n\n둘째 문단") == "첫 문단\n\n둘째 문단" + + +def test_normalize_semantic_text_does_not_embed_visual_indentation_markers() -> None: + assert normalize_semantic_text("\xa0\xa0계속되는 문장\n\xa0\xa0\xa0\xa0다음 줄") == ( + "계속되는 문장 다음 줄" + ) + + +def test_chunk_by_dom_does_not_infer_marker_depth_without_source_whitespace() -> None: + chunks = chunk_by_dom("

    1. Root
    1) Child
    - Detail

    ") + + assert [chunk.text for chunk in chunks] == ["1. Root", "1) Child", "- Detail"] + assert [chunk.indent_width for chunk in chunks] == [0, 0, 0] + + def test_chunk_by_dom_empty_html_yields_no_chunks() -> None: assert chunk_by_dom("
    ") == [] assert chunk_by_dom("") == [] +def test_chunk_by_dom_falls_back_for_inline_only_markup() -> None: + chunks = chunk_by_dom("First inline block.Second inline block.") + + assert len(chunks) == 1 + assert chunks[0].unit_type == "plain_text" + assert chunks[0].text == "First inline block.Second inline block." + + +def test_chunk_by_dom_flushes_unclosed_block_at_end_of_document() -> None: + chunks = chunk_by_dom("
    Unclosed source fragment.") + + assert len(chunks) == 1 + assert chunks[0].unit_type == "dom" + assert chunks[0].text == "Unclosed source fragment." + + def test_chunk_by_dom_interleaves_images_with_text_in_document_order() -> None: tiny_png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" html = ( @@ -107,18 +335,6 @@ def test_chunk_by_dom_skips_malformed_image_data() -> None: assert [c.unit_type for c in chunks] == ["dom"] -def test_chunk_by_dom_skips_script_and_style_images() -> None: - tiny_png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" - html = ( - f'' - f'' - "

    Visible.

    " - ) - chunks = chunk_by_dom(html) - assert [c.unit_type for c in chunks] == ["dom"] - assert chunks[0].text == "Visible." - - def test_chunk_by_conversation_turn_labels_each_chunk_with_its_sender() -> None: turns = [ ConversationTurn(sender="alice@example.com", text="Can we move the meeting?"), diff --git a/tests/test_contextual_orchestrator_start.py b/tests/test_contextual_orchestrator_start.py new file mode 100644 index 000000000..e3618aa82 --- /dev/null +++ b/tests/test_contextual_orchestrator_start.py @@ -0,0 +1,120 @@ +"""Bootstrap delegates model and provider protocol behavior to the orchestrator.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +import types +from pathlib import Path + + +def _load_start_module(): + stubs = {} + previous = {name: sys.modules.get(name) for name in stubs} + sys.modules.update(stubs) + try: + path = Path(__file__).parents[1] / "docker" / "contextual-orchestrator" / "start.py" + spec = importlib.util.spec_from_file_location("lineageweave_contextual_start", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + finally: + for name, previous_module in previous.items(): + if previous_module is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = previous_module + + +def test_bootstrap_does_not_patch_upstream_model_classes() -> None: + module = _load_start_module() + assert "ModelClient" not in module.__dict__ + assert "_apply_provider_models" not in module.__dict__ + + +def test_provider_api_url_is_canonical_over_compatibility_aliases(monkeypatch) -> None: + module = _load_start_module() + monkeypatch.setenv("LLM_GATEWAY_API_URL", "https://canonical.example/v1") + monkeypatch.setenv("LLM_GATEWAY_URL", "https://legacy.example/v1") + monkeypatch.setenv("LLM_API_GATEWAY", "https://local-alias.example/v1") + + assert module._pop_first_env("LLM_GATEWAY_API_URL", "LLM_GATEWAY_URL", "LLM_API_GATEWAY") == ( + "https://canonical.example/v1" + ) + + +def test_gateway_api_key_accepts_local_compatibility_alias(monkeypatch) -> None: + module = _load_start_module() + monkeypatch.setenv("LLM_API_KEY", "compatibility-key") + + assert module._pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY") == "compatibility-key" + + +def test_bootstrap_registers_embedding_agent_before_deleting_secrets(monkeypatch) -> None: + module = _load_start_module() + captured: dict[str, object] = {} + + class FakePath: + def __init__(self, value: str) -> None: + self.value = value + + def read_text(self, *, encoding: str) -> str: + assert self.value == "/app/agents.json" + assert encoding == "utf-8" + return json.dumps({"agents": [{}]}) + + def write_text(self, value: str, *, encoding: str) -> None: + assert self.value == "/tmp/lineageweave-agents.json" + assert encoding == "utf-8" + captured["agents"] = json.loads(value) + + credentials = types.ModuleType("contextual_orchestrator.credentials") + + def register_credential(name: str, value: str) -> None: + captured.setdefault("credentials", []).append((name, value)) + + credentials.register_credential = register_credential + server = types.ModuleType("contextual_orchestrator.__main__") + + def serve() -> None: + captured["argv"] = list(sys.argv) + + server.main = serve + package = types.ModuleType("contextual_orchestrator") + package.__path__ = [] + monkeypatch.setitem(sys.modules, "contextual_orchestrator", package) + monkeypatch.setitem(sys.modules, "contextual_orchestrator.credentials", credentials) + monkeypatch.setitem(sys.modules, "contextual_orchestrator.__main__", server) + monkeypatch.setattr(module, "Path", FakePath) + monkeypatch.setattr(sys, "argv", ["start.py"]) + monkeypatch.setenv("LLM_GATEWAY_API_KEY", "provider-key") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "orchestrator-token") + monkeypatch.setenv("LLM_GATEWAY_API_URL", "https://gateway.example") + monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "embedding-model") + + module.main() + + argv = captured["argv"] + assert isinstance(argv, list) + assert "--embedding-provider-url" not in argv + assert "--embedding-model" not in argv + assert captured["credentials"] == [ + ("NVIDIA_NIM_API_KEY", "provider-key"), + ("LLM_GATEWAY_API_KEY", "provider-key"), + ] + agents = captured["agents"] + assert isinstance(agents, dict) + embedding_agents = [agent for agent in agents["agents"] if "embedding" in agent.get("tags", [])] + assert embedding_agents == [ + { + "id": "llm_gateway_embedding_agent", + "model": "embedding-model", + "base_url": "https://gateway.example/v1", + "credential_key": "LLM_GATEWAY_API_KEY", + "provider_protocol": "auto", + "tags": ["embedding"], + "priority": 0, + } + ] diff --git a/tests/test_contextual_orchestrator_vision.py b/tests/test_contextual_orchestrator_vision.py new file mode 100644 index 000000000..0cd81c499 --- /dev/null +++ b/tests/test_contextual_orchestrator_vision.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import json + +import lineageweave.image_content as image_content +import lineageweave.http_client as http_client +from lineageweave.llm_context import use_llm_metadata + + +def test_native_vision_client_sends_multimodal_payload_through_orchestrator(monkeypatch) -> None: + captured = {} + + monkeypatch.setattr( + "lineageweave.vision_image.normalize_vision_image", + lambda image_bytes, mime_type: (image_bytes, mime_type), + ) + + def fake_request(method, url, *, body, headers, timeout): + captured["url"] = url + captured["payload"] = json.loads(body) + response = { + "choices": [{ + "message": { + "content": "TEXT: visible text\nCAPTION: a diagram\nTAGS: diagram", + } + }] + } + return 200, json.dumps(response).encode("utf-8") + + monkeypatch.setattr(http_client, "_request", fake_request) + client = image_content.OpenAiCompatibleVisionClient( + "http://orchestrator/v1", "test-key", allow_insecure_http=True + ) + + with use_llm_metadata({"lineageweave_post_session_id": "session-1"}): + result = client.describe(b"image-bytes", "image/png") + + assert result.extracted_text == "visible text" + assert captured["url"] == "http://orchestrator/v1/chat/completions" + assert captured["payload"]["mode"] == "auto" + assert captured["payload"]["reasoning_effort"] == "auto" + assert captured["payload"]["metadata"]["lineageweave_post_session_id"] == "session-1" + assert captured["payload"]["messages"][1]["content"][1]["type"] == "image_url" + + +def test_native_vision_region_locator_uses_orchestrator_auto_contract(monkeypatch) -> None: + captured = {} + + monkeypatch.setattr( + "lineageweave.vision_image.normalize_vision_image", + lambda image_bytes, mime_type: (image_bytes, mime_type), + ) + + def fake_request(method, url, *, body, headers, timeout): + captured["url"] = url + captured["payload"] = json.loads(body) + return 200, json.dumps({ + "choices": [{"message": {"content": '{"regions":[{"x":0,"y":0,"width":0.5,"height":1}]}'}}] + }).encode("utf-8") + + monkeypatch.setattr(http_client, "_request", fake_request) + client = image_content.OpenAiCompatibleVisionClient( + "http://orchestrator/v1", "test-key", allow_insecure_http=True + ) + + regions = client.locate_regions(b"image-bytes", "image/png") + + assert len(regions) == 1 + assert captured["url"] == "http://orchestrator/v1/chat/completions" + assert captured["payload"]["mode"] == "auto" + assert captured["payload"]["reasoning_effort"] == "auto" + assert captured["payload"]["response_format"] == {"type": "json_object"} + + +def test_native_vision_region_locator_accepts_single_region_object(monkeypatch) -> None: + monkeypatch.setattr( + "lineageweave.vision_image.normalize_vision_image", + lambda image_bytes, mime_type: (image_bytes, mime_type), + ) + + def fake_request(method, url, *, body, headers, timeout): + return 200, json.dumps({ + "choices": [{"message": {"content": '{"x":0.13,"y":0.545,"width":0.74,"height":0.41}'}}] + }).encode("utf-8") + + monkeypatch.setattr(http_client, "_request", fake_request) + client = image_content.OpenAiCompatibleVisionClient( + "http://orchestrator/v1", "test-key", allow_insecure_http=True + ) + + regions = client.locate_regions(b"image-bytes", "image/png") + + assert regions == (image_content.ImageRegion(0.13, 0.545, 0.74, 0.41),) diff --git a/tests/test_corporate_hierarchy_inference.py b/tests/test_corporate_hierarchy_inference.py index b8795bb0a..dd4f67f05 100644 --- a/tests/test_corporate_hierarchy_inference.py +++ b/tests/test_corporate_hierarchy_inference.py @@ -7,6 +7,7 @@ from __future__ import annotations from lineageweave.corporate_hierarchy_inference import ( + ContextualOrchestratorHierarchyInferenceClient, LEVEL_COMPANY, LEVEL_GROUP, LEVEL_PLANT, @@ -15,6 +16,31 @@ ) +def test_live_hierarchy_client_uses_adaptive_orchestrator_mode(monkeypatch) -> None: + seen: dict[str, object] = {} + + def fake_post_json(url, body, *, headers, timeout): + seen.update(url=url, body=body, headers=headers, timeout=timeout) + return { + "choices": [ + {"message": {"content": '{"level":"plant","parent_name":"Aurora Grid Power"}'}}, + ] + } + + monkeypatch.setattr("lineageweave.corporate_hierarchy_inference.post_json", fake_post_json) + client = ContextualOrchestratorHierarchyInferenceClient( + "http://orchestrator", "secret", reasoning_effort="high", timeout=13.0 + ) + + assert client.infer("Aurora Grid Power South Plant", "synthetic plant context") == HierarchyProposal( + level_code=LEVEL_PLANT, parent_name="Aurora Grid Power" + ) + assert seen["url"] == "http://orchestrator/v1/chat/completions" + assert seen["body"]["mode"] == "auto" + assert seen["body"]["reasoning_effort"] == "high" + assert seen["timeout"] == 13.0 + + def test_parses_a_plant_with_a_parent() -> None: content = '{"level": "plant", "parent_name": "Acme Electronics"}' assert parse_inference_response(content) == HierarchyProposal( diff --git a/tests/test_customer_group_tree.py b/tests/test_customer_group_tree.py deleted file mode 100644 index 852d11adb..000000000 --- a/tests/test_customer_group_tree.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Authorized customer-group forest: affiliated rows plus ancestors/descendants.""" - -from __future__ import annotations - -from lineageweave.customer_group_tree import ( - CatalogEntityRow, - TreeAbbreviation, - authorized_catalog_ids, - build_customer_group_forest, -) - -_ENTITIES = ( - CatalogEntityRow("group-id", None, "Demo Group", "group"), - CatalogEntityRow("corp-id", "group-id", "Demo Corp", "company"), - CatalogEntityRow("plant-id", "corp-id", "Demo Plant", "plant"), - CatalogEntityRow("other-id", None, "Other Corp", "group"), -) - - -def test_affiliated_company_includes_group_parent_and_plant_child() -> None: - needed = authorized_catalog_ids(_ENTITIES, ("corp-id",)) - assert needed == {"group-id", "corp-id", "plant-id"} - - -def test_unaffiliated_sibling_group_is_omitted() -> None: - forest = build_customer_group_forest(_ENTITIES, ("corp-id",)) - assert [node.entity_name for node in forest] == ["Demo Group"] - group = forest[0] - assert [child.entity_name for child in group.children] == ["Demo Corp"] - assert [child.entity_name for child in group.children[0].children] == ["Demo Plant"] - - -def test_unknown_affiliation_adds_no_invented_parent() -> None: - forest = build_customer_group_forest(_ENTITIES, ("missing-id",)) - assert forest == () - - -def test_broken_parent_pointer_stops_without_inventing_an_ancestor() -> None: - broken = ( - CatalogEntityRow("corp-id", "missing-parent", "Demo Corp", "company"), - ) - assert authorized_catalog_ids(broken, ("corp-id",)) == {"corp-id"} - - -def test_already_included_descendant_is_not_walked_twice() -> None: - needed = authorized_catalog_ids(_ENTITIES, ("plant-id", "group-id")) - assert needed == {"group-id", "corp-id", "plant-id"} - - -def test_corroborated_abbreviation_attaches_only_to_authorized_nodes() -> None: - forest = build_customer_group_forest( - _ENTITIES, - ("corp-id",), - ( - ( - "corp-id", - TreeAbbreviation("DC", "verify_corroborated", "https://example.test/demo-corp-dc"), - ), - ( - "other-id", - TreeAbbreviation("OC", "verify_corroborated", "https://example.test/other"), - ), - ), - ) - company = forest[0].children[0] - assert [alias.raw_organization_name for alias in company.abbreviations] == ["DC"] - assert forest[0].to_dict()["children"][0]["abbreviations"][0]["raw_organization_name"] == "DC" - serialized = forest[0].to_dict() - assert "Other Corp" not in str(serialized) diff --git a/tests/test_customer_group_tree_labels.py b/tests/test_customer_group_tree_labels.py deleted file mode 100644 index 592496dee..000000000 --- a/tests/test_customer_group_tree_labels.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Lookup-label helpers for the customer-group JSON forest.""" - -from __future__ import annotations - -from backend.app.customer_group_tree_ingestion import _apply_lookup_labels, _collect_level_codes - - -def test_collect_level_codes_walks_nested_children() -> None: - codes = _collect_level_codes( - [ - { - "entity_level_code": "group", - "children": [ - { - "entity_level_code": "company", - "children": [{"entity_level_code": "plant", "children": []}], - } - ], - } - ] - ) - assert codes == ["group", "company", "plant"] - - -def test_apply_lookup_labels_falls_back_to_the_code() -> None: - forest = [ - { - "entity_level_code": "group", - "children": [{"entity_level_code": "company", "children": []}], - }, - {"entity_level_code": None, "children": []}, - ] - _apply_lookup_labels(forest, {"group": "Group"}) - assert forest[0]["entity_level_label"] == "Group" - assert forest[0]["children"][0]["entity_level_label"] == "company" - assert forest[1]["entity_level_label"] is None diff --git a/tests/test_customer_hint_ingestion.py b/tests/test_customer_hint_ingestion.py new file mode 100644 index 000000000..d1f63d80f --- /dev/null +++ b/tests/test_customer_hint_ingestion.py @@ -0,0 +1,116 @@ +"""Tests for backend.app.customer_hint_ingestion. + +Deterministic FakeConnection, same style as +tests/test_organization_name_resolution_ingestion.py. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import backend.app.customer_hint_ingestion as ingestion +from lineageweave.relation_verification import STATUS_CORROBORATED, STATUS_UNCORROBORATED + + +class _Client: + available = True + + +class _UnavailableClient: + available = False + + +class _Connection: + def __init__(self, *, sample_rows=None, existing_entity=None) -> None: + self._sample_rows = sample_rows or [] + self._existing_entity = existing_entity + self.executed: list[tuple[str, tuple[object, ...]]] = [] + + async def fetch(self, query: str, *args: object): + if "from source_post" in query: + return self._sample_rows + if "update source_post" in query: + self.executed.append((query, args)) + return [{"post_id": "post-1"}, {"post_id": "post-2"}] + return [] + + async def fetchrow(self, query: str, *args: object): + if "from corporate_entity" in query: + return self._existing_entity + if "insert into corporate_entity" in query: + self.executed.append((query, args)) + return {"corporate_entity_id": "new-entity-id"} + return None + + +def _resolution(status: str): + return SimpleNamespace( + raw_organization_name="0019999999", + resolved_organization_name="Northridge Grid", + verification_status_code=status, + verification_evidence_url="https://evidence.example/result" if status == STATUS_CORROBORATED else None, + ) + + +def test_unavailable_client_resolves_nothing() -> None: + conn = _Connection() + result = asyncio.run( + ingestion.resolve_customer_hint(conn, _UnavailableClient(), _Client(), "0019999999") + ) + assert result is None + + +def test_no_sample_posts_resolves_nothing() -> None: + conn = _Connection(sample_rows=[]) + result = asyncio.run(ingestion.resolve_customer_hint(conn, _Client(), _Client(), "0019999999")) + assert result is None + + +def test_uncorroborated_resolution_does_not_create_or_link_an_entity(monkeypatch) -> None: + monkeypatch.setattr( + ingestion, "resolve_and_verify_organization_name", lambda *_args: _resolution(STATUS_UNCORROBORATED) + ) + conn = _Connection(sample_rows=[{"post_title": "Visit", "post_body": "

    Visit notes

    "}]) + result = asyncio.run(ingestion.resolve_customer_hint(conn, _Client(), _Client(), "0019999999")) + + assert result is None + assert conn.executed == [] + + +def test_corroborated_resolution_creates_and_links_a_new_entity(monkeypatch) -> None: + monkeypatch.setattr( + ingestion, "resolve_and_verify_organization_name", lambda *_args: _resolution(STATUS_CORROBORATED) + ) + conn = _Connection(sample_rows=[{"post_title": "Visit", "post_body": "

    Visit notes

    "}]) + result = asyncio.run(ingestion.resolve_customer_hint(conn, _Client(), _Client(), "0019999999")) + + assert result == { + "corporate_entity_id": "new-entity-id", + "entity_name": "Northridge Grid", + "linked_post_count": 2, + "verification_evidence_url": "https://evidence.example/result", + } + insert_calls = [call for call in conn.executed if "insert into corporate_entity" in call[0]] + assert len(insert_calls) == 1 + assert insert_calls[0][1] == ("HINT-0019999999", "Northridge Grid") + # Live-shaped bug: re-resolving the same hint_code is not guaranteed to + # get byte-identical LLM phrasing back, so the create path must key off + # corporate_entity_code (deterministic from hint_code), not rely on the + # name-based lookup alone -- otherwise a second resolve with slightly + # different wording collides on the unique code and raises uncaught. + assert "on conflict (corporate_entity_code)" in insert_calls[0][0] + + +def test_corroborated_resolution_reuses_an_existing_entity_by_name(monkeypatch) -> None: + monkeypatch.setattr( + ingestion, "resolve_and_verify_organization_name", lambda *_args: _resolution(STATUS_CORROBORATED) + ) + conn = _Connection( + sample_rows=[{"post_title": "Visit", "post_body": "

    Visit notes

    "}], + existing_entity={"corporate_entity_id": "existing-entity-id"}, + ) + result = asyncio.run(ingestion.resolve_customer_hint(conn, _Client(), _Client(), "0019999999")) + + assert result["corporate_entity_id"] == "existing-entity-id" + assert all("insert into corporate_entity" not in call[0] for call in conn.executed) diff --git a/tests/test_customer_hint_resolution.py b/tests/test_customer_hint_resolution.py new file mode 100644 index 000000000..f4b3aacf8 --- /dev/null +++ b/tests/test_customer_hint_resolution.py @@ -0,0 +1,51 @@ +"""Tests for lineageweave.customer_hint_resolution. + +Deterministic fake HTTP transport, same style as +tests/test_organization_name_resolution.py -- post_json's own HTTP +mechanics are already covered in test_http_client.py; these tests are +for this module's own prompt/response contract. +""" + +from __future__ import annotations + +from lineageweave.customer_hint_resolution import ( + ContextualOrchestratorCustomerHintResolutionClient, + NullCustomerHintResolutionClient, +) + + +def test_null_client_is_unavailable() -> None: + client = NullCustomerHintResolutionClient() + assert client.available is False + + +def test_live_client_uses_adaptive_orchestrator_mode(monkeypatch) -> None: + seen: dict[str, object] = {} + + def fake_post_json(url, body, *, headers, timeout): + seen.update(url=url, body=body, headers=headers, timeout=timeout) + return {"choices": [{"message": {"content": "Northridge Grid"}}]} + + monkeypatch.setattr("lineageweave.customer_hint_resolution.post_json", fake_post_json) + client = ContextualOrchestratorCustomerHintResolutionClient( + "http://orchestrator", "secret", reasoning_effort="high", timeout=11.0 + ) + + assert client.resolve("0019999999", "Northridge Grid visited our booth") == "Northridge Grid" + assert seen["url"] == "http://orchestrator/v1/chat/completions" + assert seen["body"]["mode"] == "auto" + assert seen["body"]["reasoning_effort"] == "high" + assert seen["timeout"] == 11.0 + # The opaque hint code is threaded into the prompt so the model knows + # which records it is naming, even though the code itself never + # appears in their text. + assert "0019999999" in seen["body"]["messages"][0]["content"] + + +def test_live_client_returns_none_when_model_declines(monkeypatch) -> None: + monkeypatch.setattr( + "lineageweave.customer_hint_resolution.post_json", + lambda *args, **kwargs: {"choices": [{"message": {"content": "UNKNOWN"}}]}, + ) + client = ContextualOrchestratorCustomerHintResolutionClient("http://orchestrator", "secret") + assert client.resolve("0019999999", "ambiguous context") is None diff --git a/tests/test_documentation_hygiene.py b/tests/test_documentation_hygiene.py index 53a93cb24..010bb5e5d 100644 --- a/tests/test_documentation_hygiene.py +++ b/tests/test_documentation_hygiene.py @@ -27,6 +27,8 @@ def test_adr_numbers_are_unique_and_documents_are_not_placeholders() -> None: numbered_paths: list[tuple[str, Path]] = [] for path in paths: + if path.name == "README.md": + continue match = _ADR_NAME.fullmatch(path.name) assert match is not None, f"ADR filename is not numbered: {path.name}" numbered_paths.append((match.group("number"), path)) @@ -72,21 +74,11 @@ def test_role_catalog_identity_migration_is_wired() -> None: ).read_text(encoding="utf-8") assert "0019_role_catalog_identity.sql" in dockerfile assert "0025_role_person_catalog_identity.sql" in dockerfile - assert "0026_report_leftover_pair.sql" in dockerfile - assert "0027_abbreviation_tree_corroboration.sql" in dockerfile assert "0019_role_catalog_identity.sql" in seed assert "0025_role_person_catalog_identity.sql" in seed - assert "0026_report_leftover_pair.sql" in seed - assert "0027_abbreviation_tree_corroboration.sql" in seed assert seed.index("0024_source_post_revision.sql") < seed.index( "0025_role_person_catalog_identity.sql" ) - assert seed.index("0025_role_person_catalog_identity.sql") < seed.index( - "0026_report_leftover_pair.sql" - ) - assert seed.index("0026_report_leftover_pair.sql") < seed.index( - "0027_abbreviation_tree_corroboration.sql" - ) assert "cataloged_person_id" in seed assert "order by created_at, person_id limit 1" in seed assert "cataloged_team_id" in migration_0019 @@ -98,3 +90,18 @@ def test_role_catalog_identity_migration_is_wired() -> None: assert "having count(*) = 1" in migration_0025 assert "distinct on" not in migration_0019.lower() assert "distinct on" not in migration_0025.lower() + + +def test_orchestrator_runtime_pin_matches_adr() -> None: + """The image pin and ADR must describe the same immutable upstream commit.""" + dockerfile = ( + _ROOT / "docker" / "contextual-orchestrator" / "Dockerfile" + ).read_text(encoding="utf-8") + adr = (_ADR_DIRECTORY / "0083-orchestrator-runtime-commit-pin.md").read_text( + encoding="utf-8" + ) + docker_match = re.search(r"archive/([0-9a-f]{40})\.tar\.gz", dockerfile) + adr_match = re.search(r"commit `([0-9a-f]{40})`", adr) + assert docker_match is not None + assert adr_match is not None + assert docker_match.group(1) == adr_match.group(1) diff --git a/tests/test_embedded_image_payload.py b/tests/test_embedded_image_payload.py deleted file mode 100644 index f57566137..000000000 --- a/tests/test_embedded_image_payload.py +++ /dev/null @@ -1,109 +0,0 @@ -from __future__ import annotations - -import base64 - -from lineageweave.embedded_image_payload import ( - decode_data_uri_image, - looks_like_raster_image, - source_offset, -) - -_TINY_PNG_B64 = ( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" -) -_TINY_PNG = base64.b64decode(_TINY_PNG_B64) -_JPEG_BYTES = b"\xff\xd8\xff\x00" -_GIF87_BYTES = b"GIF87a" + b"\x00" * 2 -_GIF89_BYTES = b"GIF89a" + b"\x00" * 2 -_WEBP_BYTES = b"RIFF\x00\x00\x00\x00WEBP" -_AVIF_BYTES = b"\x00\x00\x00\x00ftypavif\x00\x00\x00\x00" -_AVIS_BYTES = b"\x00\x00\x00\x00ftypavis\x00\x00\x00\x00" -_MIF1_BYTES = b"\x00\x00\x00\x00ftypmif1\x00\x00\x00\x00" - - -def test_looks_like_raster_image_accepts_png_signature() -> None: - assert looks_like_raster_image("image/png", _TINY_PNG) is True - - -def test_looks_like_raster_image_rejects_ascii_labeled_as_png() -> None: - assert looks_like_raster_image("image/png", b"Hello") is False - - -def test_looks_like_raster_image_rejects_empty_payload() -> None: - assert looks_like_raster_image("image/png", b"") is False - - -def test_looks_like_raster_image_accepts_jpeg_gif_webp_avif_signatures() -> None: - assert looks_like_raster_image("image/jpeg", _JPEG_BYTES) is True - assert looks_like_raster_image("image/jpg", _JPEG_BYTES) is True - assert looks_like_raster_image("image/gif", _GIF87_BYTES) is True - assert looks_like_raster_image("image/gif", _GIF89_BYTES) is True - assert looks_like_raster_image("image/webp", _WEBP_BYTES) is True - assert looks_like_raster_image("image/avif", _AVIF_BYTES) is True - assert looks_like_raster_image("image/avif", _AVIS_BYTES) is True - assert looks_like_raster_image("image/avif", _MIF1_BYTES) is True - - -def test_looks_like_raster_image_rejects_wrong_magic_and_unknown_type() -> None: - assert looks_like_raster_image("image/jpeg", b"not-a-jpeg") is False - assert looks_like_raster_image("image/gif", b"GIF8xa") is False - assert looks_like_raster_image("image/webp", b"RIFF....NOTW") is False - assert looks_like_raster_image("image/webp", b"RIFF") is False - assert looks_like_raster_image("image/avif", b"xxxxftypxxxx") is False - assert looks_like_raster_image("image/avif", b"short") is False - assert looks_like_raster_image("image/svg+xml", _TINY_PNG) is False - - -def test_decode_data_uri_image_rejects_svg_and_remote_src() -> None: - assert decode_data_uri_image("https://example.test/invoice.png") is None - assert ( - decode_data_uri_image( - "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==" - ) - is None - ) - - -def test_decode_data_uri_image_rejects_missing_comma_or_base64_marker() -> None: - assert decode_data_uri_image("data:image/png;base64") is None - assert decode_data_uri_image(f"data:image/png,{_TINY_PNG_B64}") is None - - -def test_decode_data_uri_image_rejects_unpadded_and_wrong_magic() -> None: - assert decode_data_uri_image("data:image/png;base64,YQ") is None - assert decode_data_uri_image("data:image/png;base64,AAAA") is None - - -def test_decode_data_uri_image_accepts_newlines_inside_png_payload() -> None: - wrapped = f"data:image/png;base64,{_TINY_PNG_B64[:24]}\n{_TINY_PNG_B64[24:]}" - decoded = decode_data_uri_image(wrapped) - assert decoded == ("image/png", _TINY_PNG) - - -def test_decode_data_uri_image_accepts_jpeg_alias() -> None: - encoded = base64.b64encode(_JPEG_BYTES).decode("ascii") - assert decode_data_uri_image(f"data:image/jpg;base64,{encoded}") == ( - "image/jpg", - _JPEG_BYTES, - ) - - -def test_decode_data_uri_image_accepts_gif_webp_and_avif() -> None: - for mime_type, payload in ( - ("image/gif", _GIF89_BYTES), - ("image/webp", _WEBP_BYTES), - ("image/avif", _AVIF_BYTES), - ): - encoded = base64.b64encode(payload).decode("ascii") - assert decode_data_uri_image(f"data:{mime_type};base64,{encoded}") == ( - mime_type, - payload, - ) - - -def test_source_offset_maps_htmlparser_getpos() -> None: - source = "ab\ncd" - assert source_offset(source, 1, 0) == 0 - assert source_offset(source, 2, 1) == 4 - assert source_offset(source, 0, 0) == 0 - assert source_offset(source, 9, 0) == len(source) diff --git a/tests/test_embedding_client.py b/tests/test_embedding_client.py index a399fa1f2..0ab4d0f8d 100644 --- a/tests/test_embedding_client.py +++ b/tests/test_embedding_client.py @@ -9,7 +9,10 @@ from __future__ import annotations from lineageweave.chunking import Chunk -from lineageweave.embedding_client import chunked_max_similarity +from lineageweave.embedding_client import ( + ContextualOrchestratorEmbeddingClient, + chunked_max_similarity, +) class _RecordingFakeEmbeddingClient: @@ -84,3 +87,32 @@ def test_uses_chunker_output_directly_when_it_returns_two_or_more_pieces() -> No # Both documents chunk into 2 pieces each via _chunk_to_two_pieces -- # the fallback must NOT engage, so every chunk gets its own embed call. assert len(client.embed_calls) == 4 + + +def test_orchestrator_embedding_client_submits_and_polls_batch(monkeypatch) -> None: + calls = [] + + def fake_post_json(url, payload, *, headers, timeout): + calls.append(("post", url, payload, headers)) + return {"batch_id": "synthetic-batch", "status": "queued"} + + def fake_get_json(url, *, headers, timeout): + calls.append(("get", url, headers)) + return { + "batch_id": "synthetic-batch", + "status": "completed", + "embeddings": [ + {"index": 1, "embedding": [2.0, 3.0]}, + {"index": 0, "embedding": [0.0, 1.0]}, + ], + } + + monkeypatch.setattr("lineageweave.embedding_client.post_json", fake_post_json) + monkeypatch.setattr("lineageweave.embedding_client.get_json", fake_get_json) + client = ContextualOrchestratorEmbeddingClient( + "http://orchestrator:8000", "synthetic-token", "synthetic-embedding", poll_interval=0 + ) + + assert client.embed_many(["first", "second"]) == [[0.0, 1.0], [2.0, 3.0]] + assert calls[0][1] == "http://orchestrator:8000/v1/batch/embeddings" + assert calls[0][3] == {"authorization": "Bearer synthetic-token"} diff --git a/tests/test_embedding_client_edges.py b/tests/test_embedding_client_edges.py new file mode 100644 index 000000000..a7d8a9fcb --- /dev/null +++ b/tests/test_embedding_client_edges.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import pytest + +import lineageweave.embedding_client as embedding_client + + +def test_missing_embedding_configuration_returns_null_client() -> None: + client = embedding_client.orchestrator_embedding_client("", "", "") + assert isinstance(client, embedding_client.NullEmbeddingClient) + assert client.available is False + with pytest.raises(RuntimeError, match="no embedding channel"): + client.embed("text") + + +def test_empty_batch_does_not_call_orchestrator(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(embedding_client, "post_json", lambda *_args, **_kwargs: pytest.fail("unexpected call")) + client = embedding_client.ContextualOrchestratorEmbeddingClient("http://orchestrator", "key", "model") + assert client.embed_many([]) == [] + + +def test_immediate_embedding_response_is_ordered(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + embedding_client, + "post_json", + lambda *_args, **_kwargs: { + "embeddings": [ + {"index": 1, "embedding": [2]}, + {"index": 0, "embedding": [1]}, + ] + }, + ) + client = embedding_client.ContextualOrchestratorEmbeddingClient("http://orchestrator/v1", "key", "model") + assert client.embed_many(["a", "b"]) == [[1.0], [2.0]] + + +def test_batch_response_polls_until_complete(monkeypatch: pytest.MonkeyPatch) -> None: + responses = iter([{"batch_id": "batch-1", "status": "pending"}]) + monkeypatch.setattr(embedding_client, "post_json", lambda *_args, **_kwargs: next(responses)) + monkeypatch.setattr( + embedding_client, + "get_json", + lambda *_args, **_kwargs: {"embeddings": [{"index": 0, "embedding": [0.5]}]}, + ) + monkeypatch.setattr(embedding_client.time, "sleep", lambda _seconds: None) + monkeypatch.setattr(embedding_client.time, "monotonic", lambda: 0.0) + client = embedding_client.ContextualOrchestratorEmbeddingClient("http://orchestrator", "key", "model", timeout=1) + assert client.embed_many(["a"]) == [[0.5]] + + +def test_failed_batch_raises_without_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + embedding_client, + "post_json", + lambda *_args, **_kwargs: {"batch_id": "batch-1", "status": "failed"}, + ) + client = embedding_client.ContextualOrchestratorEmbeddingClient("http://orchestrator", "key", "model") + with pytest.raises(RuntimeError, match="did not complete"): + client.embed_many(["a"]) + + +def test_batch_timeout_raises(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + embedding_client, + "post_json", + lambda *_args, **_kwargs: {"batch_id": "batch-1", "status": "pending"}, + ) + monkeypatch.setattr(embedding_client.time, "monotonic", iter([0.0, 2.0]).__next__) + client = embedding_client.ContextualOrchestratorEmbeddingClient("http://orchestrator", "key", "model", timeout=1) + with pytest.raises(TimeoutError, match="timed out"): + client.embed_many(["a"]) + + +@pytest.mark.parametrize( + "response", + [ + {}, + {"embeddings": []}, + {"embeddings": [{"index": 0, "embedding": []}]}, + {"embeddings": [{"index": 0, "embedding": [float("nan")]}]}, + {"embeddings": [{"index": 2, "embedding": [1]}]}, + ], +) +def test_invalid_embedding_vectors_are_rejected(response: dict) -> None: + assert embedding_client.ContextualOrchestratorEmbeddingClient._vectors(response, 1) is None + + +def test_legacy_client_name_delegates(monkeypatch: pytest.MonkeyPatch) -> None: + class Delegate: + def __init__(self, *_args, **_kwargs) -> None: + pass + + def embed(self, text: str) -> list[float]: + return [float(len(text))] + + monkeypatch.setattr(embedding_client, "ContextualOrchestratorEmbeddingClient", Delegate) + client = embedding_client.OpenAiCompatibleEmbeddingClient("http://orchestrator", "key", "model") + assert client.embed("abc") == [3.0] + + +def test_cosine_similarity_returns_zero_for_zero_vector() -> None: + assert embedding_client.cosine_similarity([0.0], [1.0]) == 0.0 diff --git a/tests/test_entity_relationship_ingestion.py b/tests/test_entity_relationship_ingestion.py new file mode 100644 index 000000000..81d6fff64 --- /dev/null +++ b/tests/test_entity_relationship_ingestion.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager + +from backend.app.entity_relationship_ingestion import ingest_post_entity_relationships +from lineageweave.entity_relationship_classification import OrganizationRelationship + + +class _Connection: + def __init__(self) -> None: + self.executed: list[tuple[str, tuple[object, ...]]] = [] + + @asynccontextmanager + async def transaction(self): + yield self + + async def execute(self, query: str, *args: object) -> str: + self.executed.append((" ".join(query.split()), args)) + return "OK" + + +class _Client: + available = True + + def classify(self, _title: str, _body: str, names: list[str]) -> list[OrganizationRelationship]: + return [OrganizationRelationship(names[0], "rel_voc")] if names else [] + + +def test_relationship_ingestion_replaces_stale_rows_and_filters_unknown_names() -> None: + conn = _Connection() + relationships = asyncio.run( + ingest_post_entity_relationships( + conn, + _Client(), + "post-1", + "Synthetic title", + "Synthetic body", + ["External organization"], + ) + ) + + assert relationships[0].organization_name == "External organization" + assert conn.executed[0][0] == "delete from post_counterparty_entity where post_id = $1" + assert conn.executed[1][0].startswith("insert into post_counterparty_entity") + + +def test_relationship_ingestion_clears_rows_when_no_counterparty_remains() -> None: + conn = _Connection() + assert asyncio.run( + ingest_post_entity_relationships( + conn, _Client(), "post-2", "Synthetic title", "Synthetic body", [] + ) + ) == [] + assert conn.executed == [ + ("delete from post_counterparty_entity where post_id = $1", ("post-2",)) + ] diff --git a/tests/test_five_w1h.py b/tests/test_five_w1h.py new file mode 100644 index 000000000..60b5735c7 --- /dev/null +++ b/tests/test_five_w1h.py @@ -0,0 +1,64 @@ +from lineageweave.five_w1h import assemble_five_w1h_slots, slots_payload + + +def test_five_w1h_keeps_persisted_evidence_and_leaves_unsupported_slots_empty() -> None: + slots = assemble_five_w1h_slots( + roles=[ + { + "actor_name": "Ada West", + "actor_type_code": "prov_person", + "affiliated_organization_name": "Demo Corp", + } + ], + key_events=["검사 일정 확정"], + counterparties=["Northwind Labs"], + ) + + assert [item["text"] for item in slots["who"]] == ["Ada West"] + assert [item["text"] for item in slots["what"]] == ["검사 일정 확정"] + # "when" has no persisted evidence of the narrated event's own time -- + # source_post.created_at is the record's filing time, a different + # PROV-O category (prov:generatedAtTime), and must not be shown here + # as if it answered "when did this happen" (see five_w1h.py). + assert slots["when"] == [] + assert {item["text"] for item in slots["where"]} == {"Demo Corp", "Northwind Labs"} + assert slots["why"] == [] + assert slots["how"] == [] + + +def test_five_w1h_uses_visible_lineage_title_only_as_what_fallback() -> None: + slots = assemble_five_w1h_slots( + roles=[], + key_events=[], + lineage_node_labels=["검사 후속 조치"], + ) + + payload = slots_payload(slots) + what = next(row for row in payload if row["slot_code"] == "what") + why = next(row for row in payload if row["slot_code"] == "why") + assert what["values"][0]["source"] == "post_lineage_edge" + assert why["values"] == [] + assert why["empty_next_action_code"] == "inspect_source_body_or_related_posts" + + +def test_five_w1h_uses_only_explicit_claims_for_missing_dimensions() -> None: + slots = assemble_five_w1h_slots( + roles=[], + key_events=[], + evidence_claims=[ + { + "slot_code": "when", + "value_text": "2026년 3월 4일", + "evidence_text": "3월 4일 현장 회의", + }, + { + "slot_code": "how", + "value_text": "화상 회의로", + "evidence_text": "화상으로 협의했다", + }, + ], + ) + assert slots["when"][0]["source"] == "post_summary_five_w1h" + assert slots["when"][0]["evidence_text"] == "3월 4일 현장 회의" + assert slots["how"][0]["text"] == "화상 회의로" + assert slots["where"] == [] diff --git a/tests/test_global_ask_sources.py b/tests/test_global_ask_sources.py new file mode 100644 index 000000000..9a41b507c --- /dev/null +++ b/tests/test_global_ask_sources.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +import asyncio + +from backend.app.post_chat_ingestion import gather_global_chat_sources + + +def test_global_sources_apply_visibility_before_normalization() -> None: + rows = [ + { + "post_id": "public-post", + "post_title": "Public evidence", + "post_body": "

    public body

    ", + "visibility_code": "public", + "corporate_entity_id": None, + }, + { + "post_id": "private-affiliated", + "post_title": "Affiliated evidence", + "post_body": "

    affiliated body

    ", + "visibility_code": "private", + "corporate_entity_id": "corp-demo", + }, + { + "post_id": "private-hidden", + "post_title": "Hidden evidence", + "post_body": "

    hidden body

    ", + "visibility_code": "private", + "corporate_entity_id": "corp-other", + }, + ] + + class FakeConnection: + async def fetch(self, query: str, *args): + return rows if "from source_post" in query else [] + + sources = asyncio.run( + gather_global_chat_sources( + FakeConnection(), + lambda row: row["visibility_code"] == "public" + or row["corporate_entity_id"] == "corp-demo", + {"corp-demo"}, + ) + ) + + assert [source.post_id for source in sources] == ["public-post", "private-affiliated"] + assert sources[0].post_body == "public body" + assert sources[1].post_body == "affiliated body" + + +def test_global_sources_prioritize_question_terms_and_bound_long_bodies() -> None: + rows = [ + { + "post_id": "newest-post", + "post_title": "Unrelated evidence", + "post_body": "ordinary body", + "visibility_code": "public", + "corporate_entity_id": None, + "matched_in": "body", + }, + { + "post_id": "uam-post", + "post_title": "UAM deployment evidence", + "post_body": "x" * 7000, + "visibility_code": "public", + "corporate_entity_id": None, + "matched_in": "title", + }, + ] + calls: list[tuple[str, tuple[object, ...]]] = [] + + class FakeConnection: + async def fetch(self, query: str, *args): + calls.append((query, args)) + return rows if "from source_post" in query else [] + + sources = asyncio.run( + gather_global_chat_sources( + FakeConnection(), + lambda row: True, + question="Which posts mention UAM?", + limit=8, + ) + ) + + candidate_query, candidate_args = calls[0] + source_query, source_args = next( + (query, args) for query, args in calls if "array_position($2::uuid[], post_id)" in query + ) + assert "to_tsvector('simple'" in candidate_query + assert candidate_args[0] == "mention" + assert "array_position($2::uuid[], post_id)" in source_query + assert source_args[2] == 8 + # Live bug (2026-08-19): a title match must outrank a body/source-field + # match regardless of discovery order -- "uam-post" matched in the + # title (higher weight) but was appended to candidate_rows after + # "newest-post" (a body match); the final candidate_ids array passed + # as $2 must still rank uam-post first. + assert list(source_args[1]) == ["uam-post", "newest-post"] + assert sources[1].post_body.startswith("x" * 4000) + assert "Source body truncated for Global Ask" in sources[1].post_body + + +def test_global_sources_carry_source_and_semantic_evidence() -> None: + rows = [ + { + "post_id": "semantic-post", + "post_title": "Operational note", + "post_body": "No project name in this body.", + "visibility_code": "public", + "corporate_entity_id": None, + "source_project_code": "PROJECT-HINT", + "source_record_key": "SYNTHETIC-KEY-001", + "matched_in": "source_field", + } + ] + + class FakeConnection: + async def fetch(self, query: str, *args): + if "from source_post" in query: + return rows + if "from post_project_mention" in query: + return [ + { + "post_id": "semantic-post", + "fact": "project: semantic project | ontology_iri: urn:test", + } + ] + return [] + + sources = __import__("asyncio").run( + gather_global_chat_sources( + FakeConnection(), lambda row: True, question="PROJECT-HINT", limit=1 + ) + ) + + assert len(sources) == 1 + assert any( + "source project code=PROJECT-HINT" in fact for fact in sources[0].evidence_facts + ) + assert sources[0].evidence_facts[-1].startswith("project: semantic project") + + +def test_global_sources_keep_hyphenated_source_codes_atomic() -> None: + """Live bug (2026-08-19): a hyphenated ERP-style job code such as + ``P41-4182-202405-0015`` used to be shredded into generic numeric + fragments (``P41``, ``4182``, ``202405``, ``0015``) by the search-term + tokenizer, so unrelated posts sharing only a short fragment (e.g. a + ``202405``-dated post from a different project) outranked or crowded + out the actual code match. The tokenizer must keep a hyphen-joined + code as one atomic search term. + """ + calls: list[tuple[str, tuple[object, ...]]] = [] + + class FakeConnection: + async def fetch(self, query: str, *args): + calls.append((query, args)) + return [] + + asyncio.run( + gather_global_chat_sources( + FakeConnection(), + lambda row: True, + question="P41-4182-202405-0015", + limit=4, + ) + ) + + candidate_terms = [args[0] for query, args in calls if "matched_in" in query] + assert candidate_terms == ["p41-4182-202405-0015"] + + +def test_global_sources_keep_unicode_search_terms_for_localized_buyers() -> None: + calls: list[tuple[str, tuple[object, ...]]] = [] + + class FakeConnection: + async def fetch(self, query: str, *args): + calls.append((query, args)) + return [] + + asyncio.run( + gather_global_chat_sources( + FakeConnection(), + lambda row: True, + question="无人机 ドローン dự-án", + limit=4, + ) + ) + + candidate_terms = [args[0] for query, args in calls if "matched_in" in query] + assert candidate_terms == ["无人机", "ドローン", "dự-án"] + + +def test_global_sources_keep_lineage_expansion_within_requested_limit() -> None: + matched_row = { + "post_id": "anchor-post", + "post_title": "Anchor evidence", + "post_body": "anchor body", + "visibility_code": "public", + "corporate_entity_id": None, + "matched_in": "title", + } + neighbor_ids = [f"neighbor-{index:02d}" for index in range(20)] + source_call: tuple[str, tuple[object, ...]] | None = None + + class FakeConnection: + async def fetch(self, query: str, *args): + nonlocal source_call + if "matched_in" in query: + return [matched_row] + if "post_lineage_edge" in query: + return [{"other_id": post_id} for post_id in reversed(neighbor_ids)] + if "array_position($2::uuid[], post_id)" in query: + source_call = (query, args) + rows = { + "anchor-post": matched_row, + **{ + post_id: { + "post_id": post_id, + "post_title": post_id, + "post_body": "neighbor body", + "visibility_code": "public", + "corporate_entity_id": None, + } + for post_id in neighbor_ids + }, + } + return [rows[post_id] for post_id in args[1]] + return [] + + sources = asyncio.run( + gather_global_chat_sources( + FakeConnection(), + lambda _row: True, + question="Anchor evidence", + limit=4, + ) + ) + + assert source_call is not None + _query, source_args = source_call + assert source_args[2] == 4 + assert list(source_args[1]) == [ + "anchor-post", + "neighbor-00", + "neighbor-01", + "neighbor-02", + ] + assert [source.post_id for source in sources] == list(source_args[1]) + assert len(sources) == 4 + + +def test_global_sources_return_no_evidence_for_zero_limit() -> None: + class FakeConnection: + async def fetch(self, _query: str, *_args): + raise AssertionError("zero source budget must not query evidence") + + assert ( + asyncio.run( + gather_global_chat_sources( + FakeConnection(), + lambda _row: True, + question="anything", + limit=0, + ) + ) + == [] + ) + + +def test_global_sources_expand_top_match_through_event_lineage() -> None: + """Global Ask must speak to a connected timeline, not an isolated + snapshot -- expand the single top-ranked keyword match through its + direct `post_lineage_edge` neighbors (`lineageweave.reconstruct`'s + output), mirroring the post-scoped chat flow's `find_linked_post_ids`. + """ + matched_row = { + "post_id": "event-2", + "post_title": "Northridge Grid capacity review", + "post_body": "capacity review body", + "visibility_code": "public", + "corporate_entity_id": None, + "matched_in": "title", + } + lineage_row = { + "post_id": "event-1", + "post_title": "Northridge Grid kickoff", + "post_body": "kickoff body", + "visibility_code": "public", + "corporate_entity_id": None, + } + + class FakeConnection: + async def fetch(self, query: str, *args): + if "matched_in" in query: + return [matched_row] + if "post_lineage_edge" in query: + return [{"other_id": "event-1"}] + if "array_position($2::uuid[], post_id)" in query: + return [matched_row, lineage_row] + return [] + + sources = asyncio.run( + gather_global_chat_sources( + FakeConnection(), + lambda row: True, + question="Northridge Grid capacity", + limit=4, + ) + ) + + assert [source.post_id for source in sources] == ["event-2", "event-1"] + assert sources[0].evidence_facts == () + assert any( + "Event Lineage: reconstructed timeline neighbor of post_id=event-2" in fact + for fact in sources[1].evidence_facts + ) + + +def test_global_sources_do_not_leak_lineage_anchor_id_when_anchor_is_invisible() -> None: + """If ABAC hides the top match itself, an expanded neighbor must not + cite that hidden post's id as its lineage anchor. + """ + matched_row = { + "post_id": "hidden-anchor", + "post_title": "Private kickoff", + "post_body": "private body", + "visibility_code": "private", + "corporate_entity_id": "corp-other", + "matched_in": "title", + } + lineage_row = { + "post_id": "visible-neighbor", + "post_title": "Public follow-up", + "post_body": "public body", + "visibility_code": "public", + "corporate_entity_id": None, + } + + class FakeConnection: + async def fetch(self, query: str, *args): + if "matched_in" in query: + return [matched_row] + if "post_lineage_edge" in query: + return [{"other_id": "visible-neighbor"}] + if "array_position($2::uuid[], post_id)" in query: + return [matched_row, lineage_row] + return [] + + sources = asyncio.run( + gather_global_chat_sources( + FakeConnection(), + lambda row: row["visibility_code"] == "public", + question="kickoff", + limit=4, + ) + ) + + assert [source.post_id for source in sources] == ["visible-neighbor"] + assert sources[0].evidence_facts == () diff --git a/tests/test_http_client_edges.py b/tests/test_http_client_edges.py new file mode 100644 index 000000000..8c6119fc9 --- /dev/null +++ b/tests/test_http_client_edges.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import pytest + +import lineageweave.http_client as http_client + + +def test_json_helpers_reject_non_json_and_wrong_shapes(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(http_client, "_request", lambda *_args, **_kwargs: (200, b"not-json")) + with pytest.raises(http_client.HttpClientError, match="non-JSON"): + http_client.get_json("https://gateway.example/health", timeout=1) + + monkeypatch.setattr(http_client, "_request", lambda *_args, **_kwargs: (200, b"[]")) + with pytest.raises(http_client.HttpClientError, match="JSON object"): + http_client.post_json("https://gateway.example/v1", {}, headers={}, timeout=1) + + monkeypatch.setattr(http_client, "_request", lambda *_args, **_kwargs: (200, b"{}")) + with pytest.raises(http_client.HttpClientError, match="JSON array"): + http_client.get_json_list("https://gateway.example/items", timeout=1) + + +@pytest.mark.parametrize( + "helper", + [http_client.post_json, http_client.post_form, http_client.get_json, http_client.get_json_list], +) +def test_json_helpers_raise_on_http_errors(monkeypatch: pytest.MonkeyPatch, helper) -> None: + monkeypatch.setattr(http_client, "_request", lambda *_args, **_kwargs: (503, b"{}")) + kwargs = {"timeout": 1} + if helper is http_client.post_json: + kwargs.update(payload={}, headers={}) + elif helper is http_client.post_form: + kwargs.update(fields={}, headers={}) + with pytest.raises(http_client.HttpClientError, match="HTTP 503"): + if helper in (http_client.post_json, http_client.post_form): + helper("https://gateway.example/endpoint", **kwargs) + else: + helper("https://gateway.example/endpoint", **kwargs) + + +def test_json_helpers_accept_optional_headers(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[object, ...]] = [] + + def request(*args, **kwargs): + calls.append((args, kwargs)) + return 200, b"{}" if kwargs["headers"].get("content-type") != "application/json" else b"{}" + + monkeypatch.setattr(http_client, "_request", request) + assert http_client.get_json("https://gateway.example", timeout=1) == {} + assert http_client.post_form("https://gateway.example", {}, timeout=1) == {} + assert len(calls) == 2 diff --git a/tests/test_image_content.py b/tests/test_image_content.py index 8015c1b6d..de7fc49b5 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -1,20 +1,20 @@ from __future__ import annotations import base64 -from pathlib import Path import pytest -from lineageweave.chunking import chunk_by_dom -from lineageweave.embedded_image_payload import decode_data_uri_image from lineageweave.image_content import ( ImageContentClient, ImageDescriptionParseError, NullImageContentClient, OpenAiCompatibleVisionClient, + _RESPONSE_FORMAT, + _REGION_RESPONSE_FORMAT, _parse_description, extract_base64_images, orchestrator_vision_client, + regions_cover_image, ) # A 1x1 transparent PNG, valid base64 -- enough to exercise real decoding @@ -58,57 +58,6 @@ def test_extract_base64_images_empty_document_yields_no_images() -> None: assert extract_base64_images("

    No images here.

    ") == [] -def test_extract_base64_images_skips_svg_and_unpadded_payloads() -> None: - svg = ( - '' - ) - assert extract_base64_images(svg) == [] - assert extract_base64_images('') == [] - assert decode_data_uri_image(f"data:image/png;base64,{_TINY_PNG_B64}") is not None - - -def test_extract_base64_images_skips_style_script_and_src_less_tags() -> None: - html = ( - f'' - f'' - "" - f'' - ) - images = extract_base64_images(html) - assert len(images) == 1 - assert images[0].data == base64.b64decode(_TINY_PNG_B64) - - -def test_invoice_fixture_is_one_visible_png_for_every_extractor() -> None: - """Outlook-style invoice HTML must not resurrect the base64 wall. - - The same file is read by the TypeScript popup splitter. All three - extractors must see one raster PNG, ignore the commented copy, the - remote URL, the SVG, and the CSS background, and keep surrounding - sentences readable. - """ - html = (Path(__file__).parent / "fixtures" / "synthetic_invoice_embedded_image.html").read_text( - encoding="utf-8" - ) - images = extract_base64_images(html) - chunks = chunk_by_dom(html) - image_chunks = [chunk for chunk in chunks if chunk.unit_type == "image"] - text = " ".join(chunk.text for chunk in chunks if chunk.unit_type == "dom") - - assert len(images) == 1 - assert len(image_chunks) == 1 - assert images[0].mime_type == "image/png" - assert images[0].data == base64.b64decode(_TINY_PNG_B64) - assert images[0].position == html.find(") - assert None: content = "TEXT: Quarterly Budget Report\nCAPTION: A printed report cover page.\nTAGS: document, report, text" description = _parse_description(content) @@ -136,6 +85,21 @@ def test_parse_description_preserves_multiline_ocr_text() -> None: assert description.caption == "A scanned page." +def test_parse_description_preserves_table_row_structure_in_ocr_text() -> None: + """Live gap (2026-08-19): an image containing a table used to have its + text flattened into an unstructured word list on OCR, the same + row-grouping loss chunk_by_dom had for real HTML tables. The parser + already preserves multi-line TEXT (see the sibling test above); this + confirms a table-shaped response -- one row per line, columns + delimited by " | " per the response-format prompt -- round-trips intact. + """ + content = "TEXT: No. | Company | Result\n1 | Acme Corp | Declined\n2 | Globex Corp | Interested\nCAPTION: A visit log table.\nTAGS: table, log" + description = _parse_description(content) + assert description.extracted_text == ( + "No. | Company | Result\n1 | Acme Corp | Declined\n2 | Globex Corp | Interested" + ) + + def test_parse_description_tolerates_markdown_emphasis_on_labels() -> None: """Synthetic provider drift may bold labels without changing content.""" content = "**TEXT:** LT7\n**CAPTION:** A close-up of a component.\n**TAGS:** component, close-up" @@ -239,7 +203,7 @@ def test_orchestrator_vision_client_does_not_double_v1() -> None: def test_orchestrator_vision_client_is_null_when_unconfigured() -> None: - client = orchestrator_vision_client("", "", "") + client = orchestrator_vision_client("", "") assert isinstance(client, NullImageContentClient) assert client.available is False @@ -252,6 +216,31 @@ def test_image_content_client_protocol_stub_raises() -> None: ImageContentClient.describe(None, b"", "image/png") # type: ignore[arg-type] +def test_ocr_prompt_asks_for_table_row_structure() -> None: + """Live gap (2026-08-19): the OCR prompt had no guidance for a + table-shaped image, so a real table image had its text flattened into + an unstructured list -- the same class of bug chunk_by_dom had for + real HTML tables. The prompt must tell the model to preserve rows. + """ + assert "row" in _RESPONSE_FORMAT.lower() + assert "table" in _RESPONSE_FORMAT.lower() + + +def test_region_prompt_requires_full_image_coverage() -> None: + """Live gap (2026-08-19): "distinct meaningful visual regions" alone + let the model describe only the most visually striking part of an + image and skip the rest, instead of covering the whole DOM/image area. + """ + assert "entire image" in _REGION_RESPONSE_FORMAT.lower() + + +def test_region_coverage_guard_rejects_a_salient_crop() -> None: + from lineageweave.image_content import ImageRegion + + assert not regions_cover_image((ImageRegion(0.2, 0.2, 0.3, 0.3),)) + assert not regions_cover_image((ImageRegion(0.0, 0.0, 1.0, 1.0),)) + + def test_parse_description_does_not_absorb_unknown_labels_into_tags() -> None: parsed = _parse_description( "TEXT: NONE\nCAPTION: A turbine diagram\n" diff --git a/tests/test_image_content_gateway.py b/tests/test_image_content_gateway.py new file mode 100644 index 000000000..9a1728a17 --- /dev/null +++ b/tests/test_image_content_gateway.py @@ -0,0 +1,65 @@ +import base64 +from io import BytesIO + +from PIL import Image + +import lineageweave.image_content as image_content + + +def _transparent_png() -> bytes: + image = Image.new("RGBA", (1, 1), (12, 34, 56, 0)) + output = BytesIO() + image.save(output, format="PNG") + return output.getvalue() + + +def test_vision_gateway_normalizes_image_and_preserves_structured_response(monkeypatch) -> None: + captured: dict[str, object] = {} + + def post_json(url, payload, *, headers, timeout): + captured.update(url=url, payload=payload, headers=headers, timeout=timeout) + return { + "choices": [ + { + "message": { + "content": "TEXT: synthetic label\nCAPTION: a synthetic diagram\nTAGS: diagram, test" + } + } + ] + } + + monkeypatch.setattr(image_content, "post_json", post_json) + client = image_content.OpenAiCompatibleVisionClient( + "http://orchestrator/v1", "gateway-key", "", allow_insecure_http=True + ) + + description = client.describe(_transparent_png(), "image/tiff") + + assert description.extracted_text == "synthetic label" + assert description.caption == "a synthetic diagram" + assert description.tags == ("diagram", "test") + assert captured["url"] == "http://orchestrator/v1/chat/completions" + assert captured["headers"] == {"authorization": "Bearer gateway-key"} + payload = captured["payload"] + assert "model" not in payload + assert payload["mode"] == "auto" + assert payload["reasoning_effort"] == "auto" + assert [message["role"] for message in payload["messages"]] == ["system", "user"] + user_message = payload["messages"][1] + image_url = user_message["content"][1]["image_url"]["url"] + assert image_url.startswith("data:image/png;base64,") + normalized = Image.open(BytesIO(base64.b64decode(image_url.split(",", 1)[1]))) + assert normalized.convert("RGB").getpixel((0, 0)) == (255, 255, 255) + + +def test_vision_factory_fails_closed_for_unsupported_url_scheme() -> None: + client = image_content.orchestrator_vision_client("ftp://orchestrator", "gateway-key", "vision-model") + assert isinstance(client, image_content.NullImageContentClient) + assert client.available is False + + +def test_vision_factory_allows_orchestrator_model_selection() -> None: + client = image_content.orchestrator_vision_client("http://orchestrator", "gateway-key") + + assert isinstance(client, image_content.OpenAiCompatibleVisionClient) + assert client.available is True diff --git a/tests/test_import_postgresql_posts.py b/tests/test_import_postgresql_posts.py new file mode 100644 index 000000000..c98e96531 --- /dev/null +++ b/tests/test_import_postgresql_posts.py @@ -0,0 +1,223 @@ +import uuid +from types import SimpleNamespace +from pathlib import Path + +import pytest + +from scripts.import_postgresql_posts import ( + _parser, + _normalize_voc_type, + _source_post_id, + _source_code_matches, + _validate_source_mapping, + _validate_source_rows, + _validate_corporate_entity_scope, +) + + +@pytest.mark.parametrize( + ("source_value", "expected"), + [("VOC", "voc"), ("VOCC", "vocc"), ("VOCO", "voco"), ("VOM", "vom"), ("VOP", "vop")], +) +def test_importer_preserves_source_voc_type_vocabulary(source_value: str, expected: str) -> None: + assert _normalize_voc_type(source_value, mapped=True) == expected + + +def test_importer_rejects_unknown_or_empty_mapped_voc_type() -> None: + with pytest.raises(ValueError, match="unsupported source VOC type"): + _normalize_voc_type("not-a-voc-type", mapped=True) + with pytest.raises(ValueError, match="mapped source VOC type is empty"): + _normalize_voc_type("", mapped=True) + + +def test_source_state_exclusion_uses_only_explicit_caller_values() -> None: + row = {"draft_state": " Temporary ", "deleted_state": "N"} + + assert _source_code_matches(row, "draft_state", ["temporary"]) + assert not _source_code_matches(row, "draft_state", ["draft"]) + assert not _source_code_matches(row, "deleted_state", ["Y"]) + + +def test_source_state_exclusion_does_not_guess_when_mapping_is_absent() -> None: + row = {"draft_state": "draft", "deleted_state": "Y"} + + assert not _source_code_matches(row, None, ["draft"]) + assert not _source_code_matches(row, "draft_state", []) + + +def test_importer_rejects_mapping_the_pu_column_as_sales_pool() -> None: + with pytest.raises(ValueError, match="PU is source_process_unit_code"): + _validate_source_mapping("pu_code", "pu_code") + + +def test_importer_preflights_identity_and_body_before_target_mutation() -> None: + mapping = SimpleNamespace(record_key="record_key", body="body", draft="draft_state", deleted=None) + + with pytest.raises(ValueError, match="source record key cannot be empty at source row 2"): + _validate_source_rows( + [ + {"record_key": "one", "body": "body", "draft_state": "N"}, + {"record_key": "", "body": "body", "draft_state": "N"}, + ], + mapping, + ["Y"], + [], + ) + + with pytest.raises(ValueError, match="source post body cannot be empty at source row 1"): + _validate_source_rows( + [{"record_key": "one", "body": "", "draft_state": "N"}], mapping, ["Y"], [] + ) + + +def test_importer_keeps_source_record_key_separate_from_source_uuid() -> None: + mapping = SimpleNamespace(post_id="guid_field") + source_uuid = "01234567-89ab-cdef-0123-456789abcdef" + + assert _source_post_id( + {"guid_field": source_uuid}, mapping, "source", "human-entered-source-key" + ) == uuid.UUID(source_uuid) + + +def test_importer_derives_legacy_post_uuid_without_a_post_id_mapping() -> None: + mapping = SimpleNamespace(post_id=None) + + assert _source_post_id( + {}, mapping, "source", "human-entered-source-key" + ) == uuid.uuid5(uuid.UUID("b6e4b1d6-5fd0-4ca1-92b0-8f7a4e2df83e"), "source:human-entered-source-key") + + +def test_importer_rejects_duplicate_active_source_identity() -> None: + mapping = SimpleNamespace(record_key="record_key", body="body", draft="draft_state", deleted=None) + + with pytest.raises(ValueError, match="duplicate source record key at source rows 1 and 2"): + _validate_source_rows( + [ + {"record_key": "same", "body": "first", "draft_state": "N"}, + {"record_key": "same", "body": "second", "draft_state": "N"}, + ], + mapping, + ["Y"], + [], + ) + + +def test_importer_allows_repeated_lookup_keys_when_source_uuids_are_distinct() -> None: + mapping = SimpleNamespace( + record_key="record_key", + post_id="post_id", + body="body", + draft="draft_state", + deleted=None, + ) + + _validate_source_rows( + [ + { + "record_key": "same", + "post_id": "01234567-89ab-cdef-0123-456789abcdef", + "body": "first", + "draft_state": "N", + }, + { + "record_key": "same", + "post_id": "11234567-89ab-cdef-0123-456789abcdef", + "body": "second", + "draft_state": "N", + }, + ], + mapping, + ["Y"], + [], + ) + + +def test_source_record_key_index_is_a_lookup_not_a_uniqueness_constraint() -> None: + migration = ( + Path(__file__).resolve().parents[1] / "migrations" / "0037_source_record_identity.sql" + ).read_text() + assert "create unique index" not in migration.casefold() + assert "create index if not exists source_post_source_identity_idx" in migration + + +def test_importer_always_requires_verified_publication_state() -> None: + mapping = SimpleNamespace(record_key="record_key", body="body", draft="draft_state", deleted=None) + + with pytest.raises(ValueError, match="at least one source draft value"): + _validate_source_rows( + [{"record_key": "one", "body": "body", "draft_state": "N"}], + mapping, + [], + [], + ) + + with pytest.raises(ValueError, match="publication state is unknown"): + _validate_source_rows( + [{"record_key": "one", "body": "body", "draft_state": None}], + mapping, + ["Y"], + [], + ) + + +def test_importer_has_no_unknown_publication_state_bypass() -> None: + with pytest.raises(SystemExit): + _parser().parse_args(["--allow-unknown-publication-state"]) + + +def test_importer_rejects_demo_scope_without_explicit_test_override() -> None: + with pytest.raises(ValueError, match="non-DEMO corporate entity code"): + _validate_corporate_entity_scope("DEMO-CORP-01", allow_demo=False) + _validate_corporate_entity_scope("DEMO-CORP-01", allow_demo=True) + + +def test_importer_prefers_canonical_gateway_embedding_model(monkeypatch) -> None: + monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "gateway-embedding") + monkeypatch.setenv("EMBEDDING_MODEL", "legacy-embedding") + + args = _parser().parse_args( + [ + "--source-dsn", "postgresql://source", + "--target-dsn", "postgresql://target", + "--query-file", "query.sql", + "--source-system-code", "source", + "--record-key-column", "record_key", + "--title-column", "title", + "--body-column", "body", + "--created-at-column", "created_at", + "--author-subject-id", "subject", + "--corporate-entity-code", "corp", + "--process-unit-code", "pu", + ] + ) + + assert args.embedding_model == "gateway-embedding" + + +def test_importer_accepts_explicit_source_name_mappings() -> None: + args = _parser().parse_args( + [ + "--source-dsn", "postgresql://source", + "--target-dsn", "postgresql://target", + "--query-file", "query.sql", + "--source-system-code", "source", + "--record-key-column", "record_key", + "--title-column", "title", + "--body-column", "body", + "--created-at-column", "created_at", + "--author-subject-id", "subject", + "--corporate-entity-code", "corp", + "--process-unit-code", "pu", + "--source-sales-pool-name-column", "sales_pool_name", + "--source-customer-name-column", "customer_name", + "--source-project-name-column", "project_name", + "--source-company-name-column", "company_name", + "--source-process-unit-name-column", "process_unit_name", + ] + ) + + assert args.source_sales_pool_name_column == "sales_pool_name" + assert args.source_customer_name_column == "customer_name" + assert args.source_project_name_column == "project_name" + assert args.source_company_name_column == "company_name" + assert args.source_business_unit_name_column == "process_unit_name" diff --git a/tests/test_ingestion_transaction_contracts.py b/tests/test_ingestion_transaction_contracts.py index 55a94ab4d..68441ec0c 100644 --- a/tests/test_ingestion_transaction_contracts.py +++ b/tests/test_ingestion_transaction_contracts.py @@ -19,6 +19,7 @@ ACTOR_TYPE_PERSON, ACTOR_TYPE_TEAM, PostSummary, + POST_SUMMARY_CONTRACT_VERSION, RoleResponsibility, ) from lineageweave.relation_verification import STATUS_CORROBORATED @@ -196,9 +197,12 @@ async def execute(self, query: str, *args: Any) -> str: async def fetchrow(self, query: str, *args: Any) -> dict[str, Any] | None: compact = " ".join(query.split()) self._events.append(("fetchrow", compact)) - if compact.startswith("select korean_summary from post_summary_result"): + if compact.startswith("select korean_summary, summary_contract_version from post_summary_result"): assert not self.in_transaction - return {"korean_summary": "합성 요약"} + return { + "korean_summary": "합성 요약", + "summary_contract_version": POST_SUMMARY_CONTRACT_VERSION, + } if compact.startswith("select person_id from cataloged_person"): assert self.in_transaction return None @@ -208,8 +212,12 @@ async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: compact = " ".join(query.split()) self._events.append(("fetch", compact)) assert not self.in_transaction + if "from post_summary_action" in compact: + return [] if "from post_summary_event" in compact: return [{"event_text": "검토 완료"}] + if "from post_project_mention" in compact: + return [] if "from post_summary_role" in compact: assert "entity_name" not in compact assert "cataloged_corporate_entity_id" in compact @@ -499,15 +507,22 @@ class _PersonFetchConnection: async def fetchrow(self, query: str, *args: Any) -> dict[str, Any] | None: compact = " ".join(query.split()) events.append(("fetchrow", compact)) - if compact.startswith("select korean_summary from post_summary_result"): - return {"korean_summary": "합성 요약"} + if compact.startswith("select korean_summary, summary_contract_version from post_summary_result"): + return { + "korean_summary": "합성 요약", + "summary_contract_version": POST_SUMMARY_CONTRACT_VERSION, + } raise AssertionError(f"unexpected fetchrow query: {compact}") async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: compact = " ".join(query.split()) events.append(("fetch", compact)) + if "from post_summary_action" in compact: + return [] if "from post_summary_event" in compact: return [] + if "from post_project_mention" in compact: + return [] if "from post_summary_role" in compact: assert "cataloged_person_id" in compact return [ @@ -533,6 +548,26 @@ async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: assert role["actor_name"] == "Priya Nair" +def test_stale_summary_is_not_returned_as_current_evidence() -> None: + """Legacy generic summaries must yield to current body-grounded extraction.""" + + class _StaleSummaryConnection: + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any]: + assert "summary_contract_version" in query + return { + "korean_summary": "오래된 일반화 요약", + "summary_contract_version": POST_SUMMARY_CONTRACT_VERSION - 1, + } + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + raise AssertionError("stale summary must be rejected before loading projections") + + payload = asyncio.run( + summary_ingestion.fetch_persisted_summary(_StaleSummaryConnection(), str(uuid.uuid4())) + ) + assert payload is None + + class _PersonPersistConnection(_SummaryConnection): """Resolve one existing catalog person during the write transaction.""" @@ -642,36 +677,6 @@ async def persist_edges(conn, post_id) -> list[Any]: assert mention_inserts == [] -def test_hidden_run_copy_stays_generic_and_drops_the_stale_row() -> None: - """ADR 0014/0018: a 404 must not confirm why a row is hidden.""" - - app = ( - Path(__file__).resolve().parents[1] / "frontend" / "src" / "App.tsx" - ).read_text(encoding="utf-8") - alert = ( - Path(__file__).resolve().parents[1] - / "frontend" - / "src" - / "components" - / "StatusAlert.tsx" - ).read_text(encoding="utf-8") - agents = ( - Path(__file__).resolve().parents[1] / "AGENTS.md" - ).read_text(encoding="utf-8") - assert "This analysis run is not visible." not in app - assert "This run is not on your list. Open a visible run from the home list," in app - assert ( - "or request a lineage reconstruction for a corporation you already walk." - in app - ) - assert "do not name the thread or the cutoff" in app - assert "setRuns((await fetchAnalysisRuns(accessToken)).analysis_runs)" in app - assert 'role="alert"' in alert - assert "{error}" in app - assert "re-read the authorized list" in agents - assert "do not name the thread or the cutoff" in agents - - def test_role_catalog_identity_is_stored_on_the_role_row() -> None: """ADR 0019: fetch must not reconstruct organization identity by name.""" root = Path(__file__).resolve().parents[1] @@ -701,6 +706,5 @@ def test_role_catalog_identity_is_stored_on_the_role_row() -> None: assert "cataloged_person_id" in person_upgrade assert "0019_role_catalog_identity.sql" in dockerfile assert "0025_role_person_catalog_identity.sql" in dockerfile - assert "0026_report_leftover_pair.sql" in dockerfile assert "ADR 0019" in changelog assert "ADR 0027" in changelog diff --git a/tests/test_leftover_pairs.py b/tests/test_leftover_pairs.py index 989062ab3..72dea89b6 100644 --- a/tests/test_leftover_pairs.py +++ b/tests/test_leftover_pairs.py @@ -1,4 +1,4 @@ -"""Leftover post–criterion pairs after the main-effect IRT (ADR 0028). +"""Leftover post–criterion pairs after the main-effect IRT (ADR 0017). Uses a constructed residual matrix so the closest and farthest pair are known without calling ``fit_polytomous``. Loads diff --git a/tests/test_lineage_ingestion.py b/tests/test_lineage_ingestion.py index ae728e84c..de8f289c6 100644 --- a/tests/test_lineage_ingestion.py +++ b/tests/test_lineage_ingestion.py @@ -2,9 +2,14 @@ from __future__ import annotations +import asyncio from datetime import datetime, timezone -from backend.app.lineage_ingestion import reconstruct_group_key, records_from_source_posts +from backend.app.lineage_ingestion import ( + reconstruct_group_key, + records_from_source_posts, + visible_lineage_graph, +) from lineageweave.fixtures import sample_records from lineageweave.lineage_persistence import lineage_edge_specs @@ -88,3 +93,60 @@ def test_seed_shaped_rows_rebuild_to_the_designed_a100_fork() -> None: assert ("rec-002", "rec-003") in pairs assert ("rec-002", "rec-004") in pairs assert "rec-006" not in {edge.child_id for edge in edges} + + +def test_focused_lineage_graph_includes_a_post_outside_landing_limit() -> None: + class FakeConnection: + posts = [ + { + "post_id": "post-a", + "post_title": "A", + "voc_type_code": "voc", + "visibility_code": "public", + "corporate_entity_id": "corp", + "process_unit_id": "pu", + "thread_group_key": "thread-a", + "created_at": datetime(2026, 1, 1), + }, + { + "post_id": "post-b", + "post_title": "B", + "voc_type_code": "voc", + "visibility_code": "public", + "corporate_entity_id": "corp", + "process_unit_id": "pu", + "thread_group_key": "thread-a", + "created_at": datetime(2026, 1, 2), + }, + { + "post_id": "post-c", + "post_title": "C", + "voc_type_code": "voc", + "visibility_code": "public", + "corporate_entity_id": "corp", + "process_unit_id": "pu", + "thread_group_key": "thread-c", + "created_at": datetime(2026, 1, 3), + }, + ] + edges = [ + {"parent_post_id": "post-a", "child_post_id": "post-b", "fused_score": 0.8} + ] + + async def fetch(self, query: str): + return self.edges if "post_lineage_edge" in query else self.posts + + connection = FakeConnection() + landing = asyncio.run(visible_lineage_graph(connection, lambda row: True, limit=1)) + focused = asyncio.run( + visible_lineage_graph(connection, lambda row: True, limit=1, focus_post_id="post-a") + ) + isolated = asyncio.run( + visible_lineage_graph(connection, lambda row: True, limit=1, focus_post_id="post-c") + ) + + assert [node["id"] for node in landing["nodes"]] == ["post-c"] + assert {node["id"] for node in focused["nodes"]} == {"post-a", "post-b"} + assert len(focused["edges"]) == 1 + assert focused["truncated"] is False + assert isolated == {"nodes": [], "edges": [], "truncated": False} diff --git a/tests/test_llm_context.py b/tests/test_llm_context.py new file mode 100644 index 000000000..0dc6c21d0 --- /dev/null +++ b/tests/test_llm_context.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import lineageweave.http_client as http_client +from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata + + +def test_post_metadata_is_stable_and_post_specific() -> None: + values = { + "source_process_unit_code": "PU-01", + "author_account_id": "author-1", + "corporate_entity_code": "CORP-01", + } + first = build_post_llm_metadata("post-1", values) + second = build_post_llm_metadata("post-1", values) + other = build_post_llm_metadata("post-2", values) + + assert first == second + assert first["lineageweave_post_session_id"] != other["lineageweave_post_session_id"] + assert first["lineageweave_pu"] == "PU-01" + assert first["lineageweave_author_id"] == "author-1" + assert first["lineageweave_corp_code"] == "CORP-01" + + +def test_http_transport_merges_context_metadata_without_mutating_payload(monkeypatch) -> None: + seen = {} + + def fake_request(method, url, *, body, headers, timeout): + seen["payload"] = body + return 200, b"{}" + + monkeypatch.setattr(http_client, "_request", fake_request) + payload = {"messages": [], "metadata": {"channel": "summary"}} + metadata = build_post_llm_metadata("post-1", {"source_process_unit_code": "PU-01"}) + + with use_llm_metadata(metadata): + http_client.post_json("http://orchestrator/v1/chat/completions", payload, headers={}, timeout=1) + + assert payload["metadata"] == {"channel": "summary"} + assert seen["payload"] + assert "lineageweave_post_session_id" in seen["payload"].decode("utf-8") + assert "lineageweave_pu" in seen["payload"].decode("utf-8") diff --git a/tests/test_migration_replay.py b/tests/test_migration_replay.py new file mode 100644 index 000000000..29fe1c176 --- /dev/null +++ b/tests/test_migration_replay.py @@ -0,0 +1,35 @@ +from pathlib import Path + + +def test_shared_metric_migration_does_not_narrow_later_report_dimensions() -> None: + sql = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0009_shared_metric_bank.sql" + ).read_text(encoding="utf-8") + + assert "if not exists" in sql + assert "'team'" in sql + assert "'project'" in sql + assert "'shared_metric'" in sql + + +def test_migrate_sh_replays_leftover_pair_migration_on_existing_volumes() -> None: + """migrate.sh's replay window must cover 0012 (report_leftover_pair). + + The Dockerfile only bakes migrations into a brand-new Postgres data + directory via docker-entrypoint-initdb.d; any volume created before a + migration existed never gets it unless migrate.sh replays it on every + `docker compose up`. A window starting above 0012 silently leaves + report_leftover_pair missing on such volumes -- GET + /api/reports/{grouping}/{period} then 500s on undefined_table the + first time a period actually has leftover pairs. + """ + script = ( + Path(__file__).resolve().parents[1] + / "docker" + / "postgres-init" + / "migrate.sh" + ).read_text(encoding="utf-8") + + assert "0012_*" in script diff --git a/tests/test_ontology.py b/tests/test_ontology.py index 0e611bc8e..23eb77fda 100644 --- a/tests/test_ontology.py +++ b/tests/test_ontology.py @@ -27,7 +27,7 @@ load_ontology, ontology_annotations, ) -from rdflib.namespace import RDFS, SKOS +from rdflib.namespace import OWL, RDF, RDFS, SKOS, XSD _SEED_SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "seed_demo_data.py" @@ -38,7 +38,7 @@ # prov_team), 0016 (ADR 0009: node_team/edge_mention_team/ # edge_team_affiliation/edge_mention_organization). _ADDITIONAL_LOOKUP_MIGRATION_PATHS = ( - Path(__file__).resolve().parents[1] / "migrations" / "0012_role_responsibility_agent_type.sql", + Path(__file__).resolve().parents[1] / "migrations" / "0060_role_responsibility_agent_type.sql", Path(__file__).resolve().parents[1] / "migrations" / "0014_role_responsibility_team_actor_type.sql", Path(__file__).resolve().parents[1] / "migrations" / "0016_cross_post_actor_identity.sql", ) @@ -118,7 +118,7 @@ def test_knowledge_graph_lookup_constants_are_declared_in_the_ontology() -> None def test_iri_for_lookup_code_resolves_a_real_term() -> None: - assert iri_for_lookup_code("edge_mention") == str(LW.mentions) + assert iri_for_lookup_code("edge_mention") == str(LW.mentionedIn) assert iri_for_lookup_code("rel_voc") == str(LW.hasVocRelationship) @@ -142,12 +142,12 @@ def test_ontology_annotations_are_empty_for_an_undeclared_code() -> None: assert ontology_annotations("open") == {} -def test_mentions_property_domain_and_range_match_the_schema() -> None: - """`mentions` goes Post -> Person, matching post_person_mention's - actual foreign keys -- not just any two classes.""" +def test_mentioned_in_property_matches_canonical_edge_direction() -> None: + """`mentionedIn` goes Person -> Post, matching stored KG triples.""" graph = load_ontology() - assert (LW.mentions, RDFS.domain, LW.Post) in graph - assert (LW.mentions, RDFS.range, LW.Person) in graph + assert (LW.mentionedIn, RDFS.domain, LW.Person) in graph + assert (LW.mentionedIn, RDFS.range, LW.Post) in graph + assert (LW.mentions, OWL.inverseOf, LW.mentionedIn) in graph def test_prov_agent_type_terms_resolve_and_subclass_real_prov_o() -> None: @@ -199,3 +199,15 @@ def test_actor_mentions_follow_stored_edge_direction() -> None: assert (LW.mentionsTeam, RDFS.range, LW.Post) in graph assert (LW.mentionsOrganization, RDFS.domain, LW.CorporateEntity) in graph assert (LW.mentionsOrganization, RDFS.range, LW.Post) in graph + + +def test_semantic_project_terms_preserve_post_evidence_and_confidence() -> None: + """ADR 0036's project vocabulary must remain machine-checkable.""" + graph = load_ontology() + assert (LW.Project, RDF.type, OWL.Class) in graph + assert (LW.ProjectMention, RDF.type, OWL.Class) in graph + assert (LW.mentionsProject, RDFS.domain, LW.Post) in graph + assert (LW.mentionsProject, RDFS.range, LW.Project) in graph + assert (LW.projectEvidence, RDFS.domain, LW.ProjectMention) in graph + assert (LW.projectEvidence, RDFS.range, XSD.string) in graph + assert (LW.semanticConfidence, RDFS.range, XSD.decimal) in graph diff --git a/tests/test_organization_name_resolution.py b/tests/test_organization_name_resolution.py index 7eb0a96c3..02f171e96 100644 --- a/tests/test_organization_name_resolution.py +++ b/tests/test_organization_name_resolution.py @@ -11,11 +11,31 @@ from __future__ import annotations from lineageweave.organization_name_resolution import ( + ContextualOrchestratorOrganizationNameResolutionClient, NullOrganizationNameResolutionClient, OrganizationNameResolution, parse_resolution_response, resolve_and_verify_organization_name, ) + + +def test_live_resolution_client_uses_adaptive_orchestrator_mode(monkeypatch) -> None: + seen: dict[str, object] = {} + + def fake_post_json(url, body, *, headers, timeout): + seen.update(url=url, body=body, headers=headers, timeout=timeout) + return {"choices": [{"message": {"content": "Aurora Grid Power"}}]} + + monkeypatch.setattr("lineageweave.organization_name_resolution.post_json", fake_post_json) + client = ContextualOrchestratorOrganizationNameResolutionClient( + "http://orchestrator", "secret", reasoning_effort="high", timeout=11.0 + ) + + assert client.resolve("AGP", "AGP joined the synthetic meeting") == "Aurora Grid Power" + assert seen["url"] == "http://orchestrator/v1/chat/completions" + assert seen["body"]["mode"] == "auto" + assert seen["body"]["reasoning_effort"] == "high" + assert seen["timeout"] == 11.0 from lineageweave.relation_verification import ( STATUS_CORROBORATED, STATUS_PENDING, diff --git a/tests/test_organization_name_resolution_ingestion.py b/tests/test_organization_name_resolution_ingestion.py new file mode 100644 index 000000000..afdfa8f32 --- /dev/null +++ b/tests/test_organization_name_resolution_ingestion.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import pytest + +import backend.app.organization_name_resolution_ingestion as ingestion +from lineageweave.relation_verification import STATUS_CORROBORATED, STATUS_UNCORROBORATED + + +class _Connection: + def __init__(self, cached: dict[str, str] | None = None) -> None: + self.cached = cached + self.executed: list[tuple[str, tuple[object, ...]]] = [] + + async def fetchrow(self, _query: str, _raw_name: str): + return self.cached + + async def execute(self, query: str, *args: object) -> str: + self.executed.append((query, args)) + return "OK" + + +class _Client: + available = True + + +class _UnavailableClient: + available = False + + +def _resolution(status: str): + return SimpleNamespace( + raw_organization_name="AGP", + resolved_organization_name="Aurora Grid Power", + verification_status_code=status, + verification_evidence_url="https://evidence.example/result" if status == STATUS_CORROBORATED else None, + ) + + +def test_cached_verified_name_is_returned_without_resolution() -> None: + conn = _Connection({"resolved_organization_name": "Aurora Grid Power", "verification_status_code": STATUS_CORROBORATED}) + result = asyncio.run(ingestion.resolve_organization_name(conn, _UnavailableClient(), _Client(), "AGP", "context")) + assert result == "Aurora Grid Power" + assert conn.executed == [] + + +def test_cached_unverified_name_stays_raw() -> None: + conn = _Connection({"resolved_organization_name": "Aurora Grid Power", "verification_status_code": STATUS_UNCORROBORATED}) + result = asyncio.run(ingestion.resolve_organization_name(conn, _UnavailableClient(), _Client(), "AGP", "context")) + assert result == "AGP" + + +def test_unavailable_and_no_resolution_keep_raw(monkeypatch: pytest.MonkeyPatch) -> None: + unavailable = _Connection() + assert asyncio.run(ingestion.resolve_organization_name(unavailable, _UnavailableClient(), _Client(), "AGP", "context")) == "AGP" + + monkeypatch.setattr(ingestion, "resolve_and_verify_organization_name", lambda *_args: None) + no_resolution = _Connection() + assert asyncio.run(ingestion.resolve_organization_name(no_resolution, _Client(), _Client(), "AGP", "context")) == "AGP" + assert no_resolution.executed == [] + + +@pytest.mark.parametrize("status,expected", [(STATUS_CORROBORATED, "Aurora Grid Power"), (STATUS_UNCORROBORATED, "AGP")]) +def test_new_resolution_is_persisted_but_only_verified_name_is_returned( + monkeypatch: pytest.MonkeyPatch, status: str, expected: str +) -> None: + monkeypatch.setattr(ingestion, "resolve_and_verify_organization_name", lambda *_args: _resolution(status)) + conn = _Connection() + result = asyncio.run(ingestion.resolve_organization_name(conn, _Client(), _Client(), "AGP", "context")) + + assert result == expected + assert len(conn.executed) == 1 + assert "organization_name_resolution" in conn.executed[0][0] diff --git a/tests/test_person_mention_projection.py b/tests/test_person_mention_projection.py index d24195bb9..7252fb8e8 100644 --- a/tests/test_person_mention_projection.py +++ b/tests/test_person_mention_projection.py @@ -44,7 +44,10 @@ from lineageweave.post_summary import ( ACTOR_TYPE_ORGANIZATION, ACTOR_TYPE_PERSON, + KeyEvent, + MajorEventAction, PostSummary, + ProjectMention, RoleResponsibility, ) @@ -52,6 +55,49 @@ "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" ) _MIGRATION_PATH = Path(__file__).resolve().parents[1] / "migrations" / "0001_initial_schema.sql" +_SEMANTIC_PROJECT_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0031_semantic_project_mentions.sql" +) +_POST_SUMMARY_CONTRACT_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0040_post_summary_contract.sql" +) +_SUMMARY_FIVE_W1H_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0048_post_summary_five_w1h.sql" +) +_MAJOR_EVENT_ACTION_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0100_major_event_action.sql" +) +_PROJECT_BOUND_ACTION_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0101_project_bound_major_event_action.sql" +) +_PROJECT_BOUND_EVENT_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0102_project_bound_summary_event.sql" +) +_SEMANTIC_SEARCH_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0032_semantic_search_trigram.sql" +) +_SOURCE_STATE_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0033_source_state_provenance.sql" +) +_SOURCE_CONTEXT_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0034_source_context_provenance.sql" +) +_NORMALIZED_BODY_SEARCH_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0036_normalized_body_search.sql" +) +_SOURCE_RECORD_IDENTITY_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0037_source_record_identity.sql" +) +_SOURCE_NAMED_HINTS_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0038_source_named_hints.sql" +) +_SOURCE_ORG_NAMED_HINTS_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0039_source_org_named_hints.sql" +) def _postgres_available() -> bool: @@ -106,6 +152,19 @@ def projection_database() -> str: try: with connection.cursor() as cursor: cursor.execute(_MIGRATION_PATH.read_text(encoding="utf-8")) + cursor.execute(_SEMANTIC_PROJECT_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_SEMANTIC_SEARCH_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_SOURCE_STATE_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_SOURCE_CONTEXT_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_NORMALIZED_BODY_SEARCH_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_SOURCE_RECORD_IDENTITY_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_SOURCE_NAMED_HINTS_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_SOURCE_ORG_NAMED_HINTS_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_POST_SUMMARY_CONTRACT_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_SUMMARY_FIVE_W1H_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_MAJOR_EVENT_ACTION_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_PROJECT_BOUND_ACTION_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_PROJECT_BOUND_EVENT_MIGRATION.read_text(encoding="utf-8")) cursor.execute( """ insert into common_lookup_value @@ -208,14 +267,50 @@ async def _exercise_projection_contract( post_id, PostSummary( korean_summary="합성 요약", + key_event_details=( + KeyEvent(event_text="합성 프로젝트 검토", project_key="Synthetic Project"), + ), roles_and_responsibilities=( RoleResponsibility( actor_name="Summary Person", responsibility="검토", ), ), + major_event_actions=( + MajorEventAction( + action_text="합성 프로젝트 검토 요청", + requester_actor_name="Summary Person", + processor_actor_name=None, + evidence_text="합성 본문에 프로젝트 검토 요청이 기록됨", + project_key="Synthetic Project", + ), + MajorEventAction( + action_text="연결되지 않은 프로젝트 요청", + requester_actor_name=None, + processor_actor_name=None, + evidence_text="프로젝트 연결 근거가 없음", + project_key="unsupported-project", + ), + ), + project_mentions=( + ProjectMention( + project_name="Synthetic Project", + canonical_name="Synthetic Project", + evidence="합성 본문에 프로젝트명이 있음", + confidence=0.9, + ), + ), ), ) + summary_payload = await fetch_persisted_summary(connection, post_id) + assert summary_payload is not None + assert [ + action["project_name"] + for action in summary_payload["major_event_actions"] + ] == ["Synthetic Project", None] + assert summary_payload["key_event_details"] == [ + {"event_text": "합성 프로젝트 검토", "project_name": "Synthetic Project"} + ] keyman_rows = await connection.fetch( "select person_id from post_person_mention where post_id = $1", @@ -346,8 +441,9 @@ def test_cross_post_identity_upgrade_keeps_keyman_mention_context( ), ) cursor.execute( - "insert into post_summary_result (post_id, korean_summary) values (%s, %s)", - (post_id, "합성 요약"), + "insert into post_summary_result " + "(post_id, korean_summary, summary_contract_version) values (%s, %s, %s)", + (post_id, "합성 요약", 1), ) cursor.execute( """ diff --git a/tests/test_post_chat.py b/tests/test_post_chat.py index d3603d374..f401e50d7 100644 --- a/tests/test_post_chat.py +++ b/tests/test_post_chat.py @@ -8,11 +8,13 @@ from __future__ import annotations +import asyncio import os import pytest from backend.app.post_chat_ingestion import ( + _graph_facts_for_posts, seeded_demo_chat, seeded_demo_commitment_chat, seeded_demo_exchanges, @@ -30,6 +32,8 @@ ChatSourceDocument, ContextualOrchestratorPostChatClient, NullPostChatClient, + _render_sources_block, + cited_post_evidence, cited_post_summaries, normalize_chat_question, parse_chat_response, @@ -162,6 +166,86 @@ def test_cited_post_summaries_keep_citation_order_and_drop_unknown_ids() -> None ] +def test_cited_post_evidence_hides_prompt_metadata_but_keeps_semantic_facts() -> None: + source = ChatSourceDocument( + "post-evidence", + "Evidence post", + "body", + evidence_facts=( + "project: Semantic project | evidence: Body evidence | ontology_iri: https://example.test/ontology#Project | extraction_method: contextual_orchestrator_semantic | confidence: 0.9 [provenance=post_project_mention]", + "Keyman mention: Ada West | context: account lead [provenance=post_person_mention]", + ), + ) + + evidence = cited_post_evidence((source,), ("post-evidence", "missing")) + + assert evidence == [ + { + "post_id": "post-evidence", + "facts": [ + {"kind": "semantic_project", "text": "project: Semantic project | evidence: Body evidence"}, + {"kind": "semantic_keyman", "text": "Keyman mention: Ada West | context: account lead"}, + ], + } + ] + + +def test_chat_render_includes_persisted_graph_facts_with_source_evidence() -> None: + source = ChatSourceDocument( + "post-graph", + "Graph-backed post", + "A customer asked for a revised quote.", + graph_facts=( + 'node_person "Ada West" --edge_affiliation--> ' + 'node_corporate_entity "Demo Corp" [evidence_post_id=post-graph]', + ), + evidence_facts=("source project code=PROJECT-HINT [hint_only]",), + ) + + rendered = _render_sources_block([source]) + + assert "Persisted Knowledge Graph facts" in rendered + assert "Demo Corp" in rendered + assert "evidence_post_id=post-graph" in rendered + assert "Persisted source/semantic evidence" in rendered + assert "PROJECT-HINT" in rendered + + +def test_graph_facts_are_hydrated_from_visible_evidence_posts(monkeypatch) -> None: + class _Connection: + async def fetch(self, _query, _visible_post_ids): + return [ + { + "source_node_type_code": "node_person", + "source_node_id": "person-ada", + "target_node_type_code": "node_corporate_entity", + "target_node_id": "corp-demo", + "edge_type_code": "edge_affiliation", + "edge_weight": 1.0, + "evidence_post_ids": ["post-graph"], + } + ] + + async def fake_hydrate(_conn, _node_keys): + return [ + {"node_type_code": "node_person", "node_id": "person-ada", "label": "Ada West"}, + { + "node_type_code": "node_corporate_entity", + "node_id": "corp-demo", + "label": "Demo Corp", + }, + ] + + monkeypatch.setattr("backend.app.post_chat_ingestion.hydrate_related_nodes", fake_hydrate) + facts = asyncio.run(_graph_facts_for_posts(_Connection(), ["post-graph"])) + + assert facts == ( + 'node_person "Ada West" --edge_affiliation ' + '(https://contextualwisdomlab.github.io/lineageweave/ontology#affiliatedWith)--> ' + 'node_corporate_entity "Demo Corp" [evidence_post_id=post-graph]', + ) + + def test_parses_a_well_formed_json_object() -> None: content = '{"answer_text": "The bid was submitted then revised.", "cited_source_numbers": [1, 2]}' answer = parse_chat_response(content, _SOURCES) @@ -244,3 +328,29 @@ def test_contextual_orchestrator_does_not_cite_an_irrelevant_source() -> None: assert "post-a" in answer.cited_post_ids assert "post-b" not in answer.cited_post_ids + + +def test_contextual_orchestrator_chat_requests_plain_citations(monkeypatch) -> None: + observed = {} + + def fake_post_json(url, payload, *, headers, timeout): + observed["payload"] = payload + return { + "choices": [ + { + "message": { + "content": "근거 답변\nCITED SOURCES: 1" + } + } + ] + } + + monkeypatch.setattr("lineageweave.post_chat.post_json", fake_post_json) + answer = ContextualOrchestratorPostChatClient("https://orchestrator.test", "token").answer( + "What happened?", _SOURCES + ) + + assert answer.answer_text == "근거 답변" + assert observed["payload"]["reasoning_effort"] == "auto" + assert observed["payload"]["mode"] == "auto" + assert "CITED SOURCES" in observed["payload"]["messages"][0]["content"] diff --git a/tests/test_post_chat_ingestion.py b/tests/test_post_chat_ingestion.py new file mode 100644 index 000000000..92351ebc1 --- /dev/null +++ b/tests/test_post_chat_ingestion.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +import asyncio +from threading import Event, Timer +from types import SimpleNamespace + +import pytest + +from backend.app.post_chat_ingestion import ( + LinkedPostIds, + fetch_persisted_chat, + fetch_persisted_chats, + gather_chat_sources, + normalize_chat_question, + persist_post_chat, +) +from lineageweave.post_chat import ( + ChatSourceDocument, + ContextualOrchestratorPostChatClient, + parse_chat_response, +) + + +class _Connection: + def __init__(self, *, header: dict[str, str] | None, citations: list[dict[str, str]]) -> None: + self.header = header + self.citations = citations + self.executed: list[tuple[str, tuple[object, ...]]] = [] + + async def execute(self, query: str, *args: object) -> str: + self.executed.append((query, args)) + return "OK" + + async def fetchrow(self, _query: str, *_args: object): + return self.header + + async def fetch(self, query: str, *_args: object): + if "question_norm from post_chat_result" in query: + return [{"question_norm": "question"}] + return self.citations + + +class _SourceConnection: + async def fetchrow(self, query: str, *_args: object): + if "from source_post where post_id" not in query: + return None + return { + "post_id": "post-1", + "post_title": "Public post", + "post_body": "

    Body

    ", + "source_system_code": None, + "source_record_key": None, + "source_author_code": None, + "source_author_name": None, + "source_company_code": None, + "source_company_name": None, + "source_process_unit_code": None, + "source_process_unit_name": None, + "source_sales_pool_code": None, + "source_sales_pool_name": None, + "source_customer_code": None, + "source_customer_name": None, + "source_project_code": None, + "source_project_name": None, + } + + async def fetch(self, _query: str, *_args: object): + return [] + + +def test_gather_chat_sources_keeps_the_event_loop_responsive_during_body_normalization( + monkeypatch: pytest.MonkeyPatch, +) -> None: + order: list[str] = [] + release = Event() + + def blocking_normalize(_body: str, *, vision_client: object) -> SimpleNamespace: + del vision_client + order.append("normalization_started") + assert release.wait(timeout=1.0) + order.append("normalization_finished") + return SimpleNamespace(text="normalized body") + + monkeypatch.setattr( + "backend.app.post_chat_ingestion.normalize_post_body", + blocking_normalize, + ) + + async def exercise() -> None: + loop = asyncio.get_running_loop() + timer = Timer(0.2, release.set) + timer.start() + loop.call_later(0.01, order.append, "event_loop_progress") + try: + sources = await gather_chat_sources( + _SourceConnection(), + "post-1", + lambda _row: True, + ) + await asyncio.sleep(0) + finally: + release.set() + timer.cancel() + assert sources[0].post_body == "normalized body" + + asyncio.run(exercise()) + + assert order.index("event_loop_progress") < order.index("normalization_finished") + + +def test_gather_chat_sources_bounds_and_orders_linked_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + root_id = "00000000-0000-0000-0000-000000000000" + direct_ids = frozenset( + f"00000000-0000-0000-0000-{index:012d}" for index in range(1, 21) + ) + indirect_ids = frozenset( + f"00000000-0000-0000-0001-{index:012d}" for index in range(1, 21) + ) + + async def fake_find_linked_post_ids(_conn: object, _post_id: str) -> LinkedPostIds: + return LinkedPostIds(direct=direct_ids, indirect=indirect_ids) + + monkeypatch.setattr( + "backend.app.post_chat_ingestion.find_linked_post_ids", + fake_find_linked_post_ids, + ) + + class SourceBudgetConnection: + def __init__(self) -> None: + self.candidate_ids: list[str] = [] + self.candidate_query = "" + + async def fetchrow(self, query: str, *_args: object): + if "from source_post where post_id" not in query: + return None + return { + "post_id": root_id, + "post_title": "Root post", + "post_body": "Root body", + **{ + field_name: None + for field_name in ( + "source_system_code", + "source_record_key", + "source_author_code", + "source_author_name", + "source_company_code", + "source_company_name", + "source_process_unit_code", + "source_process_unit_name", + "source_sales_pool_code", + "source_sales_pool_name", + "source_customer_code", + "source_customer_name", + "source_project_code", + "source_project_name", + ) + }, + } + + async def fetch(self, query: str, *args: object): + if "from source_post where post_id = any" not in query: + return [] + self.candidate_query = query + self.candidate_ids = list(args[0]) + return [ + { + "post_id": post_id, + "post_title": f"Post {post_id}", + "post_body": "Body", + "visibility_code": "public", + "corporate_entity_id": None, + **{ + field_name: None + for field_name in ( + "source_system_code", + "source_record_key", + "source_author_code", + "source_author_name", + "source_company_code", + "source_company_name", + "source_process_unit_code", + "source_process_unit_name", + "source_sales_pool_code", + "source_sales_pool_name", + "source_customer_code", + "source_customer_name", + "source_project_code", + "source_project_name", + ) + }, + } + for post_id in self.candidate_ids + ] + + conn = SourceBudgetConnection() + sources = asyncio.run(gather_chat_sources(conn, root_id, lambda _row: True)) + + expected_candidates = [*sorted(direct_ids), *sorted(indirect_ids)][:32] + assert conn.candidate_ids == expected_candidates + assert "array_position" in conn.candidate_query + assert [source.post_id for source in sources] == [root_id, *expected_candidates[:7]] + assert len(sources) == 8 + + +def test_normalize_question_rejects_empty_and_collapses_whitespace() -> None: + assert normalize_chat_question(" What happened? ") == "what happened between these events" + assert normalize_chat_question(" \t ") == "" + + +def test_persist_chat_deduplicates_citations_and_serializes_result() -> None: + conn = _Connection( + header={"question_text": "What happened?", "answer_text": "A synthetic answer."}, + citations=[ + {"cited_post_id": "post-a", "post_title": "Evidence A"}, + {"cited_post_id": "post-b", "post_title": "Evidence B"}, + ], + ) + + payload = asyncio.run( + persist_post_chat(conn, "post-1", " What happened? ", "A synthetic answer.", ["post-a", "post-a", "post-b"]) + ) + + assert payload["cited_post_ids"] == ["post-a", "post-b"] + assert len([query for query, _args in conn.executed if "post_chat_citation" in query]) == 2 + assert any("post_chat_result" in query and "delete" in query.lower() for query, _args in conn.executed) + + +def test_fetch_chat_handles_empty_and_missing_rows() -> None: + missing = _Connection(header=None, citations=[]) + assert asyncio.run(fetch_persisted_chat(missing, "post-1", " ")) is None + assert asyncio.run(fetch_persisted_chat(missing, "post-1", "question")) is None + assert asyncio.run(fetch_persisted_chats(missing, "post-1")) == [] + + +def test_fetch_chat_list_serializes_existing_exchange() -> None: + conn = _Connection( + header={"question_text": "Question", "answer_text": "Answer"}, + citations=[{"cited_post_id": "post-a", "post_title": "Evidence A"}], + ) + exchanges = asyncio.run(fetch_persisted_chats(conn, "post-1")) + assert len(exchanges) == 1 + assert exchanges[0]["cited_posts"][0]["post_title"] == "Evidence A" + + +def test_parse_chat_response_strips_fence_and_drops_invalid_citations() -> None: + sources = [ChatSourceDocument("post-a", "Evidence A", "body")] + answer = parse_chat_response( + '```json\n{"answer_text":" answer ","cited_source_numbers":[1, 0, 2, "bad"]}\n```', + sources, + ) + assert answer is not None + assert answer.answer_text == "answer" + assert answer.cited_post_ids == ("post-a",) + assert parse_chat_response("not json", sources) is None + assert parse_chat_response('{"answer_text":""}', sources) is None + + +def test_contextual_chat_client_uses_auto_mode_and_evidence_prompt(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} + + def fake_post_json(url: str, payload: dict, *, headers: dict[str, str], timeout: float) -> dict: + captured.update({"url": url, "payload": payload, "headers": headers, "timeout": timeout}) + return { + "choices": [ + {"message": {"content": "supported\nCITED SOURCES: 1, 9"}}, + ] + } + + monkeypatch.setattr("lineageweave.post_chat.post_json", fake_post_json) + client = ContextualOrchestratorPostChatClient("https://orchestrator", "secret", reasoning_effort="low") + answer = client.answer( + "What happened?", + [ChatSourceDocument("post-a", "Evidence A", "body", graph_facts=("fact",))], + ) + + assert answer.answer_text == "supported" + assert answer.cited_post_ids == ("post-a",) + assert captured["url"] == "https://orchestrator/v1/chat/completions" + payload = captured["payload"] + assert payload["mode"] == "auto" + assert payload["reasoning_effort"] == "low" + assert "fact" in payload["messages"][0]["content"] + + +def test_contextual_chat_client_rejects_malformed_provider_response(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "lineageweave.post_chat.post_json", + lambda *_args, **_kwargs: {"choices": [{"message": {"content": "{}"}}]}, + ) + client = ContextualOrchestratorPostChatClient("https://orchestrator", "secret") + with pytest.raises(ValueError, match="required format"): + client.answer("Question", [ChatSourceDocument("post-a", "Evidence A", "body")]) diff --git a/tests/test_post_content_normalization.py b/tests/test_post_content_normalization.py index 0428a4378..0beead6f4 100644 --- a/tests/test_post_content_normalization.py +++ b/tests/test_post_content_normalization.py @@ -9,8 +9,15 @@ from __future__ import annotations import base64 +from threading import Lock -from lineageweave.image_content import ImageDescription +from lineageweave.chunking import Chunk +from lineageweave.image_content import ( + ImageDescription, + ImageRegion, + NullImageContentClient, +) +from lineageweave.llm_context import current_llm_metadata, use_llm_metadata from lineageweave.post_content_normalization import normalize_post_body _PNG_1X1 = base64.b64decode( @@ -35,6 +42,73 @@ def describe(self, image_bytes: bytes, mime_type: str) -> ImageDescription: raise RuntimeError("provider is down") +class _MetadataCapturingVisionClient(_FakeVisionClient): + def __init__(self, description: ImageDescription) -> None: + super().__init__(description) + self._lock = Lock() + self.seen_metadata: list[dict[str, str] | None] = [] + + def describe(self, image_bytes: bytes, mime_type: str) -> ImageDescription: + with self._lock: + self.seen_metadata.append(current_llm_metadata()) + return super().describe(image_bytes, mime_type) + + +class _FullImageRegionVisionClient(_FakeVisionClient): + def locate_regions(self, image_bytes: bytes, mime_type: str) -> tuple[ImageRegion, ...]: + return (ImageRegion(0.0, 0.0, 1.0, 1.0),) + + +class _PartialRegionVisionClient(_FakeVisionClient): + def __init__(self, description: ImageDescription, fail_on_call: int | None = None) -> None: + super().__init__(description) + self.describe_calls = 0 + self.fail_on_call = fail_on_call + + def describe(self, image_bytes: bytes, mime_type: str) -> ImageDescription: + self.describe_calls += 1 + if self.describe_calls == self.fail_on_call: + raise RuntimeError("synthetic parent-image provider outage") + return super().describe(image_bytes, mime_type) + + def locate_regions(self, image_bytes: bytes, mime_type: str) -> tuple[ImageRegion, ...]: + return (ImageRegion(0.25, 0.25, 0.25, 0.25),) + + +class _MixedValidityRegionVisionClient(_PartialRegionVisionClient): + def locate_regions(self, image_bytes: bytes, mime_type: str) -> tuple[ImageRegion, ...]: + return ( + ImageRegion(0.25, 0.25, 0.25, 0.25), + ImageRegion(-0.1, 0.0, 0.5, 0.5), + ImageRegion(0.0, 0.0, float("nan"), 0.5), + ImageRegion(None, 0.0, 0.5, 0.5), # type: ignore[arg-type] + object(), # type: ignore[arg-type] + ) + + +class _LocatorFailureVisionClient(_FakeVisionClient): + def locate_regions(self, image_bytes: bytes, mime_type: str) -> tuple[ImageRegion, ...]: + raise RuntimeError("synthetic locator outage") + + +class _EmptyLocatorVisionClient(_FakeVisionClient): + def locate_regions(self, image_bytes: bytes, mime_type: str) -> tuple[ImageRegion, ...]: + return None # type: ignore[return-value] + + +class _MalformedLocatorVisionClient(_FakeVisionClient): + def locate_regions(self, image_bytes: bytes, mime_type: str) -> tuple[ImageRegion, ...]: + return object() # type: ignore[return-value] + + +class _PartialRegionFailureVisionClient(_FakeVisionClient): + def locate_regions(self, image_bytes: bytes, mime_type: str) -> tuple[ImageRegion, ...]: + return (ImageRegion(0.25, 0.25, 0.25, 0.25),) + + def describe(self, image_bytes: bytes, mime_type: str) -> ImageDescription: + raise RuntimeError("synthetic region and parent outage") + + def test_plain_text_passes_through_unchanged() -> None: result = normalize_post_body("Just a plain business record, no markup here.") assert result.text == "Just a plain business record, no markup here." @@ -42,6 +116,12 @@ def test_plain_text_passes_through_unchanged() -> None: assert result.image_descriptions == () +def test_plain_text_visual_continuation_breaks_are_normalized_for_embeddings() -> None: + result = normalize_post_body("- 요청 사항\n 후속 설명은 같은 항목에 속한다.\n· 다음 항목") + + assert result.text == "- 요청 사항 후속 설명은 같은 항목에 속한다.\n· 다음 항목" + + def test_html_tags_never_appear_in_the_normalized_text() -> None: html = '

    Confirm delivery by Friday.

    ' result = normalize_post_body(html) @@ -51,6 +131,11 @@ def test_html_tags_never_appear_in_the_normalized_text() -> None: assert "Confirm delivery by Friday." in result.text +def test_nested_html_character_references_are_decoded() -> None: + result = normalize_post_body("

    Company&nbsp;&amp;&nbsp;Product 's note

    ") + assert result.text == "Company & Product 's note" + + def test_formatting_hints_are_captured_separately_from_text() -> None: html = '

    Urgent

    Please review the attached quote.

    ' result = normalize_post_body(html) @@ -81,6 +166,208 @@ def test_image_is_described_and_placed_at_its_document_position_not_dropped() -> assert result.image_descriptions == (description,) +def test_single_full_image_locator_response_keeps_parent_evidence_without_region() -> None: + b64 = base64.b64encode(_PNG_1X1).decode("ascii") + html = f'

    Before.

    After.

    ' + description = ImageDescription( + extracted_text="panel text", caption="one visual panel", tags=("panel",) + ) + + result = normalize_post_body(html, vision_client=_FullImageRegionVisionClient(description)) + + assert result.image_results[0].status_code == "described" + assert result.image_results[0].regions == () + assert result.image_results[0].description == description + assert "panel text" in result.text + + +def test_image_without_ocr_uses_caption_only_and_preserves_image_result() -> None: + b64 = base64.b64encode(_PNG_1X1).decode("ascii") + description = ImageDescription(extracted_text="", caption="a blank chart", tags=()) + + result = normalize_post_body( + f'', + vision_client=_FakeVisionClient(description), + ) + + assert result.text == "[image: a blank chart]" + assert result.image_results[0].status_code == "described" + + +def test_unavailable_vision_channel_keeps_an_explicit_image_outcome() -> None: + b64 = base64.b64encode(_PNG_1X1).decode("ascii") + + result = normalize_post_body( + f'', + vision_client=NullImageContentClient(), + ) + + assert result.text == "[image: content unavailable]" + assert result.image_results[0].status_code == "unavailable" + + +def test_available_client_with_missing_image_bytes_keeps_unavailable_outcome() -> None: + from lineageweave.post_content_normalization import _describe_image_chunk + + result, description, placeholder = _describe_image_chunk( + Chunk(text="", unit_type="image", index=0, label="image/png", image_data=None), + _FakeVisionClient(ImageDescription(extracted_text="", caption="unused", tags=())), + ) + + assert result.status_code == "unavailable" + assert description is None + assert placeholder == "[image: content unavailable]" + + +def test_locator_failure_falls_back_to_parent_image_evidence() -> None: + b64 = base64.b64encode(_PNG_1X1).decode("ascii") + description = ImageDescription(extracted_text="parent", caption="whole image", tags=()) + + result = normalize_post_body( + f'', + vision_client=_LocatorFailureVisionClient(description), + ) + + assert result.image_results[0].status_code == "described" + assert result.image_results[0].regions == () + assert result.image_results[0].description == description + + +def test_empty_locator_result_falls_back_to_parent_image_evidence() -> None: + b64 = base64.b64encode(_PNG_1X1).decode("ascii") + description = ImageDescription(extracted_text="parent", caption="whole image", tags=()) + + result = normalize_post_body( + f'', + vision_client=_EmptyLocatorVisionClient(description), + ) + + assert result.image_results[0].status_code == "described" + assert result.image_results[0].regions == () + assert result.image_results[0].description == description + + +def test_non_iterable_locator_result_falls_back_to_parent_image_evidence() -> None: + b64 = base64.b64encode(_PNG_1X1).decode("ascii") + description = ImageDescription(extracted_text="parent", caption="whole image", tags=()) + + result = normalize_post_body( + f'', + vision_client=_MalformedLocatorVisionClient(description), + ) + + assert result.image_results[0].status_code == "described" + assert result.image_results[0].regions == () + assert result.image_results[0].description == description + + +def test_partial_locator_with_no_successful_description_fails_closed() -> None: + b64 = base64.b64encode(_PNG_1X1).decode("ascii") + + result = normalize_post_body( + f'', + vision_client=_PartialRegionFailureVisionClient( + ImageDescription(extracted_text="unused", caption="unused", tags=()) + ), + ) + + assert result.image_results[0].status_code == "failed" + assert result.text == "[image: content unavailable]" + + +def test_unknown_chunk_kinds_are_not_leaked_into_buyer_text(monkeypatch) -> None: + from lineageweave import post_content_normalization + + monkeypatch.setattr( + post_content_normalization, + "chunk_by_dom", + lambda _body: [Chunk(text="hidden", unit_type="unknown", index=0)], + ) + + result = normalize_post_body("
    ignored by the synthetic chunker
    ") + + assert result.text == "" + + +def test_image_analysis_preserves_post_scoped_llm_metadata() -> None: + b64 = base64.b64encode(_PNG_1X1).decode("ascii") + html = ( + f'' + f'' + ) + description = ImageDescription(extracted_text="Q3 2026", caption="a chart", tags=("chart",)) + client = _MetadataCapturingVisionClient(description) + metadata = { + "lineageweave_post_id": "post-1", + "lineageweave_pu": "PU-01", + } + + with use_llm_metadata(metadata): + result = normalize_post_body(html, vision_client=client) + + assert len(result.image_descriptions) == 2 + assert len(client.seen_metadata) == 2 + assert all(seen == metadata for seen in client.seen_metadata) + + +def test_partial_region_response_retains_panel_and_parent_evidence() -> None: + b64 = base64.b64encode(_PNG_1X1).decode("ascii") + html = f'' + client = _PartialRegionVisionClient( + ImageDescription(extracted_text="whole image", caption="whole", tags=()) + ) + result = normalize_post_body(html, vision_client=client) + + assert result.image_results[0].regions[0].region == ImageRegion(0.25, 0.25, 0.25, 0.25) + assert client.describe_calls == 2 + + +def test_partial_region_parent_failure_keeps_successful_panel_evidence() -> None: + b64 = base64.b64encode(_PNG_1X1).decode("ascii") + client = _PartialRegionVisionClient( + ImageDescription(extracted_text="panel", caption="panel", tags=()), + fail_on_call=2, + ) + + result = normalize_post_body( + f'', + vision_client=client, + ) + + assert result.image_results[0].status_code == "described" + assert result.image_results[0].regions[0].description is not None + assert result.image_results[0].description is not None + + +def test_partial_region_analysis_discards_unbounded_locator_regions() -> None: + b64 = base64.b64encode(_PNG_1X1).decode("ascii") + client = _MixedValidityRegionVisionClient( + ImageDescription(extracted_text="whole", caption="whole", tags=()) + ) + + result = normalize_post_body( + f'', + vision_client=client, + ) + + assert len(result.image_results[0].regions) == 1 + assert result.image_results[0].regions[0].region == ImageRegion(0.25, 0.25, 0.25, 0.25) + + +def test_non_iterable_locator_result_falls_back_to_parent_evidence() -> None: + b64 = base64.b64encode(_PNG_1X1).decode("ascii") + description = ImageDescription(extracted_text="parent", caption="whole", tags=()) + + result = normalize_post_body( + f'', + vision_client=_MalformedLocatorVisionClient(description), + ) + + assert result.image_results[0].status_code == "described" + assert result.image_results[0].regions == () + assert result.image_results[0].description == description + + def test_comparison_operators_in_plain_text_are_not_treated_as_html() -> None: body = "Need delivery if qty < 50 and price > 10." result = normalize_post_body(body) diff --git a/tests/test_post_content_persistence.py b/tests/test_post_content_persistence.py new file mode 100644 index 000000000..829d4b40b --- /dev/null +++ b/tests/test_post_content_persistence.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import asyncio + +from lineageweave.post_content_persistence import persist_post_content + + +class _Transaction: + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, traceback): + return False + + +class _Connection: + def __init__(self) -> None: + self.executed: list[tuple[str, tuple[object, ...]]] = [] + self.fetched: list[str] = [] + + def transaction(self) -> _Transaction: + return _Transaction() + + async def execute(self, query: str, *args: object) -> str: + self.executed.append((query, args)) + return "OK" + + async def fetchval(self, query: str, *args: object) -> str: + self.fetched.append(query) + if "post_content_unit" in query: + return "unit-1" + return "embedding-1" + + +class _EmbeddingClient: + available = True + + def embed_many(self, texts: list[str]) -> list[list[float]]: + return [[0.1, 0.2] for _ in texts] + + +class _FailingEmbeddingClient: + available = True + + def embed_many(self, texts: list[str]) -> list[list[float]]: + raise RuntimeError("synthetic provider outage") + + +def test_persist_post_content_writes_units_and_validated_vectors() -> None: + conn = _Connection() + + unit_count = asyncio.run( + persist_post_content( + conn, + "post-1", + "A paragraph with a meaningful retrieval unit.", + embedding_client=_EmbeddingClient(), + embedding_model_code="text-embedding-3-large", + ) + ) + + assert unit_count == 1 + assert any("delete from post_content_unit" in query for query, _args in conn.executed) + assert any("post_content_embedding_value" in query for query, _args in conn.executed) + assert any("post_content_embedding" in query for query in conn.fetched) + + +def test_persist_post_content_keeps_units_when_embedding_provider_fails() -> None: + conn = _Connection() + + unit_count = asyncio.run( + persist_post_content( + conn, + "post-1", + "A paragraph that remains searchable without a vector.", + embedding_client=_FailingEmbeddingClient(), + embedding_model_code="text-embedding-3-large", + ) + ) + + assert unit_count == 1 + assert any("post_content_unit" in query for query, _args in conn.executed) + assert not any("post_content_embedding_value" in query for query, _args in conn.executed) diff --git a/tests/test_post_content_persistence_edges.py b/tests/test_post_content_persistence_edges.py new file mode 100644 index 000000000..24bec9e6d --- /dev/null +++ b/tests/test_post_content_persistence_edges.py @@ -0,0 +1,336 @@ +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from types import SimpleNamespace + +import pytest + +from lineageweave.chunking import chunk_by_dom +from lineageweave.image_content import ImageRegion +from lineageweave.post_content_normalization import ( + FormattingHint, + ImageContentResult, + ImageRegionResult, + NormalizedPostContent, +) +from lineageweave.post_content_persistence import ( + _bounded_unit_batches, + _render_image_text, + persist_post_content, +) +from lineageweave.post_structure import StructureDecision + + +def _persist(*args: object, **kwargs: object) -> int: + return asyncio.run(persist_post_content(*args, **kwargs)) + + +class _Connection: + def __init__(self) -> None: + self.executed: list[tuple[str, tuple[object, ...]]] = [] + self.fetchvals: list[tuple[str, tuple[object, ...]]] = [] + self._next_id = 0 + + @asynccontextmanager + async def transaction(self): + yield self + + async def execute(self, query: str, *args: object) -> str: + self.executed.append((query, args)) + return "OK" + + async def fetchval(self, query: str, *args: object) -> str: + self.fetchvals.append((query, args)) + self._next_id += 1 + return f"id-{self._next_id}" + + +class _EmbedMany: + available = True + + async_calls = 0 + + def __init__(self) -> None: + self.texts: list[str] = [] + + def embed_many(self, texts: list[str]) -> list[list[float]]: + self.async_calls += 1 + self.texts.extend(texts) + return [[1.0, 2.0] for _ in texts] + + +class _LegacyEmbed: + available = True + + def __init__(self, vector: list[float]) -> None: + self.vector = vector + self.calls: list[str] = [] + + def embed(self, text: str) -> list[float]: + self.calls.append(text) + return self.vector + + +class _UnavailableEmbed: + available = False + + def embed(self, _text: str) -> list[float]: + raise AssertionError("unavailable channel must not be called") + + +class _FailingStructure: + """Represent an expected structure-channel response failure.""" + + available = True + + def infer( + self, _post_title: str, _units: list[dict[str, object]] + ) -> tuple[StructureDecision, ...]: + """Raise the response-validation error handled by persistence.""" + raise ValueError("synthetic invalid structure response") + + +class _UnexpectedChannelFailure: + """Represent a programming defect that persistence must expose.""" + + available = True + + def embed_many(self, _texts: list[str]) -> list[list[float]]: + """Raise a defect outside the expected channel-failure contract.""" + raise AssertionError("synthetic programming defect") + + def infer( + self, _post_title: str, _units: list[dict[str, object]] + ) -> tuple[StructureDecision, ...]: + """Raise the same defect from the structure-channel boundary.""" + raise AssertionError("synthetic programming defect") + + +class _ResolvedStructure: + """Return one applicable and one out-of-scope structure decision.""" + + available = True + + def infer( + self, _post_title: str, units: list[dict[str, object]] + ) -> tuple[StructureDecision, ...]: + """Return bounded synthetic decisions for persistence filtering.""" + return ( + StructureDecision( + unit_index=int(units[0]["unit_index"]), + indent_level=2, + confidence=0.9, + evidence="Synthetic semantic nesting evidence.", + ), + StructureDecision( + unit_index=999, + indent_level=9, + confidence=0.1, + evidence="Out-of-scope synthetic decision.", + ), + ) + + +def test_render_image_text_preserves_unavailable_and_caption_variants() -> None: + assert _render_image_text(None) == "[image: content unavailable]" + assert _render_image_text(ImageContentResult(0, "image/png", "failed")) == "[image: content unavailable]" + assert ( + _render_image_text( + ImageContentResult( + 0, + "image/png", + "described", + SimpleNamespace(caption="caption", extracted_text=" ", tags=()), + ) + ) + == "[image: caption]" + ) + assert ( + _render_image_text( + ImageContentResult( + 0, + "image/png", + "described", + SimpleNamespace(caption="", extracted_text=" OCR ", tags=()), + ) + ) + == "[image: no caption available | text: OCR]" + ) + + +def test_persists_image_tags_formatting_and_embeddings() -> None: + body = '

    before

    after

    ' + chunks = chunk_by_dom(body) + image_index = next(chunk.index for chunk in chunks if chunk.unit_type == "image") + first_dom = next(chunk for chunk in chunks if chunk.unit_type == "dom") + normalized = NormalizedPostContent( + text="before\n\nafter", + formatting_hints=(FormattingHint(first_dom.index, first_dom.label, "color:red"),), + image_results=( + ImageContentResult( + image_index, + "image/png", + "described", + SimpleNamespace(caption="diagram", extracted_text="OCR", tags=("one", "two")), + regions=( + ImageRegionResult( + 0, + ImageRegion(0.0, 0.0, 1.0, 1.0), + "described", + SimpleNamespace(caption="panel", extracted_text="panel OCR", tags=("panel",)), + ), + ImageRegionResult( + 1, + ImageRegion(0.1, 0.1, 0.5, 0.5), + "unavailable", + None, + ), + ), + ), + ), + ) + conn = _Connection() + embedder = _EmbedMany() + + count = _persist( + conn, + "post-1", + body, + embedding_client=embedder, + embedding_model_code="embedding-model", + normalized_result=normalized, + ) + + assert count == len(chunks) + assert embedder.async_calls == 1 + assert "[image: panel | text: panel OCR]" in embedder.texts + assert any("post_content_image" in query for query, _args in conn.fetchvals) + assert sum("post_content_image_tag" in query for query, _args in conn.executed) == 2 + assert any("post_content_image_region_embedding" in query for query, _args in conn.fetchvals) + assert sum("post_content_image_region_embedding_value" in query for query, _args in conn.executed) == 2 + assert any("post_content_embedding" in query for query, _args in conn.fetchvals) + assert sum("post_content_embedding_value" in query for query, _args in conn.executed) == 2 * len(chunks) + + +def test_legacy_embed_and_malformed_vectors_never_write_vectors() -> None: + conn = _Connection() + legacy = _LegacyEmbed([float("nan")]) + count = _persist( + conn, + "post-2", + "plain text", + embedding_client=legacy, + embedding_model_code="embedding-model", + ) + + assert count == 1 + assert legacy.calls == ["plain text"] + assert not any("post_content_embedding" in query for query, _args in conn.fetchvals) + + +def test_unavailable_embedding_channel_is_skipped_and_empty_body_is_safe() -> None: + conn = _Connection() + assert ( + _persist( + conn, + "post-3", + "", + embedding_client=_UnavailableEmbed(), + embedding_model_code="embedding-model", + ) + == 0 + ) + assert not any("post_content_embedding" in query for query, _args in conn.fetchvals) + + +def test_source_only_whitespace_is_not_persisted_as_explicit_depth() -> None: + """Presentation alignment must not become authoritative hierarchy.""" + conn = _Connection() + + assert ( + _persist( + conn, + "post-4", + "

      First item

        Second item

    ", + ) + == 2 + ) + + structure_rows = [ + args + for query, args in conn.executed + if "insert into post_content_unit_structure" in query + ] + assert [(args[1], args[2]) for args in structure_rows] == [ + (0, "unresolved"), + (0, "unresolved"), + ] + + +def test_expected_structure_failure_remains_unresolved_for_retry() -> None: + """Keep an invalid provider response absent without losing source units.""" + conn = _Connection() + + assert ( + _persist(conn, "post-5", "plain text", structure_client=_FailingStructure()) + == 1 + ) + assert any( + args[2] == "unresolved" + for query, args in conn.executed + if "insert into post_content_unit_structure" in query + ) + + +@pytest.mark.parametrize( + "channel_kwargs", + ( + { + "embedding_client": _UnexpectedChannelFailure(), + "embedding_model_code": "embedding-model", + }, + {"structure_client": _UnexpectedChannelFailure()}, + ), +) +def test_unexpected_channel_defects_propagate( + channel_kwargs: dict[str, object], +) -> None: + """Expose programming defects so the durable worker records the failure.""" + with pytest.raises(AssertionError, match="synthetic programming defect"): + _persist(_Connection(), "post-6", "plain text", **channel_kwargs) + + +def test_bounded_batches_cover_empty_count_and_character_limits() -> None: + """Preserve generic keys while enforcing both provider request bounds.""" + assert _bounded_unit_batches([]) == [] + count_bounded = _bounded_unit_batches([(str(i), "x") for i in range(33)]) + assert [len(batch) for batch in count_bounded] == [32, 1] + assert [ + len(batch) + for batch in _bounded_unit_batches([(str(i), "x" * 12_001) for i in range(3)]) + ] == [1, 1, 1] + + +def test_explicit_and_adjudicated_structure_are_persisted_by_unit() -> None: + """Persist explicit depth and only in-scope orchestrator decisions.""" + conn = _Connection() + + assert ( + _persist( + conn, + "post-7", + '

    Explicit

    Semantic

    ', + structure_client=_ResolvedStructure(), + ) + == 2 + ) + structure_rows = [ + args + for query, args in conn.executed + if "insert into post_content_unit_structure" in query + ] + assert [(args[1], args[2]) for args in structure_rows] == [ + (1, "explicit"), + (2, "llm"), + ] diff --git a/tests/test_post_content_queue.py b/tests/test_post_content_queue.py new file mode 100644 index 000000000..3f8c1398b --- /dev/null +++ b/tests/test_post_content_queue.py @@ -0,0 +1,427 @@ +"""Contracts for the durable post-content queue boundary.""" + +from __future__ import annotations + +import asyncio +import re +from datetime import timedelta +from pathlib import Path + +import pytest + +from backend.app.post_content_queue import ( + FAILED, + POST_CONTENT_RETRY_INTERVAL, + POST_CONTENT_STREAM_KEY, + STALE_RUNNING_INTERVAL, + QUEUED, + RUNNING, + SUCCEEDED, + record_post_content_backfill_success, + requeue_failed_post_content_job, + post_content_api_status, + post_content_is_complete, + post_content_stream_fields, + source_body_sha256, +) + +_ROOT = Path(__file__).resolve().parents[1] + + +def test_stream_is_a_wakeup_and_never_contains_a_body() -> None: + fields = post_content_stream_fields( + post_id="00000000-0000-0000-0000-000000000001", + source_body_digest="ab" * 32, + ) + assert POST_CONTENT_STREAM_KEY == "post-content-ingestion" + assert set(fields) == {"post_id", "source_body_sha256"} + assert "body" not in fields.values() + assert source_body_sha256("body") == source_body_sha256("body") + assert source_body_sha256("body") != source_body_sha256("changed") + + +def test_api_status_does_not_call_failed_content_ready() -> None: + assert post_content_api_status(QUEUED, content_present=False) == "processing" + assert post_content_api_status(QUEUED, content_present=True) == "processing" + assert post_content_api_status(RUNNING, content_present=False) == "processing" + assert post_content_api_status(SUCCEEDED, content_present=True) == "ready" + assert post_content_api_status(FAILED, content_present=False) == "unavailable" + assert post_content_api_status(FAILED, content_present=True) == "unavailable" + + +def test_embedding_gap_is_not_complete_content() -> None: + class FakeConnection: + async def fetchval(self, query: str, *_args: object) -> int: + assert "post_content_embedding" in query + assert "post_content_image_region_embedding" in query + assert "unit_kind_code <> 'image'" in query + return 0 + + assert ( + asyncio.run( + post_content_is_complete( + FakeConnection(), + "00000000-0000-0000-0000-000000000001", + embedding_model_code="text-embedding-3-large", + ) + ) + is False + ) + + +def test_structure_gap_is_part_of_orchestrated_completeness() -> None: + class FakeConnection: + async def fetchval(self, query: str, *_args: object) -> int: + assert "post_content_unit_structure" in query + assert "decision_source_code = 'unresolved'" in query + return 0 + + assert ( + asyncio.run( + post_content_is_complete( + FakeConnection(), + "00000000-0000-0000-0000-000000000001", + embedding_model_code="text-embedding-3-large", + require_structure=True, + ) + ) + is False + ) + + +def test_republish_query_recovers_due_queue_and_stale_running_leases() -> None: + from backend.app import post_content_queue + + class FakeConnection: + async def fetch(self, query: str, *args: object): + assert "status_code = $1" in query + assert "status_code = $3" in query + assert "started_at < now() - $4::interval" in query + assert args[0] == QUEUED + assert args[2] == RUNNING + assert args[1] == POST_CONTENT_RETRY_INTERVAL + return [ + { + "post_id": "00000000-0000-0000-0000-000000000001", + "source_body_sha256": "a" * 64, + } + ] + + class Acquire: + async def __aenter__(self): + return FakeConnection() + + async def __aexit__(self, *_args: object) -> None: + return None + + class Pool: + def acquire(self): + return Acquire() + + class Client: + pass + + published: list[tuple[str, str]] = [] + + async def publish(_client, *, post_id: str, source_body_digest: str) -> bool: + published.append((post_id, source_body_digest)) + return True + + original = post_content_queue.publish_post_content_event + post_content_queue.publish_post_content_event = publish + try: + assert asyncio.run( + post_content_queue.republish_queued_post_content_jobs(Client(), Pool()) + ) == 1 + finally: + post_content_queue.publish_post_content_event = original + assert published == [("00000000-0000-0000-0000-000000000001", "a" * 64)] + + +def test_existing_units_are_requeued_when_the_source_digest_changes() -> None: + from backend.app.post_content_queue import ensure_post_content_job + + class FakeConnection: + def __init__(self) -> None: + self.executed: list[tuple[str, tuple[object, ...]]] = [] + + async def fetchrow(self, _query: str, _post_id: str): + return { + "source_body_sha256": source_body_sha256("old body"), + "status_code": SUCCEEDED, + } + + async def fetchval(self, _query: str, _post_id: str) -> int: + return 0 + + async def execute(self, query: str, *args: object): + self.executed.append((query, args)) + + conn = FakeConnection() + job = asyncio.run( + ensure_post_content_job( + conn, + "00000000-0000-0000-0000-000000000001", + "new body", + content_complete=True, + ) + ) + + assert job.status_code == QUEUED + assert job.should_publish is True + assert any("set source_body_sha256" in query for query, _args in conn.executed) + + +def test_existing_units_register_as_succeeded_without_a_wakeup() -> None: + from backend.app.post_content_queue import ensure_post_content_job + + class FakeConnection: + def __init__(self) -> None: + self.executed: list[tuple[str, tuple[object, ...]]] = [] + + async def fetchrow(self, _query: str, _post_id: str): + return None + + async def fetchval(self, _query: str, _post_id: str) -> int: + return 0 + + async def execute(self, query: str, *args: object): + self.executed.append((query, args)) + + conn = FakeConnection() + job = asyncio.run( + ensure_post_content_job( + conn, + "00000000-0000-0000-0000-000000000001", + "existing body", + content_complete=True, + ) + ) + + assert job.status_code == SUCCEEDED + assert job.should_publish is False + assert any("insert into post_content_ingestion_job" in query for query, _args in conn.executed) + + +def test_failed_same_body_is_not_requeued_by_a_read_poll() -> None: + from backend.app.post_content_queue import ensure_post_content_job + + class FakeConnection: + async def fetchrow(self, _query: str, _post_id: str): + return { + "source_body_sha256": source_body_sha256("same body"), + "status_code": FAILED, + } + + async def fetchval(self, _query: str, _post_id: str) -> int: + return 0 + + async def execute(self, *_args: object) -> None: + raise AssertionError("a failed job must remain terminal for the same digest") + + job = asyncio.run( + ensure_post_content_job( + FakeConnection(), + "00000000-0000-0000-0000-000000000001", + "same body", + content_complete=False, + ) + ) + + assert job.status_code == FAILED + assert job.should_publish is False + + +def test_changed_body_resets_a_terminal_job_and_republishes() -> None: + from backend.app.post_content_queue import ensure_post_content_job + + class FakeConnection: + def __init__(self) -> None: + self.executed: list[tuple[str, tuple[object, ...]]] = [] + + async def fetchrow(self, _query: str, _post_id: str): + return { + "source_body_sha256": source_body_sha256("old body"), + "status_code": FAILED, + } + + async def fetchval(self, _query: str, _post_id: str) -> int: + return 0 + + async def execute(self, query: str, *args: object) -> None: + self.executed.append((query, args)) + + conn = FakeConnection() + job = asyncio.run( + ensure_post_content_job( + conn, + "00000000-0000-0000-0000-000000000001", + "new body", + content_complete=False, + ) + ) + + assert job.status_code == QUEUED + assert job.should_publish is True + assert any("attempt_count = 0" in query for query, _args in conn.executed) + + +def test_recovery_query_carries_one_bounded_retry_interval() -> None: + assert POST_CONTENT_RETRY_INTERVAL == timedelta(minutes=5) + migration = (_ROOT / "migrations" / "0050_post_content_ingestion_queue.sql").read_text() + assert "queued_at timestamptz not null" in migration + + +def test_explicit_retry_resets_only_one_failed_job() -> None: + executed: list[tuple[str, tuple[object, ...]]] = [] + + class FakeConnection: + async def fetchrow(self, query: str, *_args: object): + assert "for update" in query + return {"status_code": FAILED} + + async def fetchval(self, query: str, *_args: object) -> int: + assert "status_ordinal" in query + return 4 + + async def execute(self, query: str, *args: object) -> str: + executed.append((query, args)) + return "UPDATE 1" if query.lstrip().startswith("update") else "INSERT 0 1" + + request = asyncio.run( + requeue_failed_post_content_job( + FakeConnection(), + "00000000-0000-0000-0000-000000000001", + "current body", + ) + ) + + assert request.status_code == QUEUED + assert request.should_publish is True + assert request.source_body_sha256 == source_body_sha256("current body") + assert len(executed) == 2 + assert "attempt_count = 0" in executed[0][0] + assert executed[1][1][-1] == "operator requested an explicit post-content retry" + + +def test_explicit_retry_rejects_missing_and_nonterminal_jobs() -> None: + """The operator command cannot create a job or reset an active job.""" + + class MissingConnection: + async def fetchrow(self, _query: str, *_args: object): + return None + + with pytest.raises(ValueError, match="does not exist"): + asyncio.run( + requeue_failed_post_content_job( + MissingConnection(), + "00000000-0000-0000-0000-000000000001", + "current body", + ) + ) + + class QueuedConnection: + async def fetchrow(self, _query: str, *_args: object): + return {"status_code": QUEUED} + + with pytest.raises(ValueError, match="only a failed"): + asyncio.run( + requeue_failed_post_content_job( + QueuedConnection(), + "00000000-0000-0000-0000-000000000001", + "current body", + ) + ) + + +def test_backfill_success_clears_terminal_error_and_records_succeeded() -> None: + executed: list[tuple[str, tuple[object, ...]]] = [] + + class FakeConnection: + async def fetchrow(self, query: str, *_args: object): + assert "for update" in query + return {"status_code": FAILED} + + async def fetchval(self, query: str, *_args: object) -> int: + assert "status_ordinal" in query + return 5 + + async def execute(self, query: str, *args: object) -> str: + executed.append((query, args)) + return "UPDATE 1" if query.lstrip().startswith("update") else "INSERT 0 1" + + request = asyncio.run( + record_post_content_backfill_success( + FakeConnection(), + "00000000-0000-0000-0000-000000000001", + "current body", + ) + ) + + assert request.status_code == SUCCEEDED + assert request.should_publish is False + assert len(executed) == 2 + assert "last_error_code = null" in executed[0][0] + assert executed[1][1][-1] == "operator backfill persisted post-content evidence" + + +def test_recovery_republishes_due_rows_in_queued_at_order() -> None: + from contextlib import asynccontextmanager + + from backend.app.post_content_queue import republish_queued_post_content_jobs + + class FakeConnection: + def __init__(self) -> None: + self.query = "" + self.args: tuple[object, ...] = () + + async def fetch(self, query: str, *args: object): + self.query = query + self.args = args + return [ + {"post_id": "first", "source_body_sha256": "a" * 64}, + {"post_id": "second", "source_body_sha256": "b" * 64}, + ] + + class FakePool: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + + @asynccontextmanager + async def acquire(self): + yield self.connection + + class FakeClient: + def __init__(self) -> None: + self.events: list[tuple[str, str]] = [] + + async def xadd(self, _stream: str, fields: dict[str, str], **_kwargs: object) -> str: + self.events.append((fields["post_id"], fields["source_body_sha256"])) + return str(len(self.events)) + + connection = FakeConnection() + client = FakeClient() + published = asyncio.run( + republish_queued_post_content_jobs(client, FakePool(connection), limit=2) + ) + + assert published == 2 + assert client.events == [("first", "a" * 64), ("second", "b" * 64)] + assert "queued_at <= now() - $2::interval" in connection.query + assert "order by queued_at" in connection.query + assert connection.args == (QUEUED, POST_CONTENT_RETRY_INTERVAL, RUNNING, STALE_RUNNING_INTERVAL, 2) + + +def test_migration_contains_normalized_job_and_status_event_tables() -> None: + migration = (_ROOT / "migrations" / "0050_post_content_ingestion_queue.sql").read_text() + assert "create table if not exists post_content_ingestion_job" in migration + assert "create table if not exists post_content_ingestion_job_status_event" in migration + assert "post_body" not in migration + assert "jsonb" not in migration.casefold() + for table_name in re.findall(r"create table if not exists\s+([a-z0-9_]+)", migration): + assert len(table_name.split("_")) >= 2 + + +def test_migration_replay_window_includes_post_content_queue() -> None: + migrate = (_ROOT / "docker/postgres-init/migrate.sh").read_text() + assert "0050_*)" in migrate diff --git a/tests/test_post_content_worker.py b/tests/test_post_content_worker.py new file mode 100644 index 000000000..dddace990 --- /dev/null +++ b/tests/test_post_content_worker.py @@ -0,0 +1,285 @@ +"""Worker regressions for bounded, evidence-complete post ingestion.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from types import SimpleNamespace + +from backend.app import post_content_worker +from backend.app.post_content_queue import ( + FAILED, + POST_CONTENT_MAX_ATTEMPTS, + QUEUED, + RUNNING, + SUCCEEDED, +) + + +class _Transaction: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args: object) -> bool: + return False + + +class _Connection: + def __init__(self, row: dict[str, object] | None = None, values: list[object] | None = None): + self.row = row + self.values = list(values or []) + self.executed: list[tuple[str, tuple[object, ...]]] = [] + + def transaction(self) -> _Transaction: + return _Transaction() + + async def fetchrow(self, *_args: object): + return self.row + + async def fetchval(self, query: str, *_args: object): + if self.values: + return self.values.pop(0) + if "status_ordinal" in query: + return 0 + return False + + async def execute(self, query: str, *args: object) -> str: + self.executed.append((query, args)) + return "OK" + + +class _Pool: + def __init__(self, connection: _Connection): + self.connection = connection + + @asynccontextmanager + async def acquire(self): + yield self.connection + + +def _row(status: str, attempt_count: int, *, started_at: object = None) -> dict[str, object]: + return { + "job_status_code": status, + "job_attempt_count": attempt_count, + "job_started_at": started_at, + "job_queued_at": "queued-at", + "post_body": "A synthetic post body with a retrieval unit.", + "post_title": "Synthetic post title", + } + + +def test_worker_starts_after_historical_stream_tail() -> None: + class Client: + async def xrevrange(self, key: str, *, count: int): + assert key == post_content_worker.POST_CONTENT_STREAM_KEY + assert count == 1 + return [("123-0", {})] + + assert asyncio.run(post_content_worker._stream_tail(Client())) == "123-0" + + +def test_terminal_failed_job_ignores_a_stale_duplicate_wakeup() -> None: + connection = _Connection(_row(FAILED, POST_CONTENT_MAX_ATTEMPTS)) + + claimed = asyncio.run( + post_content_worker._claim_job( + _Pool(connection), + "00000000-0000-0000-0000-000000000001", + "a" * 64, + embedding_model_code="", + ) + ) + + assert claimed is None + assert connection.executed == [] + + +def test_duplicate_wakeup_before_retry_delay_is_not_claimable() -> None: + connection = _Connection(_row(QUEUED, 1), values=[False]) + + claimed = asyncio.run( + post_content_worker._claim_job( + _Pool(connection), + "00000000-0000-0000-0000-000000000001", + "a" * 64, + embedding_model_code="", + ) + ) + + assert claimed is None + assert connection.executed == [] + + +def test_due_retry_is_claimed_and_attempt_is_incremented() -> None: + connection = _Connection(_row(QUEUED, 1), values=[True]) + + claimed = asyncio.run( + post_content_worker._claim_job( + _Pool(connection), + "00000000-0000-0000-0000-000000000001", + "a" * 64, + embedding_model_code="", + ) + ) + + assert claimed is not None + assert any("attempt_count = attempt_count + 1" in query for query, _args in connection.executed) + assert any(args[1] == RUNNING for query, args in connection.executed if len(args) > 1 and "set status_code" in query) + + +def test_successful_job_reclaims_when_configured_evidence_is_incomplete(monkeypatch) -> None: + connection = _Connection(_row(SUCCEEDED, 0), values=[False]) + calls: list[str] = [] + + async def incomplete(*_args, **_kwargs) -> bool: + calls.append("checked") + return False + + monkeypatch.setattr(post_content_worker, "post_content_is_complete", incomplete) + claimed = asyncio.run( + post_content_worker._claim_job( + _Pool(connection), + "00000000-0000-0000-0000-000000000001", + "a" * 64, + embedding_model_code="embedding-model", + require_structure=True, + ) + ) + + assert claimed is not None + assert calls == ["checked"] + + +def test_incomplete_provider_output_is_requeued_with_a_failure_code(monkeypatch) -> None: + connection = _Connection(values=[2]) + pool = _Pool(connection) + + async def claim(*_args, **_kwargs): + return _row(RUNNING, 1) + + async def persist(*_args, **_kwargs): + return 1 + + async def incomplete(*_args, **_kwargs): + return False + + monkeypatch.setattr(post_content_worker, "_claim_job", claim) + monkeypatch.setattr(post_content_worker, "persist_post_content", persist) + monkeypatch.setattr(post_content_worker, "post_content_is_complete", incomplete) + monkeypatch.setattr( + post_content_worker, + "load_settings", + lambda: SimpleNamespace( + embedding_model="embedding-model", + orchestrator_base_url="gateway", + orchestrator_api_key="key", + ), + ) + monkeypatch.setattr(post_content_worker, "normalize_post_body", lambda *_args: object()) + client = SimpleNamespace(available=True) + + asyncio.run( + post_content_worker.process_post_content_job( + pool, + post_id="00000000-0000-0000-0000-000000000001", + source_body_digest="a" * 64, + vision_factory=lambda: client, + embedding_factory=lambda: client, + structure_factory=lambda: client, + ) + ) + + updates = [args for query, args in connection.executed if "set status_code" in query] + assert any(args[1] == QUEUED and args[6] == "post_content_ingestion_incomplete" for args in updates) + + +def test_transient_provider_error_is_requeued_before_attempt_limit(monkeypatch) -> None: + connection = _Connection(values=[2]) + pool = _Pool(connection) + + async def claim(*_args, **_kwargs): + return _row(RUNNING, 1) + + async def persist(*_args, **_kwargs): + raise TimeoutError("provider timeout") + + monkeypatch.setattr(post_content_worker, "_claim_job", claim) + monkeypatch.setattr(post_content_worker, "persist_post_content", persist) + monkeypatch.setattr( + post_content_worker, + "load_settings", + lambda: SimpleNamespace( + embedding_model="embedding-model", + orchestrator_base_url="", + orchestrator_api_key="", + ), + ) + monkeypatch.setattr(post_content_worker, "normalize_post_body", lambda *_args: object()) + client = SimpleNamespace(available=True) + + asyncio.run( + post_content_worker.process_post_content_job( + pool, + post_id="00000000-0000-0000-0000-000000000001", + source_body_digest="a" * 64, + vision_factory=lambda: client, + embedding_factory=lambda: client, + structure_factory=lambda: client, + ) + ) + + updates = [args for query, args in connection.executed if "set status_code" in query] + assert any(args[1] == QUEUED and args[6] == "post_content_ingestion_failed" for args in updates) + + +def test_failure_at_attempt_limit_is_terminal_and_visible() -> None: + connection = _Connection(values=[POST_CONTENT_MAX_ATTEMPTS]) + + asyncio.run( + post_content_worker._finish_failed_job( + _Pool(connection), + "00000000-0000-0000-0000-000000000001", + failure_code="post_content_ingestion_failed", + detail_text="provider outage", + expected_attempt_count=POST_CONTENT_MAX_ATTEMPTS, + ) + ) + + updates = [args for query, args in connection.executed if "set status_code" in query] + assert any(args[1] == FAILED and args[6] == "post_content_ingestion_attempt_limit" for args in updates) + + +def test_stale_worker_cannot_retry_after_lease_recovery() -> None: + connection = _Connection(values=[2]) + + asyncio.run( + post_content_worker._finish_failed_job( + _Pool(connection), + "00000000-0000-0000-0000-000000000001", + failure_code="post_content_ingestion_failed", + detail_text="late provider failure", + expected_attempt_count=1, + ) + ) + + assert not any("set status_code" in query for query, _args in connection.executed) + + +def test_stale_worker_cannot_mark_recovered_attempt_succeeded() -> None: + class StaleConnection(_Connection): + async def execute(self, query: str, *args: object) -> str: + self.executed.append((query, args)) + return "UPDATE 0" if "update post_content_ingestion_job" in query else "OK" + + connection = StaleConnection() + + asyncio.run( + post_content_worker._finish_job( + _Pool(connection), + "00000000-0000-0000-0000-000000000001", + SUCCEEDED, + expected_attempt_count=1, + ) + ) + + assert not any("insert into post_content_ingestion_job_status_event" in query for query, _args in connection.executed) diff --git a/tests/test_post_eligibility.py b/tests/test_post_eligibility.py new file mode 100644 index 000000000..336820e3d --- /dev/null +++ b/tests/test_post_eligibility.py @@ -0,0 +1,18 @@ +from backend.app.post_eligibility import ( + SOURCE_CONTEXT_COLUMNS, + SOURCE_POST_ELIGIBILITY_SQL, + source_context_missing_sql, + source_context_present_sql, +) + + +def test_real_source_context_hides_pure_seed_rows_at_read_boundary() -> None: + eligibility = SOURCE_POST_ELIGIBILITY_SQL.format(alias="post") + + assert "source_draft_code" in eligibility + assert "source_deleted_flag" in eligibility + assert "not ((" in eligibility + assert "exists (select 1 from source_post real_post" in eligibility + for column in SOURCE_CONTEXT_COLUMNS: + assert f"post.{column}" in source_context_missing_sql("post") + assert f"real_post.{column}" in source_context_present_sql("real_post") diff --git a/tests/test_post_evaluation_ingestion.py b/tests/test_post_evaluation_ingestion.py new file mode 100644 index 000000000..ee97376a1 --- /dev/null +++ b/tests/test_post_evaluation_ingestion.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import asyncio + +import pytest +from fast_mlsirm import LLMJudgeResult + +from backend.app.post_evaluation_ingestion import ingest_post_evaluation +from lineageweave.post_evaluation import CRITERION_CODES, IRT_CATEGORY_COUNT, RUBRIC_VERSION + + +class _Connection: + def __init__(self, rows: list[dict[str, object]]) -> None: + self.rows = rows + self.executed: list[tuple[str, tuple[object, ...]]] = [] + + async def execute(self, query: str, *args: object) -> str: + self.executed.append((query, args)) + return "OK" + + async def fetch(self, _query: str, *_args: object) -> list[dict[str, object]]: + return self.rows + + +class _Client: + available = True + + def __init__(self, result: LLMJudgeResult) -> None: + self.result = result + + def evaluate(self, _title: str, _body: str) -> LLMJudgeResult: + return self.result + + +def _result() -> LLMJudgeResult: + return LLMJudgeResult( + score=0.8, + accepted=True, + rationale="synthetic ingestion result", + criterion_scores={code: 0.8 for code in CRITERION_CODES}, + raw_output="{}", + orchestration_mode="route", + trace_step_count=0, + usage={}, + criterion_categories={code: 2 for code in CRITERION_CODES}, + category_count=IRT_CATEGORY_COUNT, + ) + + +def test_ingest_evaluation_upserts_each_criterion_and_fetches_rows() -> None: + rows = [ + { + "criterion_code": code, + "criterion_label": f"label-{code}", + "response_category": 2, + "rubric_version": RUBRIC_VERSION, + } + for code in CRITERION_CODES + ] + conn = _Connection(rows) + persisted = asyncio.run(ingest_post_evaluation(conn, _Client(_result()), "post-1", "title", "body")) + + assert [item.criterion_code for item in persisted] == list(CRITERION_CODES) + assert len(conn.executed) == len(CRITERION_CODES) + assert all("on conflict" in query.lower() for query, _args in conn.executed) + + +def test_ingest_evaluation_propagates_judge_failure_without_writes() -> None: + class FailingClient: + def evaluate(self, _title: str, _body: str) -> LLMJudgeResult: + raise RuntimeError("synthetic judge failure") + + conn = _Connection([]) + with pytest.raises(RuntimeError, match="synthetic judge failure"): + asyncio.run(ingest_post_evaluation(conn, FailingClient(), "post-1", "title", "body")) + assert conn.executed == [] diff --git a/tests/test_post_structure.py b/tests/test_post_structure.py new file mode 100644 index 000000000..89e518fda --- /dev/null +++ b/tests/test_post_structure.py @@ -0,0 +1,45 @@ +import json + +from lineageweave.post_structure import ContextualOrchestratorPostStructureClient + + +def test_structure_client_validates_complete_decisions(monkeypatch) -> None: + captured = [] + + def fake_post_json(*args, **kwargs): + captured.append(args[1]) + units = json.loads(args[1]["messages"][1]["content"])["ordered_units"] + return { + "choices": [ + { + "message": { + "content": json.dumps( + {"decisions": [ + { + "unit_index": int(unit["unit_index"]), + "indent_level": 0, + "confidence": 0.9, + "evidence": "top-level heading", + } + for unit in units + ]} + ) + } + } + ] + } + + monkeypatch.setattr( + "lineageweave.post_structure.post_json", + fake_post_json, + ) + client = ContextualOrchestratorPostStructureClient("http://orchestrator", "test-key") + + assert client.timeout == 600.0 + assert client.infer("Title", [{"unit_index": 0, "text": "1. Heading"}])[0].indent_level == 0 + assert len(captured) == 1 + response_format = captured[0]["response_format"] + assert response_format["type"] == "json_schema" + assert response_format["json_schema"]["strict"] is True + assert response_format["json_schema"]["schema"]["required"] == ["decisions"] + assert captured[0]["max_tokens"] == 4096 diff --git a/tests/test_post_summary.py b/tests/test_post_summary.py index 4f863769a..34abac38a 100644 --- a/tests/test_post_summary.py +++ b/tests/test_post_summary.py @@ -14,7 +14,7 @@ import pytest -from backend.app.post_summary_ingestion import seeded_fixture_summary +from backend.app.post_summary_ingestion import require_summary_source_body, seeded_fixture_summary from lineageweave.fixtures import ( ambiguous_commitment_post, ambiguous_keyman_post, @@ -25,6 +25,10 @@ ContextualOrchestratorPostSummaryClient, NullPostSummaryClient, RoleResponsibility, + _SUMMARY_REQUEST_PROMPT_TEMPLATE, + _parse_optional_project_key, + _parse_plain_summary_response, + _parse_plain_summary_details, parse_summary_response, ) @@ -36,6 +40,35 @@ def test_null_summary_client_is_unavailable_not_empty_summary() -> None: client.summarize("any title", "any body") +def test_summary_prompt_requires_trigger_development_conclusion_structure() -> None: + """Feature request (2026-08-19): a flat 5W1H restatement in body + order was ruled a bug -- the prompt must ask for a legible + 발단(trigger)/전개(development)/결론(conclusion) narrative arc so a + reader can tell what triggered the post, what was actually + considered, and what was decided or left open. + """ + for marker in ("발단", "전개", "결론", "다음 조치는"): + assert marker in _SUMMARY_REQUEST_PROMPT_TEMPLATE + + +def test_summary_prompt_requires_naming_actual_people_not_generic_titles() -> None: + """Live bug (2026-08-19): a real post's summary said "PM들이 + 참석했다" (a generic "PMs attended") even though the post body + literally named each attendee -- the same names a separate R&R + extraction call correctly pulled out. The summary call has no + knowledge of that separate call's output, so the summary prompt + itself must demand real names, not rely on R&R to carry them. + """ + assert "PM들이 참석했다" in _SUMMARY_REQUEST_PROMPT_TEMPLATE + assert "홍길동" in _SUMMARY_REQUEST_PROMPT_TEMPLATE + + +def test_summary_requires_imported_source_body() -> None: + assert require_summary_source_body(" body ") == " body " + with pytest.raises(ValueError, match="source post body is empty"): + require_summary_source_body("") + + def test_parses_a_well_formed_json_object() -> None: content = ( '{"korean_summary": "회의 후속 조치에 대한 요약입니다.", ' @@ -53,6 +86,106 @@ def test_parses_a_well_formed_json_object() -> None: assert role.affiliated_organization_name == "Westfield Power" +def test_parses_explicit_five_w1h_evidence_with_source_phrase() -> None: + summary = parse_summary_response( + '{"korean_summary":"요약", "key_events":[], ' + '"five_w1h_evidence":[{"slot_code":"when", "value_text":"3월 4일", ' + '"evidence_text":"3월 4일 현장 회의"}]}' + ) + assert summary is not None + assert summary.five_w1h_evidence[0].slot_code == "when" + assert summary.five_w1h_evidence[0].evidence_text == "3월 4일 현장 회의" + + +def test_parses_plain_summary_evidence_section() -> None: + details = _parse_plain_summary_details( + "ROLES:\nNONE\nPROJECTS:\nNONE\nEVIDENCE:\n" + "where | 제3공장 | 제3공장에서 협의했다" + ) + assert details is not None + assert details[3][0].value_text == "제3공장" + + +def test_parses_major_event_requester_and_processor() -> None: + details = _parse_plain_summary_details( + "ROLES:\n" + "홍길동 | 변경 요청 | person | 당사\n" + "김철수 | 도면 수정 | person | 고객사\n" + "PROJECTS:\nNONE\n" + "ACTIONS:\n" + "도면 변경 승인 | 홍길동 | 김철수 | 홍길동이 변경을 요청했고 김철수가 수정하기로 함" + ) + assert details is not None + action = details[2][0] + assert action.requester_actor_name == "홍길동" + assert action.processor_actor_name == "김철수" + + +def test_parses_project_bound_major_event_action() -> None: + details = _parse_plain_summary_details( + "ROLES:\n" + "홍길동 | 변경 요청 | person | 당사\n" + "김철수 | 도면 수정 | person | 고객사\n" + "PROJECTS:\n" + "HVDC Pilot | hvdc-pilot | 파일럿 도면 | 0.9\n" + "ACTIONS:\n" + "도면 변경 승인 | hvdc-pilot | 홍길동 | 김철수 | 프로젝트 도면 근거" + ) + assert details is not None + assert details[2][0].project_key == "hvdc-pilot" + + +def test_legacy_action_preserves_pipe_in_evidence_text() -> None: + details = _parse_plain_summary_details( + "ROLES:\n" + "Synthetic requester | 요청 | person | Synthetic organization\n" + "Synthetic processor | 처리 | person | Synthetic organization\n" + "PROJECTS:\nNONE\n" + "ACTIONS:\n" + "합성 조치 | Synthetic requester | Synthetic processor | 첫 근거 | 추가 근거" + ) + assert details is not None + assert details[2][0].project_key is None + assert details[2][0].evidence_text == "첫 근거 | 추가 근거" + + +def test_json_project_name_is_normalized_for_legacy_action_contract() -> None: + summary = parse_summary_response( + '{"korean_summary":"요약", "major_event_actions":[' + '{"action_text":"검토", "project_name":"HVDC Pilot", ' + '"evidence_text":"본문 근거"}]}' + ) + assert summary is not None + assert summary.major_event_actions[0].project_key == "hvdc-pilot" + + +def test_parses_project_bound_key_event_without_leaking_internal_key_to_text() -> None: + summary = parse_summary_response( + '{"korean_summary":"요약", "key_events":[{"event_text":"도면 검토",' + '"project_key":"HVDC Pilot"}]}' + ) + assert summary is not None + assert summary.key_events == ("도면 검토",) + assert summary.key_event_details[0].project_key == "hvdc-pilot" + + +def test_parses_project_bound_plain_key_event() -> None: + parsed = _parse_plain_summary_response( + "회의 요약\nKEY EVENTS: hvdc-pilot :: 도면 검토; NONE :: 공통 일정 확인" + ) + assert parsed is not None + _summary, events, details = parsed + assert events == ("도면 검토", "공통 일정 확인") + assert details[0].project_key == "hvdc-pilot" + assert details[1].project_key is None + + +def test_optional_project_key_normalizes_unicode_and_rejects_sentinels() -> None: + assert _parse_optional_project_key(" Project Ω ") == "project-ω" + for sentinel in (None, "", " ", "None", "N/A", "unknown", 42): + assert _parse_optional_project_key(sentinel) is None + + def test_organization_actor_is_not_forced_into_a_person_slot() -> None: """A named actor that is genuinely an organization (e.g. our own company acting in its own name, not a named individual) must parse @@ -158,6 +291,85 @@ def test_malformed_roles_entries_are_skipped_not_crashed_on() -> None: assert summary.roles_and_responsibilities == () +def test_summary_request_uses_plain_route_evidence_contract(monkeypatch) -> None: + observed: list[dict[str, object]] = [] + + def fake_post_json(url, payload, *, headers, timeout): + observed.append(payload) + prompt = payload["messages"][0]["content"] + if "ROLES:" in prompt: + content = ( + "ROLES:\n" + "Jordan Hale | 입찰 일정 안내 | Westfield Power\n" + "PROJECTS:\n" + "HVDC pilot | pilot bid workshop | 0.9\n" + "Unsupported project | NONE | 1" + ) + else: + content = "본문 근거 요약\n\nKEY EVENTS: 후속 확인" + return { + "choices": [ + { + "message": { + "content": content + } + } + ] + } + + monkeypatch.setattr("lineageweave.post_summary.post_json", fake_post_json) + summary = ContextualOrchestratorPostSummaryClient("https://orchestrator.test", "token").summarize( + "Synthetic title", "Synthetic body" + ) + + assert summary.korean_summary == "본문 근거 요약" + assert summary.key_events == ("후속 확인",) + assert len(observed) == 2 + assert all(payload["mode"] == "auto" for payload in observed) + assert "KEY EVENTS" in observed[0]["messages"][0]["content"] + details_prompt = observed[1]["messages"][0]["content"] + assert "source_process_unit_name are PU/business-unit hints only" in details_prompt + assert "must never be" in details_prompt + assert "sales-pool/order-pool value" in details_prompt + assert "source_sales_pool_name are sales-pool/order-pool hints only" in details_prompt + assert "PU/business-unit value" in details_prompt + assert summary.roles_and_responsibilities[0].actor_name == "Jordan Hale" + assert summary.project_mentions[0].canonical_name == "hvdc-pilot" + + +def test_title_match_can_supply_explicit_project_evidence_but_not_a_guess() -> None: + details = _parse_plain_summary_details( + "ROLES:\nNONE\nPROJECTS:\nNorthridge transformer bid | NONE | 1", + post_title="Follow-up after the Northridge transformer bid workshop", + ) + assert details is not None + assert details[1][0].evidence == "Follow-up after the Northridge transformer bid workshop" + + unrelated = _parse_plain_summary_details( + "ROLES:\nNONE\nPROJECTS:\nUnrelated project | NONE | 1", + post_title="Follow-up after the Northridge transformer bid workshop", + ) + assert unrelated == ((), (), (), ()) + + +def test_role_matching_the_hinted_account_name_is_dropped_not_cataloged() -> None: + """Live finding: the model wrote a ROLES row for the logged-in + account's display name (from author_account_name in the hints) + even though that name never appeared in the post text -- see + _hallucinated_account_name's docstring. + """ + details = _parse_plain_summary_details( + "ROLES:\n" + "Demo Analyst | met the customer | person | Demo Corp\n" + "Jordi Gil | approved the quote | person | Northwind Labs\n" + "PROJECTS:\nNONE", + context_hints="author_account_name=Demo Analyst [source_field=user_account.display_name]; " + "author_affiliations=Demo Corp [source_field=account_affiliation.corporate_entity_id]", + ) + assert details is not None + assert [role.actor_name for role in details[0]] == ["Jordi Gil"] + + _ORCHESTRATOR_BASE_URL = os.environ.get("LINEAGEWEAVE_TEST_ORCHESTRATOR_BASE_URL") _ORCHESTRATOR_API_KEY = os.environ.get("LINEAGEWEAVE_TEST_ORCHESTRATOR_API_KEY") diff --git a/tests/test_real_provider_integration.py b/tests/test_real_provider_integration.py index d92d39ab3..b2a151688 100644 --- a/tests/test_real_provider_integration.py +++ b/tests/test_real_provider_integration.py @@ -17,41 +17,37 @@ from lineageweave.adjudication_client import ContextualOrchestratorAdjudicationClient from lineageweave.embedding_client import ( - OpenAiCompatibleEmbeddingClient, + ContextualOrchestratorEmbeddingClient, chunked_max_similarity, cosine_similarity, ) from lineageweave.fixtures import ambiguous_keyman_post -from lineageweave.image_content import OpenAiCompatibleVisionClient +from lineageweave.image_content import orchestrator_vision_client from lineageweave.keyman_extraction import ( COUNTERPARTY, OUR_SIDE, ContextualOrchestratorKeymanExtractionClient, ) -_EMBEDDING_BASE_URL = os.environ.get("LINEAGEWEAVE_TEST_EMBEDDING_BASE_URL") -_EMBEDDING_API_KEY = os.environ.get("LINEAGEWEAVE_TEST_EMBEDDING_API_KEY") _EMBEDDING_MODEL = os.environ.get("LINEAGEWEAVE_TEST_EMBEDDING_MODEL", "text-embedding-3-large") _ORCHESTRATOR_BASE_URL = os.environ.get("LINEAGEWEAVE_TEST_ORCHESTRATOR_BASE_URL") _ORCHESTRATOR_API_KEY = os.environ.get("LINEAGEWEAVE_TEST_ORCHESTRATOR_API_KEY") -_VISION_BASE_URL = os.environ.get("LINEAGEWEAVE_TEST_VISION_BASE_URL") -_VISION_API_KEY = os.environ.get("LINEAGEWEAVE_TEST_VISION_API_KEY") _VISION_MODEL = os.environ.get("LINEAGEWEAVE_TEST_VISION_MODEL", "gpt-4.1-mini") @pytest.mark.skipif( - not (_EMBEDDING_BASE_URL and _EMBEDDING_API_KEY), - reason="set LINEAGEWEAVE_TEST_EMBEDDING_BASE_URL and LINEAGEWEAVE_TEST_EMBEDDING_API_KEY to run", + not (_ORCHESTRATOR_BASE_URL and _ORCHESTRATOR_API_KEY), + reason="set LINEAGEWEAVE_TEST_ORCHESTRATOR_BASE_URL and LINEAGEWEAVE_TEST_ORCHESTRATOR_API_KEY to run", ) -def test_openai_compatible_embedding_client_scores_similar_text_higher() -> None: +def test_contextual_orchestrator_embedding_client_returns_real_vectors() -> None: """A real embedding call, with a real, meaningful assertion: two labels about the same synthetic topic must cosine-score higher than two about unrelated synthetic topics -- not just "the call didn't crash". """ - client = OpenAiCompatibleEmbeddingClient( - base_url=_EMBEDDING_BASE_URL, api_key=_EMBEDDING_API_KEY, model=_EMBEDDING_MODEL + client = ContextualOrchestratorEmbeddingClient( + base_url=_ORCHESTRATOR_BASE_URL, api_key=_ORCHESTRATOR_API_KEY, model=_EMBEDDING_MODEL ) a = client.embed("Quarterly budget review meeting notes") @@ -64,11 +60,12 @@ def test_openai_compatible_embedding_client_scores_similar_text_higher() -> None assert 0.0 <= related_score <= 1.0 assert 0.0 <= unrelated_score <= 1.0 assert related_score > unrelated_score + assert len(a) > 8 @pytest.mark.skipif( - not (_EMBEDDING_BASE_URL and _EMBEDDING_API_KEY), - reason="set LINEAGEWEAVE_TEST_EMBEDDING_BASE_URL and LINEAGEWEAVE_TEST_EMBEDDING_API_KEY to run", + not (_ORCHESTRATOR_BASE_URL and _ORCHESTRATOR_API_KEY), + reason="set LINEAGEWEAVE_TEST_ORCHESTRATOR_BASE_URL and LINEAGEWEAVE_TEST_ORCHESTRATOR_API_KEY to run", ) def test_chunked_embedding_finds_a_relevant_unit_buried_in_a_longer_document() -> None: """The real case chunking exists for: a short relevant passage sitting @@ -76,8 +73,8 @@ def test_chunked_embedding_finds_a_relevant_unit_buried_in_a_longer_document() - embedding dilutes the relevant passage with everything around it; chunked max-pooled similarity should not. """ - client = OpenAiCompatibleEmbeddingClient( - base_url=_EMBEDDING_BASE_URL, api_key=_EMBEDDING_API_KEY, model=_EMBEDDING_MODEL + client = ContextualOrchestratorEmbeddingClient( + base_url=_ORCHESTRATOR_BASE_URL, api_key=_ORCHESTRATOR_API_KEY, model=_EMBEDDING_MODEL ) query = "Quarterly budget review meeting notes" @@ -97,8 +94,8 @@ def test_chunked_embedding_finds_a_relevant_unit_buried_in_a_longer_document() - @pytest.mark.skipif( - not (_VISION_BASE_URL and _VISION_API_KEY), - reason="set LINEAGEWEAVE_TEST_VISION_BASE_URL and LINEAGEWEAVE_TEST_VISION_API_KEY to run", + not (_ORCHESTRATOR_BASE_URL and _ORCHESTRATOR_API_KEY), + reason="set LINEAGEWEAVE_TEST_ORCHESTRATOR_BASE_URL and LINEAGEWEAVE_TEST_ORCHESTRATOR_API_KEY to run", ) def test_vision_client_performs_real_ocr_on_a_generated_image() -> None: """Generate a real PNG with real rendered text (Pillow, not a fixture @@ -115,9 +112,8 @@ def test_vision_client_performs_real_ocr_on_a_generated_image() -> None: buffer = BytesIO() image.save(buffer, format="PNG") - client = OpenAiCompatibleVisionClient( - base_url=_VISION_BASE_URL, api_key=_VISION_API_KEY, model=_VISION_MODEL - ) + client = orchestrator_vision_client(_ORCHESTRATOR_BASE_URL, _ORCHESTRATOR_API_KEY, _VISION_MODEL) + assert client.available description = client.describe(buffer.getvalue(), "image/png") assert "48213" in description.extracted_text diff --git a/tests/test_relation_verification_edges.py b/tests/test_relation_verification_edges.py new file mode 100644 index 000000000..c32ca8653 --- /dev/null +++ b/tests/test_relation_verification_edges.py @@ -0,0 +1,73 @@ +import lineageweave.relation_verification as relation_verification + + +def test_searxng_verification_drops_malformed_or_search_page_results(monkeypatch) -> None: + responses = iter( + [ + {"results": {}}, + { + "results": [ + None, + {}, + {"url": "https://www.google.example/search?q=Aurora"}, + {"url": "https://unrelated.example/page", "content": "generic result"}, + ] + }, + ] + ) + monkeypatch.setattr(relation_verification, "get_json", lambda *_args, **_kwargs: next(responses)) + client = relation_verification.SearxngRelationVerificationClient("http://searxng") + + assert client.verify("Aurora Grid Power", "customer").status_code == relation_verification.STATUS_UNCORROBORATED + assert client.verify("Aurora Grid Power", "customer").status_code == relation_verification.STATUS_UNCORROBORATED + + +def test_corroborating_evidence_requires_distinctive_org_token() -> None: + assert relation_verification.corroborating_evidence_url("Corp Ltd", {"url": "https://corp.example"}) is None + assert relation_verification.corroborating_evidence_url("Aurora Grid Power", {"url": ""}) is None + assert ( + relation_verification.corroborating_evidence_url( + "Aurora Grid Power", + {"url": "https://aurora.example/about", "content": "Aurora Grid Power"}, + ) + == "https://aurora.example/about" + ) + + +def test_generic_page_matching_one_fake_name_token_is_not_evidence() -> None: + assert ( + relation_verification.corroborating_evidence_url( + "Zzqxvthorp Fictitious Nonexistent Org 8f3e1c", + { + "url": "https://learn.microsoft.com/en-us/writing-style-guide-msft-internal/legal-content/fictitious-names-domains-and-addresses", + "content": "Use fictitious names and domains in examples.", + }, + ) + is None + ) + + +def test_corroborating_evidence_requires_a_majority_of_tokens() -> None: + """One ordinary dictionary word in an invented name must not corroborate. + + "Fictitious" and "Nonexistent" are real English words that can appear + on an unrelated page by coincidence; a single such match is not + evidence the organization itself has a real-world footprint. + """ + assert ( + relation_verification.corroborating_evidence_url( + "Zzqxvthorp Fictitious Nonexistent Org", + {"url": "https://unrelated.example/error-page", "content": "This file is fictitious or missing."}, + ) + is None + ) + assert ( + relation_verification.corroborating_evidence_url( + "Zzqxvthorp Fictitious Nonexistent Org", + { + "url": "https://zzqxvthorp.example/about", + "content": "Zzqxvthorp Fictitious Nonexistent Org is a real company.", + }, + ) + == "https://zzqxvthorp.example/about" + ) diff --git a/tests/test_relation_verification_internal.py b/tests/test_relation_verification_internal.py new file mode 100644 index 000000000..760d2ad45 --- /dev/null +++ b/tests/test_relation_verification_internal.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import asyncio + +from backend.app.relation_verification_ingestion import verify_post_relations +from lineageweave.relation_verification import ( + STATUS_CORROBORATED, + RelationVerificationResult, +) + + +class _Connection: + def __init__(self, evidence_post_id: str | None) -> None: + self.evidence_post_id = evidence_post_id + self.fetchrow_args: tuple[object, ...] | None = None + self.execute_args: tuple[object, ...] | None = None + + async def fetch(self, query: str, post_id: str): + assert "verification_status_code = 'verify_pending'" in query + return [ + { + "counterparty_entity_name": "Example Partner", + "relationship_label": "Partner", + } + ] + + async def fetchrow(self, query: str, *args: object): + assert "post_content_unit" in query + assert "visibility_code = 'public'" in query + self.fetchrow_args = args + return None if self.evidence_post_id is None else {"post_id": self.evidence_post_id} + + async def execute(self, query: str, *args: object): + assert "verification_evidence_post_id = $5" in query + self.execute_args = args + return "UPDATE 1" + + +class _Verifier: + def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: + assert (organization_name, relationship_label) == ("Example Partner", "Partner") + return RelationVerificationResult(STATUS_CORROBORATED, "https://example.test/evidence") + + +def test_relation_verification_persists_authorized_internal_evidence() -> None: + conn = _Connection("internal-post") + + verified = asyncio.run( + verify_post_relations(conn, _Verifier(), "origin-post", visible_corporate_entity_ids=("corp-a",)) + ) + + assert verified[0].verification_evidence_post_id == "internal-post" + assert conn.fetchrow_args == ("origin-post", "Example Partner", "Partner", ["corp-a"]) + assert conn.execute_args == ( + "origin-post", + "Example Partner", + STATUS_CORROBORATED, + "https://example.test/evidence", + "internal-post", + ) + + +def test_relation_verification_keeps_external_result_when_internal_search_misses() -> None: + conn = _Connection(None) + + verified = asyncio.run(verify_post_relations(conn, _Verifier(), "origin-post")) + + assert verified[0].verification_evidence_post_id is None + assert conn.execute_args is not None + assert conn.execute_args[-1] is None diff --git a/tests/test_report_team_grouping.py b/tests/test_report_team_grouping.py new file mode 100644 index 000000000..9c9734044 --- /dev/null +++ b/tests/test_report_team_grouping.py @@ -0,0 +1,49 @@ +"""Synthetic checks for N:N team period-report grouping.""" + +from backend.app.report_ingestion import GROUPING_KINDS, _groups_from_rows, grouping_value + + +def _row( + post_id: str, + team_id: str, + category: int, + project_id: str = "project-a", +) -> dict[str, object]: + return { + "post_id": post_id, + "team_id": team_id, + "secondary_grouping_key": project_id, + "criterion_code": "criterion_1", + "response_category": category, + } + + +def test_team_grouping_preserves_multiple_membership_without_intra_group_duplicates(): + rows = [ + _row("post-1", "team-a", 1), + _row("post-1", "team-b", 1), + _row("post-2", "team-a", 2), + _row("post-3", "team-b", 2), + ] + + groups = _groups_from_rows("team", rows) + + assert "team" in GROUPING_KINDS + assert grouping_value("team", rows[0]) == "team-a" + assert list(groups["team-a"][0]) == ["post-1", "post-2"] + assert list(groups["team-b"][0]) == ["post-1", "post-3"] + + +def test_project_grouping_uses_persisted_secondary_key(): + rows = [ + _row("post-1", "team-a", 1, "project-a"), + _row("post-2", "team-a", 2, "project-a"), + _row("post-3", "team-b", 1, "project-b"), + _row("post-4", "team-b", 2, "project-b"), + ] + + groups = _groups_from_rows("project", rows) + + assert "project" in GROUPING_KINDS + assert list(groups["project-a"][0]) == ["post-1", "post-2"] + assert list(groups["project-b"][0]) == ["post-3", "post-4"] diff --git a/tests/test_schema.py b/tests/test_schema.py index 3dd99b316..1e2c708a3 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -27,6 +27,22 @@ "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" ) _MIGRATION_PATH = Path(__file__).resolve().parents[1] / "migrations" / "0001_initial_schema.sql" +_MAJOR_EVENT_ACTION_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0100_major_event_action.sql" +) +_PROJECT_MENTION_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0031_semantic_project_mentions.sql" +) +_PROJECT_BOUND_ACTION_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0101_project_bound_major_event_action.sql" +) +_PROJECT_BOUND_EVENT_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0102_project_bound_summary_event.sql" +) def _postgres_available() -> bool: @@ -59,6 +75,10 @@ def schema_db(): try: with conn.cursor() as cur: cur.execute(_MIGRATION_PATH.read_text()) + cur.execute(_PROJECT_MENTION_MIGRATION.read_text()) + cur.execute(_MAJOR_EVENT_ACTION_MIGRATION.read_text()) + cur.execute(_PROJECT_BOUND_ACTION_MIGRATION.read_text()) + cur.execute(_PROJECT_BOUND_EVENT_MIGRATION.read_text()) conn.commit() yield conn finally: @@ -87,6 +107,7 @@ def test_migration_applies_cleanly(schema_db) -> None: "abac_policy", "source_post", "post_counterparty_entity", + "post_project_mention", "cataloged_person", "person_affiliation", "post_person_mention", @@ -104,13 +125,37 @@ def test_migration_applies_cleanly(schema_db) -> None: "post_summary_result", "post_summary_event", "post_summary_role", + "post_summary_action", "post_chat_result", "post_chat_citation", - "abbreviation_tree_corroboration", } assert expected <= tables +def test_major_event_action_project_reference_is_normalized(schema_db) -> None: + with schema_db.cursor() as cur: + cur.execute( + """ + select confrelid::regclass::text + from pg_constraint + where conname = 'post_summary_action_project_mention_fk' + """ + ) + assert cur.fetchone()[0] == "post_project_mention" + + +def test_summary_event_project_reference_is_normalized(schema_db) -> None: + with schema_db.cursor() as cur: + cur.execute( + """ + select confrelid::regclass::text + from pg_constraint + where conname = 'post_summary_event_project_mention_fk' + """ + ) + assert cur.fetchone()[0] == "post_project_mention" + + def test_leftover_pair_references_member_and_item_rows(schema_db) -> None: """A leftover pair cannot name a post or item from another report.""" with schema_db.cursor() as cur: diff --git a/tests/test_seed_late_demo_post.py b/tests/test_seed_late_demo_post.py deleted file mode 100644 index 14688350a..000000000 --- a/tests/test_seed_late_demo_post.py +++ /dev/null @@ -1,147 +0,0 @@ -"""Late Demo public post is the ADR 0016 own-corp cutoff counter-example.""" - -from datetime import datetime, timezone -from inspect import getsource -from pathlib import Path - -from scripts.seed_demo_data import ( - DEMO_ANALYSIS_RUN_KNOWLEDGE_CUTOFF, - DEMO_PUBLIC_POST_CREATED_AT, - DEMO_PUBLIC_POST_TITLE, - LATE_DEMO_PUBLIC_POST_BODY, - LATE_DEMO_PUBLIC_POST_CREATED_AT, - LATE_DEMO_PUBLIC_POST_TITLE, - seed, - seed_late_demo_public_post, - tepp_accepted_seed_request, - tepp_seed_outcome, - tepp_seed_request, - _ensure_demo_source_snapshot_members, - _seed_demo_run_reconstruction, -) - - -def _parse_seed_clock(value: str) -> datetime: - """Parse a seeded ISO-8601 Z clock as UTC.""" - return datetime.fromisoformat(value.replace("Z", "+00:00")) - - -class _LateDemoCursor: - """Drive ``seed_late_demo_public_post`` without a live database.""" - - def __init__(self, existing: bool = False) -> None: - self.existing = existing - self.statements: list[str] = [] - self.params: list[object] = [] - - def execute(self, sql: str, params=None) -> None: - self.statements.append(" ".join(sql.split())) - self.params.append(params) - - def fetchone(self): - last = self.statements[-1] - if last.lstrip().startswith("select") and "from source_post" in last: - return ("late-demo-id",) if self.existing else None - return None - - -def test_january_12_run_lists_demo_public_not_late_demo() -> None: - """ADR 0016 ``created_at <= knowledge_cutoff`` keeps Demo public, drops Late Demo.""" - cutoff = _parse_seed_clock(DEMO_ANALYSIS_RUN_KNOWLEDGE_CUTOFF) - listed = { - title: _parse_seed_clock(created_at) <= cutoff - for title, created_at in ( - (DEMO_PUBLIC_POST_TITLE, DEMO_PUBLIC_POST_CREATED_AT), - (LATE_DEMO_PUBLIC_POST_TITLE, LATE_DEMO_PUBLIC_POST_CREATED_AT), - ) - } - assert cutoff == datetime(2026, 1, 12, 12, 0, tzinfo=timezone.utc) - assert listed[DEMO_PUBLIC_POST_TITLE] is True - assert listed[LATE_DEMO_PUBLIC_POST_TITLE] is False - - -def test_listing_still_uses_created_at_not_a_second_cutoff() -> None: - """Visible-post SQL stays the ADR 0016 created_at gate on every scope.""" - listing = ( - Path(__file__).resolve().parents[1] - / "backend" - / "app" - / "analysis_run_ingestion.py" - ).read_text(encoding="utf-8") - start = listing.index("async def fetch_visible_scope_posts") - end = listing.index("\nclass AnalysisRunCreateError") - listing_fn = listing[start:end] - assert listing_fn.count("created_at <= $") == 4 - assert "LATE_DEMO" not in listing_fn - - -def test_reconstruction_seed_uses_the_january_12_cutoff() -> None: - """Run reconstruction persists only posts known at the same analysis clock.""" - source = getsource(_seed_demo_run_reconstruction) - assert "created_at <= %s" in source - assert "DEMO_ANALYSIS_RUN_KNOWLEDGE_CUTOFF" in source - assert LATE_DEMO_PUBLIC_POST_TITLE not in source - - -def test_snapshot_members_exclude_late_demo_created_at() -> None: - """Frozen snapshot membership uses created_at before Late Demo exists.""" - source = getsource(_ensure_demo_source_snapshot_members) - assert "created_at <= '2026-01-12T00:00:00Z'" in source - late = _parse_seed_clock(LATE_DEMO_PUBLIC_POST_CREATED_AT) - snapshot_max = datetime(2026, 1, 12, tzinfo=timezone.utc) - assert late > snapshot_max - - -def test_seed_late_demo_public_post_inserts_after_cutoff() -> None: - """Persist writes the own-corp public counter-example dated 2026-01-13.""" - cursor = _LateDemoCursor() - seed_late_demo_public_post(cursor, "account-1", "corp-1", "pu-1") - inserts = [ - (sql, params) - for sql, params in zip(cursor.statements, cursor.params, strict=True) - if "insert into source_post" in sql - ] - assert inserts, "missing Late Demo must be inserted" - sql, params = inserts[0] - assert params is not None - assert LATE_DEMO_PUBLIC_POST_TITLE in params - assert LATE_DEMO_PUBLIC_POST_BODY in params - assert LATE_DEMO_PUBLIC_POST_CREATED_AT in params - assert params.count(LATE_DEMO_PUBLIC_POST_CREATED_AT) == 2 - assert "public" in sql - assert "theta" not in sql.lower() - assert not any( - isinstance(value, str) and ("theta" in value.lower() or "θ" in value) - for value in params - ) - - -def test_seed_late_demo_public_post_skips_when_already_present() -> None: - """Re-seed must not invent a second Late Demo row.""" - cursor = _LateDemoCursor(existing=True) - seed_late_demo_public_post(cursor, "account-1", "corp-1", "pu-1") - assert not any("insert into source_post" in sql for sql in cursor.statements) - - -def test_seed_calls_late_demo_before_analysis_runs() -> None: - """``seed()`` writes Late Demo, then lineage/TEPP runs on the January 12 clock.""" - source = getsource(seed) - late_at = source.index("seed_late_demo_public_post(") - lineage_at = source.index("_seed_demo_analysis_run(") - tepp_at = source.index("_seed_demo_tepp_run(") - assert late_at < lineage_at < tepp_at - helper = getsource(seed_late_demo_public_post) - insert_sql = helper[helper.index("insert into source_post") :] - assert "created_at <= " not in insert_sql - assert "theta" not in insert_sql.lower() - - -def test_tepp_seed_keeps_the_same_january_12_cutoff() -> None: - """Late Demo does not fork TEPP arithmetic or stamp Succeeded.""" - request = tepp_seed_request() - accepted = tepp_accepted_seed_request() - assert request.knowledge_cutoff == DEMO_ANALYSIS_RUN_KNOWLEDGE_CUTOFF - assert accepted.knowledge_cutoff == DEMO_ANALYSIS_RUN_KNOWLEDGE_CUTOFF - status, failure = tepp_seed_outcome() - assert status == "analysis_status_failed" - assert failure == "tepp_not_available" diff --git a/tests/test_seed_report_run.py b/tests/test_seed_report_run.py index 9ed060fcd..b5bd5f7c6 100644 --- a/tests/test_seed_report_run.py +++ b/tests/test_seed_report_run.py @@ -91,9 +91,16 @@ def test_seed_demo_report_run_inserts_succeeded_report_without_a_theta() -> None if "insert into analysis_run_scope" in sql ] assert any( - event_params is not None and "2026-W02" in event_params + event_params is not None + and "corp-1" in event_params + and "2026-W02" not in event_params for event_params in scope_params ) + assert all( + "scope_key" not in sql + for sql in cursor.statements + if "insert into analysis_run_scope" in sql + ) assert not any( event_params is not None and any( diff --git a/tests/test_seed_tepp_run.py b/tests/test_seed_tepp_run.py index 69484862b..b25908cbe 100644 --- a/tests/test_seed_tepp_run.py +++ b/tests/test_seed_tepp_run.py @@ -1,15 +1,10 @@ """Seeded TEPP analysis runs go through tepp_client, never a local model.""" from lineageweave.tepp_client import AnalysisRunRequest, TeppClient, TeppNotAvailable -from lineageweave.tepp_result import persistable_tepp_seed_envelope from scripts.seed_demo_data import ( _ensure_demo_source_counts, - _seed_demo_accepted_tepp_run, _seed_demo_tepp_run, demo_source_snapshot_sha256, - tepp_accepted_seed_client, - tepp_accepted_seed_request, - tepp_persistable_seed_client, tepp_seed_outcome, tepp_seed_request, ) @@ -78,23 +73,6 @@ def test_tepp_seed_outcome_does_not_treat_an_empty_envelope_as_success() -> None assert failure == "tepp_result_not_persisted" -def test_tepp_seed_outcome_keeps_a_published_accepted_envelope_failed() -> None: - request = tepp_seed_request() - status, failure = tepp_seed_outcome( - tepp_accepted_seed_client(request.idempotency_key), - request, - ) - assert status == "analysis_status_failed" - assert failure == "tepp_completed_result_unsupported" - - -def test_tepp_seed_outcome_rejects_a_local_completed_envelope() -> None: - status, failure = tepp_seed_outcome(tepp_persistable_seed_client()) - assert status == "analysis_status_failed" - assert failure == "tepp_result_not_persisted" - assert persistable_tepp_seed_envelope()["affiliation_count"] == 2 - - def test_ensure_demo_source_counts_skips_insert_when_counts_exist() -> None: cursor = _CountCursor(existing_counts=True) _ensure_demo_source_counts(cursor, "snapshot-1") @@ -152,32 +130,3 @@ def test_seed_demo_tepp_run_inserts_failed_tepp_not_available() -> None: assert not any( params is not None and "analysis_status_succeeded" in params for params in status_params ) - - -def test_seed_demo_accepted_tepp_run_persists_transport_evidence() -> None: - cursor = _TeppSeedCursor() - _seed_demo_accepted_tepp_run(cursor, "account-1", "corp-1") - assert any("insert into analysis_run_tepp_accepted" in sql for sql in cursor.statements) - assert not any("insert into analysis_run_tepp_result" in sql for sql in cursor.statements) - status_params = [ - params - for sql, params in zip(cursor.statements, cursor.params, strict=True) - if "insert into analysis_run_status_event" in sql - ] - assert any( - params is not None - and "analysis_status_failed" in params - and "tepp_completed_result_unsupported" in params - for params in status_params - ) - assert not any( - params is not None and "analysis_status_succeeded" in params for params in status_params - ) - result_params = [ - params - for sql, params in zip(cursor.statements, cursor.params, strict=True) - if "insert into analysis_run_tepp_accepted" in sql - ] - assert result_params - assert all(params is not None and "theta" not in str(params).casefold() for params in result_params) - assert tepp_accepted_seed_request().idempotency_key == "demo-tepp-seed-2026-w02-succeeded" diff --git a/tests/test_semantic_author_provenance.py b/tests/test_semantic_author_provenance.py new file mode 100644 index 000000000..8575db112 --- /dev/null +++ b/tests/test_semantic_author_provenance.py @@ -0,0 +1,17 @@ +from lineageweave.semantic_hints import format_semantic_hints + + +def test_author_identity_is_a_prior_with_explicit_side_provenance() -> None: + hints = format_semantic_hints( + author_name="Synthetic Author", + author_account_id="account-1", + author_affiliations=["Synthetic Corp"], + order_pool_code=None, + order_pool_name=None, + project_field=None, + customer_name="기타", + ) + + assert "author_account_id=account-1 [source_field=source_post.author_account_id]" in hints + assert "author_side_hint=our_side_candidate" in hints + assert "customer_hint_trust=low" in hints diff --git a/tests/test_semantic_hints.py b/tests/test_semantic_hints.py new file mode 100644 index 000000000..97e180377 --- /dev/null +++ b/tests/test_semantic_hints.py @@ -0,0 +1,170 @@ +from lineageweave.semantic_hints import customer_hint_trust, format_semantic_hints + + +def test_customer_hint_trust_marks_generic_values_weak() -> None: + for value in ("기타", "기타고객", "기타 고객", "미등록", "미등록고객", "미등록 고객"): + assert customer_hint_trust(value) == "low" + assert customer_hint_trust("Named customer", "other") == "low" + assert customer_hint_trust("Named customer") == "normal" + + +def test_semantic_hints_keep_explicit_project_pool_and_author_sources() -> None: + hints = format_semantic_hints( + author_name="Synthetic Author", + author_account_id="synthetic-author-account", + author_affiliations=["Synthetic Corp"], + order_pool_code="POOL-7", + order_pool_name="Synthetic bids", + project_field="PROJECT-42", + customer_name="Synthetic Customer", + source_author_code="source-author", + source_company_code="SOURCE-COMPANY", + source_business_unit_code="SOURCE-BU", + source_customer_code="SOURCE-CUSTOMER", + source_company_catalog_name="Catalog Company", + source_process_unit_catalog_name="Catalog PU", + source_customer_catalog_name="Catalog Customer", + source_project_code="SOURCE-PROJECT", + source_company_name="Named company", + source_process_unit_name="Named PU", + ) + + assert "project_field=PROJECT-42" in hints + assert "source_field=source_post.secondary_grouping_key" in hints + assert "order_pool=POOL-7: Synthetic bids" in hints + # Real source-system fields are present, so this is implicitly a + # bulk-imported record -- the placeholder account's affiliation is + # untrustworthy here (see test_source_context_drops_account_affiliation + # _as_untrustworthy_org_identity for why). + assert "author_affiliations=none" in hints + assert "author_account_id=synthetic-author-account" in hints + assert "author_side_hint=our_side_context_only" in hints + assert "customer_hint_trust=normal" in hints + assert "source_author_code=source-author" in hints + assert "source_company_code=SOURCE-COMPANY" in hints + assert "source_customer_code=SOURCE-CUSTOMER" in hints + assert "source_project_code=SOURCE-PROJECT" in hints + assert "source_company_name=Named company [source_field=source_post.source_company_name]" in hints + assert "source_process_unit_name=Named PU [source_field=source_post.source_process_unit_name]" in hints + assert "source_company_catalog_name=Catalog Company [source_lookup=corporate_entity.corporate_entity_code]" in hints + assert "source_process_unit_catalog_name=Catalog PU [source_lookup=process_unit.process_unit_code]" in hints + assert "source_customer_catalog_name=Catalog Customer [source_lookup=corporate_entity.corporate_entity_code]" in hints + + +def test_catalog_lookup_hint_reports_a_code_without_inventing_a_name() -> None: + hints = format_semantic_hints( + author_name=None, + author_affiliations=(), + order_pool_code=None, + order_pool_name=None, + project_field=None, + customer_name=None, + source_company_code="UNRESOLVED-COMPANY", + ) + + assert "source_company_catalog_name=none [source_lookup=corporate_entity.corporate_entity_code]" in hints + + +def test_unknown_customer_is_a_weak_hint_not_project_evidence() -> None: + hints = format_semantic_hints( + author_name=None, + author_affiliations=[], + order_pool_code=None, + order_pool_name=None, + project_field=None, + customer_name="미등록고객", + ) + + assert "customer=미등록고객" in hints + assert "customer_hint_trust=low" in hints + assert "project_field=none" in hints + + +def test_source_pool_and_project_code_keep_distinct_provenance() -> None: + hints = format_semantic_hints( + author_name=None, + author_affiliations=[], + order_pool_code="SOURCE-POOL", + order_pool_name=None, + project_field="SECONDARY-PROJECT", + customer_name="Demo Corp", + source_sales_pool_code="SOURCE-POOL", + source_project_code="SOURCE-PROJECT", + source_context_present=True, + ) + + assert "order_pool=SOURCE-POOL [source_field=source_post.source_sales_pool_code]" in hints + assert "project_field=SECONDARY-PROJECT [source_field=source_post.secondary_grouping_key]" in hints + assert "source_project_code=SOURCE-PROJECT [source_field=source_post.source_project_code]" in hints + + +def test_explicit_source_names_are_hints_with_name_provenance_and_customer_trust() -> None: + hints = format_semantic_hints( + author_name=None, + author_affiliations=[], + order_pool_code=None, + order_pool_name="Named sales pool", + project_field=None, + customer_name=None, + source_sales_pool_name="Named sales pool", + source_customer_name="미등록고객", + source_project_name="Named project", + source_context_present=True, + ) + + assert "order_pool=Named sales pool [source_field=source_post.source_sales_pool_name]" in hints + assert "source_customer_name=미등록고객 [source_field=source_post.source_customer_name]" in hints + assert "source_customer_name_hint_trust=low" in hints + assert "source_project_name=Named project [source_field=source_post.source_project_name]" in hints + + +def test_source_context_drops_account_affiliation_as_untrustworthy_org_identity() -> None: + """A bulk-imported real record shares one platform placeholder account + across every record, so its `account_affiliation` names the + placeholder's own org, never the real author `source_author_code`/ + `source_company_code` actually names. Live bug (2026-08-19): asserting + the placeholder's org as "our side" context fed a wrong company name + into a real Keyman-extraction prompt and inverted the + our_side/counterparty classification. `customer_name` already gets + this same treatment for the identical reason; extend it here too. + """ + hints = format_semantic_hints( + author_name="Synthetic Analyst", + author_account_id="demo-account", + author_account_name="Synthetic Analyst", + author_affiliations=["Synthetic Corp"], + order_pool_code="DEMO-PU", + order_pool_name="Demo scope", + project_field=None, + customer_name="Demo Corp", + source_author_code="SOURCE-AUTHOR", + source_company_code="SOURCE-COMPANY", + source_context_present=True, + ) + + assert "author_account_id=demo-account" in hints + assert "author_account_name=Synthetic Analyst" in hints + assert "author_affiliations=none" in hints + assert "customer=none" in hints + assert "author_side_hint=our_side_context_only" in hints + assert "Synthetic Corp" not in hints + + +def test_without_source_context_account_affiliation_is_kept_as_keyman_hint() -> None: + """A genuine, non-bulk-imported post (no independent source-system + record) has no placeholder-account ambiguity -- the account's real + affiliation is legitimate non-binding Keyman context here. + """ + hints = format_semantic_hints( + author_name="Synthetic Analyst", + author_account_id="demo-account", + author_account_name="Synthetic Analyst", + author_affiliations=["Synthetic Corp"], + order_pool_code=None, + order_pool_name=None, + project_field=None, + customer_name=None, + ) + + assert "author_affiliations=Synthetic Corp" in hints + assert "author_side_hint=our_side_candidate" in hints diff --git a/tests/test_semantic_keyman_context.py b/tests/test_semantic_keyman_context.py new file mode 100644 index 000000000..b5f973212 --- /dev/null +++ b/tests/test_semantic_keyman_context.py @@ -0,0 +1,20 @@ +from lineageweave.keyman_extraction import ContextualOrchestratorKeymanExtractionClient + + +def test_keyman_extraction_sends_author_context_hints(monkeypatch) -> None: + captured: dict[str, str] = {} + + def fake_post_json(url, payload, *, headers, timeout): + captured["prompt"] = payload["messages"][0]["content"] + return {"choices": [{"message": {"content": "[]"}}]} + + monkeypatch.setattr("lineageweave.keyman_extraction.post_json", fake_post_json) + client = ContextualOrchestratorKeymanExtractionClient("http://orchestrator.test", "synthetic") + + assert client.extract_with_hints( + "Synthetic title", + "Synthetic body", + "author=Synthetic Author; author_affiliations=Synthetic Corp; customer_hint_trust=low", + ) == [] + assert "author=Synthetic Author" in captured["prompt"] + assert "customer_hint_trust=low" in captured["prompt"] diff --git a/tests/test_semantic_project_evidence.py b/tests/test_semantic_project_evidence.py new file mode 100644 index 000000000..9ac61343b --- /dev/null +++ b/tests/test_semantic_project_evidence.py @@ -0,0 +1,19 @@ +from lineageweave.post_summary import normalize_project_key, parse_summary_response + + +def test_project_mentions_keep_evidence_and_low_confidence() -> None: + summary = parse_summary_response( + '{"korean_summary":"synthetic summary", "project_mentions": [' + '{"project_name":"Project Delta", "canonical_name":"Project Delta", ' + '"evidence":"the Delta rollout", "confidence":0.82}, ' + '{"project_name":"maybe", "canonical_name":"maybe", ' + '"evidence":"unclear reference", "confidence":0.4}]}' + ) + + assert summary is not None + assert [mention.evidence for mention in summary.project_mentions] == [ + "the Delta rollout", + "unclear reference", + ] + assert summary.project_mentions[1].confidence == 0.4 + assert normalize_project_key("Project Delta") == "project-delta" diff --git a/tests/test_source_post_revision.py b/tests/test_source_post_revision.py index 507ea90ed..4a279c239 100644 --- a/tests/test_source_post_revision.py +++ b/tests/test_source_post_revision.py @@ -59,6 +59,3 @@ def test_revision_migration_records_title_or_body_rewrites_only() -> None: assert seed.index("0024_source_post_revision.sql") < seed.index( "0025_role_person_catalog_identity.sql" ) - assert seed.index("0025_role_person_catalog_identity.sql") < seed.index( - "0026_report_leftover_pair.sql" - ) diff --git a/tests/test_source_state_serialization.py b/tests/test_source_state_serialization.py new file mode 100644 index 000000000..4eee0f5de --- /dev/null +++ b/tests/test_source_state_serialization.py @@ -0,0 +1,36 @@ +from datetime import datetime, timezone + +from backend.app.main import _serialize_post + + +def test_source_state_codes_are_serialized_without_inference() -> None: + payload = _serialize_post( + { + "post_id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + "post_title": "Source-backed post", + "voc_type_code": "voc", + "visibility_code": "public", + "source_stage_code": "Z", + "source_detail_state_code": "A", + "source_draft_code": None, + "source_deleted_flag": None, + "source_author_code": "author-1", + "source_author_name": "Source Author", + "source_company_code": "COMPANY-1", + "source_process_unit_code": "PU-1", + "source_sales_pool_code": "POOL-1", + "source_customer_code": "CUSTOMER-1", + "source_project_code": "PROJECT-1", + "created_at": datetime(2026, 1, 1, tzinfo=timezone.utc), + }, + {"voc": "Voice of Customer", "public": "Public"}, + ) + + assert payload["source_stage_code"] == "Z" + assert payload["source_detail_state_code"] == "A" + assert payload["source_draft_code"] is None + assert payload["source_deleted_flag"] is None + assert payload["publication_state_code"] == "publication_state_unknown" + assert payload["source_author_code"] == "author-1" + assert payload["source_customer_code"] == "CUSTOMER-1" + assert payload["source_project_code"] == "PROJECT-1" diff --git a/tests/test_stale_summary_continuity.py b/tests/test_stale_summary_continuity.py new file mode 100644 index 000000000..ef4f66184 --- /dev/null +++ b/tests/test_stale_summary_continuity.py @@ -0,0 +1,36 @@ +"""Regression tests for buyer-visible stale summary continuity.""" + +import asyncio + +from backend.app.post_summary_ingestion import fetch_persisted_summary +from lineageweave.post_summary import POST_SUMMARY_CONTRACT_VERSION + + +class _StaleSummaryConnection: + """Minimal asyncpg-shaped fake containing one legacy summary header.""" + + async def fetchrow(self, query: str, post_id: str) -> dict[str, object]: + return { + "korean_summary": "Previously persisted evidence.", + "summary_contract_version": POST_SUMMARY_CONTRACT_VERSION - 1, + } + + async def fetch(self, query: str, post_id: str) -> list[dict[str, object]]: + return [] + + +def test_stale_summary_is_hidden_by_default() -> None: + """Current-contract reads must not silently present legacy semantics.""" + result = asyncio.run(fetch_persisted_summary(_StaleSummaryConnection(), "post-id")) + assert result is None + + +def test_stale_summary_can_be_returned_with_explicit_status() -> None: + """The continuity path exposes the old contract so the UI can label it.""" + result = asyncio.run( + fetch_persisted_summary(_StaleSummaryConnection(), "post-id", allow_stale=True) + ) + assert result is not None + assert result["summary_status"] == "stale" + assert result["summary_contract_version"] == POST_SUMMARY_CONTRACT_VERSION - 1 + assert result["korean_summary"] == "Previously persisted evidence." diff --git a/tests/test_static_sql_review_contracts.py b/tests/test_static_sql_review_contracts.py new file mode 100644 index 000000000..7269e6acf --- /dev/null +++ b/tests/test_static_sql_review_contracts.py @@ -0,0 +1,76 @@ +"""Static review contracts for SQL composition and protocol method stubs.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +from lineageweave.post_structure import PostStructureClient + + +ROOT = Path(__file__).resolve().parents[1] +SQL_REVIEW_PATHS = ( + "backend/app/analysis_run_ingestion.py", + "backend/app/analysis_run_start.py", + "backend/app/customer_hint_ingestion.py", + "backend/app/demo_scope.py", + "backend/app/entity_relationship_ingestion.py", + "backend/app/knowledge_graph.py", + "backend/app/main.py", + "backend/app/report_ingestion.py", + "lineageweave/synthetic_seed_cleanup.py", + "scripts/backfill_post_content.py", + "scripts/backfill_post_keymen.py", + "scripts/backfill_post_summaries.py", + "scripts/queue_post_content_backfill.py", +) +ASYNC_STATEMENT_METHODS = {"execute", "fetch", "fetchrow", "fetchval"} +SQL_REVIEW_RULE = "python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli" +EXPECTED_SQL_SUPPRESSION_COUNT = 35 + + +@pytest.mark.parametrize("relative_path", SQL_REVIEW_PATHS) +def test_reviewed_asyncpg_calls_are_literal_or_explicitly_audited(relative_path: str) -> None: + """Reviewed calls are literal or carry a precise, adjacent audit suppression.""" + source = (ROOT / relative_path).read_text(encoding="utf-8") + lines = source.splitlines() + tree = ast.parse(source, filename=relative_path) + violations: list[int] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call) or not node.args: + continue + if not isinstance(node.func, ast.Attribute) or node.func.attr not in ASYNC_STATEMENT_METHODS: + continue + statement = node.args[0] + if isinstance(statement, ast.Constant) and isinstance(statement.value, str): + continue + call_line = lines[node.lineno - 1] + preceding_line = lines[node.lineno - 2] if node.lineno > 1 else "" + if SQL_REVIEW_RULE not in call_line or "Safe SQL:" not in preceding_line: + violations.append(node.lineno) + + assert not violations, f"unaudited non-literal asyncpg statements at lines {violations} in {relative_path}" + + +def test_sql_suppressions_are_precise_adjacent_and_counted() -> None: + """Every reviewed suppression names the exact rule and has a nearby reason.""" + suppression_sites: list[tuple[str, int]] = [] + for relative_path in SQL_REVIEW_PATHS: + lines = (ROOT / relative_path).read_text(encoding="utf-8").splitlines() + for line_number, line in enumerate(lines, start=1): + if "nosemgrep:" not in line: + continue + preceding_line = lines[line_number - 2] if line_number > 1 else "" + assert SQL_REVIEW_RULE in line, f"wrong Semgrep rule at {relative_path}:{line_number}" + assert "Safe SQL:" in preceding_line, f"missing Safe SQL reason at {relative_path}:{line_number}" + suppression_sites.append((relative_path, line_number)) + + assert len(suppression_sites) == EXPECTED_SQL_SUPPRESSION_COUNT + + +def test_post_structure_protocol_stub_fails_explicitly() -> None: + """The protocol method cannot silently return ``None`` when invoked directly.""" + with pytest.raises(NotImplementedError): + PostStructureClient.infer(object(), "title", []) diff --git a/tests/test_structured_vision.py b/tests/test_structured_vision.py new file mode 100644 index 000000000..42b3ed613 --- /dev/null +++ b/tests/test_structured_vision.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import io +import json + +from PIL import Image + +from lineageweave.image_content import OpenAiCompatibleVisionClient + + +def test_vision_region_locator_uses_json_object_and_returns_regions(monkeypatch) -> None: + image = io.BytesIO() + Image.new("RGB", (8, 8), "white").save(image, format="PNG") + seen: dict[str, object] = {} + + def fake_post_json(url, body, *, headers, timeout): + seen.update(url=url, body=body, headers=headers, timeout=timeout) + return {"choices": [{"message": {"content": json.dumps({"regions": [{"x": 0.1, "y": 0.2, "width": 0.5, "height": 0.6}]})}}]} + + monkeypatch.setattr("lineageweave.image_content.post_json", fake_post_json) + client = OpenAiCompatibleVisionClient( + "http://orchestrator/v1", "secret", allow_insecure_http=True + ) + + regions = client.locate_regions(image.getvalue(), "image/png") + + assert regions[0].x == 0.1 + assert regions[0].width == 0.5 + body = seen["body"] + assert body["mode"] == "auto" + assert body["reasoning_effort"] == "auto" + assert body["response_format"]["type"] == "json_object" + assert "temperature" not in body diff --git a/tests/test_synthetic_seed_cleanup.py b/tests/test_synthetic_seed_cleanup.py new file mode 100644 index 000000000..6d2ab3405 --- /dev/null +++ b/tests/test_synthetic_seed_cleanup.py @@ -0,0 +1,284 @@ +"""Real-database test for lineageweave/synthetic_seed_cleanup.py. + +Applies every migration to a throwaway PostgreSQL database, seeds a demo +scope entangled with real source-import evidence (same shape a real +customer's first import produces against this repo's `make seed` output), +then proves the cleanup deletes only the synthetic row, leaves an +analysis-run-referenced synthetic row alone, and nulls (never drops) a real +post's optional internal-evidence citation to a removed synthetic post. + +Skipped unless a local PostgreSQL server is reachable, same convention as +tests/test_schema.py and backend/tests/test_api.py. +""" + +from __future__ import annotations + +import asyncio +import os +import subprocess +import uuid +from pathlib import Path + +import asyncpg +import psycopg2 +import pytest + +from lineageweave.synthetic_seed_cleanup import cleanup_synthetic_seed + +_ADMIN_DSN = os.environ.get( + "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" +) +_MIGRATIONS_DIR = Path(__file__).resolve().parents[1] / "migrations" + + +def _postgres_available() -> bool: + try: + conn = psycopg2.connect(_ADMIN_DSN, connect_timeout=2) + conn.close() + return True + except psycopg2.OperationalError: + return False + + +pytestmark = pytest.mark.skipif( + not _postgres_available(), + reason=f"no reachable PostgreSQL server at {_ADMIN_DSN} (set LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN)", +) + + +@pytest.fixture +def migrated_db(): + """A freshly migrated, throwaway database, dropped afterward.""" + db_name = f"lineageweave_cleanup_test_{uuid.uuid4().hex[:12]}" + admin_conn = psycopg2.connect(_ADMIN_DSN) + admin_conn.autocommit = True + with admin_conn.cursor() as cur: + cur.execute(f'create database "{db_name}"') + admin_conn.close() + + db_dsn = _ADMIN_DSN.rsplit("/", 1)[0] + f"/{db_name}" + # psql, not psycopg2 cur.execute(), matches docker/postgres-init/migrate.sh: + # a few migrations use CREATE INDEX CONCURRENTLY, which errors under + # psycopg2's implicit multi-statement transaction wrapping but not under + # psql's one-statement-at-a-time execution of a -f file. + for migration in sorted(_MIGRATIONS_DIR.glob("*.sql")): + subprocess.run( + ["psql", "-X", "-v", "ON_ERROR_STOP=1", db_dsn, "-f", str(migration)], + check=True, + ) + + yield db_dsn + + admin_conn = psycopg2.connect(_ADMIN_DSN) + admin_conn.autocommit = True + with admin_conn.cursor() as cur: + cur.execute( + "select pg_terminate_backend(pid) from pg_stat_activity where datname = %s", + (db_name,), + ) + cur.execute(f'drop database "{db_name}"') + admin_conn.close() + + +def test_cleanup_deletes_only_entangled_synthetic_rows(migrated_db: str) -> None: + async def run() -> dict[str, int]: + conn = await asyncpg.connect(migrated_db) + try: + empty_result = await cleanup_synthetic_seed(conn, apply=True) + assert empty_result == { + "candidate_posts": 0, + "blocked_posts": 0, + "deletable_posts": 0, + "deleted_posts": 0, + } + + # 'corporate_entity_level'/'company' and 'voc_type'/'voc' are already + # seeded by migrations 0016 and 0042 respectively; inserting them again + # here would violate common_lookup_value's primary key. + await conn.execute( + "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values " + "('post_visibility', 'public', 'Public'), " + "('entity_relationship_type', 'rel_voc', 'Voice of Customer')" + ) + demo_entity = await conn.fetchval( + "insert into corporate_entity (corporate_entity_code, entity_name, entity_level_code) " + "values ('DEMO-CORP-01', 'Demo Corp', 'company') returning corporate_entity_id" + ) + demo_pu = await conn.fetchval( + "insert into process_unit (corporate_entity_id, process_unit_code, process_unit_name) " + "values ($1, 'DEMO-PU-A', 'Demo Unit') returning process_unit_id", + demo_entity, + ) + account = await conn.fetchval( + "insert into user_account (external_subject_id, display_name, email_address) " + "values ('demo.analyst', 'Demo Analyst', 'demo.analyst@example.test') " + "returning user_account_id" + ) + await conn.execute( + "insert into account_affiliation (user_account_id, corporate_entity_id, process_unit_id) " + "values ($1, $2, $3)", + account, + demo_entity, + demo_pu, + ) + + # The synthetic seed post: no source_* evidence at all. + synthetic_post = await conn.fetchval( + "insert into source_post " + "(author_account_id, corporate_entity_id, process_unit_id, post_title, post_body, " + " voc_type_code, visibility_code, created_at, updated_at) " + "values ($1, $2, $3, 'Synthetic seed post', 'synthetic body', 'voc', 'public', now(), now()) " + "returning post_id", + account, + demo_entity, + demo_pu, + ) + # A real, imported post sharing the same DEMO-CORP-01 entity (the + # entangled-scope shape this repo actually hit). + real_post = await conn.fetchval( + "insert into source_post " + "(author_account_id, corporate_entity_id, process_unit_id, post_title, post_body, " + " voc_type_code, visibility_code, source_author_code, created_at, updated_at) " + "values ($1, $2, $3, 'Real imported post', 'real body', 'voc', 'public', 'REAL-AUTHOR-1', now(), now()) " + "returning post_id", + account, + demo_entity, + demo_pu, + ) + # A second synthetic post that an analysis run has already + # reconstructed over -- must be reported as blocked, never deleted. + blocked_synthetic_post = await conn.fetchval( + "insert into source_post " + "(author_account_id, corporate_entity_id, process_unit_id, post_title, post_body, " + " voc_type_code, visibility_code, created_at, updated_at) " + "values ($1, $2, $3, 'Synthetic post in a run snapshot', 'synthetic body 2', 'voc', 'public', now(), now()) " + "returning post_id", + account, + demo_entity, + demo_pu, + ) + imported_entity = await conn.fetchval( + "insert into corporate_entity (corporate_entity_code, entity_name, entity_level_code) " + "values ('IMPORTED-CORP-01', 'Imported Corp', 'company') returning corporate_entity_id" + ) + imported_pu = await conn.fetchval( + "insert into process_unit (corporate_entity_id, process_unit_code, process_unit_name) " + "values ($1, 'IMPORTED-PU-A', 'Imported Unit') returning process_unit_id", + imported_entity, + ) + non_demo_synthetic_post = await conn.fetchval( + "insert into source_post " + "(author_account_id, corporate_entity_id, process_unit_id, post_title, post_body, " + " voc_type_code, visibility_code, created_at, updated_at) " + "values ($1, $2, $3, 'Synthetic row outside Demo code', 'synthetic body 3', 'voc', 'public', now(), now()) " + "returning post_id", + account, + imported_entity, + imported_pu, + ) + await conn.execute( + "insert into source_post " + "(author_account_id, corporate_entity_id, process_unit_id, post_title, post_body, " + " voc_type_code, visibility_code, source_author_code, created_at, updated_at) " + "values ($1, $2, $3, 'Real row outside Demo code', 'real body 2', 'voc', 'public', 'REAL-AUTHOR-2', now(), now())", + account, + imported_entity, + imported_pu, + ) + snapshot = await conn.fetchval( + "insert into analysis_source_snapshot " + "(snapshot_sha256, source_contract_version, maximum_available_time, captured_at) " + "values (repeat('0', 64), 'test-v1', now() - interval '1 minute', now()) " + "returning analysis_source_snapshot_id" + ) + analysis_run = await conn.fetchval( + "insert into analysis_run " + "(analysis_source_snapshot_id, run_kind_code, requested_by_account_id, " + " idempotency_key, knowledge_cutoff, configuration_schema_version, " + " configuration_sha256, code_revision_sha) " + "values ($1, 'analysis_run_lineage', $2, 'test-idem-1', now(), " + " 'lineage-run-v1', repeat('0', 64), repeat('0', 40)) " + "returning analysis_run_id", + snapshot, + account, + ) + await conn.execute( + "insert into analysis_source_snapshot_member (analysis_source_snapshot_id, source_post_id) " + "values ($1, $2)", + snapshot, + blocked_synthetic_post, + ) + + # The real post cites the synthetic post as internal corroborating + # evidence -- deleting the synthetic post must null this citation, + # never delete the real post's counterparty row. + counterparty_relationship_type = await conn.fetchval( + "select lookup_code from common_lookup_value " + "where lookup_category = 'entity_relationship_type' limit 1" + ) + await conn.execute( + "insert into post_counterparty_entity " + "(post_id, counterparty_entity_name, relationship_type_code, " + " verification_status_code, verification_evidence_post_id) " + "values ($1, 'Some Counterparty', $2, 'verify_pending', $3)", + real_post, + counterparty_relationship_type, + synthetic_post, + ) + + dry_run_result = await cleanup_synthetic_seed(conn, apply=False) + assert dry_run_result["deleted_posts"] == 0 + assert await conn.fetchval( + "select count(*) from source_post where post_id = $1", synthetic_post + ) == 1 + + result = await cleanup_synthetic_seed(conn, apply=True) + + assert ( + await conn.fetchval("select count(*) from source_post where post_id = $1", synthetic_post) + == 0 + ), "the entangled synthetic post must be deleted" + assert ( + await conn.fetchval("select count(*) from source_post where post_id = $1", real_post) == 1 + ), "the real post must survive" + assert ( + await conn.fetchval( + "select count(*) from source_post where post_id = $1", blocked_synthetic_post + ) + == 1 + ), "a synthetic post referenced by an analysis-run snapshot must never be deleted" + assert ( + await conn.fetchval( + "select count(*) from source_post where post_id = $1", non_demo_synthetic_post + ) + == 1 + ), "non-Demo corporate entities must never be synthetic cleanup candidates" + + counterparty_row = await conn.fetchrow( + "select verification_evidence_post_id from post_counterparty_entity where post_id = $1", + real_post, + ) + assert counterparty_row is not None, ( + "the real post's counterparty row must survive -- only the citation is removed" + ) + assert counterparty_row["verification_evidence_post_id"] is None, ( + "the citation to the deleted synthetic post must be nulled, not left dangling" + ) + + assert ( + await conn.fetchval( + "select count(*) from analysis_run where analysis_run_id = $1", analysis_run + ) + == 1 + ), "cleanup must never touch the immutable analysis_run family" + + return result + + finally: + await conn.close() + + result = asyncio.run(run()) + assert result["candidate_posts"] == 2 + assert result["blocked_posts"] == 1 + assert result["deletable_posts"] == 1 + assert result["deleted_posts"] == 1 diff --git a/tests/test_tepp_client.py b/tests/test_tepp_client.py index 8c2509fb9..ea87d5558 100644 --- a/tests/test_tepp_client.py +++ b/tests/test_tepp_client.py @@ -2,6 +2,7 @@ import pytest +from backend.app.analysis_run_start import configured_tepp_client from lineageweave.tepp_client import AnalysisRunRequest, TeppClient, TeppNotAvailable @@ -49,3 +50,19 @@ def fake_transport(payload: dict) -> dict: assert result == {"status": "accepted"} assert received["contract_version"] == 1 assert received["snapshot_id"] == "demo-snapshot-1" + + +def test_configured_transport_sends_optional_bearer_key(monkeypatch: pytest.MonkeyPatch) -> None: + received = {} + + def fake_post_json(url: str, payload: dict, *, headers: dict, timeout: float) -> dict: + received.update(url=url, payload=payload, headers=headers, timeout=timeout) + return {"status": "accepted"} + + monkeypatch.setattr("backend.app.analysis_run_start.post_json", fake_post_json) + client = configured_tepp_client("https://tepp.example/v1/analysis-runs", "test-key") + + client.submit_analysis_run(_sample_request()) + + assert received["headers"] == {"authorization": "Bearer test-key"} + assert received["payload"] == _sample_request().to_json() diff --git a/tests/test_tepp_public_content.py b/tests/test_tepp_public_content.py deleted file mode 100644 index 546438553..000000000 --- a/tests/test_tepp_public_content.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Public-content denylist for the TEPP honesty correction.""" - -from __future__ import annotations - -from pathlib import Path - -_ROOT = Path(__file__).resolve().parents[1] -_SCOPED_PATHS = ( - _ROOT / "lineageweave" / "tepp_result.py", - _ROOT / "lineageweave" / "tepp_client.py", - _ROOT / "backend" / "app" / "analysis_run_start.py", - _ROOT / "backend" / "app" / "analysis_run_ingestion.py", - _ROOT / "migrations" / "0029_analysis_run_tepp_accepted.sql", - _ROOT / "docs" / "adr" / "0035-tepp-accepted-transport-evidence.md", - _ROOT / "CHANGELOG.d" / "2.12.1-tepp-accepted-transport-evidence.md", - _ROOT / "CHANGELOG.d" / "2.12.2-tepp-accepted-clocks.md", - _ROOT / "tests" / "test_tepp_result.py", - _ROOT / "tests" / "test_analysis_run_tepp_accepted_schema.py", -) -_FORBIDDEN_TABLES = ( - "document_record", - "model_artifact", - "topic_prevalence", - "membership_assignment", - "event_instance", -) -_FORBIDDEN_SECRETS = ( - "NVIDIA_NIM_API_KEY", - "postgres://", -) - - -def test_tepp_honesty_files_keep_synthetic_public_content() -> None: - """Changed TEPP files must not leak private tables or credentials.""" - for path in _SCOPED_PATHS: - text = path.read_text(encoding="utf-8") - lowered = text.casefold() - for token in _FORBIDDEN_TABLES: - assert token not in lowered, f"{token} in {path.name}" - for token in _FORBIDDEN_SECRETS: - assert token.casefold() not in lowered, f"{token} in {path.name}" - assert "is a validated multilevel estimate" not in lowered diff --git a/tests/test_tepp_result.py b/tests/test_tepp_result.py deleted file mode 100644 index 371bad5c9..000000000 --- a/tests/test_tepp_result.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Published TEPP accepted evidence is not a completed measurement.""" - -from __future__ import annotations - -from lineageweave.tepp_result import ( - accepted_tepp_seed_envelope, - parse_persistable_tepp_result, - parse_tepp_accepted_evidence, - persistable_tepp_seed_envelope, - tepp_accepted_evidence_sha256, -) - - -def test_published_accepted_envelope_is_transport_evidence() -> None: - """TEPP's AnalysisRunAccepted fields are storeable transport evidence.""" - envelope = accepted_tepp_seed_envelope(idempotency_key="demo-tepp-seed-2026-w02") - parsed = parse_tepp_accepted_evidence( - envelope, - expected_idempotency_key="demo-tepp-seed-2026-w02", - ) - assert parsed is not None - assert parsed.contract_version == 1 - assert parsed.run_state == "accepted" - assert parsed.accepted_run_id == "demo-tepp-accepted-opaque" - assert parsed.evidence_kind() == "aggregate transport evidence" - expected = tepp_accepted_evidence_sha256( - contract_version=1, - accepted_run_id="demo-tepp-accepted-opaque", - run_state="accepted", - idempotency_key="demo-tepp-seed-2026-w02", - ) - assert parsed.evidence_sha256() == expected - assert len(expected) == 64 - assert "theta" not in expected - - -def test_accepted_ack_without_published_fields_is_not_evidence() -> None: - """A bare status=accepted object is not TEPP's published envelope.""" - assert parse_tepp_accepted_evidence({"status": "accepted"}) is None - assert parse_tepp_accepted_evidence( - {"contract_version": 1, "status": "accepted"} - ) is None - - -def test_local_completed_envelope_is_not_accepted_evidence() -> None: - """The v2.12.0 LineageWeave-local shape is not a TEPP completed result.""" - local = persistable_tepp_seed_envelope() - assert parse_tepp_accepted_evidence(local) is None - assert parse_persistable_tepp_result(local) is None - - -def test_theta_and_unknown_fields_are_not_accepted_evidence() -> None: - """Unknown fields and psychometric keys fail closed.""" - base = accepted_tepp_seed_envelope(idempotency_key="k") - assert parse_tepp_accepted_evidence({**base, "theta": 0.42}) is None - assert parse_tepp_accepted_evidence({**base, "affiliation_count": 2}) is None - assert parse_tepp_accepted_evidence({**base, "extra": True}) is None - assert parse_tepp_accepted_evidence({**base, "run_state": "completed"}) is None - assert parse_tepp_accepted_evidence({**base, "contract_version": 2}) is None - assert parse_tepp_accepted_evidence( - base, - expected_idempotency_key="other-key", - ) is None - assert parse_tepp_accepted_evidence("accepted") is None - - -def test_persistable_parser_never_succeeds() -> None: - """No unpublished completed envelope becomes a persistable measurement.""" - assert parse_persistable_tepp_result(persistable_tepp_seed_envelope()) is None - assert parse_persistable_tepp_result( - accepted_tepp_seed_envelope(idempotency_key="k") - ) is None - assert parse_persistable_tepp_result({"theta": 1}) is None diff --git a/tests/test_tepp_transport_evidence.py b/tests/test_tepp_transport_evidence.py deleted file mode 100644 index 764cc16d3..000000000 --- a/tests/test_tepp_transport_evidence.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Authorized TEPP transport-evidence projection stays fail-closed.""" - -from __future__ import annotations - -from pathlib import Path - -from backend.app.analysis_run_ingestion import project_tepp_transport_evidence -from lineageweave.tepp_result import ( - accepted_tepp_seed_envelope, - parse_tepp_accepted_evidence, - tepp_accepted_evidence_sha256, -) - - -def test_project_tepp_transport_evidence_keeps_digest_independent_of_clocks() -> None: - """Clock split must not change the published accepted-field digest.""" - expected = tepp_accepted_evidence_sha256( - contract_version=1, - accepted_run_id="demo-tepp-accepted-opaque", - run_state="accepted", - idempotency_key="buyer-key", - ) - later = { - "contract_version": 1, - "accepted_run_id": "demo-tepp-accepted-opaque", - "run_state": "accepted", - "idempotency_key": "buyer-key", - "evidence_sha256": expected, - "received_at": "2026-01-12T12:45:00Z", - "recorded_at": "2026-01-12T12:46:00Z", - } - projected = project_tepp_transport_evidence(later) - assert projected is not None - assert projected["tepp_evidence_sha256"] == expected - assert projected["tepp_received_at"] == "2026-01-12T12:45:00Z" - assert projected["tepp_recorded_at"] == "2026-01-12T12:46:00Z" - - -def test_project_tepp_transport_evidence_recomputes_the_exact_digest() -> None: - """The API digest must match an independent SHA-256 recomputation.""" - parsed = parse_tepp_accepted_evidence( - accepted_tepp_seed_envelope(idempotency_key="buyer-key"), - expected_idempotency_key="buyer-key", - ) - assert parsed is not None - expected = tepp_accepted_evidence_sha256( - contract_version=1, - accepted_run_id="demo-tepp-accepted-opaque", - run_state="accepted", - idempotency_key="buyer-key", - ) - row = { - "contract_version": parsed.contract_version, - "accepted_run_id": parsed.accepted_run_id, - "run_state": parsed.run_state, - "idempotency_key": parsed.idempotency_key, - "evidence_sha256": expected, - "received_at": "2026-01-12T12:45:00Z", - "recorded_at": "2026-01-12T12:45:00Z", - } - projected = project_tepp_transport_evidence(row) - assert projected is not None - assert projected["tepp_evidence_sha256"] == expected - assert projected["tepp_evidence_kind"] == "aggregate transport evidence" - assert projected["tepp_completed_artifact_available"] is False - assert "affiliation_count" not in projected - assert "theta" not in str(projected).casefold() - - -def test_project_tepp_transport_evidence_fails_closed_on_digest_mismatch() -> None: - """A substituted digest is omitted rather than shown as evidence.""" - row = { - "contract_version": 1, - "accepted_run_id": "demo-tepp-accepted-opaque", - "run_state": "accepted", - "idempotency_key": "buyer-key", - "evidence_sha256": "0" * 64, - "received_at": "2026-01-12T12:45:00Z", - "recorded_at": "2026-01-12T12:45:00Z", - } - assert project_tepp_transport_evidence(row) is None - - -def test_persist_path_binds_received_and_recorded_as_distinct_arguments() -> None: - """Start must not write one timestamp into both accepted-evidence clocks.""" - source = ( - Path(__file__).resolve().parents[1] - / "backend" - / "app" - / "analysis_run_start.py" - ).read_text(encoding="utf-8") - assert "received_at,\n recorded_at," in source - assert source.count("recorded_at,\n recorded_at,") == 0 - - -def test_tepp_accepted_query_binds_authorized_run_ids_only() -> None: - """Hidden runs never enter the evidence query parameter list.""" - source = ( - Path(__file__).resolve().parents[1] - / "backend" - / "app" - / "analysis_run_ingestion.py" - ).read_text(encoding="utf-8") - assert "from analysis_run_tepp_accepted" in source - assert "where analysis_run_id = any($1::uuid[])" in source - assert "_tepp_accepted_by_run(conn, run_ids)" in source diff --git a/tests/test_tied_organization_no_create.py b/tests/test_tied_organization_no_create.py index c595fae39..55041715c 100644 --- a/tests/test_tied_organization_no_create.py +++ b/tests/test_tied_organization_no_create.py @@ -45,6 +45,15 @@ def verify(self, subject: str, relation: str) -> SimpleNamespace: return SimpleNamespace(status_code=STATUS_CORROBORATED) +class _TimeoutInferenceClient: + """Simulate an unavailable orchestrator during hierarchy enrichment.""" + + available = True + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal: + raise TimeoutError("synthetic orchestrator timeout") + + class _Transaction: """Minimal async transaction context manager.""" @@ -107,6 +116,22 @@ def test_initial_tie_never_reaches_live_inference_or_creation() -> None: assert verification.calls == 0 +def test_hierarchy_timeout_leaves_actor_unbound_without_raising() -> None: + """Enrichment outage must not discard a source-grounded summary.""" + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + object(), + "Unresolved Energy", + "Synthetic context", + _TimeoutInferenceClient(), + _LiveVerificationClient(), + [], + ) + ) + + assert result is None + + def test_tie_discovered_under_creation_lock_does_not_insert() -> None: """Concurrent homonyms discovered after inference still fail closed.""" connection = _ReloadTieConnection() diff --git a/tests/test_vision_image.py b/tests/test_vision_image.py new file mode 100644 index 000000000..9668618a9 --- /dev/null +++ b/tests/test_vision_image.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from io import BytesIO + +import pytest +from PIL import Image + +from lineageweave.vision_image import normalize_vision_image + + +def _encoded_image(image: Image.Image, image_format: str) -> bytes: + output = BytesIO() + image.save(output, format=image_format) + return output.getvalue() + + +def test_normalize_vision_image_flattens_transparent_png_to_white() -> None: + image = Image.new("RGBA", (2, 1), (255, 0, 0, 0)) + image.putpixel((1, 0), (255, 0, 0, 255)) + + normalized, mime_type = normalize_vision_image(_encoded_image(image, "PNG"), "image/png") + + with Image.open(BytesIO(normalized)) as result: + assert mime_type == "image/png" + assert result.mode == "RGB" + assert result.getpixel((0, 0)) == (255, 255, 255) + assert result.getpixel((1, 0)) == (255, 0, 0) + + +def test_normalize_vision_image_converts_jpeg_to_png() -> None: + normalized, mime_type = normalize_vision_image( + _encoded_image(Image.new("RGB", (1, 1), "blue"), "JPEG"), "image/jpeg" + ) + + with Image.open(BytesIO(normalized)) as result: + assert mime_type == "image/png" + assert result.format == "PNG" + + +def test_normalize_vision_image_rejects_invalid_bytes() -> None: + with pytest.raises(ValueError, match="unsupported or invalid"): + normalize_vision_image(b"not-an-image", "image/png") diff --git a/tests/test_vision_image_formats.py b/tests/test_vision_image_formats.py new file mode 100644 index 000000000..50b8c922d --- /dev/null +++ b/tests/test_vision_image_formats.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from io import BytesIO + +import pytest +from PIL import Image + +from lineageweave.vision_image import normalize_vision_image + + +@pytest.mark.parametrize("image_format", ["BMP", "TIFF"]) +def test_decodable_raster_formats_are_emitted_as_png(image_format: str) -> None: + source = Image.new("RGB", (2, 1), (12, 34, 56)) + encoded = BytesIO() + source.save(encoded, format=image_format) + + normalized, mime_type = normalize_vision_image(encoded.getvalue(), f"image/{image_format.lower()}") + + assert mime_type == "image/png" + with Image.open(BytesIO(normalized)) as output: + assert output.format == "PNG" + assert output.mode == "RGB" + assert output.size == (2, 1) + assert output.getpixel((0, 0)) == (12, 34, 56) + + +def test_transparent_raster_pixels_are_composited_to_white() -> None: + source = Image.new("RGBA", (2, 1), (255, 0, 0, 0)) + source.putpixel((1, 0), (0, 0, 255, 255)) + encoded = BytesIO() + source.save(encoded, format="TIFF") + + normalized, _mime_type = normalize_vision_image(encoded.getvalue(), "image/tiff") + + with Image.open(BytesIO(normalized)) as output: + assert output.mode == "RGB" + assert output.getpixel((0, 0)) == (255, 255, 255) + assert output.getpixel((1, 0)) == (0, 0, 255) + + +def test_large_payloads_are_dimension_and_byte_bounded() -> None: + source = Image.effect_noise((5000, 3000), 100).convert("RGB") + encoded = BytesIO() + source.save(encoded, format="PNG") + + normalized, mime_type = normalize_vision_image(encoded.getvalue(), "image/png") + + assert len(normalized) <= 8 * 1024 * 1024 + with Image.open(BytesIO(normalized)) as output: + assert max(output.size) <= 4096 + assert output.mode == "RGB" + assert mime_type in {"image/png", "image/jpeg"} diff --git a/update_app.py b/update_app.py new file mode 100644 index 000000000..fe9e7eb61 --- /dev/null +++ b/update_app.py @@ -0,0 +1,28 @@ +import re + +with open("frontend/src/App.tsx", "r") as f: + content = f.read() + +# Replace hardcoded LineageWeave and BRAND in App component +# I'll inject `const brandName = "LineageWeave"; // TODO: Fetch from admin/tenant config` +# into the App component. + +# First, find the beginning of the App component: +# export default function App() { +# const auth = useAuth(); +app_start = "export default function App() {\n const auth = useAuth();" +new_app_start = "export default function App() {\n const auth = useAuth();\n const brandName = \"LineageWeave\"; // TODO: Fetch from admin/tenant config" + +content = content.replace(app_start, new_app_start) + +# Replace

    LineageWeave

    with

    {brandName}

    +content = content.replace("

    LineageWeave

    ", "

    {brandName}

    ") +# Replace

    LineageWeave

    with

    {brandName}

    +content = content.replace('

    LineageWeave

    ', '

    {brandName}

    ') +# Replace LineageWeave with {brandName} +content = content.replace('LineageWeave', '{brandName}') +# Replace by BRAND with by {brandName} +content = content.replace('by BRAND.', 'by {brandName}.') + +with open("frontend/src/App.tsx", "w") as f: + f.write(content) diff --git a/uv.lock b/uv.lock index f80b89a70..10bcf9ff1 100644 --- a/uv.lock +++ b/uv.lock @@ -454,10 +454,11 @@ wheels = [ [[package]] name = "lineageweave" -version = "2.12.5" -source = { virtual = "." } +version = "2.12.6" +source = { editable = "." } dependencies = [ { name = "certifi" }, + { name = "pillow" }, { name = "rankweave" }, { name = "rdflib" }, { name = "threadweave" }, @@ -475,7 +476,6 @@ backend = [ dev = [ { name = "coverage" }, { name = "httpx" }, - { name = "pillow" }, { name = "psycopg2-binary" }, { name = "pyjwt", extra = ["crypto"] }, { name = "pytest" }, @@ -489,7 +489,7 @@ requires-dist = [ { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=5006c38286a4fa1d81bcf57eeed5ce27ae743f50" }, { name = "fastapi", marker = "extra == 'backend'", specifier = ">=0.115.0" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27.0" }, - { name = "pillow", marker = "extra == 'dev'", specifier = ">=12.3.0" }, + { name = "pillow", specifier = ">=12.3.0" }, { name = "psycopg2-binary", marker = "extra == 'dev'", specifier = ">=2.9.12" }, { name = "pyjwt", extras = ["crypto"], marker = "extra == 'backend'", specifier = ">=2.8.0" }, { name = "pyjwt", extras = ["crypto"], marker = "extra == 'dev'", specifier = ">=2.8.0" }, From 0d827fcf24de78b2bb93b6418ec1effad2699744 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 12:16:55 +0900 Subject: [PATCH 2/7] fix: keep provider errors behind product boundaries --- backend/app/analysis_run_start.py | 2 +- backend/app/auth.py | 8 +++---- backend/app/main.py | 12 +++++----- backend/app/post_content_worker.py | 5 +++-- backend/tests/test_api.py | 22 +++++++++++++++++++ backend/tests/test_auth_jwks.py | 19 ++++++++++++++++ .../0030-external-llm-gateway-environment.md | 6 +++++ lineageweave/image_content.py | 2 +- lineageweave/rankweave_client.py | 6 ++--- tests/test_image_content.py | 5 +++-- tests/test_post_content_worker.py | 1 + 11 files changed, 69 insertions(+), 19 deletions(-) diff --git a/backend/app/analysis_run_start.py b/backend/app/analysis_run_start.py index 2387d940b..324c6cd67 100644 --- a/backend/app/analysis_run_start.py +++ b/backend/app/analysis_run_start.py @@ -104,7 +104,7 @@ def transport(payload: dict[str, Any]) -> dict[str, Any]: headers = {"authorization": f"Bearer {api_key}"} if api_key.strip() else {} return post_json(url, payload, headers=headers, timeout=30.0) except (HttpClientError, OSError, ValueError, TypeError) as exc: - raise TeppNotAvailable(str(exc)) from exc + raise TeppNotAvailable("TEPP transport unavailable") from exc return TeppClient(transport=transport) diff --git a/backend/app/auth.py b/backend/app/auth.py index 155974d52..cc19cc807 100644 --- a/backend/app/auth.py +++ b/backend/app/auth.py @@ -56,7 +56,7 @@ def _jwks(settings: Settings, *, force_refresh: bool = False) -> dict: except (HttpClientError, OSError, ValueError) as exc: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, - f"could not fetch OIDC JWKS for {settings.oidc_issuer}: {exc}", + "could not fetch OIDC JWKS from the configured identity provider", ) from exc _jwks_cache[cache_key] = cached return cached @@ -91,7 +91,7 @@ def _signing_key_from_jwks(jwks: dict, token: str): return RSAAlgorithm.from_jwk(json.dumps(key)) except (KeyError, TypeError, ValueError) as exc: raise HTTPException(status.HTTP_401_UNAUTHORIZED, "matching JWKS key is invalid") from exc - raise HTTPException(status.HTTP_401_UNAUTHORIZED, f"no JWKS key matched kid={kid!r}") + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "access token signing key is not recognized") def _signing_key(settings: Settings, token: str): @@ -99,7 +99,7 @@ def _signing_key(settings: Settings, token: str): try: return _signing_key_from_jwks(_jwks(settings), token) except HTTPException as exc: - if not str(exc.detail).startswith("no JWKS key matched kid="): + if str(exc.detail) != "access token signing key is not recognized": raise return _signing_key_from_jwks(_jwks(settings, force_refresh=True), token) @@ -134,7 +134,7 @@ def _decode_access_token(token: str, settings: Settings) -> dict: except HTTPException: raise except jwt.PyJWTError as exc: - raise HTTPException(status.HTTP_401_UNAUTHORIZED, f"invalid token: {exc}") from exc + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "invalid access token") from exc subject = claims.get("sub") if not isinstance(subject, str) or not subject.strip(): raise HTTPException(status.HTTP_401_UNAUTHORIZED, "access token has no subject") diff --git a/backend/app/main.py b/backend/app/main.py index 374b97ea2..9a2aad40d 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1034,7 +1034,7 @@ async def resolve_customer_master_hint( _relation_verification_client(), request.hint_code, ) - except (HttpClientError, OSError) as exc: + except (HttpClientError, OSError, KeyError, TypeError, ValueError, RuntimeError) as exc: # resolve_and_verify_organization_name's resolution/verification # calls raise on a failed request rather than silently returning # "unresolved" -- a failed call is not the same claim as "the @@ -1967,7 +1967,7 @@ async def verify_post_entity_relationships( post_id, visible_corporate_entity_ids=account.corporate_entity_ids, ) - except (HttpClientError, OSError) as exc: + except (HttpClientError, OSError, KeyError, TypeError, ValueError, RuntimeError) as exc: # verify_post_relations() deliberately raises on a failed search # (a failed search is not "searched and found nothing" -- see # its docstring); this is the one caller, so it is the right @@ -2407,7 +2407,7 @@ async def read_post_summary( ) else: summary = await asyncio.to_thread(client.summarize, post["post_title"], normalized_body) - except (HttpClientError, KeyError, OSError, TypeError, ValueError) as exc: + except (HttpClientError, KeyError, OSError, TypeError, ValueError, RuntimeError) as exc: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, "Post summary is unavailable: contextual-orchestrator returned no complete evidence object", @@ -2541,7 +2541,7 @@ async def chat_about_post( try: with use_llm_metadata(post_metadata): answer = await asyncio.to_thread(client.answer, question, sources) - except (HttpClientError, KeyError, OSError, ValueError) as exc: + except (HttpClientError, KeyError, OSError, TypeError, ValueError, RuntimeError) as exc: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, "Post chat is unavailable: contextual-orchestrator returned no complete evidence object", @@ -2600,10 +2600,10 @@ async def ask_agent( } try: answer = await asyncio.to_thread(client.answer, question, sources) - except (HttpClientError, KeyError, OSError, ValueError) as exc: + except (HttpClientError, KeyError, OSError, TypeError, ValueError, RuntimeError) as exc: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, - f"Ask Agent is unavailable: {exc}", + "Ask Agent is unavailable: contextual-orchestrator returned no complete evidence object", ) from exc cited_ids = list(answer.cited_post_ids) return { diff --git a/backend/app/post_content_worker.py b/backend/app/post_content_worker.py index 458b9021f..873294746 100644 --- a/backend/app/post_content_worker.py +++ b/backend/app/post_content_worker.py @@ -37,6 +37,7 @@ _RECOVERY_INTERVAL_SECONDS = 30.0 _INCOMPLETE_FAILURE_CODE = "post_content_ingestion_incomplete" _ATTEMPT_LIMIT_FAILURE_CODE = "post_content_ingestion_attempt_limit" +_UNEXPECTED_FAILURE_DETAIL = "post-content ingestion failed; retry is scheduled" async def _stream_tail(client: redis.Redis) -> str: @@ -261,13 +262,13 @@ async def process_post_content_job( expected_attempt_count=attempt_count, ) return - except Exception as exc: # noqa: BLE001 - durable failure is recorded for retry. + except Exception: # noqa: BLE001 - durable failure is recorded for retry. _logger.exception("post content ingestion failed for post_id=%s", post_id) await _finish_failed_job( pool, post_id, failure_code="post_content_ingestion_failed", - detail_text=str(exc)[:1000], + detail_text=_UNEXPECTED_FAILURE_DETAIL, expected_attempt_count=attempt_count, ) return diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 91be1dcb4..8c0b71df1 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -3293,6 +3293,28 @@ def answer(self, question: str, sources) -> ChatAnswer: assert "What happened here that no seed already answers?" in events[0]["summary"] +def test_live_chat_provider_error_does_not_leak_raw_error( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """A provider exception becomes a stable 503 without its raw message.""" + class _FailingChatClient: + available = True + + def answer(self, question: str, sources) -> object: + raise RuntimeError("raw-provider-secret") + + monkeypatch.setattr("backend.app.main._post_chat_client", lambda: _FailingChatClient()) + + response = client.post( + f"/api/posts/{seeded_db['own_private_post_id']}/chat", + json={"question": "What happened in this provider failure case?"}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + + assert response.status_code == 503 + assert "raw-provider-secret" not in response.text + + def test_evaluate_is_unavailable_without_orchestrator(client, demo_analyst_token, seeded_db) -> None: os.environ.pop("ORCHESTRATOR_BASE_URL", None) os.environ.pop("ORCHESTRATOR_API_KEY", None) diff --git a/backend/tests/test_auth_jwks.py b/backend/tests/test_auth_jwks.py index 709d2c16e..f6aa64352 100644 --- a/backend/tests/test_auth_jwks.py +++ b/backend/tests/test_auth_jwks.py @@ -116,6 +116,25 @@ def fake_jwks(settings, *, force_refresh=False): assert calls == [False, True] +def test_jwks_provider_error_does_not_leak_raw_response(monkeypatch: pytest.MonkeyPatch) -> None: + """Identity-provider response text stays on the server-side exception chain.""" + def fail(*_args: object, **_kwargs: object) -> dict: + raise auth.HttpClientError("raw identity-provider response") + + monkeypatch.setattr(auth, "get_json", fail) + settings = SimpleNamespace( + oidc_issuer="https://id.example", + oidc_discovery_uri="https://id.example/.well-known/openid-configuration", + oidc_jwks_uri_override="https://id.example/jwks", + ) + + with pytest.raises(HTTPException) as error: + auth._jwks(settings) + + assert error.value.status_code == 503 + assert "raw identity-provider response" not in str(error.value.detail) + + def test_signing_key_does_not_refresh_for_invalid_header( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/docs/adr/0030-external-llm-gateway-environment.md b/docs/adr/0030-external-llm-gateway-environment.md index e518ee114..b372e75bb 100644 --- a/docs/adr/0030-external-llm-gateway-environment.md +++ b/docs/adr/0030-external-llm-gateway-environment.md @@ -94,6 +94,12 @@ content. A text-only gateway response such as `Only 'text' content type is supported` is a provider capability failure, not a successful Vision result; the channel remains unavailable and no OCR/caption is fabricated. +All provider and identity-boundary exceptions follow the same rule: retain the +original exception only in server-side logs and exception chaining, while +persisted job details and buyer-facing HTTP responses use stable, actionable +failure categories. Raw provider response bodies, token-validation details, +and transport exception messages must never cross the product boundary. + Before the image content block is built, the backend decodes supported raster formats, applies EXIF orientation, composites transparent pixels onto white, and encodes the payload as PNG. Invalid or undecodable image bytes fail closed; diff --git a/lineageweave/image_content.py b/lineageweave/image_content.py index 3aa2c9038..9cf13970c 100644 --- a/lineageweave/image_content.py +++ b/lineageweave/image_content.py @@ -258,7 +258,7 @@ def _parse_description(content: str) -> ImageDescription: if not fields["TEXT"] and not fields["CAPTION"]: raise ImageDescriptionParseError( - f"vision response had neither TEXT nor CAPTION content: {content!r}" + "vision response had neither TEXT nor CAPTION content" ) extracted_text = "\n".join(fields["TEXT"]).strip() diff --git a/lineageweave/rankweave_client.py b/lineageweave/rankweave_client.py index eb0b3358b..1e5f77eb2 100644 --- a/lineageweave/rankweave_client.py +++ b/lineageweave/rankweave_client.py @@ -224,11 +224,11 @@ def __call__( ) except Exception as exc: raise RankWeaveNotAvailable( - f"rankweave_not_available: weighted_reciprocal_rank_fuse failed ({exc})" + "rankweave_not_available: weighted_reciprocal_rank_fuse failed" ) from exc except Exception as exc: raise RankWeaveNotAvailable( - f"rankweave_not_available: weighted_reciprocal_rank_fuse failed ({exc})" + "rankweave_not_available: weighted_reciprocal_rank_fuse failed" ) from exc projected: list[dict[str, Any]] = [] for hit in hits: @@ -268,7 +268,7 @@ def fuse_rankings( raise except Exception as exc: raise RankWeaveNotAvailable( - f"rankweave_not_available: ranking transport failed ({exc})" + "rankweave_not_available: ranking transport failed" ) from exc return project_ranking_list(raw, titles_by_id) diff --git a/tests/test_image_content.py b/tests/test_image_content.py index f694635a4..5b31917c7 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -74,8 +74,9 @@ def test_parse_description_none_text_becomes_empty_string() -> None: def test_parse_description_unexpected_format_raises_instead_of_losing_content() -> None: - with pytest.raises(ImageDescriptionParseError): - _parse_description("unexpected format") + with pytest.raises(ImageDescriptionParseError) as error: + _parse_description("raw provider response must not cross the boundary") + assert "raw provider response" not in str(error.value) def test_parse_description_preserves_multiline_ocr_text() -> None: diff --git a/tests/test_post_content_worker.py b/tests/test_post_content_worker.py index dddace990..d92ed728f 100644 --- a/tests/test_post_content_worker.py +++ b/tests/test_post_content_worker.py @@ -230,6 +230,7 @@ async def persist(*_args, **_kwargs): updates = [args for query, args in connection.executed if "set status_code" in query] assert any(args[1] == QUEUED and args[6] == "post_content_ingestion_failed" for args in updates) + assert all("provider timeout" not in str(args) for args in updates) def test_failure_at_attempt_limit_is_terminal_and_visible() -> None: From d58edc8ad0bf9ffdd578b31e064b5e8f2f721191 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 12:25:37 +0900 Subject: [PATCH 3/7] fix: close provider error leaks at write endpoints --- backend/app/main.py | 98 ++++++++++++++++++++++++--------------- backend/tests/test_api.py | 73 +++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 37 deletions(-) diff --git a/backend/app/main.py b/backend/app/main.py index 9a2aad40d..93818e9a7 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -2029,22 +2029,28 @@ async def extract_post_keymen( # tags dilute the model's attention and a base64 payload sent as # literal text either blows the token budget or is silently # ignored (see lineageweave/post_content_normalization.py). - post_body = ( - await asyncio.to_thread(normalize_post_body, raw_body, _vision_client()) - ).text context_hints = await _load_post_semantic_hints(conn, post_id) - mentions = await ingest_post_keymen( - conn, - keyman_client, - post_id, - post["post_title"], - post_body, - resolution_client=_organization_name_resolution_client(), - verification_client=_relation_verification_client(), - hierarchy_inference_client=_corporate_hierarchy_inference_client(), - context_hints=context_hints, - persist_graph=False, - ) + try: + post_body = ( + await asyncio.to_thread(normalize_post_body, raw_body, _vision_client()) + ).text + mentions = await ingest_post_keymen( + conn, + keyman_client, + post_id, + post["post_title"], + post_body, + resolution_client=_organization_name_resolution_client(), + verification_client=_relation_verification_client(), + hierarchy_inference_client=_corporate_hierarchy_inference_client(), + context_hints=context_hints, + persist_graph=False, + ) + except (HttpClientError, KeyError, OSError, TypeError, ValueError, RuntimeError) as exc: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Keymen extraction is unavailable: contextual-orchestrator or corroboration provider returned no complete evidence object", + ) from exc # Live bug (2026-08-19): an organization affiliated ONLY with an # our_side person (our own factory, our own affiliate) got fed # into the counterparty-relationship classifier the same as any @@ -2060,9 +2066,15 @@ async def extract_post_keymen( for name in mention.affiliated_organization_names } ) - relationships = await ingest_post_entity_relationships( - conn, relationship_client, post_id, post["post_title"], post_body, organization_names - ) + try: + relationships = await ingest_post_entity_relationships( + conn, relationship_client, post_id, post["post_title"], post_body, organization_names + ) + except (HttpClientError, KeyError, OSError, TypeError, ValueError, RuntimeError) as exc: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Keymen extraction is unavailable: contextual-orchestrator or corroboration provider returned no complete evidence object", + ) from exc async with conn.transaction(): await persist_edges_for_post(conn, post_id) await publish_activity_event( @@ -2190,17 +2202,23 @@ async def evaluate_post( ) async with pool.acquire() as conn: body_row = await conn.fetchrow("select post_body from source_post where post_id = $1", post_id) - normalized_body = ( - await asyncio.to_thread( - normalize_post_body, - "" if body_row is None else body_row["post_body"], - _vision_client(), - ) - ).text - async with pool.acquire() as conn: - rows = await ingest_post_evaluation( - conn, client, post_id, post["post_title"], normalized_body - ) + try: + normalized_body = ( + await asyncio.to_thread( + normalize_post_body, + "" if body_row is None else body_row["post_body"], + _vision_client(), + ) + ).text + async with pool.acquire() as conn: + rows = await ingest_post_evaluation( + conn, client, post_id, post["post_title"], normalized_body + ) + except (HttpClientError, KeyError, OSError, TypeError, ValueError, RuntimeError) as exc: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Post evaluation is unavailable: contextual-orchestrator returned no complete evidence object", + ) from exc await publish_activity_event( valkey, post_id, @@ -2834,14 +2852,20 @@ async def derive_post_commitment( ) async with pool.acquire() as conn: body_row = await conn.fetchrow("select post_body from source_post where post_id = $1", post_id) - normalized_body = ( - await asyncio.to_thread(normalize_post_body, body_row["post_body"], _vision_client()) - ).text - # TimeML/TempEval document creation time, not wall-clock now: "by next - # Friday" in a January post must resolve to that January, not to the - # Friday after the operator clicked Derive. - reference_date = post["created_at"].date().isoformat() - commitment = client.extract(post["post_title"], normalized_body, reference_date) + try: + normalized_body = ( + await asyncio.to_thread(normalize_post_body, body_row["post_body"], _vision_client()) + ).text + # TimeML/TempEval document creation time, not wall-clock now: "by next + # Friday" in a January post must resolve to that January, not to the + # Friday after the operator clicked Derive. + reference_date = post["created_at"].date().isoformat() + commitment = client.extract(post["post_title"], normalized_body, reference_date) + except (HttpClientError, KeyError, OSError, TypeError, ValueError, RuntimeError) as exc: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Commitment derivation is unavailable: contextual-orchestrator returned no complete evidence object", + ) from exc if not commitment.has_commitment: return {"post_id": str(post["post_id"]), "has_commitment": False, "ticket": None} async with pool.acquire() as conn: diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 8c0b71df1..337979ac1 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -3315,6 +3315,79 @@ def answer(self, question: str, sources) -> object: assert "raw-provider-secret" not in response.text +def test_keymen_provider_error_does_not_leak_raw_error( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """Keymen provider failures become a stable 503 at the API boundary.""" + _grant_post_admin(seeded_db["dsn"]) + + class _FailingKeymanClient: + available = True + + def extract(self, post_title: str, post_body: str) -> object: + raise RuntimeError("raw-keyman-provider-secret") + + monkeypatch.setattr("backend.app.main._keyman_extraction_client", lambda: _FailingKeymanClient()) + + response = client.post( + f"/api/posts/{seeded_db['own_private_post_id']}/extract-keymen", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + + assert response.status_code == 503 + assert "raw-keyman-provider-secret" not in response.text + + +def test_evaluation_provider_error_does_not_leak_raw_error( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """Evaluation provider failures become a stable 503 at the API boundary.""" + _grant_post_admin(seeded_db["dsn"]) + + class _FailingEvaluationClient: + available = True + + def evaluate(self, post_title: str, post_body: str) -> object: + raise RuntimeError("raw-evaluation-provider-secret") + + monkeypatch.setattr( + "backend.app.main._post_evaluation_client", lambda: _FailingEvaluationClient() + ) + + response = client.post( + f"/api/posts/{seeded_db['own_private_post_id']}/evaluate", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + + assert response.status_code == 503 + assert "raw-evaluation-provider-secret" not in response.text + + +def test_commitment_provider_error_does_not_leak_raw_error( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """Commitment provider failures become a stable 503 at the API boundary.""" + _grant_post_admin(seeded_db["dsn"]) + + class _FailingCommitmentClient: + available = True + + def extract(self, post_title: str, post_body: str, reference_date: str) -> object: + raise RuntimeError("raw-commitment-provider-secret") + + monkeypatch.setattr( + "backend.app.main._commitment_extraction_client", lambda: _FailingCommitmentClient() + ) + + response = client.post( + f"/api/posts/{seeded_db['own_private_post_id']}/derive-commitment", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + + assert response.status_code == 503 + assert "raw-commitment-provider-secret" not in response.text + + def test_evaluate_is_unavailable_without_orchestrator(client, demo_analyst_token, seeded_db) -> None: os.environ.pop("ORCHESTRATOR_BASE_URL", None) os.environ.pop("ORCHESTRATOR_API_KEY", None) From 678df0db8a9210b21f068376eb544df16ac3db43 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 12:32:05 +0900 Subject: [PATCH 4/7] fix: keep unauthenticated admin controls out of login --- CHANGELOG.d/2.12.6-frontend-build-gate.md | 3 +++ frontend/src/App.tsx | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 CHANGELOG.d/2.12.6-frontend-build-gate.md diff --git a/CHANGELOG.d/2.12.6-frontend-build-gate.md b/CHANGELOG.d/2.12.6-frontend-build-gate.md new file mode 100644 index 000000000..75d7d6590 --- /dev/null +++ b/CHANGELOG.d/2.12.6-frontend-build-gate.md @@ -0,0 +1,3 @@ +## Fixed + +- Keep the unauthenticated login surface free of authenticated admin controls and remove unused OIDC imports so TypeScript production builds pass. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6fba0dd41..666888a4d 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -101,7 +101,6 @@ import { tf, useLocale, } from "./i18n"; -import { rememberOidcReturnUrl, returnUrlFromLocation } from "./oidcReturnUrl"; import "./App.css"; function orchestratorUnavailableMessage(err: unknown, action: string): string { @@ -4620,7 +4619,6 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean Enterprise SSO Authentication
    - {destination === "admin" ? : null}