diff --git a/.github/workflows/prov-o-contract.yml b/.github/workflows/prov-o-contract.yml new file mode 100644 index 000000000..eae453153 --- /dev/null +++ b/.github/workflows/prov-o-contract.yml @@ -0,0 +1,94 @@ +name: PROV-O contract + +on: + pull_request: + branches: [main] + paths: + - "lineageweave/prov_o.py" + - "tests/test_prov_o.py" + - "tests/test_prov_o_schema.py" + - "migrations/0017_prov_o_standard_relations.sql" + - "docs/ontology/prov-o-support-profile.ttl" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/prov-o-contract.yml" + push: + branches: [main] + paths: + - "lineageweave/prov_o.py" + - "tests/test_prov_o.py" + - "tests/test_prov_o_schema.py" + - "migrations/0017_prov_o_standard_relations.sql" + - "docs/ontology/prov-o-support-profile.ttl" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/prov-o-contract.yml" + +permissions: + contents: read + +concurrency: + group: prov-o-contract-${{ github.ref }} + cancel-in-progress: true + +jobs: + standards-contract: + name: Registry, inference, coverage, PostgreSQL + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN: postgresql://postgres:postgres@localhost:5432/postgres + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Set up locked Python dependency manager + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "0.11.28" + enable-cache: false + + - name: Install the committed universal lock + run: uv sync --frozen --extra dev + + - name: Verify complete relation behavior and 100 percent coverage + run: | + uv run --frozen python -m coverage run --branch --source=lineageweave.prov_o \ + -m pytest -q tests/test_prov_o.py + uv run --frozen python -m coverage report --fail-under=100 lineageweave/prov_o.py + + - name: Require a reachable PostgreSQL service + run: | + set -euo pipefail + for _ in $(seq 1 30); do + if pg_isready -h localhost -p 5432 -U postgres; then + exit 0 + fi + sleep 2 + done + echo "PostgreSQL service is unreachable; the schema contract would silently skip." >&2 + exit 1 + + - name: Verify normalized PostgreSQL contracts + run: uv run --frozen python -m pytest -q tests/test_prov_o_schema.py + + - name: Compile owned Python surface + run: uv run --frozen python -m compileall -q lineageweave/prov_o.py tests/test_prov_o.py tests/test_prov_o_schema.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cb4c7f951..e78d36254 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: [main] permissions: contents: read @@ -17,6 +16,20 @@ jobs: pytest: name: Full test suite runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN: postgresql://postgres:postgres@localhost:5432/postgres steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 @@ -28,19 +41,22 @@ jobs: with: python-version: "3.12" - - name: Install pinned Rust toolchain + - name: Set up locked dependency manager + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "0.11.28" + enable-cache: false + + - name: Select pinned Rust toolchain run: | - # Same pin as backend/Dockerfile: fast-mlsirm's PyO3/maturin - # core has no wheel, so pip install -e ".[backend]" compiles it. - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ - sh -s -- -y --profile minimal --default-toolchain 1.97.1 - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + rustup toolchain install 1.97.1 --profile minimal + rustup default 1.97.1 - - name: Install package and test dependencies - run: python -m pip install -e ".[dev,backend]" + - name: Install the committed universal lock + run: uv sync --frozen --extra dev --extra backend - - name: Run full test suite - run: python -m pytest -q + - name: Run full test suite against PostgreSQL + run: uv run --frozen python -m pytest -q frontend: name: Frontend lint, test, build @@ -54,7 +70,6 @@ jobs: - name: Set up Node uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # actions/setup-node@v5 with: - # Matches frontend/mise.toml's pin (setup-node doesn't parse mise.toml). node-version: "24" - name: Enable Corepack diff --git a/AGENTS.md b/AGENTS.md index 735988f09..dba1c4b42 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,7 +55,10 @@ summary/chat, or invented commitment. A missing signal and a confidently-negative signal are different things. Keyman extraction, entity-relationship classification, post summary, in-popup chat, and commitment derivation go through contextual-orchestrator the same way -adjudication does -- never a raw LLM API. +adjudication does -- never a raw LLM API. Demo TEPP seed goes through +`tepp_client` the same way: a missing transport or an unused accepted +envelope is Failed (`tepp_not_available` / `tepp_result_not_persisted`), +never a fabricated theta or a local psychometric substitute. ## Tests @@ -88,3 +91,13 @@ pnpm run lint && pnpm run test && pnpm run build Do not weaken, skip, or `continue-on-error` a failing check -- fix the underlying cause or, for a genuine false positive in a third-party scanner, add a narrow, documented suppression referencing the specific finding. +## W3C PROV-O boundary + +- Add standard provenance through `lineageweave.prov_o` and the + normalized `provenance_*` schema, never by inventing another + `edge_type` alias for a W3C property. +- Qualified relations retain their Influence resource and imply the + corresponding unqualified relation. +- Appendix B inverse names normalize to the preferred W3C direction; + do not proliferate private inverse vocabulary. +- Keep `knowledge_graph_edge` an explicit navigation projection. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 617b8b95d..305361bd6 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -64,7 +64,7 @@ flowchart LR | `chunking.py` | Splits a document into meaning-identifiable units (paragraph, sentence, DOM, conversation-turn) plus embedded-image extraction, in document order | | `embedding_client.py` | Pluggable text-embedding channel (`Null` default, `OpenAiCompatible` real impl) + `chunked_max_similarity` | | `adjudication_client.py` | Pluggable LLM-judgment channel (`Null` default, `ContextualOrchestrator` real impl) | -| `image_content.py` | Pluggable vision channel: OCR + object recognition/tagging for embedded images (`Null` default, `OpenAiCompatibleVisionClient` real impl) | +| `image_content.py` | Pluggable vision channel: OCR + object recognition/tagging for embedded images (`Null` default, `OpenAiCompatibleVisionClient` real impl). The product popup (`frontend/src/PostBody.tsx`) renders each `data:image` payload in document order so the buyer sees the picture, not the base64 string. Tag-only, charset, unquoted, or undecodable bodies never fall back to the raw source. GET does not call the vision client. | | `tepp_client.py` | TEPP's published `AnalysisRunRequest` wire contract, pluggable transport | | `reconstruct.py` | The pipeline: group → candidate window → score → fuse → thread | | `lineage_persistence.py` | Flattens reconstruct trees into `post_lineage_edge` row specs (parent, child, fused_score) | @@ -136,6 +136,8 @@ identities and content) and `migrations/0001_initial_schema.sql` for the `role_permission` / `account_role_assignment`, `abac_policy`, `post` / `post_counterparty_entity`, `person` / `person_affiliation` / `post_person_mention`, `knowledge_graph_edge`, `issue_ticket`, + +Keyman and R&R person mentions are separate replaceable projections (`post_person_mention` and `post_summary_person_mention`). The read-only `combined_post_person_mention` view feeds lineage discovery. Materialized KG edges are unique and carry normalized `knowledge_graph_edge_evidence`; only evidence from an ABAC-visible post participates in RWR. `post_lineage_edge`). Real-database tests: `tests/test_schema.py` (skipped without a reachable PostgreSQL server, same pattern as the real-provider LLM tests). @@ -254,11 +256,14 @@ pattern and then hide the action button so it cannot 503 again. `find_linked_post_ids` first expands to every post sharing a mentioned person before calling `backend/app/knowledge_graph.py::load_visible_subgraph` -- that function -only loads edges among an *already-known* post set (its other caller, -`related_for_person`, pre-resolves the full set itself), it does not -discover new posts on its own; a real bug from calling it with only the -single starting post was caught while building this and is now -regression-tested (`test_post_chat_cites_a_post_linked_only_via_a_shared_keyman`). +only loads edges among an *already-known* post set (its other callers, +`related_for_person` / `related_for_entity` / `related_for_team`, +pre-resolve the full set themselves), it does not discover new posts on +its own; a real bug from calling it with only the single starting post +was caught while building this and is now regression-tested +(`test_post_chat_cites_a_post_linked_only_via_a_shared_keyman`). +Person, team, and organization mention channels load independently +(ADR 0018): a team-only or organization-only post still walks. ### Frontend (`frontend/`) @@ -274,8 +279,9 @@ summary/key-events/R&R, VOC evidence excerpts, an Event Lineage panel affiliate tree (resolved ancestors plus unresolved org roots), Keyman + counterparty panels (a Keyman click loads RWR related nodes; a related corporate-entity node, a resolved Keyman affiliation, -or a classified name that resolves to a cataloged org continues -the same walk via `GET /api/corporate-entities/{id}/related`; +a classified name that resolves to a cataloged org, or an R&R team +continues the same walk via `GET /api/corporate-entities/{id}/related` +or `GET /api/teams/{id}/related`; `post_admin` can extract), and an in-popup chat whose cited sources open a sliding evidence panel (`EvidencePanel`, CSS @@ -320,7 +326,9 @@ is the same never-guess-a-parent rule `corporate_hierarchy_resolution` already applies. Entity levels and Keyman sides are labeled from `common_lookup_value` (`Our side`, `Plant`, `Company`) so the popup never shows raw `our_side` / `plant` -codes when a label exists. +codes when a label exists. Related-node person chips use the same +side lookup label (for example, `Our side` or `Counterparty`) rather +than exposing the generic PROV-O `Person` class as business context. `GET /api/posts` and `GET /api/posts/{post_id}` include `voc_type_label` / `visibility_label` from `common_lookup_value` so @@ -450,6 +458,48 @@ lists the same dated tickets the period-report members already show. Re-seed is idempotent. The empty-state copy is only for accounts that truly have no dated open tickets. +## Phase 6-M2: authorized analysis-run evidence (read projection) + +Issue #79's first buyer-visible Milestone 2 slice is a source-redacting +read of the #89 registry. `GET /api/analysis-runs` and +`GET /api/analysis-runs/{id}` require `post_read` and apply the scope +in SQL: the requester always sees their own run; a corporate-entity or +process-unit scope is visible only to affiliated accounts; a +thread-group scope is visible only when the account can already see a +post in that group; `all_visible` is requester-only. Hidden runs 404. Detail also lists ABAC-visible post titles in the +run's scope whose `created_at` is at or before `knowledge_cutoff` +(ADR 0016) so a buyer can open a post the run was allowed to know +without seeing later live rows or hidden bodies. Detail also returns +revision and configuration digest prefixes. +`POST /api/analysis-runs` records a Pending run on a new authorized +cutoff capture (ADR 0017): snapshot, counts, run, scope, and the first +status in one transaction. It does not reconstruct lineage and does not +invent a TEPP score. Request a lineage reconstruction from the home +list, then open the Pending row to confirm the cutoff corpus. +`make seed` also records a TEPP measurement run through +`tepp_client` on that same snapshot; the default transport is +unavailable, so that run is Failed rather than a fabricated score. +The home list is clickable: `GET /api/analysis-runs/{id}` fills a +labeled detail (cutoff, requested date, 12-character digest prefixes +with full digests on hover, counts, status history) +without exposing a DSN or raw record. Opening a cutoff title warns +that the live body may have changed after the run. Status history is detail-only +and uses lookup labels plus occurrence times; a failure event keeps +its machine `failure_code` rather than an invented caption. Failed +TEPP list rows add a next-action line (open the run, then connect the +measurement service) so `tepp_not_available` is not mistaken for a +calibrated negative result. A failed lineage row tells the operator +to retry reconstruction, not to connect TEPP. A failed period-report +row tells the operator to rebuild the report. A pending TEPP row +does not claim a calibrated measurement. A pending lineage row +says reconstruction has not started yet. The +payload is lookup labels plus non-negative aggregate counts -- never +source SQL, a DSN, a raw record, or a provider body. After `make seed`, +Demo Analyst and Demo Admin see "Lineage reconstruction · Succeeded · +Demo Corp" with "3 documents" and Pending / Running / Succeeded times, +and "TEPP measurement · Failed · Demo Corp" whose detail history ends +in Failed / `tepp_not_available`. + ## Phase 6a: fast-mlsirm dependency + Rust toolchain (infra only) First of three staged slices toward the brief's weekly/monthly @@ -682,3 +732,183 @@ checks a well-known public foundation name ("Mozilla Foundation") against a deliberately fabricated one in the same request, asserting the former comes back `verify_corroborated` with a real evidence URL and the latter `verify_uncorroborated` with none. + +## Phase 7: R&R's named actor is a PROV-O Agent, not always a person + +`post_summary.py`'s R&R extraction forced every named actor into a +person slot, but business correspondence routinely names an +organization acting in its own name ("당사" [our company], +"Demo Corp"), not an individual. See +[ADR 0006](docs/adr/0006-role-responsibility-agent-ontology.md). + +Grounded in W3C PROV-O (Lebo, Sahoo, & McGuinness, 2013): +`RoleResponsibility` (renamed field `actor_name`, was `person_name` -- +the field can hold an organization's name now, so "person" in the name +would be wrong) gains `actor_type_code` (`prov_person` / +`prov_organization`, defaulting to person when the model omits it) and +`affiliated_organization_name` (an LLM-inferred affiliation for a +person actor, since a bare name without an employer is hard to place). +The ontology gains `:RoleActorPerson rdfs:subClassOf prov:Person` and +`:RoleActorOrganization rdfs:subClassOf prov:Organization` -- genuine +subclasses of the real external PROV-O classes (imported via the +`prov:` namespace), kept distinct from the ontology's existing `:Person` +(node_type's cataloged Keyman with a stable `person_id`) since an R&R +actor is a free-text name with no cataloged identity of its own. +`migrations/0012_role_responsibility_agent_type.sql` renames the +`post_summary_role` column via `RENAME COLUMN` (preserves existing +rows) rather than a drop/recreate. The popup's R&R list shows a +Person/Organization badge and the inferred affiliation; only a person +actor still links to the Keyman panel. + +## Phase 8: same-name Keymen are not silently merged; titles are captured + +Two different real people can share a name -- `keyman_extraction.py` +never captured a stated job title/position, so nothing distinguished +"Kim Cheolsu, sales manager" from an unrelated "Kim Cheolsu, purchasing +lead" beyond the bare name. `PersonMention` gains `job_title: str | +None`, and the extraction prompt now explicitly asks for one when the +text states it (never left out as a same-name disambiguation signal). + +Persistence, in two places for a reason: `person_affiliation.role_title` +(a schema column that already existed, previously never populated) for +a title tied to a specific organization, and a new +`cataloged_person.last_known_job_title` (`migrations/0013_person_job_title.sql`) +for a title stated without a named organization to attach it to (e.g. +"our legal counsel, Sam Okonkwo" -- `fixtures.ambiguous_keyman_post()`'s +own real example, which has zero affiliated organizations for Sam). +Both feed `_upsert_person`'s disambiguation check +(`backend/app/keyman_ingestion.py`): a same person_name+person_side_code +match is only reused when the new mention's stated title, if any, does +not conflict with a title already on file -- a genuine stated conflict +creates a fresh `cataloged_person` row instead of merging two different +people. A missing title on either side is not treated as a conflict +(titles legitimately change -- a promotion -- and most mentions state no +title at all), so this only splits on an actual stated disagreement, +verified by a real test that two posts naming the same name with +genuinely different stated titles produce two distinct person rows. + +## Phase 9: an R&R actor can be a team, meso-level between person and organization + +Real post text named "설계팀" (design team) -- neither a person nor the +company itself, but a sub-unit of one. See +[ADR 0007](docs/adr/0007-team-actor-type.md). `actor_type_code` gains a +third value, `prov_team`, grounded in the W3C Organization Ontology's +`org:OrganizationalUnit` (Reynolds, 2014) -- a different, complementary +W3C vocabulary from PROV-O (which models "who acted," not "how a +company is internally structured"). The prompt now offers three actor +types and requires `affiliated_organization_name` for a team actor too +(not just a person): a team's own name never answers "which company," +unlike an organization actor's. `migrations/0014_role_responsibility_team_actor_type.sql` +adds the lookup row -- purely additive, no schema change, since +`actor_type_code` already stores an arbitrary FK'd code. + +## Phase 10: an abbreviated organization name is resolved and search-verified, not left opaque + +Real post text names organizations by abbreviation ("AGP" for +"Aurora Grid Power") that character-similarity matching +(`corporate_hierarchy_resolution`) structurally cannot bridge -- an +initialism shares almost no substring with its expansion. See +[ADR 0008](docs/adr/0008-organization-abbreviation-resolution.md). + +New module `lineageweave/organization_name_resolution.py`: an LLM +proposes the full name from context (or declines with `UNKNOWN`), then +the *existing* `relation_verification` Searxng client cross-verifies +the specific raw/resolved pairing (no second web-search integration +built). Only a search-corroborated resolution is ever substituted in +for `resolve_corporate_entity` -- an unresolved or unverified name +still flows through unchanged. Cached in a new +`organization_name_resolution` table +(`migrations/0015_organization_name_resolution.sql`) keyed by the raw +name, so the same abbreviation across many posts is resolved once. +Grounded in SKOS `skos:altLabel`/`skos:prefLabel` (Miles & Bechhofer, +2009). Wired into `backend/app/keyman_ingestion.py`'s affiliation loop +and the offline synthetic-batch script's paced re-implementation of it +(the batch script's own copy was also missing `role_title` persistence +entirely -- fixed alongside this). + +Also fixed while running this against synthetic embedded-image fixtures: +`image_content.py`'s `_parse_description` required an exact single-pass +`TEXT:`/`CAPTION:`/`TAGS:` match, which was rejecting real vision +responses whose formatting was close but not exact (markdown-bolded +labels, reordered labels, a missing TAGS line) -- silently producing +the same "content unavailable" placeholder as a genuinely unconfigured +vision channel. Fields are now recovered independently per label line; +only a response with neither TEXT nor CAPTION is treated as unusable. + +## Phase 11: R&R team/organization actors get a shared cross-post identity + +Extraction runs per-post; a team's or organization's identity did not +survive across posts the way a Keyman's already did via +`cataloged_person`. See +[ADR 0009](docs/adr/0009-cross-post-actor-identity.md). New +`cataloged_team` catalog (`migrations/0016_cross_post_actor_identity.sql`, +identity key `(team_name, affiliated_organization_name)` -- a bare team +name like "설계팀" is not by itself identifying) plus two mention join +tables (`post_team_mention`, `post_organization_mention`); an +organization actor reuses the existing `corporate_entity` catalog, no +new table needed. `lineageweave/knowledge_graph.py`'s +`knowledge_graph_edges_for_post` extended with three new edge kinds +(`edge_mention_team`, `edge_team_affiliation`, `edge_mention_organization`); +`backend/app/post_summary_ingestion.py`'s `persist_post_summary` now +resolves each R&R actor's identity, stores that id on +`post_summary_role` (ADR 0019 — `entity_name` is not unique), and calls +the same `persist_edges_for_post` Keyman ingestion already uses. A person R&R +actor is opportunistically joined to an existing `cataloged_person` row +by name (never originated by R&R itself -- documented gap in the ADR: +`cataloged_person` needs `person_side_code`, which R&R's prompt does +not currently capture). + +## Phase 12: a real counterparty organization is auto-created, not left permanently unresolved + +`corporate_hierarchy_resolution`'s similarity matching only ever finds +an ALREADY-cataloged entity. Real Milestone 2 data confirmed the actual +gap: 0 of 4,154 person affiliations and 0 of 9,852 R&R organization +mentions ever resolved -- the standing "통합 고객사 계열 tree AI" +requirement was never actually populated. See +[ADR 0010](docs/adr/0010-corporate-hierarchy-auto-creation.md). + +New `lineageweave/corporate_hierarchy_inference.py` proposes a +Group/Company/Plant placement from context; new +`backend/app/corporate_entity_ingestion.py`'s +`get_or_create_corporate_entity` tries similarity matching first, then +creates a real new `corporate_entity` row once the proposal is +Searxng-corroborated (reusing `relation_verification`, no new search +integration), recursing up a bounded parent chain so the whole +hierarchy gets real links. Auto-created rows get a deterministic +`AUTO-`-prefixed code so they can never collide with a real login corp +code. Wired into both `keyman_ingestion.py`'s affiliation loop and +`post_summary_ingestion.py`'s R&R organization-actor loop. +## Standards-complete W3C PROV-O provenance layer + +ADR 0011 separates standards-complete provenance from the compact +buyer-facing navigation graph. `lineageweave/prov_o.py` validates +and materializes all 50 normative PROV-O properties, including +literal-valued times/values and qualified Influence resources. +`migrations/0017_prov_o_standard_relations.sql` stores definitions, +class/property hierarchies, domains, ranges, qualification maps, +inverse names, typed resources, literals, assertions, and inference +premises in third normal form. Existing product nodes cross the +boundary only through `provenance_resource_binding`; projection to +`knowledge_graph_edge` is explicit and reversible. + +See `docs/PROV_O_IMPLEMENTATION.md`, the complete implementation +matrix, and `docs/adr/0011-prov-o-standard-relations.md`. + +## Phase 13: corporate-entity creation is serialized against a real observed deadlock + +Phase 12's creation path made real concurrent writes for the first +time. A synthetic regression corpus batch run under real concurrency surfaced a +genuine `DeadlockDetectedError`: two concurrent transactions each +creating a different new entity, mentioned in opposite order across +two different posts, took row-level locks in opposite order and +deadlocked. See [ADR 0012](docs/adr/0012-corporate-entity-creation-lock.md). + +`get_or_create_corporate_entity` now takes a single named Postgres +advisory transaction lock (`pg_advisory_xact_lock`) immediately before +the write -- never held across the slow LLM inference/Searxng +verification calls that precede it -- and re-checks candidates fresh +under the lock before inserting. The lock key is fixed, not per-name, +so it also covers the multi-entity opposite-order case a per-name lock +would still deadlock on. Every already-cataloged entity still resolves +through the unchanged, lock-free similarity-matching fast path; only +the rare creation branch serializes. diff --git a/CHANGELOG.d/0.76.0.md b/CHANGELOG.d/0.76.0.md new file mode 100644 index 000000000..a16208167 --- /dev/null +++ b/CHANGELOG.d/0.76.0.md @@ -0,0 +1,3 @@ +# 0.76.0 — W3C PROV-O standard relations + +This release fragment is the machine-local source for the root changelog entry added by the one-shot documentation workflow. It records complete support for the Recommendation's 30 classes, 50 normative properties, 14 qualification patterns, Appendix B inverse names, normalized PostgreSQL persistence, exact RDF serialization, and 100% owned-module statement/branch coverage. diff --git a/CHANGELOG.d/0.77.0-review-hardening.md b/CHANGELOG.d/0.77.0-review-hardening.md new file mode 100644 index 000000000..873e93450 --- /dev/null +++ b/CHANGELOG.d/0.77.0-review-hardening.md @@ -0,0 +1,21 @@ +# 0.77.0 — Review hardening follow-up + +- Vision response parsing now treats caption and tag values as single-line + fields, strips balanced outer Markdown emphasis, and prevents trailing model + commentary from entering searchable tags or captions. +- The dedicated PROV-O workflow now fails when its PostgreSQL service is + unreachable instead of allowing the database contract module to skip. +- PROV-O database fixtures preserve DSN query options and quote generated + database identifiers through `psycopg2.sql.Identifier`. +- PostgreSQL lexical `xsd:dateTime` validation now rejects offsets outside + the XSD range of `Z` / `±hh:mm` with a maximum of `±14:00`, so + `+14:01` fails closed instead of being accepted as `timestamptz`. +- Corporate-entity inference, external verification, and the short advisory- + lock creation transaction now finish before the atomic post-summary + replacement transaction begins, so network latency never extends summary + write locks while post-owned rows still commit or roll back together. +- Keyman and R&R person mentions now replace separate source-owned projections, + while one canonical graph edge retains normalized per-post evidence. This + prevents a summary refresh from deleting Keyman facts or leaving removed R&R + actors visible in buyer navigation. +- The implementation matrix follows portable Markdown table spacing. diff --git a/CHANGELOG.d/0.79.0-analysis-run-authorized-read.md b/CHANGELOG.d/0.79.0-analysis-run-authorized-read.md new file mode 100644 index 000000000..7233020b2 --- /dev/null +++ b/CHANGELOG.d/0.79.0-analysis-run-authorized-read.md @@ -0,0 +1,9 @@ +# 0.79.0 — Authorized analysis-run read projection + +## Added + +- `GET /api/analysis-runs` and `GET /api/analysis-runs/{id}` expose + source-redacting registry evidence to `post_read` accounts. +- Home-page Analysis runs panel shows the seeded Demo Corp lineage run + after `make seed`. Hidden scopes 404. No raw source, DSN, or provider + payload is returned. diff --git a/CHANGELOG.d/0.80.0-analysis-run-detail-click.md b/CHANGELOG.d/0.80.0-analysis-run-detail-click.md new file mode 100644 index 000000000..090ebb824 --- /dev/null +++ b/CHANGELOG.d/0.80.0-analysis-run-detail-click.md @@ -0,0 +1,5 @@ +# 0.80.0 analysis-run detail click + +Home Analysis runs rows open `GET /api/analysis-runs/{id}`. The +detail shows labeled aggregates and dates only. Hidden runs stay +404 / "not visible". Synthetic Demo Corp seed only. diff --git a/CHANGELOG.d/0.81.0-analysis-run-status-history.md b/CHANGELOG.d/0.81.0-analysis-run-status-history.md new file mode 100644 index 000000000..9fa4b8706 --- /dev/null +++ b/CHANGELOG.d/0.81.0-analysis-run-status-history.md @@ -0,0 +1,5 @@ +# 0.81.0 analysis-run status history + +Detail of `GET /api/analysis-runs/{id}` shows the labeled append-only +lifecycle. The list stays latest-status only. Hidden runs 404. +Synthetic Demo Corp seed only. diff --git a/CHANGELOG.d/0.82.0-analysis-run-post-clickthrough.md b/CHANGELOG.d/0.82.0-analysis-run-post-clickthrough.md new file mode 100644 index 000000000..1802a5644 --- /dev/null +++ b/CHANGELOG.d/0.82.0-analysis-run-post-clickthrough.md @@ -0,0 +1,4 @@ +# 0.82.0 analysis-run post click-through + +Detail lists ABAC-visible post titles in the run scope. Hidden +other-corp private posts never appear. Synthetic titles only. diff --git a/CHANGELOG.d/0.83.0-analysis-run-cutoff-and-keyman-upgrade.md b/CHANGELOG.d/0.83.0-analysis-run-cutoff-and-keyman-upgrade.md new file mode 100644 index 000000000..f2de7cfad --- /dev/null +++ b/CHANGELOG.d/0.83.0-analysis-run-cutoff-and-keyman-upgrade.md @@ -0,0 +1,2 @@ +Analysis-run detail applies knowledge_cutoff to visible posts. Migration +0016 no longer deletes overlapping Keyman mention_context. diff --git a/CHANGELOG.d/0.84.0-tepp-analysis-run.md b/CHANGELOG.d/0.84.0-tepp-analysis-run.md new file mode 100644 index 000000000..df127a9d4 --- /dev/null +++ b/CHANGELOG.d/0.84.0-tepp-analysis-run.md @@ -0,0 +1,8 @@ +# 0.84.0 TEPP analysis-run seed + +Seed writes `analysis_run_tepp` via `tepp_client` on the shared Demo +Corp snapshot. The home list shows Failed and a kind-specific next +action; detail history keeps `tepp_not_available`. Missing transport +is not a fake measurement. A failed lineage row does not mention TEPP. +A failed period-report row rebuilds the report. A pending TEPP row +does not claim a calibrated measurement. diff --git a/CHANGELOG.d/0.84.1-analysis-run-digest-a11y.md b/CHANGELOG.d/0.84.1-analysis-run-digest-a11y.md new file mode 100644 index 000000000..213eb5451 --- /dev/null +++ b/CHANGELOG.d/0.84.1-analysis-run-digest-a11y.md @@ -0,0 +1,5 @@ +# 0.84.1 Analysis-run digest a11y and live-body warning + +Detail prefixes stay audible and hoverable. Open a cutoff title only +after reading that the live body may have changed since the run. +The list stays aggregates-only. diff --git a/CHANGELOG.d/0.85.0-analysis-run-create.md b/CHANGELOG.d/0.85.0-analysis-run-create.md new file mode 100644 index 000000000..505320a19 --- /dev/null +++ b/CHANGELOG.d/0.85.0-analysis-run-create.md @@ -0,0 +1,5 @@ +# 0.85.0 authorized analysis-run create + +`POST /api/analysis-runs` records Pending on an authorized cutoff +capture. Request a lineage reconstruction from the home list. This +write does not invent a measurement. diff --git a/CHANGELOG.d/0.86.0-related-nodes-team-org-walk.md b/CHANGELOG.d/0.86.0-related-nodes-team-org-walk.md new file mode 100644 index 000000000..6e4605d5a --- /dev/null +++ b/CHANGELOG.d/0.86.0-related-nodes-team-org-walk.md @@ -0,0 +1,5 @@ +Related-node walks include team and organization mention edges. Click an +R&R team to open sibling posts. Thread-group run lists honor knowledge_cutoff. +Failed period-report rows rebuild the report; a pending TEPP corpus +does not claim a calibrated measurement. A pending lineage row says +reconstruction has not started yet. diff --git a/CHANGELOG.d/0.86.2-role-catalog-identity.md b/CHANGELOG.d/0.86.2-role-catalog-identity.md new file mode 100644 index 000000000..a56b3d47e --- /dev/null +++ b/CHANGELOG.d/0.86.2-role-catalog-identity.md @@ -0,0 +1,3 @@ +R&R organization buttons walk the catalog id stored on the role row. A +shared display name no longer attaches a homonym. Team related matches +person/entity 403/404. diff --git a/CHANGELOG.d/0.86.3-embedded-image-fallback.md b/CHANGELOG.d/0.86.3-embedded-image-fallback.md new file mode 100644 index 000000000..93a9b2224 --- /dev/null +++ b/CHANGELOG.d/0.86.3-embedded-image-fallback.md @@ -0,0 +1,2 @@ +Tag-only or charset/unquoted data-URI bodies no longer dump raw source. +Re-export with the picture embedded. This screen does not OCR the picture. diff --git a/CHANGELOG.d/milestone2-analysis-run-registry.md b/CHANGELOG.d/milestone2-analysis-run-registry.md new file mode 100644 index 000000000..5d7e0288f --- /dev/null +++ b/CHANGELOG.d/milestone2-analysis-run-registry.md @@ -0,0 +1,21 @@ +## Added + +- Added a normalized PostgreSQL registry for immutable source snapshots, + aggregate reconciliation counts, authenticated analysis requests, product + scopes, and append-only lifecycle evidence. +- Added a run-owned knowledge cutoff and snapshot-owned evidence-availability + clock so one capture can support multiple historically valid analyses without + future-information leakage. +- Added canonical account-scoped idempotency, immutable request and scope evidence, + deletion resistance, serialized count/run locking, scope-required request-time- + ordered lifecycle transitions, database-owned record time, and a derived + current-status view. +- Added fail-closed rollback, real-PostgreSQL contract tests, ADR 0013, and APA + 7th standards traceability. + +## Security + +- The registry deliberately excludes source SQL, DSNs, raw records, inline + images, provider payloads, credentials, private source identifiers, and raw + exceptions. Necessary PII remains in purpose-bound authorized product/source + contexts rather than being copied into audit metadata or blanket-masked. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0096828a2..e2b30b214 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,301 @@ All notable changes to this project are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.86.3] - 2026-08-16 + +### Fixed + +- Opening a post whose body is only a remote `http(s)` image, a + `charset=` data-URI, an unquoted `src`, or undecodable base64 no + longer dumps the raw source string. Re-export the source with the + picture embedded, then open the post again. A valid 1x1 PNG still + renders as a picture. This screen does not read text inside the + picture; Extract Keyman and Ask stay on their own channels and stay + silent when the vision client is Null. + +## [0.86.2] - 2026-08-16 + +### Fixed + +- An R&R organization button now walks the catalog id stored on that + role row (ADR 0019). Two catalog orgs can share a display name; open + the post, click the name, and you stay on the resolved org — not a + homonym. `GET /api/teams/{id}/related` matches person/entity authz: + another corp's private-only team is 403; an unknown UUID is 404. + +## [0.86.1] - 2026-08-16 + +### Changed + +- Opening a post or its evidence panel now shows each embedded + `data:image` picture in document order, with the surrounding sentences + as text. The raw base64 string is no longer dumped into the popup. + Remote `http(s)` image URLs stay unloaded. After `make seed`, a post + whose body includes a data-URI image shows the picture. + +### Fixed + +- Opening a Pending lineage run repeats that reconstruction has not + started. Pending next-action copy is pinned to the registered run + kinds, so a Pending TEPP row does not say reconstruction. + +## [0.86.0] - 2026-08-16 + +### Added + +- Related-node walks now include team and organization mention edges. + After `make seed` and a summary that names 설계팀 on two posts, open + either post, click the R&R team, and open the sibling post (ADR 0018). + A team-only follow-up is no longer an island. +- `GET /api/teams/{team_id}/related` starts the same RWR walk Keyman + and corporate-entity related already use. Related team chips are + buttons. + +### Fixed + +- Thread-group analysis-run *lists* now require an in-cutoff visible + post. A later public post in that thread group no longer surfaces a + January run the account was not allowed to know. +- Failed period-report rows tell the operator to rebuild the report. + Next-action copy is pinned to the registered run kinds. A pending + TEPP corpus does not claim a calibrated measurement. + +## [0.85.0] - 2026-08-16 + +### Added + +- `POST /api/analysis-runs` records a Pending lineage or TEPP run on an + authorized cutoff capture (ADR 0017). The home panel's **Request a + lineage reconstruction** button writes that row so an operator can + confirm the cutoff corpus immediately. Reconstruction and live TEPP + execution stay later slices — this write never invents a theta. +- Failed lineage rows tell the operator to retry reconstruction; only + Failed TEPP rows mention the measurement service. Pending rows say + reconstruction has not started yet. + +## [0.84.1] - 2026-08-16 + +### Fixed + +- Analysis-run detail keeps 12-character digest prefixes as visible + text (so assistive technology hears `Code` / `Config` values) and + puts the full digest on hover. Open the Demo Corp lineage run, hover + a prefix, and match it to the API payload. The home list still hides + digests even when the list JSON includes them. +- Opening a cutoff title now says the live body may have changed after + that run. Compare the opened post with the cutoff date before you + treat it as reconstructed evidence (ADR 0016). + +## [0.84.0] - 2026-08-16 + +### Added + +- `make seed` records a Demo Corp TEPP measurement run through + `tepp_client` on the same snapshot as the lineage run (ADR 0013). + The default transport is unavailable, so the home list shows + "TEPP measurement · Failed · Demo Corp" and tells the operator to + open the run, then connect the measurement service. Detail history + keeps `tepp_not_available` -- never a fabricated theta. TEPP stays + a wire client, not a local psychometric engine. `make seed` skips + snapshot-count inserts once counts exist so a re-run does not hit + the freeze trigger. A failed lineage row tells the operator to retry + reconstruction; only a failed TEPP row mentions the measurement + service. A failed period-report row tells the operator to rebuild + the report from a current snapshot. A pending TEPP row does not + claim a calibrated measurement. Stacked PRs now run the same + GitHub Checks as PRs to main. + +## [0.83.0] - 2026-08-16 + +### Fixed + +- Analysis-run detail now lists only ABAC-visible posts whose + `created_at` is at or before that run's `knowledge_cutoff`. After + `make seed`, open the Demo Corp lineage run: Demo public post is + there; a later own-corp follow-up is not. The live post list is + unchanged. Click a listed title to inspect what that cutoff + reconstructed (ADR 0016). +- Upgrading through `0016_cross_post_actor_identity.sql` copies R&R + person names into `post_summary_person_mention` and leaves Keyman + `post_person_mention.mention_context` in place. Re-run Keyman only + when you want a new Keyman set -- a later summary no longer erases + the stolen row. + +## [0.82.0] - 2026-08-16 + +### Added + +- Analysis-run detail lists ABAC-visible posts in the run's scope. + After `make seed`, the Demo Corp lineage run opens the Demo public + post. Hidden other-corp private posts never appear. List payloads + stay aggregates-only. + +## [0.81.0] - 2026-08-16 + +### Added + +- Analysis-run detail shows the labeled lifecycle: Pending, Running, + then Succeeded, with occurrence times from `analysis_run_status_event`. + The list stays latest-status only. Hidden runs still 404 and never + leak events. Failure codes stay machine tokens -- no invented label. + Synthetic Demo Corp seed only. + +## [0.80.0] - 2026-08-16 + +### Added + +- Home Analysis runs rows are buttons. Clicking the seeded Demo Corp + lineage run opens `GET /api/analysis-runs/{id}` and shows cutoff, + requested date, and document count. A hidden run is "This analysis + run is not visible." -- never a raw 404 or a DSN. Still synthetic + aggregates only. + +## [0.79.0] - 2026-08-16 + +### Added + +- Authorized analysis-run evidence on the product home page. After + `make seed`, Demo Analyst sees "Lineage reconstruction · Succeeded · + Demo Corp" with the synthetic document count. `GET /api/analysis-runs` + is scoped in SQL: another tenant's run 404s and never appears in the + list. The payload is labels and aggregates -- never source SQL, a DSN, + or a raw record. TEPP stays behind `tepp_client`; Null channels are + unchanged. + +## [0.78.0] - 2026-08-15 + +### Changed + +- Related-node person chips now use the localized `person_side` lookup label + supplied by the authorized API payload. Users see business context such as + `Our side` or `Counterparty`, while ontology class metadata remains available + separately for semantic processing and provenance. +- Related-person buttons now expose that same caption in the accessible name, so + assistive technology hears `Related nodes for Priya Nair (Counterparty)` + instead of the name alone. +- Structured extraction, summarization, commitment, relationship-classification, + and LLM-as-a-Judge consumers now request contextual-orchestrator `auto` mode + so the orchestration plane can meet the quality requirement and then minimize + known execution cost. Explicit checked `verify` paths remain unchanged + (ADR 0015). + +## [0.77.0] - 2026-08-14 + +### Fixed + +- Keyman and R&R person mentions now replace independent source projections. Knowledge Graph edges have one canonical identity plus post-level evidence, so removed actors and concurrent writes cannot leave stale or duplicate buyer-visible relationships. +- Vision-response parsing now strips balanced outer Markdown emphasis from field values + while still accepting emphasized field labels, so OCR such as + ``TEXT: **LT7**`` is not truncated. +- A real live synthetic regression batch run surfaced a genuine + `DeadlockDetectedError` from concurrent corporate-entity creation: + two concurrent transactions each creating a different new entity, + mentioned in opposite order across two different posts, took + row-level locks in opposite order and deadlocked. Entity *creation* + (the rare, first-mention-only branch) now serializes through a + single named Postgres advisory transaction lock, taken only right + before the write and auto-released at commit/rollback -- the + lock-free similarity-matching fast path every already-cataloged + entity resolves through is unaffected. See ADR 0012. + +## [0.76.0] - 2026-08-14 + +### Added + +- Standards-complete W3C PROV-O support: all 30 classes, all 50 + normative properties, both qualification tables, qualified-to- + unqualified implication, property hierarchy, defined inverses, + Appendix B inverse-name normalization, RDF serialization, and a + normalized PostgreSQL assertion store with fail-closed domain, + range, object-kind, and datatype enforcement (ADR 0011). +- A dedicated exact-head PROV-O contract workflow runs the complete + registry/inference suite, real PostgreSQL migration tests, public + docstring checks, and 100% statement/branch coverage for the owned + runtime module. + +### Changed + +- The product navigation graph remains an explicit projection; + literal-valued and qualified provenance is no longer forced into + `knowledge_graph_edge`. + +### Fixed + +- Review hardening verifies complete hierarchy placement, rejects parent + failures and cycles, propagates canonical affiliations, replaces stale + actor projections, enforces atomic team identity, validates timezone-aware + `xsd:dateTime` literals (including the XSD `±14:00` offset bound), and + protects referenced provenance rows. + +## [0.75.0] - 2026-08-14 + +### Added + +- A real counterparty organization mentioned for the first time now + gets auto-created into the corporate hierarchy, not left permanently + unresolved. Synthetic regression fixtures prove the first-mention gap. + An LLM proposes a + Group/Company/Plant placement from context; a real new + `corporate_entity` row is only created once the proposal is + search-corroborated (reusing the existing Searxng verification + client, no new search integration). Auto-created rows get a + deterministic `AUTO-`-prefixed code, kept structurally separate from + the real login corp-code namespace. Wired into both Keyman + affiliation resolution and R&R organization-actor resolution. + +## [0.74.0] - 2026-08-14 + +### Added + +- R&R team and organization actors now get a shared identity across + posts, not just per-post free text -- the same "설계팀" (design + team) named in ten posts resolves to one `cataloged_team` row + (identity key: team name + parent org, since a bare team name is not + by itself identifying), and an organization actor resolves against + the existing `corporate_entity` catalog. Each resolved actor gets a + real Knowledge Graph mention edge (`edge_mention_team`, + `edge_team_affiliation`, `edge_mention_organization`), so extraction + results now genuinely become cross-post lineage clues instead of + per-post islands. A person R&R actor is opportunistically joined to + an existing Keyman-cataloged person by name (documented gap: R&R does + not yet originate new person identities itself -- see ADR 0009). + +## [0.73.0] - 2026-08-14 + +### Fixed + +- Image OCR/caption parsing no longer discards a real vision response + just because its formatting was close but not exact (bolded labels + like `**TEXT:**`, reordered labels, or a missing TAGS line) -- + observed live against synthetic embedded-image fixtures in the synthetic regression batch + in format-variation fixtures. Fields are now recovered independently; only a + response with neither TEXT nor CAPTION content is treated as + unusable. A strict format mismatch was silently producing the same + "[image: content unavailable]" placeholder as a genuinely unavailable + vision channel, discarding real, already-paid-for content. + +## [0.72.0] - 2026-08-14 + +### Added + +- Abbreviated/slang organization names (e.g. "AGP") are now resolved + to their canonical name ("Aurora Grid Power") via LLM context, then + cross-verified against external search before being trusted -- new + `lineageweave/organization_name_resolution.py`, reusing the existing + Searxng verification client rather than a second web-search + integration. Cached in a new `organization_name_resolution` table + keyed by the raw name. +- Wired into Keyman affiliation ingestion (both the API path and the + offline synthetic-batch script): a search-corroborated resolution feeds + `resolve_corporate_entity`, an unverified one leaves the raw name + unchanged. + +### Fixed + +- The offline synthetic-batch script's own re-implementation of Keyman + affiliation persistence was missing `role_title` entirely (a stale + copy that predated that feature) -- fixed alongside this change. + ## [0.71.0] - 2026-08-14 ### Added @@ -14,6 +309,61 @@ All notable changes to this project are documented here. Format follows above the names. Unassigned excerpts stay in the list; a post with no named organization still says so. +## [0.70.0] - 2026-08-14 + +### Added + +- R&R's `actor_type_code` gains `prov_team`, a meso-level actor type for + a named sub-unit of a company (e.g. "설계팀"/design team) -- distinct + from both a person and the company itself. Grounded in the W3C + Organization Ontology's `org:OrganizationalUnit` (Reynolds, 2014). +- A team actor now requires `affiliated_organization_name` in the same + way a person actor does -- a team's own name never answers "which + company." +- R&R badge shows a distinct "Team" label/color, not the prior binary + Person/Organization ternary (which would have mislabeled a team as + "Organization"). + +## [0.69.0] - 2026-08-14 + +### Added + +- Keyman extraction now captures a stated job title/position + (`PersonMention.job_title`), since two different real people can + share a name and a title is real evidence for telling them apart. + Persisted to `person_affiliation.role_title` (an existing schema + column, previously never populated) and a new + `cataloged_person.last_known_job_title` for a title stated without a + named organization to attach it to. +- `_upsert_person` no longer blindly merges a same-name+side match: a + genuinely conflicting stated title creates a fresh person row instead + of reusing one, verified by a real test with two posts naming the + same name and different titles. +- Keyman panel shows the person's title next to their name. After + `make seed`, Ada West is "Account manager" and Priya Nair is + "Procurement lead" so the title is visible without a live LLM. + +### Fixed + +- `scripts/seed_demo_data.py` no longer embeds the local Keycloak admin + password. `make seed` still supplies the compose default via + `KEYCLOAK_ADMIN_PASSWORD`; a direct script run requires that env var + or `--keycloak-admin-password`. + +## [0.68.0] - 2026-08-14 + +### Changed + +- R&R's named actor is no longer forced into a person slot. A + business post can name an organization acting in its own name + ("당사," "Demo Corp"), not an individual -- + `RoleResponsibility.actor_name` (renamed from `person_name`) now + carries `actor_type_code` (Person / Organization, W3C PROV-O + grounded) and an LLM-inferred `affiliated_organization_name` for + person actors. The popup's R&R list shows a Person/Organization + badge and the inferred affiliation; only a person actor still links + to the Keyman panel. See ADR 0006. + ## [0.67.0] - 2026-08-14 ### Added @@ -1097,7 +1447,7 @@ All notable changes to this project are documented here. Format follows embedding, against the live embedding provider. - `docs/lineage-bi-research-notes.md`: new "Chunking" section with the four units' grounding and an explicit, honest note that this project's - real dataset's only free-text field is too short to need chunking in + unseen dataset's only free-text field is too short to need chunking in practice -- the module exists for richer content sources (e.g. the raw MHTML artifacts that dataset's records were derived from). - `lineageweave/image_content.py`: pluggable vision channel for base64 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..c5a1828f2 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,23 @@ +# CLAUDE.md + +Tool-specific pointer. Policy lives in [AGENTS.md](AGENTS.md) and the +ADRs under `docs/adr/`. Do not fork those rules here. + +## Analysis-run seed (v0.85.0) + +`make seed` writes a Demo Corp lineage run and a TEPP run on the same +snapshot (ADR 0013). The TEPP path goes through `tepp_client`. A missing +transport or an unused accepted envelope is Failed +(`tepp_not_available` / `tepp_result_not_persisted`). Do not invent a +theta or a local psychometric substitute. The home list caption stays +`kind · status · entity`; the machine failure code is detail-only +(ADR 0014). Open a Failed TEPP row, then connect a live TEPP +transport. A failed lineage row retries reconstruction -- it does not +mention TEPP. A failed period-report row rebuilds the report. A +pending TEPP row does not claim a calibrated measurement. A pending +lineage row says reconstruction has not started yet. +Digest prefixes stay audible; hover a prefix to read the full digest. +Opening a cutoff title shows the live post -- compare it with the +cutoff before treating the body as reconstructed evidence (ADR 0016). +`POST /api/analysis-runs` records Pending on an authorized +cutoff capture (ADR 0017) and does not reconstruct lineage. diff --git a/Makefile b/Makefile index 68ee850eb..82aa30faf 100644 --- a/Makefile +++ b/Makefile @@ -23,4 +23,5 @@ smoke: # users' real subject ids, plus Valkey ticket_created events so Activity # is not empty (see scripts/seed_demo_data.py). Run after `up`. seed: + @test -n "$${KEYCLOAK_ADMIN_PASSWORD:-}" || { echo "KEYCLOAK_ADMIN_PASSWORD is required" >&2; exit 1; }; \ python3 scripts/seed_demo_data.py diff --git a/backend/Dockerfile b/backend/Dockerfile index b0c504cc8..eb6b86288 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,33 +1,37 @@ FROM python:3.12-slim@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36 WORKDIR /app -# rankweave and fast-mlsirm install from a git URL (see pyproject.toml) -- -# neither has a PyPI release yet. fast-mlsirm additionally ships a -# PyO3/maturin Rust core with no fallback wheel (ADR 0003), so this build -# needs a real Rust toolchain, not just Python -- build-essential supplies -# the C linker maturin needs on Linux. Create the runtime user here so the -# later USER instruction is not a no-op against a missing account -# (DS-0002: explicit non-root USER). -RUN apt-get update && apt-get install -y --no-install-recommends git build-essential curl ca-certificates \ +# Hash-pinned bootstrap input is copied before dependency installation so the +# build cannot resolve a different uv artifact for the same version. +COPY backend/uv-bootstrap-requirements.txt /tmp/uv-bootstrap-requirements.txt + +# rankweave and fast-mlsirm install from immutable git commit references. +# fast-mlsirm builds a PyO3/maturin core, so the image needs a C linker and +# the repository-pinned Rust toolchain. The runtime user is created before +# dependencies so no application process runs as root. +RUN apt-get update && apt-get install -y --no-install-recommends \ + git build-essential curl ca-certificates \ && rm -rf /var/lib/apt/lists/* \ && groupadd --gid 1000 appuser \ - && useradd --uid 1000 --gid appuser --create-home appuser + && useradd --uid 1000 --gid appuser --create-home appuser \ + && python -m pip install --no-cache-dir --no-deps \ + --require-hashes --only-binary=:all: \ + -r /tmp/uv-bootstrap-requirements.txt -# Pinned, non-interactive rustup install (minimal profile: no docs/clippy, -# just rustc+cargo) -- same "pinned, reproducible install" discipline this -# project already applies to rankweave/fast-mlsirm's own commit pins. RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ sh -s -- -y --profile minimal --default-toolchain 1.97.1 -ENV PATH="/root/.cargo/bin:${PATH}" +ENV PATH="/app/.venv/bin:/root/.cargo/bin:${PATH}" -COPY pyproject.toml ./ +COPY pyproject.toml uv.lock README.md ./ COPY lineageweave ./lineageweave COPY backend ./backend # lineageweave/ontology.py resolves this path relative to itself # (parents[1] = /app) -- ADR 0004. COPY docs/ontology ./docs/ontology -RUN pip install --no-cache-dir ".[backend]" \ +# Install exactly the committed universal lock. --no-editable prevents a +# runtime dependency on source-tree editability while retaining package data. +RUN uv sync --frozen --no-dev --extra backend --no-editable \ && chown -R appuser:appuser /app USER appuser diff --git a/backend/app/analysis_run_ingestion.py b/backend/app/analysis_run_ingestion.py new file mode 100644 index 000000000..d26eb6f6e --- /dev/null +++ b/backend/app/analysis_run_ingestion.py @@ -0,0 +1,650 @@ +"""Authorized, source-redacting reads of the Milestone 2 analysis-run registry. + +The registry itself is issue #89 / migration 0018. This module is the +product projection: an account sees only runs they requested or whose +scope they already have ABAC authority to walk. Aggregate counts and +lookup labels come back; source SQL, DSNs, raw records, and provider +payloads never do. + +``create_pending_analysis_run`` (ADR 0017) writes snapshot, counts, run, +scope, and the first Pending event atomically. It does not reconstruct +lineage or invent a TEPP score. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any +from uuid import UUID + +import asyncpg + +from backend.app.knowledge_graph import labels_for_codes +from lineageweave import __version__ as PACKAGE_VERSION + +_ALLOWED_CREATE_KINDS = frozenset({"analysis_run_lineage", "analysis_run_tepp"}) +_CORPORATE_SCOPE = "analysis_scope_corporate_entity" +_CAPTURE_CONTRACT_VERSION = "analysis-run-capture-v1" +_KIND_SCHEMA_VERSION = { + "analysis_run_lineage": "lineage-run-v1", + "analysis_run_tepp": "tepp-run-v1", +} + +_VISIBLE_RUN_SQL = """ + run.requested_by_account_id = $1 + or ( + scope.scope_kind_code = 'analysis_scope_corporate_entity' + and scope.corporate_entity_id = any($2::uuid[]) + ) + or ( + scope.scope_kind_code = 'analysis_scope_process_unit' + and exists ( + select 1 from account_affiliation aff + where aff.user_account_id = $1 + and aff.process_unit_id = scope.process_unit_id + ) + ) + or ( + scope.scope_kind_code = 'analysis_scope_thread_group' + and exists ( + select 1 from source_post p + where p.thread_group_key = scope.scope_key + and p.created_at <= run.knowledge_cutoff + and ( + p.visibility_code = 'public' + or p.corporate_entity_id = any($2::uuid[]) + ) + ) + ) +""" + +_RUN_SELECT = f""" + select + run.analysis_run_id, + run.run_kind_code, + run.knowledge_cutoff, + run.requested_at, + run.configuration_schema_version, + run.configuration_sha256, + run.code_revision_sha, + scope.scope_kind_code, + scope.corporate_entity_id, + scope.process_unit_id, + scope.scope_key, + corp.entity_name as scope_entity_name, + status.status_code, + status.failure_code + from analysis_run run + join analysis_run_scope scope on scope.analysis_run_id = run.analysis_run_id + left join analysis_run_current_status status + on status.analysis_run_id = run.analysis_run_id + left join corporate_entity corp + on corp.corporate_entity_id = scope.corporate_entity_id + where {{where}} + order by run.requested_at desc +""" + + +def _iso(value: Any) -> str: + """Serialize a timestamptz the same way post payloads do.""" + return value.isoformat() if hasattr(value, "isoformat") else str(value) + + +async def _counts_by_run( + conn: asyncpg.Connection, + run_ids: list[str], +) -> dict[str, list[asyncpg.Record]]: + """Load aggregate snapshot counts for the given runs.""" + if not run_ids: + return {} + rows = await conn.fetch( + """ + select run.analysis_run_id, counts.count_type_code, counts.count_value + from analysis_run run + join analysis_source_count counts + on counts.analysis_source_snapshot_id = run.analysis_source_snapshot_id + where run.analysis_run_id = any($1::uuid[]) + order by counts.count_type_code + """, + run_ids, + ) + grouped: dict[str, list[asyncpg.Record]] = {} + for row in rows: + grouped.setdefault(str(row["analysis_run_id"]), []).append(row) + return grouped + + +async def _status_history( + conn: asyncpg.Connection, + analysis_run_id: str, +) -> list[dict[str, Any]]: + """Labeled append-only lifecycle for one already-visible run.""" + rows = await conn.fetch( + """ + select status_ordinal, status_code, occurred_at, failure_code + from analysis_run_status_event + where analysis_run_id = $1::uuid + order by status_ordinal + """, + analysis_run_id, + ) + labels = await labels_for_codes(conn, [row["status_code"] for row in rows]) + history: list[dict[str, Any]] = [] + for row in rows: + item: dict[str, Any] = { + "status_ordinal": int(row["status_ordinal"]), + "status_code": row["status_code"], + "status_label": labels.get(row["status_code"], row["status_code"]), + "occurred_at": _iso(row["occurred_at"]), + } + if row["failure_code"]: + item["failure_code"] = row["failure_code"] + history.append(item) + return history + + +async def _serialize_runs( + conn: asyncpg.Connection, + rows: list[asyncpg.Record], +) -> list[dict[str, Any]]: + """Project registry rows into the authorized buyer-facing payload.""" + if not rows: + return [] + count_rows = await _counts_by_run(conn, [str(row["analysis_run_id"]) for row in rows]) + labels = await labels_for_codes( + conn, + [row["run_kind_code"] for row in rows] + + [row["scope_kind_code"] for row in rows] + + [row["status_code"] for row in rows if row["status_code"]] + + [ + count["count_type_code"] + for counts in count_rows.values() + for count in counts + ], + ) + payload: list[dict[str, Any]] = [] + for row in rows: + run_id = str(row["analysis_run_id"]) + kind = row["run_kind_code"] + scope = row["scope_kind_code"] + status = row["status_code"] + item: dict[str, Any] = { + "analysis_run_id": run_id, + "run_kind_code": kind, + "run_kind_label": labels.get(kind, kind), + "scope_kind_code": scope, + "scope_kind_label": labels.get(scope, scope), + "status_code": status, + "status_label": labels.get(status, status) if status else None, + "knowledge_cutoff": _iso(row["knowledge_cutoff"]), + "requested_at": _iso(row["requested_at"]), + "source_counts": [ + { + "count_type_code": count["count_type_code"], + "count_type_label": labels.get( + count["count_type_code"], count["count_type_code"] + ), + "count_value": int(count["count_value"]), + } + for count in count_rows.get(run_id, []) + ], + } + if row["scope_entity_name"]: + item["scope_entity_name"] = row["scope_entity_name"] + payload.append(item) + return payload + + +async def fetch_visible_analysis_runs( + conn: asyncpg.Connection, + account_id: str, + affiliated_entity_ids: list[str], +) -> list[dict[str, Any]]: + """Runs the account requested or whose scope they may already walk.""" + rows = await conn.fetch( + _RUN_SELECT.format(where=_VISIBLE_RUN_SQL), + account_id, + affiliated_entity_ids, + ) + return await _serialize_runs(conn, rows) + + +async def fetch_visible_analysis_run( + conn: asyncpg.Connection, + analysis_run_id: str, + account_id: str, + affiliated_entity_ids: list[str], +) -> dict[str, Any] | None: + """One visible run, or None when it is missing or hidden.""" + rows = await conn.fetch( + _RUN_SELECT.format( + where=f"run.analysis_run_id = $3 and ({_VISIBLE_RUN_SQL})" + ), + account_id, + affiliated_entity_ids, + analysis_run_id, + ) + payload = await _serialize_runs(conn, rows) + if not payload: + return None + detail = payload[0] + row = rows[0] + detail["configuration_schema_version"] = row["configuration_schema_version"] + detail["configuration_sha256"] = row["configuration_sha256"] + detail["code_revision_sha"] = row["code_revision_sha"] + if row["failure_code"]: + detail["failure_code"] = row["failure_code"] + detail["status_history"] = await _status_history(conn, analysis_run_id) + detail["visible_posts"] = await fetch_visible_scope_posts( + conn, + row["scope_kind_code"], + row["corporate_entity_id"], + row["process_unit_id"], + row["scope_key"], + affiliated_entity_ids, + row["knowledge_cutoff"], + ) + return detail + + +async def fetch_visible_scope_posts( + conn: asyncpg.Connection, + scope_kind_code: str, + corporate_entity_id: Any, + process_unit_id: Any, + scope_key: str | None, + affiliated_entity_ids: list[str], + knowledge_cutoff: Any, +) -> list[dict[str, str]]: + """ABAC-visible post titles known at the run cutoff -- never a hidden body. + + ``knowledge_cutoff`` is the analysis clock (W3C Time / ISO 8601-1:2019; + ADR 0013/0016). A later live post must not appear inside an earlier run. + """ + if scope_kind_code == "analysis_scope_corporate_entity" and corporate_entity_id: + rows = await conn.fetch( + "select post_id, post_title, visibility_code, corporate_entity_id " + "from source_post where corporate_entity_id = $1 " + "and created_at <= $2 " + "order by created_at, post_title", + corporate_entity_id, + knowledge_cutoff, + ) + elif scope_kind_code == "analysis_scope_process_unit" and process_unit_id: + rows = await conn.fetch( + "select post_id, post_title, visibility_code, corporate_entity_id " + "from source_post where process_unit_id = $1 " + "and created_at <= $2 " + "order by created_at, post_title", + process_unit_id, + knowledge_cutoff, + ) + elif scope_kind_code == "analysis_scope_thread_group" and scope_key: + rows = await conn.fetch( + "select post_id, post_title, visibility_code, corporate_entity_id " + "from source_post where thread_group_key = $1 " + "and created_at <= $2 " + "order by created_at, post_title", + scope_key, + knowledge_cutoff, + ) + elif scope_kind_code == "analysis_scope_all_visible": + rows = await conn.fetch( + "select post_id, post_title, visibility_code, corporate_entity_id " + "from source_post where created_at <= $1 " + "order by created_at, post_title", + knowledge_cutoff, + ) + else: + return [] + affiliated = {str(entity_id) for entity_id in affiliated_entity_ids} + posts: list[dict[str, str]] = [] + for row in rows: + visible = row["visibility_code"] == "public" or str(row["corporate_entity_id"]) in affiliated + if not visible: + continue + posts.append({"post_id": str(row["post_id"]), "post_title": row["post_title"]}) + return posts + + +class AnalysisRunCreateError(Exception): + """Fail-closed create: HTTP status plus a next-action detail string.""" + + def __init__(self, status_code: int, detail: str) -> None: + super().__init__(detail) + self.status_code = status_code + self.detail = detail + + +@dataclass(frozen=True) +class AnalysisRunCapture: + """Immutable capture plan for one authorized create (no source rows).""" + + snapshot_sha256: str + maximum_available_time: datetime + document_count: int + thread_count: int + configuration_sha256: str + configuration_schema_version: str + code_revision_sha: str + + +def utc_iso(value: datetime) -> str: + """Normalize a timestamp to UTC ISO-8601 for digest stability.""" + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + + +def plan_analysis_run_capture( + *, + run_kind_code: str, + scope_kind_code: str, + corporate_entity_id: str, + knowledge_cutoff: datetime, + idempotency_key: str, + post_ids: list[str], + thread_keys: list[str], + latest_post_created_at: datetime | None, + cutoff_explicit: bool = True, +) -> AnalysisRunCapture: + """Hash the authorized cutoff bag. Never stores a post body or DSN. + + An omitted cutoff is hashed as ``unspecified`` so a retry of the same + client key does not 409 just because the clock moved. + """ + cutoff_token = utc_iso(knowledge_cutoff) if cutoff_explicit else "unspecified" + snapshot_material = json.dumps( + { + "scope_kind_code": scope_kind_code, + "corporate_entity_id": corporate_entity_id, + "knowledge_cutoff": cutoff_token, + "post_ids": sorted(post_ids), + }, + separators=(",", ":"), + sort_keys=True, + ) + configuration_material = json.dumps( + { + "run_kind_code": run_kind_code, + "scope_kind_code": scope_kind_code, + "corporate_entity_id": corporate_entity_id, + "knowledge_cutoff": cutoff_token, + "idempotency_key": idempotency_key, + "configuration_schema_version": _KIND_SCHEMA_VERSION[run_kind_code], + }, + separators=(",", ":"), + sort_keys=True, + ) + available = latest_post_created_at if latest_post_created_at is not None else knowledge_cutoff + return AnalysisRunCapture( + snapshot_sha256=hashlib.sha256(snapshot_material.encode()).hexdigest(), + maximum_available_time=available, + document_count=len(post_ids), + thread_count=len(set(thread_keys)), + configuration_sha256=hashlib.sha256(configuration_material.encode()).hexdigest(), + configuration_schema_version=_KIND_SCHEMA_VERSION[run_kind_code], + code_revision_sha=hashlib.sha256(f"lineageweave-{PACKAGE_VERSION}".encode()).hexdigest(), + ) + + +def _canonical_idempotency_key(raw: str) -> str: + """Trim and reject empty or control-bearing client keys.""" + key = raw.strip() + if not key or len(key) > 256 or any(ord(char) < 32 for char in key): + raise AnalysisRunCreateError( + 422, + "Use a 1–256 character idempotency key without control characters, then retry.", + ) + return key + + +def _resolve_corporate_entity_id( + corporate_entity_id: str | None, + affiliated_entity_ids: list[str], +) -> str: + """Return the affiliated corp this run may cover, or a next-action error.""" + affiliated = [entity_id for entity_id in affiliated_entity_ids if entity_id] + if corporate_entity_id: + try: + UUID(corporate_entity_id) + except ValueError as exc: + raise AnalysisRunCreateError( + 404, + "This corporate entity is not visible to this account.", + ) from exc + if corporate_entity_id not in affiliated: + raise AnalysisRunCreateError( + 404, + "This corporate entity is not visible to this account.", + ) + return corporate_entity_id + if len(affiliated) != 1: + raise AnalysisRunCreateError( + 422, + "Choose the corporate entity this run should cover.", + ) + return affiliated[0] + + +async def create_pending_analysis_run( + conn: asyncpg.Connection, + *, + account_id: str, + affiliated_entity_ids: list[str], + run_kind_code: str, + scope_kind_code: str, + corporate_entity_id: str | None, + knowledge_cutoff: datetime | None, + idempotency_key: str, +) -> dict[str, Any]: + """Insert snapshot, counts, run, scope, and Pending in one transaction. + + Does not reconstruct lineage and does not call TEPP. A missing + measurement stays a later worker slice; this write only records the + request. Idempotent retries compare ``configuration_sha256``. + """ + if run_kind_code not in _ALLOWED_CREATE_KINDS: + raise AnalysisRunCreateError( + 422, + "Request a lineage reconstruction or a TEPP measurement. Other kinds are not available yet.", + ) + if scope_kind_code != _CORPORATE_SCOPE: + raise AnalysisRunCreateError( + 422, + "Request a corporate-entity run. Other scopes are not available yet.", + ) + cutoff_explicit = knowledge_cutoff is not None + if knowledge_cutoff is None: + knowledge_cutoff = datetime.now(timezone.utc) + elif knowledge_cutoff.tzinfo is None: + knowledge_cutoff = knowledge_cutoff.replace(tzinfo=timezone.utc) + now = datetime.now(timezone.utc) + if knowledge_cutoff > now: + raise AnalysisRunCreateError( + 422, + "Choose a knowledge cutoff at or before now, then request the run again.", + ) + key = _canonical_idempotency_key(idempotency_key) + corp_id = _resolve_corporate_entity_id(corporate_entity_id, affiliated_entity_ids) + + existing = await conn.fetchrow( + """ + select analysis_run_id, configuration_sha256 + from analysis_run + where requested_by_account_id = $1 and idempotency_key = $2 + """, + account_id, + key, + ) + + rows = await conn.fetch( + """ + select post_id, post_title, thread_group_key, created_at, + visibility_code, corporate_entity_id + from source_post + where corporate_entity_id = $1 and created_at <= $2 + order by created_at, post_title + """, + corp_id, + knowledge_cutoff, + ) + affiliated = {str(entity_id) for entity_id in affiliated_entity_ids} + visible_rows = [ + row + for row in rows + if row["visibility_code"] == "public" or str(row["corporate_entity_id"]) in affiliated + ] + post_ids = [str(row["post_id"]) for row in visible_rows] + thread_keys = [row["thread_group_key"] for row in visible_rows] + latest = max((row["created_at"] for row in visible_rows), default=None) + capture = plan_analysis_run_capture( + run_kind_code=run_kind_code, + scope_kind_code=scope_kind_code, + corporate_entity_id=corp_id, + knowledge_cutoff=knowledge_cutoff, + idempotency_key=key, + post_ids=post_ids, + thread_keys=thread_keys, + latest_post_created_at=latest, + cutoff_explicit=cutoff_explicit, + ) + if existing is not None: + if existing["configuration_sha256"] != capture.configuration_sha256: + raise AnalysisRunCreateError( + 409, + "This request does not match the earlier run with the same key. " + "Open that run, or retry with a new idempotency key.", + ) + replayed = await fetch_visible_analysis_run( + conn, + str(existing["analysis_run_id"]), + account_id, + affiliated_entity_ids, + ) + if replayed is None: + raise AnalysisRunCreateError(404, "This analysis run is not visible.") + return replayed + + snapshot_id = await conn.fetchval( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at, created_at) + values ($1, $2, $3, $4, $4) + on conflict (snapshot_sha256) do nothing + returning analysis_source_snapshot_id + """, + capture.snapshot_sha256, + _CAPTURE_CONTRACT_VERSION, + capture.maximum_available_time, + now, + ) + if snapshot_id is None: + snapshot_id = await conn.fetchval( + """ + select analysis_source_snapshot_id + from analysis_source_snapshot + where snapshot_sha256 = $1 + for update + """, + capture.snapshot_sha256, + ) + count_exists = await conn.fetchval( + """ + select 1 from analysis_source_count + where analysis_source_snapshot_id = $1 + limit 1 + """, + snapshot_id, + ) + if count_exists is None: + await conn.execute( + """ + insert into analysis_source_count + (analysis_source_snapshot_id, count_type_code, count_value) + values + ($1, 'analysis_count_document', $2), + ($1, 'analysis_count_thread', $3) + """, + snapshot_id, + capture.document_count, + capture.thread_count, + ) + try: + run_id = await conn.fetchval( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9) + returning analysis_run_id + """, + snapshot_id, + run_kind_code, + key, + account_id, + knowledge_cutoff, + capture.configuration_schema_version, + capture.configuration_sha256, + capture.code_revision_sha, + now, + ) + except asyncpg.UniqueViolationError: + raced = await conn.fetchrow( + """ + select analysis_run_id, configuration_sha256 + from analysis_run + where requested_by_account_id = $1 and idempotency_key = $2 + """, + account_id, + key, + ) + if raced is None or raced["configuration_sha256"] != capture.configuration_sha256: + raise AnalysisRunCreateError( + 409, + "This request does not match the earlier run with the same key. " + "Open that run, or retry with a new idempotency key.", + ) from None + replayed = await fetch_visible_analysis_run( + conn, + str(raced["analysis_run_id"]), + account_id, + affiliated_entity_ids, + ) + if replayed is None: + raise AnalysisRunCreateError(404, "This analysis run is not visible.") + return replayed + await conn.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values ($1, $2, $3) + """, + run_id, + scope_kind_code, + corp_id, + ) + await conn.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values ($1, 1, 'analysis_status_pending', $2) + """, + run_id, + now, + ) + created = await fetch_visible_analysis_run( + conn, + str(run_id), + account_id, + affiliated_entity_ids, + ) + if created is None: + raise AnalysisRunCreateError(404, "This analysis run is not visible.") + return created diff --git a/backend/app/corporate_entity_ingestion.py b/backend/app/corporate_entity_ingestion.py new file mode 100644 index 000000000..57baadc5e --- /dev/null +++ b/backend/app/corporate_entity_ingestion.py @@ -0,0 +1,193 @@ + +"""Resolve an organization mention to the corporate hierarchy catalog. + +Existing similarity matches are reused. A previously unseen entity is +created only after inference proposes its complete hierarchy placement +and external verification corroborates that placement. Parent failure, +cycles, and excessive depth all fail closed. See ADR 0010. + +Creation writes take one named Postgres advisory transaction lock +(``pg_advisory_xact_lock``) after network inference/verification, then +reload catalog candidates before inserting. See ADR 0012. +""" + +from __future__ import annotations + +import asyncio +import hashlib + +import asyncpg + +from lineageweave.corporate_hierarchy_inference import ( + CorporateHierarchyInferenceClient, + HierarchyProposal, +) +from lineageweave.corporate_hierarchy_resolution import ( + CorporateEntityCandidate, + resolve_corporate_entity, +) +from lineageweave.relation_verification import ( + STATUS_CORROBORATED, + RelationVerificationClient, +) + +_AUTO_CODE_PREFIX = "AUTO-" +_MAX_HIERARCHY_DEPTH = 4 +_CREATION_LOCK_KEY = "lineageweave:corporate_entity_creation" + + +def _auto_entity_code(organization_name: str) -> str: + """Return a deterministic, namespace-separated code.""" + digest = hashlib.sha256(organization_name.encode("utf-8")).hexdigest()[:16] + return f"{_AUTO_CODE_PREFIX}{digest}" + + +def _hierarchy_verification_label(proposal: HierarchyProposal) -> str: + """Describe every persisted hierarchy field in one claim.""" + parent = proposal.parent_name if proposal.parent_name is not None else "NO_PARENT" + return f"corporate hierarchy level={proposal.level_code}; immediate_parent={parent}" + + +async def _create_entity( + conn: asyncpg.Connection, + organization_name: str, + level_code: str, + parent_entity_id: str | None, +) -> str: + """Insert one entity atomically and return its catalog id.""" + row = await conn.fetchrow( + """ + insert into corporate_entity + (parent_entity_id, corporate_entity_code, entity_name, entity_level_code) + values ($1, $2, $3, $4) + on conflict (corporate_entity_code) do update set + entity_name = excluded.entity_name, + entity_level_code = excluded.entity_level_code, + parent_entity_id = excluded.parent_entity_id + returning corporate_entity_id + """, + parent_entity_id, + _auto_entity_code(organization_name), + organization_name, + level_code, + ) + return str(row["corporate_entity_id"]) + + +async def _reload_candidates(conn: asyncpg.Connection) -> list[CorporateEntityCandidate]: + """Read every cataloged entity after the creation lock is held.""" + rows = await conn.fetch("select corporate_entity_id, entity_name from corporate_entity") + return [ + CorporateEntityCandidate(str(row["corporate_entity_id"]), row["entity_name"]) + for row in rows + ] + + +def _remember_candidate( + candidates: list[CorporateEntityCandidate], + corporate_entity_id: str, + entity_name: str, +) -> None: + """Keep the caller's in-memory snapshot aligned with a resolved id.""" + if any(candidate.corporate_entity_id == corporate_entity_id for candidate in candidates): + return + candidates.append( + CorporateEntityCandidate( + corporate_entity_id=corporate_entity_id, + entity_name=entity_name, + ) + ) + + +async def get_or_create_corporate_entity( + conn: asyncpg.Connection, + organization_name: str, + context_text: str, + inference_client: CorporateHierarchyInferenceClient, + verification_client: RelationVerificationClient, + candidates: list[CorporateEntityCandidate], + *, + _depth: int = 0, + _visited_names: frozenset[str] = frozenset(), +) -> str | None: + """Return a verified catalog id, otherwise ``None``. + + A proposed parent must independently corroborate and resolve before + the child can be inserted. Repeated names in the recursion path are + cycles, including multi-node cycles such as A -> B -> A. + """ + normalized_name = organization_name.strip() + if not normalized_name: + return None + visit_key = normalized_name.casefold() + if visit_key in _visited_names: + return None + + existing_id = resolve_corporate_entity(normalized_name, candidates) + if existing_id is not None: + return existing_id + if _depth >= _MAX_HIERARCHY_DEPTH or not inference_client.available: + return None + + proposal = await asyncio.to_thread( + inference_client.infer, + normalized_name, + context_text, + ) + if proposal is None or not verification_client.available: + return None + + placement_result = await asyncio.to_thread( + verification_client.verify, + normalized_name, + _hierarchy_verification_label(proposal), + ) + if placement_result.status_code != STATUS_CORROBORATED: + return None + + visited_names = _visited_names | {visit_key} + parent_entity_id: str | None = None + if proposal.parent_name is not None: + normalized_parent = proposal.parent_name.strip() + if not normalized_parent or normalized_parent.casefold() in visited_names: + return None + parent_result = await asyncio.to_thread( + verification_client.verify, + normalized_parent, + f"immediate parent of {normalized_name}", + ) + if parent_result.status_code != STATUS_CORROBORATED: + return None + parent_entity_id = await get_or_create_corporate_entity( + conn, + normalized_parent, + context_text, + inference_client, + verification_client, + candidates, + _depth=_depth + 1, + _visited_names=visited_names, + ) + if parent_entity_id is None: + return None + + async with conn.transaction(): + await conn.execute( + "select pg_advisory_xact_lock(hashtext($1))", + _CREATION_LOCK_KEY, + ) + fresh_existing_id = resolve_corporate_entity( + normalized_name, + await _reload_candidates(conn), + ) + if fresh_existing_id is not None: + _remember_candidate(candidates, fresh_existing_id, normalized_name) + return fresh_existing_id + new_id = await _create_entity( + conn, + normalized_name, + proposal.level_code, + parent_entity_id, + ) + _remember_candidate(candidates, new_id, normalized_name) + return new_id diff --git a/backend/app/entity_relationship_ingestion.py b/backend/app/entity_relationship_ingestion.py index 30fd62361..091e58f40 100644 --- a/backend/app/entity_relationship_ingestion.py +++ b/backend/app/entity_relationship_ingestion.py @@ -6,6 +6,7 @@ from __future__ import annotations +import asyncio from collections.abc import Mapping, Sequence from typing import Any @@ -38,7 +39,9 @@ async def ingest_post_entity_relationships( if not organization_names: return [] - relationships = client.classify(post_title, post_body, organization_names) + relationships = await asyncio.to_thread( + client.classify, post_title, post_body, organization_names + ) for relationship in relationships: await conn.execute( diff --git a/backend/app/keyman_ingestion.py b/backend/app/keyman_ingestion.py index e5451a9bc..906442ba4 100644 --- a/backend/app/keyman_ingestion.py +++ b/backend/app/keyman_ingestion.py @@ -6,23 +6,63 @@ `person_affiliation` (N:N, matched to a real `corporate_entity` via similarity-based resolution -- see `lineageweave.corporate_hierarchy_resolution`, so an abbreviation or -trailing legal suffix still resolves, not just an exact string match), -and `post_person_mention`. Finishes by calling -`knowledge_graph.persist_edges_for_post` so the Knowledge Graph edges are -computed from the same write, not a separate manual step. +trailing legal suffix still resolves, not just an exact string match -- +plus `role_title`, a schema column that already existed and was +previously never populated by this pipeline), and `post_person_mention`. +Finishes by calling `knowledge_graph.persist_edges_for_post` so the +Knowledge Graph edges are computed from the same write, not a separate +manual step. + +Same-name disambiguation: `_upsert_person`'s name+side match is a real, +known simplification (documented above), but a stated job title is real +evidence a same-name match should NOT blindly trust -- when the new +mention names a title that conflicts with a title already on file for +that name+side (both stated, genuinely different), a fresh +`cataloged_person` row is created rather than merging two people who +happen to share a name. A person's title legitimately changes over time +(a promotion), so this only splits on an actual stated conflict, never +on a missing title on either side. + +Abbreviation resolution (ADR 0008): before matching against +`corporate_entity`, each affiliated organization name is run through +`organization_name_resolution_ingestion.resolve_organization_name` -- +character-similarity matching alone cannot bridge an initialism like +"AGP" to its expansion "Aurora Grid Power". Only a search-corroborated +resolution is substituted in; an unresolved or unverified name still +flows through unchanged. + +Hierarchy auto-creation (ADR 0010): a unseen dataset's first mention of +any new counterparty organization has no existing `corporate_entity` +candidate for similarity matching to find at all -- matching alone can +only ever locate an already-cataloged entity. `get_or_create_corporate_entity` +tries similarity matching first, then falls back to an LLM-proposed, +search-corroborated hierarchy placement (level + parent) before +creating a real new row, so the "통합 고객사 계열 tree AI" requirement +is actually populated from real extraction, not left permanently empty. """ from __future__ import annotations +import asyncio +from dataclasses import replace + import asyncpg -from lineageweave.corporate_hierarchy_resolution import ( - CorporateEntityCandidate, - resolve_corporate_entity, +from lineageweave.corporate_hierarchy_inference import ( + CorporateHierarchyInferenceClient, + NullCorporateHierarchyInferenceClient, ) +from lineageweave.corporate_hierarchy_resolution import CorporateEntityCandidate from lineageweave.keyman_extraction import KeymanExtractionClient, PersonMention +from lineageweave.organization_name_resolution import ( + NullOrganizationNameResolutionClient, + OrganizationNameResolutionClient, +) +from lineageweave.relation_verification import NullRelationVerificationClient, RelationVerificationClient +from .corporate_entity_ingestion import get_or_create_corporate_entity from .knowledge_graph import persist_edges_for_post +from .organization_name_resolution_ingestion import resolve_organization_name async def _load_corporate_entity_candidates(conn: asyncpg.Connection) -> list[CorporateEntityCandidate]: @@ -34,60 +74,187 @@ async def _load_corporate_entity_candidates(conn: asyncpg.Connection) -> list[Co async def _upsert_person(conn: asyncpg.Connection, mention: PersonMention) -> str: - """Reuse a same-name, same-side row so re-extraction does not duplicate.""" - row = await conn.fetchrow( - "select person_id from cataloged_person where person_name = $1 and person_side_code = $2", + """Reuse a same-name, same-side row so re-extraction does not duplicate + -- unless the new mention's stated job title conflicts with a title + already on file for that name+side (`last_known_job_title`, checked + even when this mention names no affiliated organization -- a title + is real same-name-disambiguation evidence on its own, see module + docstring), in which case a same name is not trusted as the same + real person. + """ + candidates = await conn.fetch( + "select person_id, last_known_job_title from cataloged_person " + "where person_name = $1 and person_side_code = $2", mention.person_name, mention.person_side_code, ) - if row is not None: - return str(row["person_id"]) + if candidates and mention.job_title: + for candidate in candidates: + on_file = candidate["last_known_job_title"] + if on_file is not None and on_file != mention.job_title: + continue # stated title conflicts -- do not reuse this row + if on_file is None: + await conn.execute( + "update cataloged_person set last_known_job_title = $1 where person_id = $2", + mention.job_title, + candidate["person_id"], + ) + return str(candidate["person_id"]) + elif candidates: + return str(candidates[0]["person_id"]) + row = await conn.fetchrow( - "insert into cataloged_person (person_name, person_side_code) values ($1, $2) returning person_id", + "insert into cataloged_person (person_name, person_side_code, last_known_job_title) " + "values ($1, $2, $3) returning person_id", mention.person_name, mention.person_side_code, + mention.job_title, ) return str(row["person_id"]) + +async def _upsert_affiliation( + conn: asyncpg.Connection, + person_id: str, + raw_name: str, + resolved_name: str, + corporate_entity_id: str | None, + role_title: str | None, +) -> None: + """Promote a raw affiliation row into one canonical identity.""" + await conn.execute( + """ + with legacy_affiliation as ( + select affiliated_corporate_entity_id, role_title + from person_affiliation + where person_id = $1 + and affiliated_organization_name = $2 + ), + canonical_affiliation as ( + insert into person_affiliation + (person_id, affiliated_organization_name, + affiliated_corporate_entity_id, role_title) + values ( + $1, + $3, + coalesce($4, (select affiliated_corporate_entity_id from legacy_affiliation)), + coalesce($5, (select role_title from legacy_affiliation)) + ) + on conflict (person_id, affiliated_organization_name) + do update set + affiliated_corporate_entity_id = coalesce( + excluded.affiliated_corporate_entity_id, + person_affiliation.affiliated_corporate_entity_id + ), + role_title = coalesce( + excluded.role_title, + person_affiliation.role_title + ) + returning person_affiliation_id + ) + delete from person_affiliation + where person_id = $1 + and affiliated_organization_name = $2 + and $2 <> $3 + """, + person_id, + raw_name, + resolved_name, + corporate_entity_id, + role_title, + ) + + async def ingest_post_keymen( conn: asyncpg.Connection, client: KeymanExtractionClient, post_id: str, post_title: str, post_body: str, + *, + resolution_client: OrganizationNameResolutionClient | None = None, + verification_client: RelationVerificationClient | None = None, + hierarchy_inference_client: CorporateHierarchyInferenceClient | None = None, + persist_graph: bool = True, ) -> list[PersonMention]: """Extracts, persists, and returns the `PersonMention`s found in one post. + `resolution_client`/`verification_client`/`hierarchy_inference_client` + default to the unavailable Null clients -- callers that don't pass + real ones get the exact same behavior as before ADR 0008/0010 (raw + affiliation names, unresolved). + + Organization resolution and hierarchy creation finish before the Keyman + write transaction. Callers must not wrap this function in an outer + transaction: that would turn ``pg_advisory_xact_lock`` into a savepoint + and hold the creation lock across later LLM work. The post's prior + Keyman mention set is replaced atomically after enrichment. + ``persist_graph=False`` lets a larger caller persist edges in its own + short write transaction after this function returns. + Raises whatever `client.extract` raises (e.g. a `NullKeymanExtractionClient` would raise `RuntimeError`) -- callers should check `client.available` first, same discipline as every other pluggable channel in this repo. """ - mentions = client.extract(post_title, post_body) + resolution_client = resolution_client or NullOrganizationNameResolutionClient() + verification_client = verification_client or NullRelationVerificationClient() + hierarchy_inference_client = hierarchy_inference_client or NullCorporateHierarchyInferenceClient() + mentions = await asyncio.to_thread(client.extract, post_title, post_body) candidates = await _load_corporate_entity_candidates(conn) - + resolved_by_mention: list[tuple[PersonMention, list[tuple[str, str, str | None]]]] = [] for mention in mentions: - person_id = await _upsert_person(conn, mention) + resolved_orgs: list[tuple[str, str, str | None]] = [] + for organization_name in mention.affiliated_organization_names: + resolved_name = await resolve_organization_name( + conn, + resolution_client, + verification_client, + organization_name, + post_body, + ) + corporate_entity_id = await get_or_create_corporate_entity( + conn, + resolved_name, + post_body, + hierarchy_inference_client, + verification_client, + candidates, + ) + resolved_orgs.append((organization_name, resolved_name, corporate_entity_id)) + resolved_by_mention.append((mention, resolved_orgs)) + + normalized_mentions: list[PersonMention] = [] + async with conn.transaction(): await conn.execute( - "insert into post_person_mention (post_id, person_id) values ($1, $2) on conflict do nothing", - post_id, - person_id, + "delete from post_person_mention where post_id = $1", post_id ) - for organization_name in mention.affiliated_organization_names: - corporate_entity_id = resolve_corporate_entity(organization_name, candidates) + for mention, resolved_orgs in resolved_by_mention: + person_id = await _upsert_person(conn, mention) await conn.execute( - """ - insert into person_affiliation (person_id, affiliated_organization_name, affiliated_corporate_entity_id) - values ($1, $2, $3) - on conflict (person_id, affiliated_organization_name) - do update set affiliated_corporate_entity_id = excluded.affiliated_corporate_entity_id - """, + "insert into post_person_mention (post_id, person_id) values ($1, $2) on conflict do nothing", + post_id, person_id, - organization_name, - corporate_entity_id, ) + resolved_names: list[str] = [] + for organization_name, resolved_name, corporate_entity_id in resolved_orgs: + await _upsert_affiliation( + conn, + person_id, + organization_name, + resolved_name, + corporate_entity_id, + mention.job_title, + ) + if resolved_name not in resolved_names: + resolved_names.append(resolved_name) + normalized_mentions.append( + replace( + mention, + affiliated_organization_names=tuple(resolved_names), + ) + ) + if persist_graph: + await persist_edges_for_post(conn, post_id) - if mentions: - await persist_edges_for_post(conn, post_id) - - return mentions + return normalized_mentions diff --git a/backend/app/knowledge_graph.py b/backend/app/knowledge_graph.py index bb398d141..ce7289bb5 100644 --- a/backend/app/knowledge_graph.py +++ b/backend/app/knowledge_graph.py @@ -18,9 +18,13 @@ EDGE_AFFILIATION, EDGE_CO_MENTION, EDGE_MENTION, + EDGE_MENTION_ORGANIZATION, + EDGE_MENTION_TEAM, + EDGE_TEAM_AFFILIATION, NODE_CORPORATE_ENTITY, NODE_PERSON, NODE_POST, + NODE_TEAM, KnowledgeGraphEdgeSpec, adjacency_from_edges, knowledge_graph_edges_for_post, @@ -31,6 +35,9 @@ ) +_GRAPH_PROJECTION_LOCK_KEY = "lineageweave:knowledge_graph_projection" + + def edge_spec_from_row(row: asyncpg.Record) -> KnowledgeGraphEdgeSpec: """Map one ``knowledge_graph_edge`` row onto the library spec.""" return KnowledgeGraphEdgeSpec( @@ -63,7 +70,7 @@ async def fetch_post_keymen(conn: asyncpg.Connection, post_id: str) -> list[dict """Load mentioned people and their affiliations for one post.""" person_rows = await conn.fetch( """ - select p.person_id, p.person_name, p.person_side_code, ppm.mention_context + select p.person_id, p.person_name, p.person_side_code, p.last_known_job_title, ppm.mention_context from post_person_mention ppm join cataloged_person p on p.person_id = ppm.person_id where ppm.post_id = $1 @@ -105,16 +112,34 @@ async def fetch_post_keymen(conn: asyncpg.Connection, post_id: str) -> list[dict "person_side_code": row["person_side_code"], "person_side_label": side_labels.get(row["person_side_code"], row["person_side_code"]), "mention_context": row["mention_context"], + "last_known_job_title": row["last_known_job_title"], "affiliations": affiliations_by_person.get(str(row["person_id"]), []), } for row in person_rows ] -async def persist_edges_for_post(conn: asyncpg.Connection, post_id: str) -> list[KnowledgeGraphEdgeSpec]: - """Insert mention, affiliation, and co-mention edges for one post.""" +async def persist_edges_for_post( + conn: asyncpg.Connection, post_id: str +) -> list[KnowledgeGraphEdgeSpec]: + """Reconcile one post's evidence-backed navigation projection. + + Callers own the surrounding transaction. A transaction-scoped + advisory lock serializes the small materialized projection so two + writers cannot interleave evidence deletion and orphan pruning. + Keyman and R&R person sources stay distinct in their writable tables; + ``combined_post_person_mention`` is used only to derive graph edges. + """ + await conn.execute( + "select pg_advisory_xact_lock(hashtext($1))", + _GRAPH_PROJECTION_LOCK_KEY, + ) + await conn.execute( + "delete from knowledge_graph_edge_evidence where evidence_post_id = $1", + post_id, + ) mention_rows = await conn.fetch( - "select person_id from post_person_mention where post_id = $1", + "select person_id from combined_post_person_mention where post_id = $1", post_id, ) affiliation_rows = await conn.fetch( @@ -126,6 +151,23 @@ async def persist_edges_for_post(conn: asyncpg.Connection, post_id: str) -> list """, [row["person_id"] for row in mention_rows], ) + team_mention_rows = await conn.fetch( + "select team_id from post_team_mention where post_id = $1", + post_id, + ) + team_affiliation_rows = await conn.fetch( + """ + select team_id, affiliated_corporate_entity_id + from cataloged_team + where team_id = any($1::uuid[]) + and affiliated_corporate_entity_id is not null + """, + [row["team_id"] for row in team_mention_rows], + ) + organization_mention_rows = await conn.fetch( + "select corporate_entity_id from post_organization_mention where post_id = $1", + post_id, + ) edges = knowledge_graph_edges_for_post( post_id, [str(row["person_id"]) for row in mention_rows], @@ -133,24 +175,27 @@ async def persist_edges_for_post(conn: asyncpg.Connection, post_id: str) -> list (str(row["person_id"]), str(row["affiliated_corporate_entity_id"])) for row in affiliation_rows ], + [str(row["team_id"]) for row in team_mention_rows], + [ + (str(row["team_id"]), str(row["affiliated_corporate_entity_id"])) + for row in team_affiliation_rows + ], + [str(row["corporate_entity_id"]) for row in organization_mention_rows], ) for edge in edges: - await conn.execute( + await conn.fetchrow( """ insert into knowledge_graph_edge ( source_node_type_code, source_node_id, target_node_type_code, target_node_id, edge_type_code, edge_weight - ) - select $1, $2::uuid, $3, $4::uuid, $5, $6 - where not exists ( - select 1 from knowledge_graph_edge - where source_node_type_code = $1 - and source_node_id = $2::uuid - and target_node_type_code = $3 - and target_node_id = $4::uuid - and edge_type_code = $5 - ) + ) values ($1, $2::uuid, $3, $4::uuid, $5, $6) + on conflict ( + source_node_type_code, source_node_id, + target_node_type_code, target_node_id, + edge_type_code + ) do update set edge_weight = excluded.edge_weight + returning knowledge_graph_edge_id """, edge.source_node_type_code, edge.source_node_id, @@ -159,9 +204,19 @@ async def persist_edges_for_post(conn: asyncpg.Connection, post_id: str) -> list edge.edge_type_code, edge.edge_weight, ) + await conn.execute( + """ + delete from knowledge_graph_edge edge_row + where not exists ( + select 1 + from knowledge_graph_edge_evidence evidence + where evidence.knowledge_graph_edge_id = + edge_row.knowledge_graph_edge_id + ) + """ + ) return edges - async def person_exists(conn: asyncpg.Connection, person_id: str) -> bool: """True when ``person_id`` is a UUID that exists in ``cataloged_person``.""" try: @@ -184,83 +239,189 @@ async def corporate_entity_exists(conn: asyncpg.Connection, entity_id: str) -> b return row is not None +async def team_exists(conn: asyncpg.Connection, team_id: str) -> bool: + """True when ``team_id`` is a UUID that exists in ``cataloged_team``.""" + try: + UUID(team_id) + except ValueError: + return False + row = await conn.fetchrow("select 1 from cataloged_team where team_id = $1", team_id) + return row is not None + + async def visible_mention_post_ids( conn: asyncpg.Connection, person_id: str, can_see_post, ) -> list[str]: - """Post ids that mention ``person_id`` and pass the caller's ABAC check.""" + """Visible post ids supported by Keyman or R&R person evidence.""" rows = await conn.fetch( """ - select p.post_id, p.visibility_code, p.corporate_entity_id - from post_person_mention ppm - join source_post p on p.post_id = ppm.post_id - where ppm.person_id = $1 + select post.post_id, post.visibility_code, post.corporate_entity_id + from combined_post_person_mention mention + join source_post post on post.post_id = mention.post_id + where mention.person_id = $1 + order by post.created_at, post.post_id """, person_id, ) return [str(row["post_id"]) for row in rows if can_see_post(row)] - async def visible_affiliation_post_ids( conn: asyncpg.Connection, entity_id: str, can_see_post, ) -> list[str]: - """Post ids that mention someone affiliated with ``entity_id`` and pass ABAC.""" + """Visible posts that mention an entity via a person or a direct org mention.""" rows = await conn.fetch( """ - select distinct p.post_id, p.visibility_code, p.corporate_entity_id - from person_affiliation pa - join post_person_mention ppm on ppm.person_id = pa.person_id - join source_post p on p.post_id = ppm.post_id - where pa.affiliated_corporate_entity_id = $1 + select distinct post.post_id, post.visibility_code, + post.corporate_entity_id, post.created_at + from source_post post + where post.post_id in ( + select mention.post_id + from person_affiliation affiliation + join combined_post_person_mention mention + on mention.person_id = affiliation.person_id + where affiliation.affiliated_corporate_entity_id = $1 + union + select org_mention.post_id + from post_organization_mention org_mention + where org_mention.corporate_entity_id = $1 + ) + order by post.created_at, post.post_id """, entity_id, ) return [str(row["post_id"]) for row in rows if can_see_post(row)] +async def visible_team_mention_post_ids( + conn: asyncpg.Connection, + team_id: str, + can_see_post, +) -> list[str]: + """Visible post ids supported by a cataloged team mention.""" + rows = await conn.fetch( + """ + select post.post_id, post.visibility_code, post.corporate_entity_id + from post_team_mention mention + join source_post post on post.post_id = mention.post_id + where mention.team_id = $1 + order by post.created_at, post.post_id + """, + team_id, + ) + return [str(row["post_id"]) for row in rows if can_see_post(row)] + async def load_visible_subgraph( conn: asyncpg.Connection, visible_post_ids: list[str], ) -> list[KnowledgeGraphEdgeSpec]: - """Edges whose endpoints the account can already see via those posts.""" + """Edges supported by at least one post the account may already see. + + Person, team, and organization mention channels are independent. A + team-only or organization-only post must still walk (ADR 0018). + """ if not visible_post_ids: return [] person_rows = await conn.fetch( - "select distinct person_id from post_person_mention where post_id = any($1::uuid[])", + "select distinct person_id from combined_post_person_mention " + "where post_id = any($1::uuid[])", visible_post_ids, ) person_ids = [row["person_id"] for row in person_rows] - if not person_ids: + team_rows = await conn.fetch( + "select distinct team_id from post_team_mention " + "where post_id = any($1::uuid[])", + visible_post_ids, + ) + team_ids = [row["team_id"] for row in team_rows] + organization_rows = await conn.fetch( + "select distinct corporate_entity_id from post_organization_mention " + "where post_id = any($1::uuid[])", + visible_post_ids, + ) + organization_ids = [row["corporate_entity_id"] for row in organization_rows] + if not person_ids and not team_ids and not organization_ids: return [] rows = await conn.fetch( """ - select source_node_type_code, source_node_id, - target_node_type_code, target_node_id, - edge_type_code, edge_weight - from knowledge_graph_edge - where + select distinct edge.source_node_type_code, edge.source_node_id, + edge.target_node_type_code, edge.target_node_id, + edge.edge_type_code, edge.edge_weight + from knowledge_graph_edge edge + join knowledge_graph_edge_evidence evidence + on evidence.knowledge_graph_edge_id = edge.knowledge_graph_edge_id + and evidence.evidence_post_id = any($1::uuid[]) + where ( - edge_type_code = $3 + edge.edge_type_code = $3 and ( - (source_node_type_code = $4 and source_node_id = any($1::uuid[])) - or (target_node_type_code = $4 and target_node_id = any($1::uuid[])) + (edge.source_node_type_code = $4 + and edge.source_node_id = any($1::uuid[])) + or + (edge.target_node_type_code = $4 + and edge.target_node_id = any($1::uuid[])) ) ) or ( - edge_type_code = $5 - and source_node_type_code = $6 - and target_node_type_code = $6 - and source_node_id = any($2::uuid[]) - and target_node_id = any($2::uuid[]) + edge.edge_type_code = $5 + and edge.source_node_type_code = $6 + and edge.target_node_type_code = $6 + and edge.source_node_id = any($2::uuid[]) + and edge.target_node_id = any($2::uuid[]) ) or ( - edge_type_code = $7 + edge.edge_type_code = $7 and ( - (source_node_type_code = $6 and source_node_id = any($2::uuid[])) - or (target_node_type_code = $6 and target_node_id = any($2::uuid[])) + (edge.source_node_type_code = $6 + and edge.source_node_id = any($2::uuid[])) + or + (edge.target_node_type_code = $6 + and edge.target_node_id = any($2::uuid[])) + ) + ) + or ( + edge.edge_type_code = $8 + and ( + (edge.source_node_type_code = $4 + and edge.source_node_id = any($1::uuid[])) + or + (edge.target_node_type_code = $4 + and edge.target_node_id = any($1::uuid[])) + or + (edge.source_node_type_code = $9 + and edge.source_node_id = any($10::uuid[])) + or + (edge.target_node_type_code = $9 + and edge.target_node_id = any($10::uuid[])) + ) + ) + or ( + edge.edge_type_code = $11 + and ( + (edge.source_node_type_code = $9 + and edge.source_node_id = any($10::uuid[])) + or + (edge.target_node_type_code = $9 + and edge.target_node_id = any($10::uuid[])) + ) + ) + or ( + edge.edge_type_code = $12 + and ( + (edge.source_node_type_code = $4 + and edge.source_node_id = any($1::uuid[])) + or + (edge.target_node_type_code = $4 + and edge.target_node_id = any($1::uuid[])) + or + (edge.source_node_type_code = $13 + and edge.source_node_id = any($14::uuid[])) + or + (edge.target_node_type_code = $13 + and edge.target_node_id = any($14::uuid[])) ) ) """, @@ -271,10 +432,16 @@ async def load_visible_subgraph( EDGE_CO_MENTION, NODE_PERSON, EDGE_AFFILIATION, + EDGE_MENTION_TEAM, + NODE_TEAM, + team_ids, + EDGE_TEAM_AFFILIATION, + EDGE_MENTION_ORGANIZATION, + NODE_CORPORATE_ENTITY, + organization_ids, ) return [edge_spec_from_row(row) for row in rows] - async def hydrate_related_nodes( conn: asyncpg.Connection, related: list[tuple[str, float]], @@ -287,6 +454,7 @@ async def hydrate_related_nodes( person_ids: list[str] = [] post_ids: list[str] = [] corp_ids: list[str] = [] + team_ids: list[str] = [] parsed: list[tuple[str, str, float]] = [] for key, score in related: node_type_code, node_id = parse_node_key(key) @@ -297,6 +465,8 @@ async def hydrate_related_nodes( post_ids.append(node_id) elif node_type_code == NODE_CORPORATE_ENTITY: corp_ids.append(node_id) + elif node_type_code == NODE_TEAM: + team_ids.append(node_id) people = { str(row["person_id"]): row @@ -319,6 +489,17 @@ async def hydrate_related_nodes( corp_ids, ) } if corp_ids else {} + teams = { + str(row["team_id"]): row + for row in await conn.fetch( + "select team_id, team_name from cataloged_team where team_id = any($1::uuid[])", + team_ids, + ) + } if team_ids else {} + + side_labels = await labels_for_codes( + conn, [row["person_side_code"] for row in people.values()] + ) payload: list[dict[str, Any]] = [] for node_type_code, node_id, score in parsed: @@ -329,12 +510,16 @@ async def hydrate_related_nodes( **ontology_annotations(node_type_code), } if node_type_code == NODE_PERSON and node_id in people: + side = people[node_id]["person_side_code"] item["label"] = people[node_id]["person_name"] - item["person_side_code"] = people[node_id]["person_side_code"] + item["person_side_code"] = side + item["person_side_label"] = side_labels.get(side, side) elif node_type_code == NODE_POST and node_id in posts: item["label"] = posts[node_id]["post_title"] elif node_type_code == NODE_CORPORATE_ENTITY and node_id in corps: item["label"] = corps[node_id]["entity_name"] + elif node_type_code == NODE_TEAM and node_id in teams: + item["label"] = teams[node_id]["team_name"] else: continue payload.append(item) @@ -371,3 +556,12 @@ async def related_for_entity( ) -> list[dict[str, Any]]: """Run RWR from ``entity_id`` over the account's visible subgraph.""" return await related_for_start(conn, NODE_CORPORATE_ENTITY, entity_id, visible_post_ids) + + +async def related_for_team( + conn: asyncpg.Connection, + team_id: str, + visible_post_ids: list[str], +) -> list[dict[str, Any]]: + """Run RWR from ``team_id`` over the account's visible subgraph.""" + return await related_for_start(conn, NODE_TEAM, team_id, visible_post_ids) diff --git a/backend/app/main.py b/backend/app/main.py index c69609a93..adb7a20a8 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -19,8 +19,11 @@ from __future__ import annotations +import asyncio from contextlib import asynccontextmanager +from datetime import datetime from typing import Any +from uuid import UUID import asyncpg import redis.asyncio as redis @@ -37,10 +40,18 @@ NullEntityRelationshipClient, ) from lineageweave.image_content import orchestrator_vision_client +from lineageweave.corporate_hierarchy_inference import ( + ContextualOrchestratorHierarchyInferenceClient, + NullCorporateHierarchyInferenceClient, +) from lineageweave.keyman_extraction import ( ContextualOrchestratorKeymanExtractionClient, NullKeymanExtractionClient, ) +from lineageweave.organization_name_resolution import ( + ContextualOrchestratorOrganizationNameResolutionClient, + NullOrganizationNameResolutionClient, +) from lineageweave.post_chat import ( ContextualOrchestratorPostChatClient, NullPostChatClient, @@ -55,6 +66,12 @@ from lineageweave.post_summary import ContextualOrchestratorPostSummaryClient, NullPostSummaryClient from lineageweave.relation_verification import NullRelationVerificationClient, SearxngRelationVerificationClient +from backend.app.analysis_run_ingestion import ( + AnalysisRunCreateError, + create_pending_analysis_run, + fetch_visible_analysis_run, + fetch_visible_analysis_runs, +) from backend.app.activity_stream import ( create_valkey_client, get_valkey, @@ -96,10 +113,14 @@ fetch_post_keymen, labels_for_codes, person_exists, + persist_edges_for_post, related_for_entity, related_for_person, + related_for_team, + team_exists, visible_affiliation_post_ids, visible_mention_post_ids, + visible_team_mention_post_ids, ) from backend.app.lineage_ingestion import rebuild_lineage, visible_lineage_graph from backend.app.post_chat_ingestion import ( @@ -178,6 +199,26 @@ def _relation_verification_client(): return SearxngRelationVerificationClient(base_url=settings.searxng_base_url) +def _organization_name_resolution_client(): + """Live orchestrator client when configured; otherwise the unavailable null.""" + settings = load_settings() + if not (settings.orchestrator_base_url and settings.orchestrator_api_key): + return NullOrganizationNameResolutionClient() + return ContextualOrchestratorOrganizationNameResolutionClient( + base_url=settings.orchestrator_base_url, api_key=settings.orchestrator_api_key + ) + + +def _corporate_hierarchy_inference_client(): + """Live orchestrator client when configured; otherwise the unavailable null.""" + settings = load_settings() + if not (settings.orchestrator_base_url and settings.orchestrator_api_key): + return NullCorporateHierarchyInferenceClient() + return ContextualOrchestratorHierarchyInferenceClient( + base_url=settings.orchestrator_base_url, api_key=settings.orchestrator_api_key + ) + + def _post_summary_client(): """Live orchestrator client when configured; otherwise the unavailable null.""" settings = load_settings() @@ -435,6 +476,34 @@ async def read_related_corporate_entity( } +@app.get("/api/teams/{team_id}/related") +async def read_related_team( + team_id: str, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """RWR-ranked related nodes from one cataloged team, hiding unseen posts.""" + _require_post_read(account) + async with pool.acquire() as conn: + if not await team_exists(conn, team_id): + raise HTTPException(status.HTTP_404_NOT_FOUND, "team not found") + visible_post_ids = await visible_team_mention_post_ids( + conn, team_id, lambda row: _can_see_post(account, row) + ) + if not visible_post_ids: + raise HTTPException(status.HTTP_403_FORBIDDEN, "not authorized to view this team") + team = await conn.fetchrow( + "select team_id, team_name from cataloged_team where team_id = $1", + team_id, + ) + related = await related_for_team(conn, team_id, visible_post_ids) + return { + "team_id": str(team["team_id"]), + "team_name": team["team_name"], + "related": related, + } + + @app.get("/api/posts/{post_id}/counterparties") async def read_post_counterparties( post_id: str, @@ -553,17 +622,28 @@ async def extract_post_keymen( # literal text either blows the token budget or is silently # ignored (see lineageweave/post_content_normalization.py). post_body = normalize_post_body(raw_body, vision_client=_vision_client()).text + mentions = await ingest_post_keymen( + conn, + keyman_client, + post_id, + post["post_title"], + post_body, + resolution_client=_organization_name_resolution_client(), + verification_client=_relation_verification_client(), + hierarchy_inference_client=_corporate_hierarchy_inference_client(), + persist_graph=False, + ) + organization_names = sorted( + {name for mention in mentions for name in mention.affiliated_organization_names} + ) + # relationship_client is gated by the same settings check as + # keyman_client above (both read ORCHESTRATOR_BASE_URL/_API_KEY), + # so reaching here means it is available too. + relationships = await ingest_post_entity_relationships( + conn, relationship_client, post_id, post["post_title"], post_body, organization_names + ) async with conn.transaction(): - mentions = await ingest_post_keymen(conn, keyman_client, post_id, post["post_title"], post_body) - organization_names = sorted( - {name for mention in mentions for name in mention.affiliated_organization_names} - ) - # relationship_client is gated by the same settings check as - # keyman_client above (both read ORCHESTRATOR_BASE_URL/_API_KEY), - # so reaching here means it is available too. - relationships = await ingest_post_entity_relationships( - conn, relationship_client, post_id, post["post_title"], post_body, organization_names - ) + await persist_edges_for_post(conn, post_id) return { "post_id": str(post["post_id"]), "extracted_count": len(mentions), @@ -814,8 +894,17 @@ async def read_post_summary( ) body_row = await conn.fetchrow("select post_body from source_post where post_id = $1", post_id) normalized_body = normalize_post_body(body_row["post_body"], vision_client=_vision_client()).text - summary = client.summarize(post["post_title"], normalized_body) - return await persist_post_summary(conn, post_id, summary) + summary = await asyncio.to_thread( + client.summarize, post["post_title"], normalized_body + ) + return await persist_post_summary( + conn, + post_id, + summary, + post_body=normalized_body, + hierarchy_inference_client=_corporate_hierarchy_inference_client(), + verification_client=_relation_verification_client(), + ) class ChatRequest(BaseModel): @@ -1098,6 +1187,99 @@ async def derive_post_commitment( return {"post_id": str(post["post_id"]), "has_commitment": True, "ticket": ticket} +@app.get("/api/analysis-runs") +async def list_analysis_runs( + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """Authorized analysis-run list: aggregates and labels only. + + Hidden scopes 404 at the item path and never appear here. The + payload has no source SQL, DSN, raw record, or provider body. + """ + _require_post_read(account) + async with pool.acquire() as conn: + runs = await fetch_visible_analysis_runs( + conn, + account.user_account_id, + list(account.corporate_entity_ids), + ) + return {"analysis_runs": runs} + + +class CreateAnalysisRunRequest(BaseModel): + """JSON body for ``POST /api/analysis-runs``. + + Omitting ``corporate_entity_id`` uses the account's sole affiliation. + Reconstruction and TEPP execution stay later slices; this write + records Pending only. + """ + + run_kind_code: str = "analysis_run_lineage" + scope_kind_code: str = "analysis_scope_corporate_entity" + corporate_entity_id: str | None = None + knowledge_cutoff: datetime | None = None + idempotency_key: str + + +@app.post("/api/analysis-runs", status_code=status.HTTP_201_CREATED) +async def create_analysis_run( + request: CreateAnalysisRunRequest, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """Record a Pending analysis run on an authorized cutoff capture. + + post_read is enough: the caller requests a run of a corp they + already walk. The payload is the same authorized detail as GET. + Hidden scopes 404. A matching idempotent retry returns the same run. + """ + _require_post_read(account) + async with pool.acquire() as conn: + async with conn.transaction(): + try: + created = await create_pending_analysis_run( + conn, + account_id=account.user_account_id, + affiliated_entity_ids=list(account.corporate_entity_ids), + run_kind_code=request.run_kind_code, + scope_kind_code=request.scope_kind_code, + corporate_entity_id=request.corporate_entity_id, + knowledge_cutoff=request.knowledge_cutoff, + idempotency_key=request.idempotency_key, + ) + except AnalysisRunCreateError as exc: + raise HTTPException(exc.status_code, exc.detail) from exc + return created + + +@app.get("/api/analysis-runs/{analysis_run_id}") +async def read_analysis_run( + analysis_run_id: str, + account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, Any]: + """One authorized analysis-run projection, or 404 when hidden. + + Detail adds the labeled status history. Hidden runs never leak events. + """ + _require_post_read(account) + try: + UUID(analysis_run_id) + except ValueError: + raise HTTPException(status.HTTP_404_NOT_FOUND, "analysis run not found") from None + async with pool.acquire() as conn: + run = await fetch_visible_analysis_run( + conn, + analysis_run_id, + account.user_account_id, + list(account.corporate_entity_ids), + ) + if run is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "analysis run not found") + return run + + @app.get("/api/calendar") async def read_calendar( account: CurrentAccount = Depends(get_current_account), diff --git a/backend/app/organization_name_resolution_ingestion.py b/backend/app/organization_name_resolution_ingestion.py new file mode 100644 index 000000000..9300586c4 --- /dev/null +++ b/backend/app/organization_name_resolution_ingestion.py @@ -0,0 +1,73 @@ + +"""Cache and persist verified organization-name normalization.""" + +from __future__ import annotations + +import asyncio + +import asyncpg + +from lineageweave.organization_name_resolution import ( + OrganizationNameResolutionClient, + resolve_and_verify_organization_name, +) +from lineageweave.relation_verification import ( + STATUS_CORROBORATED, + RelationVerificationClient, +) + + +async def resolve_organization_name( + conn: asyncpg.Connection, + resolution_client: OrganizationNameResolutionClient, + verification_client: RelationVerificationClient, + raw_name: str, + context_text: str, +) -> str: + """Return the corroborated canonical name, otherwise ``raw_name``. + + Synchronous network adapters run in a worker thread so this async + ingestion path does not block unrelated requests. + """ + cached = await conn.fetchrow( + "select resolved_organization_name, verification_status_code " + "from organization_name_resolution where raw_organization_name = $1", + raw_name, + ) + if cached is not None: + if cached["verification_status_code"] == STATUS_CORROBORATED: + return cached["resolved_organization_name"] + return raw_name + if not resolution_client.available: + return raw_name + + resolution = await asyncio.to_thread( + resolve_and_verify_organization_name, + raw_name, + context_text, + resolution_client, + verification_client, + ) + if resolution is None: + return raw_name + + await conn.execute( + """ + insert into organization_name_resolution + (raw_organization_name, resolved_organization_name, + verification_status_code, verification_evidence_url) + values ($1, $2, $3, $4) + on conflict (raw_organization_name) do update set + resolved_organization_name = excluded.resolved_organization_name, + verification_status_code = excluded.verification_status_code, + verification_evidence_url = excluded.verification_evidence_url, + resolved_at = now() + """, + resolution.raw_organization_name, + resolution.resolved_organization_name, + resolution.verification_status_code, + resolution.verification_evidence_url, + ) + if resolution.verification_status_code == STATUS_CORROBORATED: + return resolution.resolved_organization_name + return raw_name diff --git a/backend/app/post_chat_ingestion.py b/backend/app/post_chat_ingestion.py index 2f9fe77ff..794faa10d 100644 --- a/backend/app/post_chat_ingestion.py +++ b/backend/app/post_chat_ingestion.py @@ -66,13 +66,14 @@ async def find_linked_post_ids(conn: asyncpg.Connection, post_id: str) -> Linked # Discover them here first: every post that mentions any person this # post itself mentions, then load the subgraph over that expanded set. person_rows = await conn.fetch( - "select distinct person_id from post_person_mention where post_id = $1", post_id + "select distinct person_id from combined_post_person_mention where post_id = $1", post_id ) person_ids = [row["person_id"] for row in person_rows] sibling_post_ids = [post_id] if person_ids: sibling_rows = await conn.fetch( - "select distinct post_id from post_person_mention where person_id = any($1::uuid[])", + "select distinct post_id from combined_post_person_mention " + "where person_id = any($1::uuid[])", person_ids, ) sibling_post_ids = list({str(row["post_id"]) for row in sibling_rows} | {post_id}) diff --git a/backend/app/post_summary_ingestion.py b/backend/app/post_summary_ingestion.py index 7e7569d68..3febf9b21 100644 --- a/backend/app/post_summary_ingestion.py +++ b/backend/app/post_summary_ingestion.py @@ -1,4 +1,28 @@ -"""Persist and load the popup's Korean summary / key events / R&R.""" +"""Persist and load the popup's Korean summary / key events / R&R. + +ADR 0009 / 0019: an R&R actor is not just per-post free text -- when it +is a team or organization, it is resolved to a shared catalog identity +(``cataloged_team`` / ``corporate_entity``) stored on the role row and +a Knowledge Graph mention edge is written, so the same "설계팀" or +organization named across two posts becomes one linkable node. Fetch +never reconstructs that id by ``entity_name``; that column is not unique. +A person actor is opportunistically joined to an *existing* +``cataloged_person`` row by name when Keyman extraction has already +cataloged that name. The R&R evidence is written to +``post_summary_person_mention`` rather than Keyman's +``post_person_mention`` so either extractor can replace its own result +without leaving or deleting the other's evidence. + +ADR 0010: an organization actor's name is resolved via +``get_or_create_corporate_entity`` -- similarity matching first, then +an LLM-proposed, search-corroborated hierarchy placement before +creating a real new row, so a real dataset's first mention of a +counterparty organization actually populates the corporate hierarchy +tree instead of staying permanently unresolved. Inference, +verification, and the short advisory-lock creation transaction finish +before the summary-replacement transaction begins; slow external work +therefore cannot extend the lock or the atomic replacement window. +""" from __future__ import annotations @@ -6,12 +30,40 @@ import asyncpg +from lineageweave.corporate_hierarchy_inference import ( + CorporateHierarchyInferenceClient, + NullCorporateHierarchyInferenceClient, +) from lineageweave.fixtures import fixture_thread_cast -from lineageweave.post_summary import PostSummary, RoleResponsibility +from lineageweave.knowledge_graph import NODE_CORPORATE_ENTITY, NODE_TEAM +from lineageweave.ontology import ontology_annotations +from lineageweave.post_summary import ( + ACTOR_TYPE_ORGANIZATION, + ACTOR_TYPE_PERSON, + ACTOR_TYPE_TEAM, + PostSummary, + RoleResponsibility, +) +from lineageweave.relation_verification import ( + NullRelationVerificationClient, + RelationVerificationClient, +) + +from .corporate_entity_ingestion import get_or_create_corporate_entity +from .keyman_ingestion import _load_corporate_entity_candidates +from .knowledge_graph import persist_edges_for_post +from .team_ingestion import upsert_team -async def fetch_persisted_summary(conn: asyncpg.Connection, post_id: str) -> dict[str, Any] | None: - """Return the stored summary payload, or None when none has been written.""" +async def fetch_persisted_summary( + conn: asyncpg.Connection, post_id: str +) -> dict[str, Any] | None: + """Return the stored summary payload, or None when none has been written. + + ``catalog_node_id`` comes from the role row's catalog foreign keys + (ADR 0019). This function does not join ``corporate_entity`` by + ``entity_name``. + """ header = await conn.fetchrow( "select korean_summary from post_summary_result where post_id = $1", post_id, @@ -23,23 +75,127 @@ async def fetch_persisted_summary(conn: asyncpg.Connection, post_id: str) -> dic post_id, ) roles = await conn.fetch( - "select person_name, responsibility from post_summary_role " - "where post_id = $1 order by person_name", + """ + select role.actor_name, role.responsibility, role.actor_type_code, + role.affiliated_organization_name, + role.cataloged_team_id, + role.cataloged_corporate_entity_id + from post_summary_role role + where role.post_id = $1 + order by role.actor_name + """, post_id, ) + payload_roles: list[dict[str, Any]] = [] + for row in roles: + catalog_node_id = None + catalog_node_type_code = None + if row["cataloged_team_id"] is not None: + catalog_node_id = str(row["cataloged_team_id"]) + catalog_node_type_code = NODE_TEAM + elif row["cataloged_corporate_entity_id"] is not None: + catalog_node_id = str(row["cataloged_corporate_entity_id"]) + catalog_node_type_code = NODE_CORPORATE_ENTITY + payload_roles.append( + { + "actor_name": row["actor_name"], + "responsibility": row["responsibility"], + "actor_type_code": row["actor_type_code"], + "affiliated_organization_name": row["affiliated_organization_name"], + "catalog_node_id": catalog_node_id, + "catalog_node_type_code": catalog_node_type_code, + **ontology_annotations(row["actor_type_code"]), + } + ) return { "post_id": post_id, "korean_summary": header["korean_summary"], "key_events": [row["event_text"] for row in events], - "roles_and_responsibilities": [ - {"person_name": row["person_name"], "responsibility": row["responsibility"]} - for row in roles - ], + "roles_and_responsibilities": payload_roles, } -async def persist_post_summary(conn: asyncpg.Connection, post_id: str, summary: PostSummary) -> dict[str, Any]: - """Replace the stored summary for ``post_id`` and return the public payload.""" +async def persist_post_summary( + conn: asyncpg.Connection, + post_id: str, + summary: PostSummary, + *, + post_body: str | None = None, + hierarchy_inference_client: CorporateHierarchyInferenceClient | None = None, + verification_client: RelationVerificationClient | None = None, +) -> dict[str, Any]: + """Replace the stored summary for ``post_id`` and return the public payload. + + ``post_body`` is the context an organization-actor hierarchy proposal + is inferred from (ADR 0010); it falls back to the summary's own Korean + text when not given. The pluggable clients default to unavailable Null + clients, so an organization actor then only resolves against an existing + ``corporate_entity``. + + Organization inference, verification, and any lock-protected catalog + creation complete before the atomic summary transaction. The catalog is + an idempotent shared identity registry; keeping that enrichment separate + prevents network latency and ``pg_advisory_xact_lock`` from extending the + summary replacement transaction while all post-owned rows still commit or + roll back together. + """ + hierarchy_inference_client = ( + hierarchy_inference_client or NullCorporateHierarchyInferenceClient() + ) + verification_client = verification_client or NullRelationVerificationClient() + + context_text = post_body if post_body is not None else summary.korean_summary + candidates = ( + await _load_corporate_entity_candidates(conn) + if summary.roles_and_responsibilities + else [] + ) + resolved_organization_ids: dict[int, str] = {} + for role_index, role in enumerate(summary.roles_and_responsibilities): + if role.actor_type_code != ACTOR_TYPE_ORGANIZATION: + continue + corporate_entity_id = await get_or_create_corporate_entity( + conn, + role.actor_name, + context_text, + hierarchy_inference_client, + verification_client, + candidates, + ) + if corporate_entity_id is not None: + resolved_organization_ids[role_index] = corporate_entity_id + + async with conn.transaction(): + await _replace_summary_projection( + conn, + post_id, + summary, + candidates, + resolved_organization_ids, + ) + + payload = await fetch_persisted_summary(conn, post_id) + if payload is None: + raise RuntimeError("persist_post_summary wrote no row") + return payload + + +async def _replace_summary_projection( + conn: asyncpg.Connection, + post_id: str, + summary: PostSummary, + candidates: list[Any], + resolved_organization_ids: dict[int, str], +) -> None: + """Write one atomic replacement using pre-resolved shared identities.""" + # Summary replacement owns only R&R projections. Keyman mentions remain + # independent and are combined only by the graph read/derivation view. + await conn.execute( + "delete from post_summary_person_mention where post_id = $1", + post_id, + ) + await conn.execute("delete from post_team_mention where post_id = $1", post_id) + await conn.execute("delete from post_organization_mention where post_id = $1", post_id) await conn.execute("delete from post_summary_result where post_id = $1", post_id) await conn.execute( "insert into post_summary_result (post_id, korean_summary) values ($1, $2)", @@ -48,22 +204,70 @@ async def persist_post_summary(conn: asyncpg.Connection, post_id: str, summary: ) for ordinal, event_text in enumerate(summary.key_events): await conn.execute( - "insert into post_summary_event (post_id, event_ordinal, event_text) values ($1, $2, $3)", + "insert into post_summary_event (post_id, event_ordinal, event_text) " + "values ($1, $2, $3)", post_id, ordinal, event_text, ) - for role in summary.roles_and_responsibilities: + # ADR 0009 / 0019: resolve catalog identity before writing the role + # row so fetch never reconstructs it by a non-unique name. + for role_index, role in enumerate(summary.roles_and_responsibilities): + cataloged_team_id = None + cataloged_corporate_entity_id = None + if role.actor_type_code == ACTOR_TYPE_TEAM: + cataloged_team_id = await upsert_team( + conn, + role.actor_name, + role.affiliated_organization_name, + candidates, + ) + elif role.actor_type_code == ACTOR_TYPE_ORGANIZATION: + cataloged_corporate_entity_id = resolved_organization_ids.get( + role_index + ) await conn.execute( - "insert into post_summary_role (post_id, person_name, responsibility) values ($1, $2, $3)", + "insert into post_summary_role " + "(post_id, actor_name, responsibility, actor_type_code, " + "affiliated_organization_name, cataloged_team_id, " + "cataloged_corporate_entity_id) values " + "($1, $2, $3, $4, $5, $6, $7)", post_id, - role.person_name, + role.actor_name, role.responsibility, + role.actor_type_code, + role.affiliated_organization_name, + cataloged_team_id, + cataloged_corporate_entity_id, ) - payload = await fetch_persisted_summary(conn, post_id) - if payload is None: - raise RuntimeError("persist_post_summary wrote no row") - return payload + if cataloged_team_id is not None: + await conn.execute( + "insert into post_team_mention (post_id, team_id) values ($1, $2) " + "on conflict do nothing", + post_id, + cataloged_team_id, + ) + elif cataloged_corporate_entity_id is not None: + await conn.execute( + "insert into post_organization_mention " + "(post_id, corporate_entity_id) values ($1, $2) " + "on conflict do nothing", + post_id, + cataloged_corporate_entity_id, + ) + elif role.actor_type_code == ACTOR_TYPE_PERSON: + person_row = await conn.fetchrow( + "select person_id from cataloged_person where person_name = $1 limit 1", + role.actor_name, + ) + if person_row is not None: + await conn.execute( + "insert into post_summary_person_mention (post_id, person_id) " + "values ($1, $2) on conflict do nothing", + post_id, + str(person_row["person_id"]), + ) + await persist_edges_for_post(conn, post_id) def seeded_demo_summary() -> PostSummary: @@ -75,8 +279,21 @@ def seeded_demo_summary() -> PostSummary: ), key_events=("출하 지연 후속 연락",), roles_and_responsibilities=( - RoleResponsibility(person_name="Ada West", responsibility="일정 확인 후속"), - RoleResponsibility(person_name="Priya Nair", responsibility="고객 측 수신"), + RoleResponsibility( + actor_name="Ada West", + responsibility="일정 확인 후속", + affiliated_organization_name="Demo Corp", + ), + RoleResponsibility( + actor_name="Priya Nair", + responsibility="고객 측 수신", + affiliated_organization_name="Northridge Grid", + ), + RoleResponsibility( + actor_name="당사", + responsibility="출하 일정 확정", + actor_type_code=ACTOR_TYPE_ORGANIZATION, + ), ), ) @@ -108,7 +325,11 @@ def _roles_for_fixture(post_title: str) -> tuple[RoleResponsibility, ...]: if cast is None or not cast.person_names: return () return tuple( - RoleResponsibility(person_name=name, responsibility=responsibility) + RoleResponsibility( + actor_name=name, + responsibility=responsibility, + affiliated_organization_name=_FIXTURE_ROLE_AFFILIATION.get(name), + ) for name in cast.person_names if (responsibility := _FIXTURE_ROLE_RESPONSIBILITY.get(name)) ) @@ -120,8 +341,15 @@ def _roles_for_fixture(post_title: str) -> tuple[RoleResponsibility, ...]: "Jordan Hale": "사양 검토", } +_FIXTURE_ROLE_AFFILIATION = { + "Ada West": "Demo Corp", + "Priya Nair": "Northridge Grid", + "Jordan Hale": "Westfield Power", +} + def _summary(korean: str, *events: str) -> PostSummary: + """Create one compact synthetic fixture summary.""" return PostSummary(korean_summary=korean, key_events=events) diff --git a/backend/app/team_ingestion.py b/backend/app/team_ingestion.py new file mode 100644 index 000000000..2d0c8787d --- /dev/null +++ b/backend/app/team_ingestion.py @@ -0,0 +1,48 @@ + +"""Resolve an R&R team actor to one shared cross-post identity.""" + +from __future__ import annotations + +import asyncpg + +from lineageweave.corporate_hierarchy_resolution import ( + CorporateEntityCandidate, + resolve_corporate_entity, +) + + +async def upsert_team( + conn: asyncpg.Connection, + team_name: str, + affiliated_organization_name: str | None, + candidates: list[CorporateEntityCandidate], +) -> str: + """Atomically return the unique team identity for the pair. + + ``UNIQUE NULLS NOT DISTINCT`` makes NULL affiliations participate + in the same conflict rule. One upsert removes the prior + read-then-insert race. + """ + corporate_entity_id = ( + resolve_corporate_entity(affiliated_organization_name, candidates) + if affiliated_organization_name + else None + ) + row = await conn.fetchrow( + """ + insert into cataloged_team + (team_name, affiliated_organization_name, + affiliated_corporate_entity_id) + values ($1, $2, $3) + on conflict (team_name, affiliated_organization_name) do update set + affiliated_corporate_entity_id = coalesce( + excluded.affiliated_corporate_entity_id, + cataloged_team.affiliated_corporate_entity_id + ) + returning team_id + """, + team_name, + affiliated_organization_name, + corporate_entity_id, + ) + return str(row["team_id"]) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 74ab44701..21c71bc9a 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -22,6 +22,7 @@ import redis from lineageweave.http_client import HttpClientError, get_json, post_form +from lineageweave.knowledge_graph import knowledge_graph_edges_for_post _POSTGRES_ADMIN_DSN = os.environ.get( "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://lineageweave:lineageweave_dev_only@localhost:15432/lineageweave" @@ -30,6 +31,7 @@ _VALKEY_URL = os.environ.get("LINEAGEWEAVE_TEST_VALKEY_URL", "redis://localhost:16379/0") _REALM = "lineageweave-demo" _MIGRATION_PATH = Path(__file__).resolve().parents[2] / "migrations" / "0001_initial_schema.sql" +_REGISTRY_MIGRATION = Path(__file__).resolve().parents[2] / "migrations" / "0018_analysis_run_registry.sql" def _postgres_available() -> bool: @@ -112,10 +114,12 @@ def seeded_db(demo_analyst_token): try: with conn.cursor() as cur: cur.execute(_MIGRATION_PATH.read_text()) + cur.execute(_REGISTRY_MIGRATION.read_text()) cur.execute( "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values " "('corporate_entity_level', 'group', 'Group'), " "('corporate_entity_level', 'company', 'Company'), " + "('corporate_entity_level', 'plant', 'Plant'), " "('post_visibility', 'public', 'Public'), " "('post_visibility', 'private', 'Private'), " "('voc_type', 'voc', 'Voice of Customer'), " @@ -125,9 +129,13 @@ def seeded_db(demo_analyst_token): "('node_type', 'node_person', 'Person'), " "('node_type', 'node_corporate_entity', 'Corporate entity'), " "('node_type', 'node_post', 'Post'), " + "('node_type', 'node_team', 'Team'), " "('edge_type', 'edge_mention', 'Mentioned in'), " "('edge_type', 'edge_affiliation', 'Affiliated with'), " "('edge_type', 'edge_co_mention', 'Co-mentioned'), " + "('edge_type', 'edge_mention_team', 'Team mentioned in'), " + "('edge_type', 'edge_team_affiliation', 'Team affiliated with'), " + "('edge_type', 'edge_mention_organization', 'Organization mentioned in'), " "('entity_relationship_type', 'rel_voc', 'Voice of Customer'), " "('entity_relationship_type', 'rel_vom', 'Voice of Market'), " "('entity_relationship_type', 'rel_vop', 'Voice of Partner'), " @@ -142,7 +150,10 @@ def seeded_db(demo_analyst_token): "('relation_verification_status', 'verify_uncorroborated', 'No corroborating evidence found'), " "('evaluation_criterion', 'general_sentiment_positive', 'Constructive stance'), " "('evaluation_criterion', 'general_sentiment_negative', 'Negative stance'), " - "('evaluation_criterion', 'sales_lead_specificity', 'Sales-lead specificity')" + "('evaluation_criterion', 'sales_lead_specificity', 'Sales-lead specificity'), " + "('prov_agent_type', 'prov_person', 'Person'), " + "('prov_agent_type', 'prov_organization', 'Organization'), " + "('prov_agent_type', 'prov_team', 'Team')" ) cur.execute( "insert into corporate_entity (corporate_entity_code, entity_name, entity_level_code) " @@ -179,16 +190,124 @@ def seeded_db(demo_analyst_token): "insert into role_permission (access_role_id, permission_code) values (%s, 'post_read')", (role_id,), ) + + def _seed_analysis_run( + digest: str, + idempotency_key: str, + requester_id, + scope_kind: str, + corp_id=None, + ) -> str: + cur.execute( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', + '2026-01-12T00:00:00Z', '2026-01-12T00:05:00Z') + returning analysis_source_snapshot_id + """, + (digest,), + ) + snapshot_id = cur.fetchone()[0] + cur.execute( + """ + insert into analysis_source_count + (analysis_source_snapshot_id, count_type_code, count_value) + values (%s, 'analysis_count_document', 3) + """, + (snapshot_id,), + ) + cur.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_lineage', %s, %s, + '2026-01-12T12:00:00Z', 'lineage-run-v1', %s, %s, + '2026-01-12T12:30:00Z') + returning analysis_run_id + """, + (snapshot_id, idempotency_key, requester_id, "b" * 64, "c" * 40), + ) + run_id = str(cur.fetchone()[0]) + if scope_kind == "analysis_scope_corporate_entity": + cur.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values (%s, %s, %s) + """, + (run_id, scope_kind, corp_id), + ) + else: + cur.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code) + values (%s, %s) + """, + (run_id, scope_kind), + ) + for ordinal, status, occurred in ( + (1, "analysis_status_pending", "2026-01-12T12:31:00Z"), + (2, "analysis_status_running", "2026-01-12T12:32:00Z"), + (3, "analysis_status_succeeded", "2026-01-12T12:33:00Z"), + ): + cur.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values (%s, %s, %s, %s) + """, + (run_id, ordinal, status, occurred), + ) + return run_id + + cur.execute( + "insert into user_account (external_subject_id, display_name, email_address) " + "values (%s, 'Other Analyst', 'other.analyst@example.test') returning user_account_id", + (f"other-{uuid.uuid4()}",), + ) + other_account_id = cur.fetchone()[0] + visible_run_id = _seed_analysis_run( + "a" * 64, + "visible-own-corp", + account_id, + "analysis_scope_corporate_entity", + own_corp_id, + ) + hidden_run_id = _seed_analysis_run( + "d" * 64, + "hidden-other-corp", + other_account_id, + "analysis_scope_corporate_entity", + other_corp_id, + ) + hidden_all_visible_id = _seed_analysis_run( + "e" * 64, + "hidden-all-visible", + other_account_id, + "analysis_scope_all_visible", + ) cur.execute( "insert into account_role_assignment (user_account_id, access_role_id) values (%s, %s)", (account_id, role_id), ) - def _insert_post(title: str, corporate_entity_id, visibility_code: str, body: str = "body") -> str: + def _insert_post( + title: str, + corporate_entity_id, + visibility_code: str, + body: str = "body", + created_at: str = "2026-01-10T12:00:00Z", + ) -> str: cur.execute( - "insert into source_post (author_account_id, corporate_entity_id, post_title, post_body, voc_type_code, visibility_code) " - "values (%s, %s, %s, %s, 'voc', %s) returning post_id", - (account_id, corporate_entity_id, title, body, visibility_code), + "insert into source_post (author_account_id, corporate_entity_id, post_title, post_body, voc_type_code, visibility_code, created_at) " + "values (%s, %s, %s, %s, 'voc', %s, %s) returning post_id", + (account_id, corporate_entity_id, title, body, visibility_code, created_at), ) return str(cur.fetchone()[0]) @@ -201,6 +320,13 @@ def _insert_post(title: str, corporate_entity_id, visibility_code: str, body: st "The weather in Gwangju was irrelevant.", ) other_private_post_id = _insert_post("Other-corp private post", other_corp_id, "private") + late_own_private_post_id = _insert_post( + "Late own-corp private post", + own_corp_id, + "private", + "A follow-up written after the January 2026 run cutoff.", + created_at="2026-01-20T12:00:00Z", + ) cur.execute( "insert into cataloged_person (person_name, person_side_code) values " @@ -286,10 +412,14 @@ def _insert_post(title: str, corporate_entity_id, visibility_code: str, body: st "own_corp_id": str(own_corp_id), "other_corp_id": str(other_corp_id), "own_private_post_id": own_private_post_id, + "late_own_private_post_id": late_own_private_post_id, "other_private_post_id": other_private_post_id, "our_person_id": our_person_id, "counterpart_person_id": counterpart_person_id, "hidden_person_id": hidden_person_id, + "visible_run_id": visible_run_id, + "hidden_run_id": hidden_run_id, + "hidden_all_visible_id": hidden_all_visible_id, } finally: conn.close() @@ -312,6 +442,137 @@ def client(seeded_db): yield test_client +def test_analysis_runs_are_labeled_aggregates_and_hide_other_scopes( + client, demo_analyst_token, seeded_db +) -> None: + """Demo analyst sees the Test Corp run, never the Other Corp or outsider run.""" + listed = client.get("/api/analysis-runs", headers={"Authorization": f"Bearer {demo_analyst_token}"}) + assert listed.status_code == 200 + runs = listed.json()["analysis_runs"] + ids = {run["analysis_run_id"] for run in runs} + assert seeded_db["visible_run_id"] in ids + assert seeded_db["hidden_run_id"] not in ids + assert seeded_db["hidden_all_visible_id"] not in ids + visible = next(run for run in runs if run["analysis_run_id"] == seeded_db["visible_run_id"]) + assert visible["run_kind_label"] == "Lineage reconstruction" + assert visible["status_label"] == "Succeeded" + assert visible["scope_kind_label"] == "Corporate entity" + assert visible["scope_entity_name"] == "Test Corp" + assert visible["source_counts"] == [ + { + "count_type_code": "analysis_count_document", + "count_type_label": "Documents", + "count_value": 3, + } + ] + dumped = str(visible) + assert "postgresql://" not in dumped + assert "select " not in dumped.lower() + assert "status_history" not in visible + + detail = client.get( + f"/api/analysis-runs/{seeded_db['visible_run_id']}", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert detail.status_code == 200 + body = detail.json() + assert body["configuration_schema_version"] == "lineage-run-v1" + assert "snapshot_sha256" not in body + history = body["status_history"] + assert [event["status_label"] for event in history] == [ + "Pending", + "Running", + "Succeeded", + ] + assert [event["occurred_at"][:16] for event in history] == [ + "2026-01-12T12:31", + "2026-01-12T12:32", + "2026-01-12T12:33", + ] + assert all("failure_code" not in event for event in history) + titles = {post["post_title"] for post in body["visible_posts"]} + assert "Own-corp private post" in titles + assert "Late own-corp private post" not in titles + assert "Other-corp private post" not in titles + assert "postgresql://" not in str(body) + assert "visible_posts" not in visible + + hidden = client.get( + f"/api/analysis-runs/{seeded_db['hidden_run_id']}", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert hidden.status_code == 404 + + unauthenticated = client.get("/api/analysis-runs") + assert unauthenticated.status_code == 401 + + +def test_create_analysis_run_records_pending_without_inventing_a_score( + client, demo_analyst_token, seeded_db +) -> None: + """POST /api/analysis-runs writes Pending on the authorized cutoff bag.""" + created = client.post( + "/api/analysis-runs", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + json={ + "run_kind_code": "analysis_run_lineage", + "corporate_entity_id": seeded_db["own_corp_id"], + "idempotency_key": "buyer-create-2026-w02", + }, + ) + assert created.status_code == 201 + body = created.json() + assert body["run_kind_label"] == "Lineage reconstruction" + assert body["status_label"] == "Pending" + assert body["status_history"][0]["status_label"] == "Pending" + assert all(event["status_label"] != "Succeeded" for event in body["status_history"]) + titles = {post["post_title"] for post in body["visible_posts"]} + assert "Own-corp private post" in titles + assert "Other-corp private post" not in titles + assert "theta" not in str(body).lower() + assert "postgresql://" not in str(body) + + replay = client.post( + "/api/analysis-runs", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + json={ + "run_kind_code": "analysis_run_lineage", + "corporate_entity_id": seeded_db["own_corp_id"], + "idempotency_key": "buyer-create-2026-w02", + }, + ) + assert replay.status_code == 201 + assert replay.json()["analysis_run_id"] == body["analysis_run_id"] + + conflict = client.post( + "/api/analysis-runs", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + json={ + "run_kind_code": "analysis_run_tepp", + "corporate_entity_id": seeded_db["own_corp_id"], + "idempotency_key": "buyer-create-2026-w02", + }, + ) + assert conflict.status_code == 409 + + hidden = client.post( + "/api/analysis-runs", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + json={ + "run_kind_code": "analysis_run_lineage", + "corporate_entity_id": seeded_db["other_corp_id"], + "idempotency_key": "buyer-create-hidden-corp", + }, + ) + assert hidden.status_code == 404 + + unauthenticated = client.post( + "/api/analysis-runs", + json={"idempotency_key": "buyer-create-unauthenticated"}, + ) + assert unauthenticated.status_code == 401 + + def test_me_reflects_the_authenticated_account(client, demo_analyst_token) -> None: response = client.get("/api/me", headers={"Authorization": f"Bearer {demo_analyst_token}"}) assert response.status_code == 200 @@ -324,7 +585,7 @@ def test_post_list_includes_public_and_own_corp_but_excludes_other_corp(client, response = client.get("/api/posts", headers={"Authorization": f"Bearer {demo_analyst_token}"}) assert response.status_code == 200 titles = {post["post_title"] for post in response.json()} - assert titles == {"Public post", "Own-corp private post"} + assert titles == {"Public post", "Own-corp private post", "Late own-corp private post"} public = next(post for post in response.json() if post["post_title"] == "Public post") assert public["voc_type_label"] == "Voice of Customer" assert public["visibility_label"] == "Public" @@ -363,8 +624,9 @@ def test_persisted_summary_is_returned_without_an_llm(client, demo_analyst_token (seeded_db["public_post_id"],), ) cur.execute( - "insert into post_summary_role (post_id, person_name, responsibility) " - "values (%s, 'Ada West', '후속 연락')", + "insert into post_summary_role " + "(post_id, actor_name, responsibility, actor_type_code, affiliated_organization_name) " + "values (%s, 'Ada West', '후속 연락', 'prov_person', 'Demo Corp')", (seeded_db["public_post_id"],), ) finally: @@ -378,9 +640,13 @@ def test_persisted_summary_is_returned_without_an_llm(client, demo_analyst_token body = response.json() assert body["korean_summary"] == "저장된 한국어 요약입니다." assert body["key_events"] == ["저장된 이벤트"] - assert body["roles_and_responsibilities"] == [ - {"person_name": "Ada West", "responsibility": "후속 연락"} - ] + assert len(body["roles_and_responsibilities"]) == 1 + role = body["roles_and_responsibilities"][0] + assert role["actor_name"] == "Ada West" + assert role["responsibility"] == "후속 연락" + assert role["actor_type_code"] == "prov_person" + assert role["affiliated_organization_name"] == "Demo Corp" + assert role["ontology_label"] == "Role actor (person)" def test_seed_demo_summary_surfaces_on_get_summary(client, demo_analyst_token, seeded_db) -> None: @@ -407,7 +673,10 @@ def test_seed_demo_summary_surfaces_on_get_summary(client, demo_analyst_token, s body = response.json() assert "에이다" in body["korean_summary"] assert body["key_events"] - assert any(role["person_name"] == "Ada West" for role in body["roles_and_responsibilities"]) + roles = {role["actor_name"]: role for role in body["roles_and_responsibilities"]} + assert roles["Ada West"]["actor_type_code"] == "prov_person" + assert roles["당사"]["actor_type_code"] == "prov_organization" + assert roles["당사"]["ontology_label"] == "Role actor (organization)" def test_seed_fixture_summaries_surface_on_get_summary(client, demo_analyst_token, seeded_db) -> None: @@ -466,7 +735,7 @@ def test_seed_fixture_summaries_surface_on_get_summary(client, demo_analyst_toke assert fork.status_code == 200, fork.text assert "재협상" in fork.json()["korean_summary"] assert fork.json()["key_events"] - fork_roles = {role["person_name"] for role in fork.json()["roles_and_responsibilities"]} + fork_roles = {role["actor_name"] for role in fork.json()["roles_and_responsibilities"]} assert fork_roles == {"Ada West", "Priya Nair"} calendar = client.get( @@ -883,6 +1152,8 @@ def test_related_keymen_use_rwr_and_hide_invisible_posts(client, demo_analyst_to counterpart = by_id[seeded_db["counterpart_person_id"]] assert counterpart["ontology_label"] == "Person" assert counterpart["ontology_iri"].endswith("#Person") + assert counterpart["person_side_code"] == "counterparty" + assert counterpart["person_side_label"] == "Counterparty" own_post = by_id[seeded_db["own_private_post_id"]] assert own_post["ontology_label"] == "Post" @@ -902,6 +1173,9 @@ def test_related_corporate_entity_uses_rwr_and_hides_invisible_posts( assert body["entity_name"] == "Test Corp" related_ids = {node["node_id"] for node in body["related"]} assert seeded_db["our_person_id"] in related_ids + our_person = next(node for node in body["related"] if node["node_id"] == seeded_db["our_person_id"]) + assert our_person["person_side_code"] == "our_side" + assert our_person["person_side_label"] == "Our side" assert seeded_db["other_private_post_id"] not in related_ids assert seeded_db["hidden_person_id"] not in related_ids @@ -927,6 +1201,44 @@ def test_unknown_corporate_entity_related_is_not_found( assert response.status_code == 404 +def test_team_only_on_other_corp_private_post_is_forbidden( + client, demo_analyst_token, seeded_db +) -> None: + """A team mentioned only on another corp's private post must 403.""" + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into cataloged_team (team_name, affiliated_organization_name) " + "values ('비공개 설계팀', 'Other Corp') returning team_id" + ) + team_id = str(cur.fetchone()[0]) + cur.execute( + "insert into post_team_mention (post_id, team_id) values (%s, %s)", + (seeded_db["other_private_post_id"], team_id), + ) + finally: + admin_conn.close() + + response = client.get( + f"/api/teams/{team_id}/related", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 403 + + +def test_unknown_team_related_is_not_found(client, demo_analyst_token) -> None: + """An unknown team UUID must 404, matching person and entity related.""" + + response = client.get( + f"/api/teams/{uuid.uuid4()}/related", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 404 + + def test_keyman_only_on_other_corp_private_post_is_forbidden(client, demo_analyst_token, seeded_db) -> None: response = client.get( f"/api/keymen/{seeded_db['hidden_person_id']}/related", @@ -947,6 +1259,552 @@ def test_extract_keymen_requires_post_admin(client, demo_analyst_token, seeded_d _ORCHESTRATOR_API_KEY = os.environ.get("LINEAGEWEAVE_TEST_ORCHESTRATOR_API_KEY") +def test_extract_keymen_does_not_merge_same_name_people_with_conflicting_titles( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """Two different real people can share a name -- extracting a second + post that names the same person_name+side but a genuinely different + stated job_title must NOT reuse the first post's cataloged_person row. + A deterministic fake client (not a real orchestrator call) so this + is CI-stable: the point under test is `_upsert_person`'s own SQL + logic, not LLM extraction quality. + """ + from lineageweave.keyman_extraction import COUNTERPARTY, PersonMention + + _grant_post_admin(seeded_db["dsn"]) + + class _FakeClient: + available = True + + def __init__(self, job_title: str) -> None: + self._job_title = job_title + + def extract(self, post_title: str, post_body: str) -> list[PersonMention]: + return [PersonMention(person_name="Kim Cheolsu", person_side_code=COUNTERPARTY, job_title=self._job_title)] + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + post_ids = [] + for title in ("Sales follow-up", "Purchasing follow-up"): + cur.execute( + "insert into source_post (author_account_id, corporate_entity_id, post_title, post_body, voc_type_code, visibility_code) " + "select author_account_id, corporate_entity_id, %s, %s, 'voc', 'public' " + "from source_post where post_id = %s " + "returning post_id", + (title, "placeholder body", seeded_db["own_private_post_id"]), + ) + post_ids.append(str(cur.fetchone()[0])) + finally: + admin_conn.close() + + monkeypatch.setattr("backend.app.main._entity_relationship_client", lambda: _FakeClient("unused")) + + monkeypatch.setattr("backend.app.main._keyman_extraction_client", lambda: _FakeClient("Sales Manager")) + response_a = client.post( + f"/api/posts/{post_ids[0]}/extract-keymen", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response_a.status_code == 200, response_a.text + + monkeypatch.setattr("backend.app.main._keyman_extraction_client", lambda: _FakeClient("Purchasing Lead")) + response_b = client.post( + f"/api/posts/{post_ids[1]}/extract-keymen", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response_b.status_code == 200, response_b.text + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "select count(distinct person_id) from cataloged_person where person_name = 'Kim Cheolsu'" + ) + distinct_people = cur.fetchone()[0] + finally: + admin_conn.close() + + assert distinct_people == 2, "conflicting stated job titles for the same name must not be merged into one person" + + +def test_extract_keymen_resolves_and_caches_an_abbreviated_organization_name( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """ADR 0008: an affiliated organization named by abbreviation + ("AGP") must be resolved to its canonical name + ("Aurora Grid Power") and cross-verified before that name is trusted -- + deterministic fake resolution/verification clients (not a real LLM + or Searxng call) so this is CI-stable; the point under test is the + resolve-then-persist wiring, not model/search quality. + """ + from lineageweave.keyman_extraction import COUNTERPARTY, PersonMention + from lineageweave.relation_verification import STATUS_CORROBORATED, RelationVerificationResult + + _grant_post_admin(seeded_db["dsn"]) + + class _FakeKeymanClient: + available = True + + def extract(self, post_title: str, post_body: str) -> list[PersonMention]: + return [ + PersonMention( + person_name="Kim Cheolsu", + person_side_code=COUNTERPARTY, + affiliated_organization_names=("AGP",), + ) + ] + + class _FakeRelationshipClient: + available = True + + def classify(self, post_title: str, post_body: str, organization_names: list[str]): + return [] + + class _FakeResolutionClient: + available = True + + def resolve(self, raw_name: str, context_text: str) -> str | None: + assert raw_name == "AGP" + return "Aurora Grid Power" + + class _FakeVerificationClient: + available = True + + def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: + assert organization_name == "Aurora Grid Power" + assert relationship_label == "AGP" + return RelationVerificationResult( + status_code=STATUS_CORROBORATED, evidence_url="https://example.org/khnp" + ) + + monkeypatch.setattr("backend.app.main._keyman_extraction_client", lambda: _FakeKeymanClient()) + monkeypatch.setattr("backend.app.main._entity_relationship_client", lambda: _FakeRelationshipClient()) + monkeypatch.setattr( + "backend.app.main._organization_name_resolution_client", lambda: _FakeResolutionClient() + ) + monkeypatch.setattr("backend.app.main._relation_verification_client", lambda: _FakeVerificationClient()) + + response = client.post( + f"/api/posts/{seeded_db['own_private_post_id']}/extract-keymen", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "select resolved_organization_name, verification_status_code, verification_evidence_url " + "from organization_name_resolution where raw_organization_name = 'AGP'" + ) + cached = cur.fetchone() + cur.execute( + "select pa.affiliated_organization_name from person_affiliation pa " + "join cataloged_person cp on cp.person_id = pa.person_id " + "where cp.person_name = 'Kim Cheolsu'" + ) + affiliation_name = cur.fetchone()[0] + finally: + admin_conn.close() + + assert cached == ("Aurora Grid Power", STATUS_CORROBORATED, "https://example.org/khnp") + assert affiliation_name == "Aurora Grid Power", "a corroborated resolution must be the stored affiliation name" + + +def test_same_team_named_in_two_posts_resolves_to_one_cataloged_team( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """ADR 0009: extraction runs per-post, but "설계팀" (design team) at + the same company named in two different posts must resolve to the + same cataloged_team row -- otherwise every extraction is an island + and can never become a cross-post Knowledge Graph clue. A + deterministic fake summary client (not a real LLM call) so this is + CI-stable; the point under test is the upsert-then-dedupe wiring. + """ + from lineageweave.post_summary import ACTOR_TYPE_TEAM, PostSummary, RoleResponsibility + + class _FakeSummaryClient: + available = True + + def summarize(self, post_title: str, post_body: str) -> PostSummary: + return PostSummary( + korean_summary="설계팀이 도면을 검토했다.", + roles_and_responsibilities=( + RoleResponsibility( + actor_name="설계팀", + responsibility="도면 검토", + actor_type_code=ACTOR_TYPE_TEAM, + affiliated_organization_name="Demo Corp", + ), + ), + ) + + monkeypatch.setattr("backend.app.main._post_summary_client", lambda: _FakeSummaryClient()) + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + post_ids = [] + for title in ("설계 검토 회의 1", "설계 검토 회의 2"): + cur.execute( + "insert into source_post (author_account_id, corporate_entity_id, post_title, post_body, voc_type_code, visibility_code) " + "select author_account_id, corporate_entity_id, %s, %s, 'voc', 'public' " + "from source_post where post_id = %s returning post_id", + (title, "placeholder body", seeded_db["own_private_post_id"]), + ) + post_ids.append(str(cur.fetchone()[0])) + finally: + admin_conn.close() + + headers = {"Authorization": f"Bearer {demo_analyst_token}"} + for post_id in post_ids: + response = client.get(f"/api/posts/{post_id}/summary", headers=headers) + assert response.status_code == 200, response.text + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute("select count(*), count(distinct team_id) from cataloged_team where team_name = '설계팀'") + team_row_count, distinct_team_count = cur.fetchone() + cur.execute( + "select count(distinct pt.post_id) from post_team_mention pt " + "join cataloged_team ct on ct.team_id = pt.team_id " + "where ct.team_name = '설계팀'" + ) + mentioning_post_count = cur.fetchone()[0] + cur.execute( + "select count(*) from knowledge_graph_edge " + "where source_node_type_code = 'node_team' and edge_type_code = 'edge_mention_team'" + ) + team_mention_edge_count = cur.fetchone()[0] + finally: + admin_conn.close() + + assert (team_row_count, distinct_team_count) == (1, 1), "the same team+org pair must dedupe to one row" + assert mentioning_post_count == 2, "both posts must link to the single cataloged team" + assert team_mention_edge_count == 2, "each post's mention must become a real KG edge" + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + try: + with admin_conn.cursor() as cur: + cur.execute("select team_id from cataloged_team where team_name = '설계팀'") + team_id = str(cur.fetchone()[0]) + finally: + admin_conn.close() + + related = client.get( + f"/api/teams/{team_id}/related", + headers=headers, + ) + assert related.status_code == 200, related.text + related_ids = {node["node_id"] for node in related.json()["related"]} + assert set(post_ids) <= related_ids + summaries = [ + client.get(f"/api/posts/{post_id}/summary", headers=headers).json() + for post_id in post_ids + ] + for body in summaries: + role = body["roles_and_responsibilities"][0] + assert role["catalog_node_id"] == team_id + assert role["catalog_node_type_code"] == "node_team" + + +def test_organization_mention_only_posts_appear_in_entity_related( + client, demo_analyst_token, seeded_db +) -> None: + """An org mentioned with no affiliated person must still start a related walk.""" + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into source_post (author_account_id, corporate_entity_id, post_title, post_body, voc_type_code, visibility_code) " + "select author_account_id, corporate_entity_id, %s, %s, 'voc', 'public' " + "from source_post where post_id = %s returning post_id", + ("Org-only mention", "Test Corp was named without a person.", seeded_db["own_private_post_id"]), + ) + org_only_post_id = str(cur.fetchone()[0]) + cur.execute( + "insert into post_organization_mention (post_id, corporate_entity_id) values (%s, %s)", + (org_only_post_id, seeded_db["own_corp_id"]), + ) + for edge in knowledge_graph_edges_for_post( + org_only_post_id, + [], + organization_corporate_entity_ids=[seeded_db["own_corp_id"]], + ): + cur.execute( + "insert into knowledge_graph_edge (" + "source_node_type_code, source_node_id, target_node_type_code, " + "target_node_id, edge_type_code, edge_weight" + ") values (%s, %s, %s, %s, %s, %s) " + "on conflict do nothing", + ( + edge.source_node_type_code, + edge.source_node_id, + edge.target_node_type_code, + edge.target_node_id, + edge.edge_type_code, + edge.edge_weight, + ), + ) + finally: + admin_conn.close() + + response = client.get( + f"/api/corporate-entities/{seeded_db['own_corp_id']}/related", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + related_ids = {node["node_id"] for node in response.json()["related"]} + assert org_only_post_id in related_ids + + +def test_private_other_corp_organization_mention_does_not_leak( + client, demo_analyst_token, seeded_db +) -> None: + """The org-mention UNION must still apply ABAC per post.""" + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into post_organization_mention (post_id, corporate_entity_id) " + "values (%s, %s) on conflict do nothing", + (seeded_db["other_private_post_id"], seeded_db["own_corp_id"]), + ) + cur.execute( + "insert into post_organization_mention (post_id, corporate_entity_id) " + "values (%s, %s) on conflict do nothing", + (seeded_db["other_private_post_id"], seeded_db["other_corp_id"]), + ) + for entity_id in (seeded_db["own_corp_id"], seeded_db["other_corp_id"]): + for edge in knowledge_graph_edges_for_post( + seeded_db["other_private_post_id"], + [], + organization_corporate_entity_ids=[entity_id], + ): + cur.execute( + "insert into knowledge_graph_edge (" + "source_node_type_code, source_node_id, target_node_type_code, " + "target_node_id, edge_type_code, edge_weight" + ") values (%s, %s, %s, %s, %s, %s) " + "on conflict do nothing", + ( + edge.source_node_type_code, + edge.source_node_id, + edge.target_node_type_code, + edge.target_node_id, + edge.edge_type_code, + edge.edge_weight, + ), + ) + finally: + admin_conn.close() + + headers = {"Authorization": f"Bearer {demo_analyst_token}"} + own = client.get( + f"/api/corporate-entities/{seeded_db['own_corp_id']}/related", + headers=headers, + ) + assert own.status_code == 200, own.text + own_ids = {node["node_id"] for node in own.json()["related"]} + assert seeded_db["other_private_post_id"] not in own_ids + + hidden = client.get( + f"/api/corporate-entities/{seeded_db['other_corp_id']}/related", + headers=headers, + ) + assert hidden.status_code == 403 + + +def test_thread_group_run_list_honors_knowledge_cutoff( + client, demo_analyst_token, seeded_db +) -> None: + """A later public post must not surface a previously hidden thread-group run.""" + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "insert into source_post (author_account_id, corporate_entity_id, post_title, post_body, voc_type_code, visibility_code, thread_group_key, created_at) " + "select author_account_id, corporate_entity_id, %s, %s, 'voc', 'public', %s, %s " + "from source_post where post_id = %s", + ( + "Late thread-group post", + "Written after the January cutoff.", + "late-thread-group", + "2026-01-20T12:00:00Z", + seeded_db["own_private_post_id"], + ), + ) + cur.execute( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', + '2026-01-12T00:00:00Z', '2026-01-12T00:05:00Z') + returning analysis_source_snapshot_id + """, + ("f" * 64,), + ) + snapshot_id = cur.fetchone()[0] + cur.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_lineage', %s, + (select user_account_id from user_account + where email_address = 'other.analyst@example.test'), + '2026-01-12T12:00:00Z', 'lineage-run-v1', %s, %s, + '2026-01-12T12:30:00Z') + returning analysis_run_id + """, + (snapshot_id, "hidden-late-thread", "b" * 64, "c" * 40), + ) + run_id = str(cur.fetchone()[0]) + cur.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, scope_key) + values (%s, 'analysis_scope_thread_group', 'late-thread-group') + """, + (run_id,), + ) + cur.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values (%s, 1, 'analysis_status_succeeded', '2026-01-12T12:33:00Z') + """, + (run_id,), + ) + finally: + admin_conn.close() + + listed = client.get( + "/api/analysis-runs", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert listed.status_code == 200 + ids = {run["analysis_run_id"] for run in listed.json()["analysis_runs"]} + assert run_id not in ids + assert seeded_db["visible_run_id"] in ids + + +def test_first_mention_of_a_new_counterparty_creates_a_real_corporate_entity( + client, demo_analyst_token, seeded_db, monkeypatch +) -> None: + """ADR 0010: a person's affiliation to an organization with no + existing corporate_entity candidate must not stay permanently + unresolved -- an LLM-proposed, search-corroborated hierarchy + placement creates a real new row, closing the "통합 고객사 계열 + tree AI" gap synthetic regression corpus data confirmed (0 of thousands of + real affiliations ever resolved before this). Deterministic fake + clients, CI-stable -- the point under test is the create-then-link + wiring, not model/search quality. + """ + from lineageweave.corporate_hierarchy_inference import HierarchyProposal + from lineageweave.keyman_extraction import COUNTERPARTY, PersonMention + from lineageweave.relation_verification import STATUS_CORROBORATED, RelationVerificationResult + + _grant_post_admin(seeded_db["dsn"]) + + class _FakeKeymanClient: + available = True + + def extract(self, post_title: str, post_body: str) -> list[PersonMention]: + return [ + PersonMention( + person_name="Priya Sharma", + person_side_code=COUNTERPARTY, + affiliated_organization_names=("Northwind Turbines Gwangju Plant",), + ) + ] + + class _FakeRelationshipClient: + available = True + + def classify(self, post_title: str, post_body: str, organization_names: list[str]): + return [] + + class _FakeHierarchyInferenceClient: + available = True + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal | None: + if organization_name == "Northwind Turbines Gwangju Plant": + return HierarchyProposal(level_code="plant", parent_name="Northwind Turbines") + if organization_name == "Northwind Turbines": + return HierarchyProposal(level_code="company", parent_name=None) + return None + + class _FakeVerificationClient: + available = True + + def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: + return RelationVerificationResult( + status_code=STATUS_CORROBORATED, evidence_url=f"https://example.org/{organization_name}" + ) + + monkeypatch.setattr("backend.app.main._keyman_extraction_client", lambda: _FakeKeymanClient()) + monkeypatch.setattr("backend.app.main._entity_relationship_client", lambda: _FakeRelationshipClient()) + monkeypatch.setattr( + "backend.app.main._corporate_hierarchy_inference_client", lambda: _FakeHierarchyInferenceClient() + ) + monkeypatch.setattr("backend.app.main._relation_verification_client", lambda: _FakeVerificationClient()) + + response = client.post( + f"/api/posts/{seeded_db['own_private_post_id']}/extract-keymen", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + + admin_conn = psycopg2.connect(seeded_db["dsn"]) + admin_conn.autocommit = True + try: + with admin_conn.cursor() as cur: + cur.execute( + "select corporate_entity_id, entity_level_code, parent_entity_id, corporate_entity_code " + "from corporate_entity where entity_name = 'Northwind Turbines Gwangju Plant'" + ) + plant_row = cur.fetchone() + cur.execute( + "select corporate_entity_id, entity_level_code " + "from corporate_entity where entity_name = 'Northwind Turbines'" + ) + company_row = cur.fetchone() + cur.execute( + "select pa.affiliated_corporate_entity_id from person_affiliation pa " + "join cataloged_person cp on cp.person_id = pa.person_id " + "where cp.person_name = 'Priya Sharma'" + ) + affiliation_entity_id = cur.fetchone()[0] + finally: + admin_conn.close() + + assert plant_row is not None, "the plant-level entity must be created" + plant_entity_id, plant_level_code, plant_parent_id, plant_code = plant_row + assert plant_level_code == "plant" + assert plant_code.startswith("AUTO-"), "an auto-created code must never collide with a real login corp code" + assert company_row is not None, "the inferred parent company must also be created, not left dangling" + company_entity_id, company_level_code = company_row + assert company_level_code == "company" + assert str(plant_parent_id) == str(company_entity_id), "the plant's parent must be the real created company" + assert str(affiliation_entity_id) == str(plant_entity_id), "the affiliation must link to the real created plant" + + @pytest.mark.skipif( not (_ORCHESTRATOR_BASE_URL and _ORCHESTRATOR_API_KEY), reason="set LINEAGEWEAVE_TEST_ORCHESTRATOR_BASE_URL and LINEAGEWEAVE_TEST_ORCHESTRATOR_API_KEY to run", diff --git a/backend/uv-bootstrap-requirements.txt b/backend/uv-bootstrap-requirements.txt new file mode 100644 index 000000000..c981ffe9c --- /dev/null +++ b/backend/uv-bootstrap-requirements.txt @@ -0,0 +1,14 @@ +# uv bootstrap dependency for backend/Dockerfile. +# +# All hashes are the PyPI trusted-published uv 0.11.28 binary wheels for the +# Linux architectures supported by the pinned python:3.12-slim base manifest. +# --only-binary=:all: in the Dockerfile makes unsupported platforms fail closed +# instead of falling back to an unhashed source build. +uv==0.11.28 \ + --hash=sha256:49fe42df9f42056037473f3876adec1615709b57d3470ed39178ff420f3afb9f \ + --hash=sha256:041e4b80bebc58d7142ac9394370cacd73185fd8d066d6675d14707d83408f6d \ + --hash=sha256:185416a5316df8c5442b47178349f1f27fc1034468670ac1fb499eae3b25bd68 \ + --hash=sha256:a4a9fe246cb2882532277f5d5e5bd8a59462981462a2f98426f35ecfca82460e \ + --hash=sha256:6f7ce6f6015a3e857bc6a663514afa62856b669ee5c1bd120e4c58ac2ef5513d \ + --hash=sha256:6b3d0ea11e83b373a2166b82dd0864f5677fbadf98db64541ab2e59c42968905 \ + --hash=sha256:8c60294e3be4fa203a04015fc02ac8a31d936e86fde06dcb43c7f8f22661dfff diff --git a/docker/postgres-init/Dockerfile b/docker/postgres-init/Dockerfile index 0c6323a92..e394d376e 100644 --- a/docker/postgres-init/Dockerfile +++ b/docker/postgres-init/Dockerfile @@ -3,10 +3,9 @@ FROM postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f5 # Keycloak-database bootstrap and the product schema can be copied from # their single sources of truth. COPY docker/postgres-init/01-create-keycloak-db.sql /docker-entrypoint-initdb.d/01-create-keycloak-db.sql -# The exact same migration file tests/test_schema.py applies -- single -# source of truth, no re-typed copy. Runs against POSTGRES_DB (the "app" -# database) because docker-entrypoint-initdb.d executes each *.sql file -# with that database already selected. +# The exact same migration files the PostgreSQL contract tests apply -- single +# source of truth, no re-typed copy. docker-entrypoint-initdb.d executes each +# file against POSTGRES_DB in lexical order. COPY migrations/0001_initial_schema.sql /docker-entrypoint-initdb.d/02-app-schema.sql COPY migrations/0002_thread_grouping_keys.sql /docker-entrypoint-initdb.d/03-thread-grouping-keys.sql COPY migrations/0003_ticket_commitment_calendar.sql /docker-entrypoint-initdb.d/04-ticket-commitment-calendar.sql @@ -18,6 +17,14 @@ COPY migrations/0008_post_summary_result.sql /docker-entrypoint-initdb.d/09-post COPY migrations/0009_shared_metric_bank.sql /docker-entrypoint-initdb.d/10-shared-metric-bank.sql COPY migrations/0010_report_item_information.sql /docker-entrypoint-initdb.d/11-report-item-information.sql COPY migrations/0011_post_chat_result.sql /docker-entrypoint-initdb.d/12-post-chat-result.sql +COPY migrations/0012_role_responsibility_agent_type.sql /docker-entrypoint-initdb.d/13-role-responsibility-agent-type.sql +COPY migrations/0013_person_job_title.sql /docker-entrypoint-initdb.d/14-person-job-title.sql +COPY migrations/0014_role_responsibility_team_actor_type.sql /docker-entrypoint-initdb.d/15-role-responsibility-team-actor-type.sql +COPY migrations/0015_organization_name_resolution.sql /docker-entrypoint-initdb.d/16-organization-name-resolution.sql +COPY migrations/0016_cross_post_actor_identity.sql /docker-entrypoint-initdb.d/17-cross-post-actor-identity.sql +COPY migrations/0017_prov_o_standard_relations.sql /docker-entrypoint-initdb.d/18-prov-o-standard-relations.sql +COPY migrations/0018_analysis_run_registry.sql /docker-entrypoint-initdb.d/19-analysis-run-registry.sql +COPY migrations/0019_role_catalog_identity.sql /docker-entrypoint-initdb.d/20-role-catalog-identity.sql # Official image already drops to this account at runtime; declare it so # the Dockerfile itself satisfies DS-0002 (explicit non-root USER). USER postgres diff --git a/docs/PROV_O_IMPLEMENTATION.md b/docs/PROV_O_IMPLEMENTATION.md new file mode 100644 index 000000000..96b471e96 --- /dev/null +++ b/docs/PROV_O_IMPLEMENTATION.md @@ -0,0 +1,102 @@ +# W3C PROV-O implementation + +## Requirement + +LineageWeave must accept, validate, persist, infer, and serialize every normative relation in *PROV-O: The PROV Ontology* without flattening qualified influences or literal-valued properties into the existing navigation graph. + +## Runtime architecture + +```mermaid +flowchart LR + A[Product records and external RDF] --> B[PROV-O canonicalizer] + B --> C[Domain/range and literal validator] + C --> D[Explicit provenance assertions] + D --> E[Deterministic materializer] + E --> F[Qualified-to-unqualified implications] + E --> G[Property hierarchy and inverse closure] + E --> H[Qualified event-time shortcuts] + D --> I[(Normalized PostgreSQL provenance store)] + E --> J[RDF/Turtle/JSON-LD via rdflib] + I --> K[Explicit projection] + K --> L[(knowledge_graph_edge navigation graph)] +``` + +## Supported standard surface + +The machine-verifiable inventory is in [`PROV_O_IMPLEMENTATION_MATRIX.md`](PROV_O_IMPLEMENTATION_MATRIX.md): + +- all 30 PROV-O classes; +- all 50 normative properties; +- exact object/datatype distinction; +- direct domains, resource ranges, and `xsd:dateTime` ranges; +- class and property hierarchies; +- both normative qualification tables; +- every Appendix B recommended inverse name. + +## Canonicalization contract + +Inputs may use a local name, `prov:` compact name, full W3C IRI, or a reserved Appendix B inverse name. A canonical property name always retains its standard direction. A reserved inverse name that is not itself one of the 50 normative properties reverses subject and object into the preferred relation. + +```text +source prov:hadDerivation derived + ↓ canonicalize and reverse +derived prov:wasDerivedFrom source +``` + +No inverse alias is accepted for datatype properties because reversing a literal cannot produce a valid RDF subject. + +## Qualification contract + +For each normative mapping: + +```text +influenced --qualifiedRelation--> influence +influence --influencerProperty--> influencer +``` + +LineageWeave materializes: + +```text +influenced --unqualifiedRelation--> influencer +``` + +This applies to Generation, Derivation, Attribution, Usage, Communication, Association, Delegation, generic Influence, PrimarySource, Quotation, Revision, Invalidation, Start, and End. + +## Persistence contract + +`migrations/0017_prov_o_standard_relations.sql` creates a third-normal-form catalog and assertion store. A PostgreSQL trigger recursively checks subject domains and resource ranges through the class hierarchy and checks datatype-property literals before insertion. One assertion has exactly one resource or literal object. Inference provenance is represented by the many-to-many `provenance_assertion_derivation` table. + +## Security and tenancy boundary + +- External IRIs and lexical values are data, never executable instructions. +- RDF serialization performs no external fetch. +- The support profile uses `owl:imports` as metadata; runtime code does not dereference it. +- Assertions are rejected if resources are undeclared or incorrectly typed. +- The migration does not weaken existing row-level access decisions. API exposure must apply the same authenticated product boundary before binding product nodes to provenance resources. + +## Operability + +- Definitions are idempotently seeded. +- Exact W3C IRIs are stable; relational codes are multiword snake case. +- Standard definitions and runtime data are separate, so ontology upgrades can be reviewed without rewriting assertions. +- `provenance_resource_binding` is the only bridge to LineageWeave node identifiers; projections remain reproducible and removable. + +## Acceptance evidence + +```bash +pytest -q tests/test_prov_o.py +coverage run --branch -m pytest -q tests/test_prov_o.py +coverage report -m lineageweave/prov_o.py +python -m compileall -q lineageweave tests +``` + +Expected focused result: all tests pass and `lineageweave/prov_o.py` reports 100% statements and branches. + +## OWL 2 RL compatibility domains are not universal permissions + +Appendix A also publishes broad `prov:Influence` domains for +`prov:hadActivity` and `prov:hadRole` as OWL 2 RL compatibility aids. +The Recommendation explicitly warns that these broad domains must not be +read as permission to use either property on every Influence. Runtime and +database validation therefore enforce the normative union members rather +than weakening the contract. diff --git a/docs/PROV_O_IMPLEMENTATION_MATRIX.md b/docs/PROV_O_IMPLEMENTATION_MATRIX.md new file mode 100644 index 000000000..8a3c28617 --- /dev/null +++ b/docs/PROV_O_IMPLEMENTATION_MATRIX.md @@ -0,0 +1,67 @@ +# PROV-O implementation matrix + +LineageWeave implements the W3C PROV-O Recommendation as a separate standards-complete provenance layer. The product-specific `knowledge_graph_edge` remains a compact navigation projection; it is not used to flatten literal-valued or qualified PROV-O assertions. + +## Coverage contract + +- 30 normative classes. +- 50 normative properties: 44 object properties and 6 datatype properties. +- 14 qualified influence mappings from Tables 2 and 3. +- Qualified forms imply their unqualified forms. +- Transitive subproperty closure, defined inverses, `alternateOf` symmetry, and qualified event-time shortcuts are materialized deterministically. +- All 44 Appendix B inverse names are cataloged; non-canonical reserved names are accepted by reversing into the preferred PROV-O direction. + +## Property matrix + +| PROV-O property | Kind | Domain | Range / datatype | Superproperty | Qualification | Appendix B inverse | +|---|---|---|---|---|---|---| +| `prov:wasGeneratedBy` | object | Entity | Activity | wasInfluencedBy | qualifiedGeneration → Generation.activity | `prov:generated` | +| `prov:wasDerivedFrom` | object | Entity | Entity | wasInfluencedBy | qualifiedDerivation → Derivation.entity | `prov:hadDerivation` | +| `prov:wasAttributedTo` | object | Entity | Agent | wasInfluencedBy | qualifiedAttribution → Attribution.agent | `prov:contributed` | +| `prov:startedAtTime` | datatype | Activity | http://www.w3.org/2001/XMLSchema#dateTime | — | — | — | +| `prov:used` | object | Activity | Entity | wasInfluencedBy | qualifiedUsage → Usage.entity | `prov:wasUsedBy` | +| `prov:wasInformedBy` | object | Activity | Activity | wasInfluencedBy | qualifiedCommunication → Communication.activity | `prov:informed` | +| `prov:endedAtTime` | datatype | Activity | http://www.w3.org/2001/XMLSchema#dateTime | — | — | — | +| `prov:wasAssociatedWith` | object | Activity | Agent | wasInfluencedBy | qualifiedAssociation → Association.agent | `prov:wasAssociateFor` | +| `prov:actedOnBehalfOf` | object | Agent | Agent | wasInfluencedBy | qualifiedDelegation → Delegation.agent | `prov:hadDelegate` | +| `prov:alternateOf` | object | Entity | Entity | — | — | `prov:alternateOf` | +| `prov:specializationOf` | object | Entity | Entity | alternateOf | — | `prov:generalizationOf` | +| `prov:generatedAtTime` | datatype | Entity | http://www.w3.org/2001/XMLSchema#dateTime | — | — | — | +| `prov:hadPrimarySource` | object | Entity | Entity | wasDerivedFrom | qualifiedPrimarySource → PrimarySource.entity | `prov:wasPrimarySourceOf` | +| `prov:value` | datatype | Entity | RDF literal | — | — | — | +| `prov:wasQuotedFrom` | object | Entity | Entity | wasDerivedFrom | qualifiedQuotation → Quotation.entity | `prov:quotedAs` | +| `prov:wasRevisionOf` | object | Entity | Entity | wasDerivedFrom | qualifiedRevision → Revision.entity | `prov:hadRevision` | +| `prov:invalidatedAtTime` | datatype | Entity | http://www.w3.org/2001/XMLSchema#dateTime | — | — | — | +| `prov:wasInvalidatedBy` | object | Entity | Activity | wasInfluencedBy | qualifiedInvalidation → Invalidation.activity | `prov:invalidated` | +| `prov:hadMember` | object | Collection | Entity | wasInfluencedBy | — | `prov:wasMemberOf` | +| `prov:wasStartedBy` | object | Activity | Entity | wasInfluencedBy | qualifiedStart → Start.entity | `prov:started` | +| `prov:wasEndedBy` | object | Activity | Entity | wasInfluencedBy | qualifiedEnd → End.entity | `prov:ended` | +| `prov:invalidated` | object | Activity | Entity | influenced | — | `prov:wasInvalidatedBy` | +| `prov:influenced` | object | Entity / Activity / Agent | Entity / Activity / Agent | — | — | `prov:wasInfluencedBy` | +| `prov:atLocation` | object | Activity / Agent / Entity / InstantaneousEvent | Location | — | — | `prov:locationOf` | +| `prov:generated` | object | Activity | Entity | influenced | — | `prov:wasGeneratedBy` | +| `prov:wasInfluencedBy` | object | Entity / Activity / Agent | Entity / Activity / Agent | — | qualifiedInfluence → Influence.influencer | `prov:influenced` | +| `prov:qualifiedInfluence` | object | Entity / Activity / Agent | Influence | — | — | `prov:qualifiedInfluenceOf` | +| `prov:qualifiedGeneration` | object | Entity | Generation | qualifiedInfluence | — | `prov:qualifiedGenerationOf` | +| `prov:qualifiedDerivation` | object | Entity | Derivation | qualifiedInfluence | — | `prov:qualifiedDerivationOf` | +| `prov:qualifiedPrimarySource` | object | Entity | PrimarySource | qualifiedInfluence | — | `prov:qualifiedSourceOf` | +| `prov:qualifiedQuotation` | object | Entity | Quotation | qualifiedInfluence | — | `prov:qualifiedQuotationOf` | +| `prov:qualifiedRevision` | object | Entity | Revision | qualifiedInfluence | — | `prov:revisedEntity` | +| `prov:qualifiedAttribution` | object | Entity | Attribution | qualifiedInfluence | — | `prov:qualifiedAttributionOf` | +| `prov:qualifiedInvalidation` | object | Entity | Invalidation | qualifiedInfluence | — | `prov:qualifiedInvalidationOf` | +| `prov:qualifiedStart` | object | Activity | Start | qualifiedInfluence | — | `prov:qualifiedStartOf` | +| `prov:qualifiedUsage` | object | Activity | Usage | qualifiedInfluence | — | `prov:qualifiedUsingActivity` | +| `prov:qualifiedCommunication` | object | Activity | Communication | qualifiedInfluence | — | `prov:qualifiedCommunicationOf` | +| `prov:qualifiedAssociation` | object | Activity | Association | qualifiedInfluence | — | `prov:qualifiedAssociationOf` | +| `prov:qualifiedEnd` | object | Activity | End | qualifiedInfluence | — | `prov:qualifiedEndOf` | +| `prov:qualifiedDelegation` | object | Agent | Delegation | qualifiedInfluence | — | `prov:qualifiedDelegationOf` | +| `prov:influencer` | object | Influence | Entity / Activity / Agent | — | — | `prov:hadInfluence` | +| `prov:entity` | object | EntityInfluence | Entity | influencer | — | `prov:entityOfInfluence` | +| `prov:hadUsage` | object | Derivation | Usage | — | — | `prov:wasUsedInDerivation` | +| `prov:hadGeneration` | object | Derivation | Generation | — | — | `prov:generatedAsDerivation` | +| `prov:activity` | object | ActivityInfluence | Activity | influencer | — | `prov:activityOfInfluence` | +| `prov:agent` | object | AgentInfluence | Agent | influencer | — | `prov:agentOfInfluence` | +| `prov:hadPlan` | object | Association | Plan | — | — | `prov:wasPlanOf` | +| `prov:hadActivity` | object | Delegation / Derivation / End / Start | Activity | — | — | `prov:wasActivityOfInfluence` | +| `prov:atTime` | datatype | InstantaneousEvent | http://www.w3.org/2001/XMLSchema#dateTime | — | — | — | +| `prov:hadRole` | object | Association / InstantaneousEvent | Role | — | — | `prov:wasRoleIn` | diff --git a/docs/adr/0006-role-responsibility-agent-ontology.md b/docs/adr/0006-role-responsibility-agent-ontology.md new file mode 100644 index 000000000..8ded02b81 --- /dev/null +++ b/docs/adr/0006-role-responsibility-agent-ontology.md @@ -0,0 +1,107 @@ +# ADR 0006 — R&R's named actor is a PROV-O Agent, not always a person + +**Decision status:** Accepted +**Date:** 2026-08-14 + +## Context + +`post_summary.py`'s R&R (roles & responsibilities) extraction treats +the acting party a post's text names as if it were always a person. +Business correspondence routinely names an organization acting in its +own name -- "당사" (our company), "Demo Corp" -- not a named +individual. The product requirement is to handle that with a general +standard ontology, and to infer a person actor's affiliation so a +bare name is not left unplaced. + +Before this change, `RoleResponsibility.person_name` had no way to +express "this actor is an organization" -- every entry was forced into +a person slot, and an organization actor's name would sit +indistinguishable from an unresolved person. + +## Decision + +Ground the distinction in W3C PROV-O (Lebo, Sahoo, & McGuinness, 2013): +`prov:Agent` is the general acting-party class, with `prov:Person` and +`prov:Organization` as its two recognized subclasses -- an existing, +widely-adopted standard for exactly this "who/what acted" provenance +question, not a bespoke local invention. + +`RoleResponsibility` (`lineageweave/post_summary.py`) gains: +- `actor_name` (renamed from `person_name` -- the field can now hold an + organization's name too, so "person" in the field name would be + actively wrong). +- `actor_type_code`: `prov_person` / `prov_organization` + (`common_lookup_value` category `prov_agent_type`), defaulting to + `prov_person` when the LLM's response omits the field, matching this + repo's existing degrade-gracefully-not-fail discipline. +- `affiliated_organization_name`: for a person actor, the organization + the text names or clearly implies they work for, inferred by the same + LLM call rather than left for a human to cross-reference against the + Keyman panel separately. `None` when the text gives nothing to infer, + or when the actor is itself an organization (its own name already + answers "which organization"). + +The LLM prompt now explicitly instructs the model to decide +person-vs-organization per actor rather than defaulting every named +actor to a person, and to give an affiliation when the text supports +one. + +Ontology (`docs/ontology/lineageweave-kg.ttl`, extending +[ADR 0004](0004-knowledge-graph-ontology.md)'s vocabulary): +`:RoleActorPerson rdfs:subClassOf prov:Person` and +`:RoleActorOrganization rdfs:subClassOf prov:Organization`, each +carrying the `:lookupCode` annotation linking it to the matching +`common_lookup_value` row -- these are genuinely subclasses of the real +external PROV-O classes (imported via the `prov:` prefix), not +same-named local terms that merely resemble the standard. Kept distinct +from the ontology's existing `:Person` (node_type's `node_person`, +i.e. a cataloged Keyman with a stable `person_id`): an R&R actor is a +free-text name with no cataloged identity of its own, and may not even +resolve to a Keyman row. + +Persistence: `post_summary_role` gains `actor_type_code` (FK to +`common_lookup_value`, default `prov_person`) and +`affiliated_organization_name`; `person_name` is renamed to +`actor_name` via `migrations/0012_role_responsibility_agent_type.sql`'s +`ALTER TABLE ... RENAME COLUMN` (preserves every existing row's data, +unlike a drop/recreate) plus the two new `ADD COLUMN IF NOT EXISTS` +statements, with `migrations/0001_initial_schema.sql` updated directly +for a fresh install, matching this repo's established pattern (e.g. +ADR 0005's `verification_status_code` additions). + +UI: the popup's R&R list (`frontend/src/App.tsx`) shows a +Person/Organization badge per actor and the inferred affiliation in +parentheses; only a person actor is still linked to the Keyman panel +(an organization actor has no `person_id` to link to). + +## Consequences + +- `RoleResponsibility.person_name` is a breaking rename to `actor_name` + across the JSON wire contract (`GET /api/posts/{id}/summary`), the + DB column, and every call site. Accepted because the field's old name + was actively misleading once an organization actor is an intended + value, not a hypothetical edge case. +- `prov_agent_type` is a `common_lookup_value` category seeded by its + own migration file (0012), not literally embedded in + `scripts/seed_demo_data.py`'s SQL string the way ADR 0004's original + five covered categories are -- `tests/test_ontology.py`'s round-trip + check reads 0012's file content alongside the seed script's own text + so this still closes the loop, rather than being silently excluded + the way `evaluation_criterion` / `relation_verification_status` + currently are. +- The affiliation inference is opportunistic, not authoritative: it is + a same-request LLM guess from the post's own text, not resolved + against `corporate_entity` the way Keyman affiliations are (see + `lineageweave/corporate_hierarchy_resolution.py`). A future slice + could route it through the same resolver if real usage shows the + free-text name needs matching back to a cataloged organization. + +## Related + +Extends [ADR 0004](0004-knowledge-graph-ontology.md)'s Ontology/ +Semantic-Layer vocabulary and reuses its round-trip enforcement +mechanism (`tests/test_ontology.py`). + +## References (APA 7th) + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ diff --git a/docs/adr/0007-team-actor-type.md b/docs/adr/0007-team-actor-type.md new file mode 100644 index 000000000..1da18883b --- /dev/null +++ b/docs/adr/0007-team-actor-type.md @@ -0,0 +1,90 @@ +# ADR 0007 — R&R's named actor can be a team, a meso-level unit, not just a person/organization + +**Decision status:** Accepted +**Date:** 2026-08-14 + +## Context + +ADR 0006 gave R&R's `actor_type_code` two values: `prov_person` and +`prov_organization`, grounded in W3C PROV-O's `prov:Agent` subclasses. +Real post text surfaced a third, distinct case those two do not cover: +a named sub-unit of a company -- e.g. "설계팀" (design team) -- acting +in the text. A team is not a person, and forcing it into +`prov_organization` is wrong for the same reason ADR 0006 rejected +forcing an organization into a person slot: it collapses a real, +useful distinction. A team is meso-level -- part of a company, not the +company itself, and not an individual either. + +PROV-O has no sub-organization concept to reuse here; `prov:Agent`'s +two subclasses are exhaustive for PROV-O's own purposes (an +organization's internal structure is out of PROV-O's scope). + +## Decision + +Ground the team case in the W3C Organization Ontology (Reynolds, 2014): +`org:OrganizationalUnit`, defined for exactly this -- representing the +division of an organization into sub-organizational units, linked to +its parent via `org:unitOf`/`org:subOrganizationOf`. This is a +different, complementary W3C vocabulary from PROV-O, not a conflicting +one: PROV-O models "who/what acted," ORG models "how an organization is +structured internally" -- a team acting in a post's text needs both a +`prov:Agent`-shaped role (it does something) and an +`org:OrganizationalUnit`-shaped identity (it belongs to a company). +`:RoleActorTeam` is declared `rdfs:subClassOf org:OrganizationalUnit` +for that reason, parallel to how `:RoleActorPerson`/ +`:RoleActorOrganization` subclass PROV-O's classes. + +`post_summary.py` gains `ACTOR_TYPE_TEAM = "prov_team"` +(`common_lookup_value` category `prov_agent_type`, extending ADR +0006's two existing values). The LLM prompt now offers three actor +types (person / organization / team) and explicitly requires a team +actor to also carry `affiliated_organization_name` -- unlike an +organization actor (whose own name already answers "which +organization"), a team's name alone does not identify a company, so +the field is not optional in the same "opportunistic" sense ADR 0006 +described for a person actor; a team is always someone's team, and the +prompt asks the model to infer the parent company from context when +the text supports it. + +No new `RoleResponsibility` field is needed: +`affiliated_organization_name` already exists (ADR 0006) and applies +unchanged to this actor type -- only its *meaning* extends from +"the person's employer" to "the person's or team's parent +organization," which the dataclass docstring now says explicitly. + +Persistence: `migrations/0014_role_responsibility_team_actor_type.sql` +inserts the `prov_team` lookup row -- purely additive +(`insert ... on conflict (lookup_code) do nothing`), no column or +constraint change, since `actor_type_code` already stores an arbitrary +FK'd lookup code and needs no schema change to accept a third value. + +## Consequences + +- `_VALID_ACTOR_TYPE_CODES` in `post_summary.py` now has three members; + any code elsewhere that pattern-matches strictly on the first two + (rather than treating an unrecognized/future code as "not this one") + needs review. Found and fixed one: the frontend badge's CSS class name + (`actor-type-${code}`) was already generic, but its *label text* was a + binary person/organization ternary that would have mislabeled a team + actor as "Organization" -- now a three-way check. +- A team actor is never linked to the Keyman panel (same as an + organization actor in ADR 0006) -- it has no `person_id`. +- Distinguishing "설계팀" (a team) from "Design Corp" (an organization) + is a real LLM judgment call with no hard syntactic rule; the prompt + gives the model the concept and an example, matching this repo's + existing degrade-gracefully discipline for judgment-call extraction + fields (a wrong guess is a labeling error on one row, not lost data -- + the raw `actor_name` string is preserved regardless of which type it + is filed under). + +## Related + +Extends [ADR 0006](0006-role-responsibility-agent-ontology.md), which +itself extends [ADR 0004](0004-knowledge-graph-ontology.md)'s Ontology/ +Semantic-Layer vocabulary. + +## References (APA 7th) + +Reynolds, D. (Ed.). (2014). *The organization ontology* (W3C Recommendation). World Wide Web Consortium. https://www.w3.org/TR/vocab-org/ + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ diff --git a/docs/adr/0008-organization-abbreviation-resolution.md b/docs/adr/0008-organization-abbreviation-resolution.md new file mode 100644 index 000000000..72b121253 --- /dev/null +++ b/docs/adr/0008-organization-abbreviation-resolution.md @@ -0,0 +1,120 @@ +# ADR 0008 — Abbreviated organization names are resolved and search-verified, not left opaque + +**Decision status:** Accepted +**Date:** 2026-08-14 + +## Context + +Real post text names organizations by abbreviated or slang forms a +human reader immediately recognizes but a string-matching pipeline +cannot -- e.g. "AGP," a synthetic contraction of "Aurora Grid Power". +`lineageweave.corporate_hierarchy_resolution` +already resolves near-matches (a trailing legal suffix, a minor +abbreviation) via character-sequence similarity +(`difflib.SequenceMatcher`, grounded in Bhattacharya & Getoor, 2007's +candidate-generation stage), but an initialism/contraction like "AGP" +shares almost no character substring with its expansion -- no +similarity threshold recovers it, because the two strings are not +similar, they are *related by real-world knowledge* the text or an +external source has to supply. + +Left unresolved, every mention of the same real organization under its +abbreviated name creates its own unmatched, un-linkable free-text +string in `person_affiliation`/R&R -- the same organization looks like +N different unknown entities across N posts, each failing to link into +the corporate hierarchy a human reader would recognize instantly. + +## Decision + +A two-stage pipeline, reusing infrastructure this repo already has for +a structurally identical problem (ADR: `lineageweave.relation_verification`, +FEVER-style claim verification) rather than building a second web-search +integration: + +1. **LLM context resolution** + (`lineageweave.organization_name_resolution.ContextualOrchestratorOrganizationNameResolutionClient`): + given the raw abbreviated name and the post's own text as context, + ask the model for the organization's full real-world name, or + `UNKNOWN` when the text gives no real basis to determine one -- + never inventing an expansion from the abbreviation's letters alone. +2. **External search cross-verification** + (reusing `lineageweave.relation_verification.RelationVerificationClient` + as-is, not a new client class): the proposed full name plus the raw + abbreviation together become the search query (e.g. "Aurora Grid Power + AGP") -- a real page mentioning both together is strong + corroboration the specific pairing is correct, not just that the + full name exists as *some* organization. + +Grounded in SKOS (Miles & Bechhofer, 2009): `skos:prefLabel` (a +resource's one preferred/canonical label) and `skos:altLabel` (an +alternative label -- exactly the abbreviation/synonym relationship) is +the standard vocabulary for a raw-name/canonical-name pair. This is a +different, complementary standard from ADR 0006/0007's PROV-O/ORG +classes: SKOS here labels the *string identity* relationship between +two names for the same thing, not the *type* of the named actor. + +Only a search-corroborated resolution is ever substituted in for +downstream entity matching (`resolve_corporate_entity`) -- an +LLM-proposed name with no corroboration, or with verification itself +unavailable, leaves the raw name flowing unchanged. This is the same +never-trust-an-unverified-guess discipline `relation_verification` +itself already established: a wrong resolution corrupts every +downstream Knowledge Graph link through it, so "did not resolve" must +stay a real, distinguishable outcome from "resolved to X." + +Persistence: a new `organization_name_resolution` cache table +(`migrations/0015_organization_name_resolution.sql`), keyed by +`raw_organization_name` -- the same abbreviation is resolved once, not +re-queried on every one of its (potentially many) mentions across +posts. `verification_status_code` reuses the existing +`relation_verification_status` lookup category rather than a +near-duplicate one: a resolved name is corroborated/uncorroborated the +exact same way a classified relationship already is. + +Wired into `backend/app/keyman_ingestion.py`'s affiliation loop (the +concrete case real data surfaced): each affiliated organization name is +resolved before corporate-entity matching and creation. A corroborated +canonical name is returned to the caller as part of the normalized +`PersonMention`, so the same request's relationship classifier uses the +canonical form too rather than reintroducing the raw abbreviation. + +## Consequences + +- The raw abbreviated form is not duplicated onto every row that + mentions it (e.g. `person_affiliation.affiliated_organization_name` + stores the resolved canonical name once corroborated) -- it remains + fully recoverable via a join against `organization_name_resolution`, + which is the authoritative raw-form/canonical-form/evidence record. + This is 3NF-motivated, not a loss: repeating the raw-to-canonical + mapping per affiliation row would be the actual redundancy. +- Resolution availability can improve between extraction runs. When a + prior raw affiliation later resolves, `_upsert_affiliation` promotes + it transactionally: the canonical row is inserted or updated while + preserving any previously resolved corporate-entity link and role + title, and the obsolete raw-name row is deleted when the two names + differ. This avoids leaving duplicate raw and canonical identities for + the same person. +- `ingest_post_keymen` returns the normalized mentions it persisted. + `extract_post_keymen` therefore passes canonical organization names to + entity-relationship classification and returns those same names on + the API response; affiliation persistence, relationship classification, + and the caller-visible payload agree within one transaction. +- Every channel here follows the existing pluggable-client discipline: + `NullOrganizationNameResolutionClient`/an unavailable verification + client degrade to "use the raw name," never a fabricated resolution. + +## Related + +Complements [ADR 0006](0006-role-responsibility-agent-ontology.md) and +[ADR 0007](0007-team-actor-type.md) (actor *type*), and reuses +`lineageweave.relation_verification` (ADR-less, predates this file, see +its own module docstring for FEVER grounding) for the verification +stage rather than duplicating it. + +## References (APA 7th) + +Miles, A., & Bechhofer, S. (Eds.). (2009). *SKOS simple knowledge organization system reference*. World Wide Web Consortium. https://www.w3.org/TR/skos-reference/ + +Bhattacharya, I., & Getoor, L. (2007). Collective entity resolution in relational data. *ACM Transactions on Knowledge Discovery from Data*, 1(1), Article 5. https://doi.org/10.1145/1217299.1217304 + +Thorne, J., Vlachos, A., Christodoulopoulos, C., & Mittal, A. (2018). FEVER: A large-scale dataset for fact extraction and VERification. In *Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies* (pp. 809–819). Association for Computational Linguistics. https://doi.org/10.18653/v1/N18-1074 diff --git a/docs/adr/0009-cross-post-actor-identity.md b/docs/adr/0009-cross-post-actor-identity.md new file mode 100644 index 000000000..7bdbfa091 --- /dev/null +++ b/docs/adr/0009-cross-post-actor-identity.md @@ -0,0 +1,130 @@ +# ADR 0009 — R&R team/organization actors get a shared cross-post identity, not just per-post text + +**Decision status:** Accepted +**Date:** 2026-08-14 + +## Context + +Extraction (Keyman, R&R) runs per-post. For a person actor, this was +already not a dead end: Keyman extraction upserts into +`cataloged_person`, so the same name across posts (mostly) resolves to +one row with a stable `person_id` the Knowledge Graph can link through. +R&R's team actor (ADR 0007, `prov_team`) and organization actor +(ADR 0006, `prov_organization`) had no equivalent -- each post's +extraction produced a bare `actor_name` string in `post_summary_role` +with no catalog entry and no Knowledge Graph mention edge. The same +"설계팀" (design team) named in ten different posts was ten unrelated +strings, not one entity a Keyman/team panel could click through to see +every post it appears in -- exactly the "extraction results must +themselves become cross-post lineage clues, not just per-post +artifacts" requirement this product exists to satisfy for people, but +was not yet satisfying for teams or organizations. + +## Decision + +**Team**: a new `cataloged_team` catalog table +(`migrations/0016_cross_post_actor_identity.sql`), the same +catalog-then-mention shape `cataloged_person`/`post_person_mention` +already establishes. Identity key is `(team_name, +affiliated_organization_name)`, not the bare name alone -- "설계팀" +exists at many real companies, so the pair is what is actually +identifying (`backend/app/team_ingestion.py`'s `upsert_team`, mirroring +`keyman_ingestion.py`'s `_upsert_person`). The team's own parent +organization is resolved to a real `corporate_entity` via the *same* +`resolve_corporate_entity` collective-entity-resolution matching +(Bhattacharya & Getoor, 2007) Keyman affiliations already use -- not a +second matching algorithm. + +**Organization**: an R&R organization actor's name is run through the +same `resolve_corporate_entity` matching; a resolved match writes a +`post_organization_mention` row (no new catalog needed -- `corporate_entity` +already is the shared, cross-post organization catalog every VOC +counterparty and Keyman affiliation already resolves against). + +**Person** (an R&R actor, not a Keyman): opportunistically joined to an +*existing* `cataloged_person` row by exact name match, when Keyman +extraction has already cataloged that name on this or another post. +R&R does not create a new person identity itself -- `cataloged_person` +requires `person_side_code` (our-side vs. counterparty), which R&R's +prompt does not currently ask for and Keyman's does; inventing one here +risked a wrong side assignment. Documented as a real, deliberate scope +boundary below, not silently half-done. +Person evidence sources remain separate: Keyman extraction replaces +`post_person_mention`; R&R replacement writes +`post_summary_person_mention`. `combined_post_person_mention` is a +read-only union used for lineage and KG derivation. This prevents a new +summary from deleting Keyman evidence and prevents removed R&R actors +from surviving as stale Keymen. Migration 0016 copies matching R&R +actor names into `post_summary_person_mention` and must not delete +overlapping Keyman rows -- `mention_context` has no R&R column, and a +later summary replacement would otherwise erase the only remaining +person evidence. + + +Each resolved actor gets a real Knowledge Graph mention edge (new +`edge_mention_team` / `edge_team_affiliation` / `edge_mention_organization` +lookup codes, `lineageweave/knowledge_graph.py`'s +`knowledge_graph_edges_for_post` extended, not a second edge-writing +path), reusing the same `persist_edges_for_post` entry point Keyman +ingestion already calls -- one function computes a post's whole edge +set regardless of which extraction step triggered it. + +`knowledge_graph_edge` is a deduplicated materialized registry. +`knowledge_graph_edge_evidence` records every post that currently supports an +edge; readers require support from an ABAC-visible post. Writers reconcile one +post under a transaction-scoped advisory lock, and unsupported registry rows +are pruned. Edge identity therefore cannot duplicate under concurrency, and a +replacement cannot leave a buyer-visible orphan edge. + +Ontology (`docs/ontology/lineageweave-kg.ttl`): `:Team a owl:Class ; +rdfs:subClassOf org:OrganizationalUnit` (same W3C ORG grounding as +ADR 0007's `:RoleActorTeam`, but a distinct term -- `:Team` is a +`cataloged_team` row with a stable identity, `:RoleActorTeam` is the +per-row `actor_type_code` classification, the same +`:Person`/`:RoleActorPerson` split ADR 0006 already established). +`:mentionsTeam` / `:teamAffiliatedWith` / `:mentionsOrganization` are +new, distinct object properties rather than widening `:mentions`'s +domain/range -- stating `rdfs:domain :mentions` twice (once `:Person`, +once `:Team`) would let RDFS entail every `:mentions` subject is BOTH, +which is false. + +## Consequences + +- Team/organization mention persistence only runs when + `summary.roles_and_responsibilities` is non-empty (a real, cheap + guard, not a correctness gap) -- a post with no R&R never touches + `cataloged_team`/`post_organization_mention` at all. +- **Documented, deliberate gap**: R&R never *creates* a new + `cataloged_person` row, only joins to an existing one by exact name. + Two failure modes follow from this, both accepted for now: (1) a + person named only in R&R (never by Keyman on any post) gets no + catalog identity at all until/unless Keyman also names them; (2) an + exact-name join has the same same-name-collision risk + `keyman_ingestion._upsert_person`'s job-title disambiguation exists + to catch, but R&R's join here does not run that check (R&R's own + prompt does not currently capture a job title). A future slice could + extend the R&R prompt to also ask for `person_side_code` (and + optionally a title) so R&R could safely originate new person + identities the same way Keyman does, closing this gap properly rather + than working around it with a guess. +- `cataloged_team` uses PostgreSQL's + `UNIQUE NULLS NOT DISTINCT (team_name, affiliated_organization_name)`. + NULL affiliation therefore participates in the identity key: two + bare-team rows with the same name conflict and the atomic upsert + returns one shared `team_id`. This database constraint, not a + read-before-insert application check, closes the concurrent duplicate + race for both affiliated and unplaced teams. + +## Related + +Depends on [ADR 0006](0006-role-responsibility-agent-ontology.md) and +[ADR 0007](0007-team-actor-type.md) (actor *type*) and +`lineageweave.corporate_hierarchy_resolution` (Bhattacharya & Getoor, +2007, cited there) for the organization-matching this ADR reuses rather +than re-deriving. + +## References (APA 7th) + +Bhattacharya, I., & Getoor, L. (2007). Collective entity resolution in relational data. *ACM Transactions on Knowledge Discovery from Data*, 1(1), Article 5. https://doi.org/10.1145/1217299.1217304 + +Reynolds, D. (Ed.). (2014). *The organization ontology* (W3C Recommendation). World Wide Web Consortium. https://www.w3.org/TR/vocab-org/ diff --git a/docs/adr/0010-corporate-hierarchy-auto-creation.md b/docs/adr/0010-corporate-hierarchy-auto-creation.md new file mode 100644 index 000000000..d034fe9b2 --- /dev/null +++ b/docs/adr/0010-corporate-hierarchy-auto-creation.md @@ -0,0 +1,100 @@ +# ADR 0010 — a real counterparty organization is auto-created into the corporate hierarchy, not left permanently unresolved + +**Decision status:** Accepted +**Date:** 2026-08-14 + +## Context + +`lineageweave.corporate_hierarchy_resolution`'s similarity-based +matching only ever locates an *already-cataloged* `corporate_entity` +row -- it has no path to create one. This was fine while the only +`corporate_entity` catalog was synthetic demo fixtures with a handful +of names extraction would naturally already know. A synthetic batch +where the catalog holds only the employer's own two-row hierarchy +exposes the same gap: every counterparty named in a post is, by +definition, outside that catalog. Similarity matching then resolves +**0 affiliation rows and 0 R&R organization-actor mentions** -- the +standing integrated customer-affiliate tree requirement (Harbor Group +-> Harbor Devices Korea -> ... in the synthetic brief) stays empty +until a verified creation path exists. + +## Decision + +`get_or_create_corporate_entity` (`backend/app/corporate_entity_ingestion.py`) +extends the existing resolution pipeline with a creation fallback, not +a competing algorithm: + +1. Try `resolve_corporate_entity` (similarity matching, unchanged, + Bhattacharya & Getoor, 2007) first -- an already-cataloged entity + still resolves exactly as before. +2. On a miss, ask an LLM + (`lineageweave.corporate_hierarchy_inference.CorporateHierarchyInferenceClient`) + to propose this organization's place in the Group -> Company -> + Plant hierarchy (`corporate_entity_level`, ADR 0004's existing SKOS + `skos:broader`/`skos:narrower` structure) from the post's own text -- + never inventing a hierarchy the text gives no evidence for; the + model may decline with `UNKNOWN`. +3. The proposal is only trusted after + `lineageweave.relation_verification`'s existing Searxng + corroboration (the same reused verification client + `organization_name_resolution`/ADR 0008 already established this + pattern for) -- an uncorroborated or unavailable-channel proposal + creates nothing, same never-trust-an-unverified-guess discipline as + every other channel here. +4. Only then is a real new `corporate_entity` row inserted. A proposed + parent organization is itself resolved-or-created first (bounded to + 4 levels of recursion, so a misbehaving response chain cannot spin + into unbounded row creation), so the whole chain gets real + `parent_entity_id` links, not an orphaned single-level row. + +**Auto-created code namespace**: `corporate_entity_code` is also the +real login "corp code" attribute Keycloak issues via the `corp_code` +token claim (`docker/keycloak/realm-export.json`) -- an auto-created +counterparty row must never collide with that namespace. Every +auto-created code is prefixed `AUTO-` followed by a deterministic hash +of the entity name (same name -> same code, so a genuine concurrent +duplicate-creation race collides on the real SQL `unique` constraint +and self-resolves via `on conflict`, rather than creating two rows for +one organization under two different codes). + +Wired into both existing organization-resolution call sites -- +`keyman_ingestion.py`'s person-affiliation loop and +`post_summary_ingestion.py`'s R&R organization-actor loop -- rather +than a third, separate code path, so both routes to `corporate_entity` +share one creation policy. + +## Consequences + +- A wrong hierarchy placement (level or parent) is a real risk this + design accepts, bounded by the same LLM-judgment-call discipline + ADR 0007's team-vs-organization classification already accepts: the + raw name is never lost regardless (it is the `entity_name` itself), + so a wrong placement is a correctable graph-structure error, not lost + data. +- The `AUTO-` code namespace is a real, deliberate simplification: an + operator wanting a genuinely curated corp-code scheme for these + entities later would need to re-code them, not just re-run + extraction -- accepted because the alternative (leaving every + counterparty unresolved forever) is strictly worse for this + product's actual purpose. +- Every LLM/search call in this path already existed for a different + purpose (`organization_name_resolution`'s resolution call shape, + `relation_verification`'s verification client) -- no new provider + integration was built, keeping this consistent with the project's + standing discipline of reusing an existing channel over adding a new + one wherever the shape already fits. + +## Related + +Extends [ADR 0008](0008-organization-abbreviation-resolution.md)'s +reuse-the-verification-client pattern and +[ADR 0009](0009-cross-post-actor-identity.md)'s cross-post identity +work -- an R&R organization actor's identity is now genuinely resolved +to a real corporate hierarchy node, not left as free text even when no +prior mention of it existed anywhere in the dataset. + +## References (APA 7th) + +Bhattacharya, I., & Getoor, L. (2007). Collective entity resolution in relational data. *ACM Transactions on Knowledge Discovery from Data*, 1(1), Article 5. https://doi.org/10.1145/1217299.1217304 + +Miles, A., & Bechhofer, S. (Eds.). (2009). *SKOS simple knowledge organization system reference*. World Wide Web Consortium. https://www.w3.org/TR/skos-reference/ diff --git a/docs/adr/0011-prov-o-standard-relations.md b/docs/adr/0011-prov-o-standard-relations.md new file mode 100644 index 000000000..8d214a224 --- /dev/null +++ b/docs/adr/0011-prov-o-standard-relations.md @@ -0,0 +1,79 @@ +# ADR 0011: Preserve W3C PROV-O as a standards-complete provenance layer + +- **Status:** Accepted +- **Date:** 2026-08-14 +- **Decision owners:** ContextualWisdomLab / LineageWeave +- **Standard:** W3C PROV-O Recommendation, 30 April 2013 + +## Context + +PR #74 introduced PROV-O-grounded actor categories, but LineageWeave's existing `knowledge_graph_edge` table only represents a compact binary navigation graph. It cannot faithfully represent PROV-O datatype properties such as `prov:startedAtTime` and `prov:value`, nor the intermediate `prov:Influence` resources required by qualified relations. Adding every standard property as another product edge code would therefore flatten the standard and lose the very provenance details it is intended to preserve. + +The Recommendation defines 30 classes and 50 normative properties grouped into Starting Point, Expanded, and Qualified terms. Tables 2 and 3 define 14 qualification patterns, and consuming applications should treat each qualified form as implying the corresponding unqualified form. Appendix B reserves interoperable inverse names while intentionally preferring the standard property direction. + +## Decision + +1. Add a separate `lineageweave.prov_o` runtime containing the complete normative class/property registry. +2. Validate object-versus-datatype shape, domain, range, subclass membership, timezone-aware `xsd:dateTime`, and Appendix B inverse aliases before accepting an assertion. +3. Deterministically materialize: + - transitive property hierarchy; + - defined inverse properties; + - `prov:alternateOf` symmetry; + - all 14 qualified-to-unqualified implications; + - qualified Generation/Invalidation/Start/End `prov:atTime` shortcuts. +4. Serialize with the exact `http://www.w3.org/ns/prov#` namespace through rdflib. +5. Store standards-complete provenance in normalized `provenance_*` tables. Keep `knowledge_graph_edge` as a buyer-facing navigation projection and bridge existing nodes through `provenance_resource_binding` rather than conflating the two models. +6. Catalog every Appendix B inverse name. Names that are not normative properties are accepted only as import aliases and rewritten by reversing endpoints into the preferred PROV-O relation. +7. Map LineageWeave `Post`, `Person`, `CorporateEntity`, and `Team` classes to PROV-O in a separate support profile that imports rather than redefines the W3C ontology. + +## Relational model + +```mermaid +erDiagram + provenance_class_definition ||--o{ provenance_class_hierarchy : child + provenance_class_definition ||--o{ provenance_class_hierarchy : parent + provenance_relation_definition ||--o{ provenance_relation_domain : has + provenance_class_definition ||--o{ provenance_relation_domain : constrains + provenance_relation_definition ||--o{ provenance_relation_resource_range : has + provenance_class_definition ||--o{ provenance_relation_resource_range : constrains + provenance_relation_definition ||--o{ provenance_relation_hierarchy : child + provenance_relation_definition ||--o{ provenance_relation_hierarchy : parent + provenance_relation_definition ||--|| provenance_inverse_definition : documents + provenance_relation_definition ||--o| provenance_qualification_definition : qualifies + provenance_resource ||--o{ provenance_resource_type : typed_as + provenance_class_definition ||--o{ provenance_resource_type : classifies + provenance_resource ||--o{ provenance_assertion : subject + provenance_relation_definition ||--o{ provenance_assertion : predicate + provenance_resource ||--o{ provenance_assertion : resource_object + provenance_literal_value ||--o{ provenance_assertion : literal_object + provenance_assertion ||--o{ provenance_assertion_derivation : derived + provenance_assertion ||--o{ provenance_assertion_derivation : premise +``` + +## Consequences + +### Positive + +- Complete PROV-O interchange without lossy custom edge codes. +- Qualified provenance retains role, plan, activity, usage, generation, time, and location detail. +- Existing LineageWeave navigation and RWR behavior remains stable. +- Database and runtime share stable multiword snake-case codes while preserving exact W3C IRIs. +- Assertions fail closed in both Python and PostgreSQL. + +### Costs + +- The product now has a standards graph and a navigation projection; projection logic must remain explicit. +- Full OWL reasoning is not embedded. The runtime intentionally materializes only the Recommendation rules needed for deterministic product behavior. +- Bundle serialization remains RDF-technology-specific; the relational layer stores the bundle resource without prescribing TriG. + +## Rejected alternatives + +- **Add 50 `edge_type` lookup rows:** rejected because literal and qualified relations cannot be represented. +- **Store arbitrary RDF triples only:** rejected because domain/range and relational integrity would be deferred to callers. +- **Define every inverse as another preferred property:** rejected because Appendix B explicitly warns that unconstrained inverse proliferation reduces interoperability. + +## Verification + +- Exact registry tests for 30 classes, 50 properties, 6 datatype properties, 14 qualification mappings, and all 44 object-property inverse names. +- Behavior-sensitive tests for validation, subclass domains, every qualification implication, superproperty closure, inverse/symmetry, direct time inference, RDF serialization, SQL seed completeness, support-profile mapping, and public docstrings. +- Owned production module statement and branch coverage: 100%. diff --git a/docs/adr/0012-corporate-entity-creation-lock.md b/docs/adr/0012-corporate-entity-creation-lock.md new file mode 100644 index 000000000..f5a01d859 --- /dev/null +++ b/docs/adr/0012-corporate-entity-creation-lock.md @@ -0,0 +1,45 @@ +# ADR 0012 — corporate-entity creation is serialized with a Postgres advisory transaction lock, not split into separate read/write databases + +**Decision status:** Accepted +**Date:** 2026-08-14 + +## Context + +ADR 0010's `get_or_create_corporate_entity` introduced concurrent writes on the organization-creation path. A deterministic synthetic regression fixture reproduced a `DeadlockDetectedError`: two transactions created different `corporate_entity` rows in opposite order, so each transaction waited for a row-level lock held by the other. + +This repository records the reproducible concurrency shape rather than customer, organization, batch, or production-log details. The architectural defect is independent of any particular dataset: multi-worker extraction can encounter child and parent organizations in different orders. + +## Decision + +Serialize only the *creation* write path with one named Postgres advisory transaction lock: + +```sql +pg_advisory_xact_lock( + hashtext('lineageweave:corporate_entity_creation') +) +``` + +The transaction-scoped lock is acquired immediately before persistence and is released automatically by the enclosing transaction's commit or rollback (PostgreSQL Global Development Group, 2024). + +1. The lock is acquired only after inference and verification complete. Holding it across network I/O would unnecessarily serialize unrelated workers. +2. Under the lock, candidates are reloaded and similarity matching is repeated. Another transaction may have committed the same entity after the caller's original snapshot was read. +3. The key is one fixed creation-path key rather than a per-name key. Per-name locking still permits the opposite-order multi-entity deadlock shape `A: [X, Y]` versus `B: [Y, X]`. +4. The already-cataloged resolution path remains lock-free. + +Splitting the system into separate read and write databases was considered and rejected. Replica separation does not resolve a write-write lock-ordering defect, while a transaction-scoped advisory lock directly enforces one global creation order. + +## Consequences + +- New entity creation is serialized cluster-wide. This is accepted because creation is the uncommon branch and correctness dominates throughput for this path. +- Resolution of existing entities remains concurrent and does not acquire the advisory lock. +- The lock is re-entrant within the same PostgreSQL session and transaction, so bounded parent-chain recursion does not self-deadlock. +- No schema split or additional service is required. +- The regression suite must retain concurrent opposite-order creation coverage. + +## Related + +This decision extends [ADR 0010](0010-corporate-hierarchy-auto-creation.md) with an explicit concurrency-safety property. + +## References — APA 7th + +PostgreSQL Global Development Group. (2024). *PostgreSQL 17 documentation: Advisory lock functions*. https://www.postgresql.org/docs/current/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS diff --git a/docs/adr/0013-normalized-analysis-run-registry.md b/docs/adr/0013-normalized-analysis-run-registry.md new file mode 100644 index 000000000..b60164dc6 --- /dev/null +++ b/docs/adr/0013-normalized-analysis-run-registry.md @@ -0,0 +1,273 @@ +# ADR 0013 — Milestone 2 uses a normalized, additive analysis-run registry + +**Decision status:** Accepted on this active PR; not protected-main truth until merge +**Date:** 2026-08-15 +**Depends on:** ADR 0011 standards-complete provenance separation and ADR 0012 corporate-entity creation locking + +## Context + +LineageWeave has a reviewed React/FastAPI/PostgreSQL product, compact lineage +navigation, normalized actor identity, report persistence, and a separate +standards-complete PROV-O layer. Milestone 2 must analyze operator-authorized +PostgreSQL evidence without replacing that product, duplicating cross-service +databases, or committing private source identity and content to a public +repository. + +A retained experiment proved that direct PostgreSQL analysis is feasible, but +its parallel application and denormalized run record cannot become product +truth. The product needs a small durable root that answers: + +- which immutable capture was used; +- which evidence was available by the run's knowledge cutoff; +- which authenticated account requested the work; +- which product scope and reproducibility digests governed the run; +- which aggregate counts reconcile the capture; +- which legal lifecycle transitions occurred. + +The registry does not store source SQL, DSNs, raw posts, inline images, provider +payloads, credentials, raw exceptions, or another service's application rows. + +## Decision + +Migration `0018_analysis_run_registry.sql` introduces five normalized relations +and one read projection. + +```mermaid +erDiagram + ANALYSIS_SOURCE_SNAPSHOT ||--o{ ANALYSIS_SOURCE_COUNT : reconciles + ANALYSIS_SOURCE_SNAPSHOT ||--o{ ANALYSIS_RUN : anchors + USER_ACCOUNT ||--o{ ANALYSIS_RUN : requests + ANALYSIS_RUN ||--o| ANALYSIS_RUN_SCOPE : limits + CORPORATE_ENTITY |o--o{ ANALYSIS_RUN_SCOPE : scopes + PROCESS_UNIT |o--o{ ANALYSIS_RUN_SCOPE : scopes + ANALYSIS_RUN ||--o{ ANALYSIS_RUN_STATUS_EVENT : records + + ANALYSIS_SOURCE_SNAPSHOT { + uuid analysis_source_snapshot_id PK + text snapshot_sha256 UK + text source_contract_version + timestamptz maximum_available_time + timestamptz captured_at + } + ANALYSIS_SOURCE_COUNT { + uuid analysis_source_snapshot_id PK,FK + text count_type_code PK,FK + bigint count_value + } + ANALYSIS_RUN { + uuid analysis_run_id PK + uuid analysis_source_snapshot_id FK + uuid requested_by_account_id FK + text idempotency_key UK + timestamptz knowledge_cutoff + text configuration_sha256 + text model_contract_sha256 + text prompt_bundle_sha256 + text code_revision_sha + } + ANALYSIS_RUN_SCOPE { + uuid analysis_run_id PK,FK + text scope_kind_code FK + uuid corporate_entity_id FK + uuid process_unit_id FK + text scope_key + } + ANALYSIS_RUN_STATUS_EVENT { + uuid analysis_run_id PK,FK + int status_ordinal PK + text status_code FK + timestamptz occurred_at + timestamptz recorded_at + text failure_code + boolean retryable + } +``` + +### Temporal ownership + +`analysis_source_snapshot.maximum_available_time` is an evidence fact: the +latest time at which any admitted fact became available. `analysis_run.knowledge_cutoff` +is an analysis fact: the latest information that this particular run may use. +A reusable capture therefore does **not** own one knowledge cutoff. + +Run creation locks the snapshot and requires: + +```text +maximum_available_time <= knowledge_cutoff <= requested_at +captured_at <= requested_at +``` + +This aggregate guard complements TEPP's finer event, assertion, document, +system, availability, and cutoff clocks. It does not replace TEPP temporal or +psychometric computation. + +### Identity and idempotency + +Every run references a real `user_account`. `requested_by_account_id` is not +nullable. Idempotency keys are trimmed, control-free canonical values and are +unique per authenticated account rather than +globally, because independent callers may legitimately choose the same opaque +client key. A later repository must compare request digests on retry and return +a conflict when the same account/key names different evidence or configuration. + +### Immutability and concurrency + +Snapshot identity and availability reject updates. Aggregate count values reject +updates. Count insert/delete and first run creation acquire the same snapshot-row +lock before checking whether a run exists. This shared lock order closes the +race in which a count set and first derivation could otherwise both commit. +After the first run, the complete count set is frozen. + +The analysis request and its authorization scope reject updates and deletes. +Lifecycle changes are represented only by append-only status events, so a cascade +cannot erase the derivation root or its access boundary. + +### Lifecycle state machine + +The parent run row serializes status appends. Events require contiguous +ordinals, monotonic occurrence time, and these transitions: + +```text +pending -> running | cancelled +running -> succeeded | failed | cancelled +succeeded | failed | cancelled -> terminal +``` + +The first event must be `pending`, requires an immutable scope, and cannot predate +the run request. Failed events require a lowercase machine-code identifier; raw +exception text is prohibited. `recorded_at` is overwritten with database system +time on every insert and cannot precede `occurred_at`. +`analysis_run_current_status` is a view, not a second mutable state authority. + +### Authorization scope + +`analysis_run_scope` stores one immutable all-visible, corporate-entity, +process-unit, or thread-group scope. Its shape is database constrained and the +first lifecycle event is rejected until it exists. The next repository/API slice +must insert run, scope, and first status in one +transaction and apply the existing RBAC/ABAC contract when listing or reading +runs. This migration does not claim that an API or UI exists. + +### Service boundaries + +- **LineageWeave** owns product run identity, authorized scope, lifecycle, + aggregate reconciliation, and product-visible derivation references. +- **TEPP** owns exact evidence spans, temporal/event measurement, + multilevel/multiple-membership psychometrics, calibration, and semantic-span + budgeting through a versioned import or REST contract. +- **contextual-orchestrator** owns provider-neutral model routing and bounded + single-model versus multi-agent test-time compute allocation through its + reviewed API. +- **fast-mlsirm** owns Rust psychometric arithmetic and calibration interfaces. +- **Valkey** remains the event queue. Durable registry truth remains in + PostgreSQL; a later outbox slice bridges the two. + +No component reads another service's private application tables. + +## Alternatives considered + +### Merge the parallel experiment unchanged + +Rejected. It replaces reviewed product history, duplicates web and identity +surfaces, and creates a second database authority. + +### Store one JSON document per run + +Rejected. Signed external manifests may be JSON artifacts, but relational +identity, scope, counts, clocks, and lifecycle need independent constraints, +authorization, and query plans. + +### Put the registry only in Valkey + +Rejected. Queue state is transient and replayable. Audit identity, +idempotency, temporal eligibility, and retention evidence require PostgreSQL. + +### Store knowledge cutoff on the snapshot + +Rejected. One immutable capture can support multiple analysis requests with +different historical cutoffs. Putting the cutoff on the snapshot violates the +functional dependency and forces duplicate snapshots. + +## Security, privacy, and compliance consequences + +- Necessary PII remains in its authorized source/product tables rather than + being blanket-masked into operational uselessness. +- This registry stores opaque UUIDs, digests, bounded machine codes, aggregate + counts, and clocks only. +- Logs and public acceptance evidence must not include SQL, DSNs, raw source + text, images, secrets, provider payloads, or private source identifiers. +- Artifact bodies remain in access-controlled deployment storage and are linked + later by content digest and policy-bound reference. +- The design supports SOC 2 and CSAP evidence collection through explicit actor, + configuration, status, retention, and rollback contracts; it does not claim + certification. +- Database RLS is deferred because the current API uses one pooled service + identity and application-level RBAC/ABAC. Adopting actor-bound RLS requires a + separate ADR and transaction-scoped identity propagation. + +## Failure and rollback + +Migration replay is idempotent and rejects lookup-category collisions. The +rollback refuses to remove non-empty registry relations. Evidence must first be +exported or explicitly deleted under an approved retention procedure. An empty +rollback removes the view, tables, functions, and lookup rows and is itself +replayable. + +## Verification + +Acceptance requires: + +- real-PostgreSQL migration and replay; +- valid snapshot, aggregate, scope, and lifecycle persistence; +- distinct cutoffs over one snapshot; +- rejection of future-information leakage; +- account-scoped idempotency; +- snapshot, count, run, and authorization-scope immutability; +- deletion resistance for request and scope audit evidence; +- scope-required lifecycle, request-time ordering, and database-owned record time; +- canonical idempotency and bounded machine-code failure identifiers; +- count/run concurrency serialization; +- pending-first, contiguous, monotonic, legal status transitions; +- append-only status evidence; +- fail-closed rollback; +- two-or-more-word `snake_case` database-object names; +- complete repository, security, SAST, documentation, and public-content gates + on the exact merge head. + +## Follow-up sequence + +1. Add a transaction repository that creates snapshot, counts, run, scope, and + first status atomically and compares request digests on idempotent retries. + `POST /api/analysis-runs` now records that Pending write (ADR 0017); + reconstruction and live TEPP execution remain later slices. +2. Add RBAC/ABAC-protected run list/detail endpoints and the DB-grounded + read-only administrator surface. +3. Add a normalized PostgreSQL outbox and Valkey delivery worker. +4. Add TEPP and contextual-orchestrator adapters only after their versioned + contracts are present on reviewed main branches. Seed now records a + Failed TEPP run through `tepp_client` on the shared Demo Corp snapshot; + a live transport remains a later slice. A missing or unused TEPP + envelope must stay Failed (`tepp_not_available` / + `tepp_result_not_persisted`) and must not write a local psychometric + substitute. +5. Execute private actual-data analysis and store only signed aggregate and + reproducibility manifests outside public source control. +6. Run browser E2E through real OIDC, product navigation, and evidence drill-down. + +## References — APA 7th + +International Organization for Standardization. (2019). *ISO 8601-1:2019: Date +and time—Representations for information interchange—Part 1: Basic rules* +(confirmed 2024; Amendment 1:2022). + +Moreau, L., & Missier, P. (Eds.). (2013). *PROV-DM: The PROV data model*. +World Wide Web Consortium. https://www.w3.org/TR/prov-dm/ + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: +5.5. Constraints*. https://www.postgresql.org/docs/current/ddl-constraints.html + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology* (W3C +Recommendation). https://www.w3.org/TR/prov-o/ + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C Recommendation). +https://www.w3.org/TR/owl-time/ diff --git a/docs/adr/0014-authorized-analysis-run-read.md b/docs/adr/0014-authorized-analysis-run-read.md new file mode 100644 index 000000000..500c2bc2a --- /dev/null +++ b/docs/adr/0014-authorized-analysis-run-read.md @@ -0,0 +1,67 @@ +# ADR 0014 — Analysis-run evidence is an authorized, source-redacting read + +**Decision status:** Accepted on this active PR; not protected-main truth until merge +**Date:** 2026-08-16 +**Depends on:** ADR 0013 normalized analysis-run registry +**Refs:** Issue #79 (Milestone 2 parent); closed PR #77 is read-only evidence + +## Context + +PR #89 persists analysis-run identity, aggregate reconciliation, scope, +and lifecycle without exposing a product API. Buyers still cannot see +whether a lineage reconstruction ran, succeeded, or reconciled how many +documents. Closed PR #77 exposed analysis records through a parallel +application that also stored raw metadata payloads -- that shape cannot +become protected product truth. + +## Decision + +LineageWeave owns a fail-closed read projection of the #89 registry: + +- `GET /api/analysis-runs` and `GET /api/analysis-runs/{id}` require + `post_read`. +- Visibility is evaluated in SQL. A run is visible when the caller + requested it, or the scope is a corporate entity / process unit / + thread group the caller may already walk. `all_visible` stays + requester-only so it cannot broaden another tenant's evidence. +- Hidden runs return 404, not 403, and never appear in the list. +- The payload carries lookup labels and non-negative aggregate counts. + It does not carry source SQL, DSNs, raw records, image bytes, provider + payloads, credentials, or another service's table names. +- `GET /api/analysis-runs/{id}` also returns the append-only labeled + `status_history`. The list does not. A failed event may include the + stored machine `failure_code`; this slice does not invent a label. +- TEPP remains a versioned `AnalysisRunRequest` consumer + (`lineageweave.tepp_client`). This slice does not fork TEPP arithmetic. +- contextual-orchestrator remains the only LLM path. This slice does not + call a raw model API. + +## Consequences + +`make seed` writes one synthetic Demo Corp lineage run and one TEPP +run on the same snapshot so the existing React home page can show both +kinds without a second application. The TEPP run is Failed / +`tepp_not_available` when the default transport is missing -- the list +keeps that machine code off the caption (this decision) and instead +tells the operator to open the TEPP run, then connect the measurement +service. A failed lineage row tells the operator to retry +reconstruction, not to connect TEPP. A failed period-report row +tells the operator to rebuild the report from a current snapshot. +A pending or running TEPP row must not claim a calibrated +measurement. A pending lineage row says reconstruction has not +started yet. The detail now shows the legal +lifecycle the registry already stored. `POST /api/analysis-runs` now +records a Pending run on an authorized cutoff capture (ADR 0017). +Reconstruction, a live TEPP transport, and a fuller Analysis Run +Console remain later slices. + +## References + +American Educational Research Association, American Psychological +Association, & National Council on Measurement in Education. (2014). +*Standards for educational and psychological testing*. American +Educational Research Association. + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV +ontology* (W3C Recommendation). World Wide Web Consortium. +https://www.w3.org/TR/2013/REC-prov-o-20130430/ diff --git a/docs/adr/0013-adaptive-contextual-orchestrator-default.md b/docs/adr/0015-adaptive-contextual-orchestrator-default.md similarity index 96% rename from docs/adr/0013-adaptive-contextual-orchestrator-default.md rename to docs/adr/0015-adaptive-contextual-orchestrator-default.md index ee0402075..433432fbb 100644 --- a/docs/adr/0013-adaptive-contextual-orchestrator-default.md +++ b/docs/adr/0015-adaptive-contextual-orchestrator-default.md @@ -1,4 +1,4 @@ -# ADR-0013: Adaptive contextual-orchestrator mode is the default +# ADR-0015: Adaptive contextual-orchestrator mode is the default - Status: Accepted - Date: 2026-08-16 diff --git a/docs/adr/0016-analysis-run-knowledge-cutoff-posts.md b/docs/adr/0016-analysis-run-knowledge-cutoff-posts.md new file mode 100644 index 000000000..089443374 --- /dev/null +++ b/docs/adr/0016-analysis-run-knowledge-cutoff-posts.md @@ -0,0 +1,67 @@ +# ADR 0016 — Analysis-run visible posts honor the run knowledge cutoff + +**Decision status:** Accepted +**Date:** 2026-08-16 + +## Context + +ADR 0013 stores `analysis_run.knowledge_cutoff` as the analysis clock: +what that run was allowed to know. The registry trigger already refuses +a cutoff earlier than `analysis_source_snapshot.maximum_available_time`. +The home-page detail, however, listed every ABAC-visible title in the +run's scope from live `source_post` rows. Fixture and seed posts that +defaulted to `created_at = now()` therefore appeared inside a January +2026 run, including a later own-corp follow-up the buyer would treat as +part of that reconstruction. + +W3C Time Ontology in OWL (Hobbs & Pan, 2017) and ISO 8601-1:2019 keep +distinct clocks from collapsing. A knowledge cutoff is not "posts the +account can see today." + +## Decision + +`fetch_visible_scope_posts` filters `created_at <= knowledge_cutoff` on +every scope branch (corporate entity, process unit, thread group, and +all-visible). ABAC visibility is applied after that temporal gate. +Click-through still opens the live post body -- post versioning is a +later slice -- but the run list itself must not advertise a post the +run was not allowed to know. The detail must say that next action +plainly: compare the opened body with this cutoff before treating it +as reconstructed evidence. + +Reproducibility digests on the same detail use a labeled group whose +accessible name does not replace the visible prefixes (W3C Accessible +Name and Description Computation 1.1). Full digests stay on `title` +for hover verification and on the API payload; the home list stays +aggregates-only. + +Seed and API fixtures backdate in-cutoff posts. A late own-corp private +post remains on the live post list and stays out of the January 2026 +run. + +## Consequences + +- After `make seed`, the Demo Corp lineage run lists Demo public post + and other in-cutoff Demo Corp titles. The later fixture account-review + post (2026-02-10) does not appear. +- Open the run, read the live-body warning, then open a listed post + and compare it with the cutoff date. +- Hover a digest prefix to read the full code or configuration digest + when you need to match the API payload. +- Post-body versioning at the cutoff remains future work. +- Thread-group *run list* visibility now uses the same cutoff + (ADR 0018). A later public post cannot surface a previously hidden + thread-group run. + +## References + +International Organization for Standardization. (2019). *ISO 8601-1:2019: +Date and time—Representations for information interchange—Part 1: Basic +rules* (confirmed 2024; Amendment 1:2022). + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C +Recommendation). https://www.w3.org/TR/owl-time/ + +World Wide Web Consortium. (2018). *Accessible name and description +computation 1.1* (W3C Recommendation). +https://www.w3.org/TR/accname-1.1/ diff --git a/docs/adr/0017-authorized-analysis-run-create.md b/docs/adr/0017-authorized-analysis-run-create.md new file mode 100644 index 000000000..e3a535a18 --- /dev/null +++ b/docs/adr/0017-authorized-analysis-run-create.md @@ -0,0 +1,56 @@ +# ADR 0017 — Operators request an analysis run through the product API + +**Decision status:** Accepted on this active PR; not protected-main truth until merge +**Date:** 2026-08-16 +**Depends on:** ADR 0013 normalized analysis-run registry; ADR 0014 authorized +analysis-run read; ADR 0016 knowledge-cutoff posts +**Refs:** Issue #79 (Milestone 2 parent); ADR 0013 follow-up 1 + +## Context + +Home Analysis runs could list seeded lineage and TEPP rows, but a buyer +could not request a new run. Seed-only evidence is a demo, not a product. +ADR 0013 already required a transaction that creates snapshot, counts, +run, scope, and the first status atomically. Follow-up 3 (outbox / worker) +still owns reconstruction and live TEPP execution. + +## Decision + +`POST /api/analysis-runs` is the authorized write: + +- `post_read` is enough. The caller may only cover a corporate entity + they already walk. An unaffiliated corp is 404, not 403. +- The capture digest hashes scope, entity, cutoff, and authorized post + ids — never a post body, DSN, or source SQL. +- The write inserts snapshot, aggregate counts, `analysis_run`, + `analysis_run_scope`, and `analysis_status_pending` in one transaction. +- The first status is Pending. This slice does not reconstruct lineage + and does not call TEPP. A missing measurement stays Failed only on the + seed path that already goes through `tepp_client`. +- Account-scoped idempotency compares `configuration_sha256`. An omitted + cutoff is hashed as `unspecified` so a retry of the same client key + does not conflict because the clock moved. +- The response is the same authorized detail as `GET /api/analysis-runs/{id}`. + +## Consequences + +The home panel's **Request a lineage reconstruction** button records a +Pending row the operator can open immediately. Reconstruction, TEPP +transport, and the outbox worker remain later slices. Do not stamp +Succeeded or invent a theta from this write. + +## References — APA 7th + +International Organization for Standardization. (2019). *ISO 8601-1:2019: +Date and time—Representations for information interchange—Part 1: Basic +rules* (confirmed 2024; Amendment 1:2022). + +Jensen, C. S., & Snodgrass, R. T. (1999). Temporal data management. +*IEEE Transactions on Knowledge and Data Engineering, 11*(1), 36–44. +https://doi.org/10.1109/69.755613 + +Moreau, L., & Missier, P. (Eds.). (2013). *PROV-DM: The PROV data model*. +World Wide Web Consortium. https://www.w3.org/TR/prov-dm/ + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C +Recommendation). https://www.w3.org/TR/owl-time/ diff --git a/docs/adr/0018-related-nodes-team-org-walk.md b/docs/adr/0018-related-nodes-team-org-walk.md new file mode 100644 index 000000000..ae0a1c331 --- /dev/null +++ b/docs/adr/0018-related-nodes-team-org-walk.md @@ -0,0 +1,68 @@ +# ADR 0018 — Related-node walks include team and organization mention edges + +**Decision status:** Accepted +**Date:** 2026-08-16 + +## Context + +ADR 0009 persists `edge_mention_team`, `edge_team_affiliation`, and +`edge_mention_organization` so a cataloged team or organization can +become a cross-post Knowledge Graph clue. The buyer-visible related-node +walk (`load_visible_subgraph` + Tong et al., 2006 random walk with +restart) still loaded only person mention, co-mention, and affiliation +edges, and returned an empty graph when a visible post had no people. +A team-only follow-up therefore never appeared as a related node, and +clicking an R&R team name had no catalog id to start a walk. + +The same temporal honesty ADR 0016 applied to run *detail* posts was +still missing from thread-group *run list* visibility: a later public +post in that thread group could surface a run the account was not +allowed to know at `knowledge_cutoff`. + +ADR 0017 already records an authorized Pending analysis-run write. +This decision is the related-node walk, not that create path. + +## Decision + +`load_visible_subgraph` loads person, team, and organization mention +channels independently. Empty person evidence is not a reason to drop +team or organization edges. `hydrate_related_nodes` labels +`cataloged_team` rows. `GET /api/teams/{team_id}/related` starts the +same RWR walk Keyman and corporate-entity related already use. +`visible_affiliation_post_ids` unions direct `post_organization_mention` +rows with person-affiliation posts so an org-only mention can start a +walk. + +The summary payload exposes `catalog_node_id` / `catalog_node_type_code` +from the catalog foreign keys stored on `post_summary_role` (ADR 0019). +The popup turns that name into a related-node button. Do not reconstruct +the id by `corporate_entity.entity_name`. + +Thread-group run list visibility requires at least one ABAC-visible +`source_post` whose `created_at` is at or before `knowledge_cutoff`. + +## Consequences + +- Open a post whose R&R names 설계팀, then click the team. Sibling posts + that mention the same cataloged team appear as related nodes. +- Click a related team chip the same way you already click a person or + organization chip. +- A later public post in a thread group no longer lists a January run + that could not have known that post. + +## References + +International Organization for Standardization. (2019). *ISO 8601-1:2019: +Date and time—Representations for information interchange—Part 1: Basic +rules* (confirmed 2024; Amendment 1:2022). + +Reynolds, D. (Ed.). (2014). *The organization ontology*. World Wide Web +Consortium. https://www.w3.org/TR/vocab-org/ + +Tong, H., Faloutsos, C., & Pan, J.-Y. (2006). Fast random walk with +restart and its applications. *Proceedings of the Sixth International +Conference on Data Mining (ICDM'06)*, 613–622. +https://doi.org/10.1109/ICDM.2006.70 + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C +Recommendation). https://www.w3.org/TR/owl-time/ diff --git a/docs/adr/0019-role-catalog-identity.md b/docs/adr/0019-role-catalog-identity.md new file mode 100644 index 000000000..32b5d0a09 --- /dev/null +++ b/docs/adr/0019-role-catalog-identity.md @@ -0,0 +1,65 @@ +# ADR 0019 — R&R catalog identity lives on the role row + +**Decision status:** Accepted +**Date:** 2026-08-16 + +## Context + +ADR 0009 writes `post_team_mention` and `post_organization_mention` so a +cataloged team or organization can start a related-node walk. ADR 0018 +exposes `catalog_node_id` on the summary payload by joining those +mentions back to `post_summary_role` through `actor_name`. + +`corporate_entity.entity_name` is not unique. Two catalog rows can share +a display name (different `corporate_entity_code`, different parents). +A fetch join on name therefore: + +- attaches a homonym that this post never resolved, or +- duplicates the role when more than one same-named row exists. + +Mention tables are post-scoped, not role-scoped. They cannot reconstruct +which catalog id was chosen for a specific R&R row. That reconstruction +is a transitive dependency on a non-key attribute, so it is not third +normal form (Codd, 1970; Date, 2019). + +Team identity is already unique on +`(team_name, affiliated_organization_name)`. Organization identity is +not. + +## Decision + +`post_summary_role` stores the resolved catalog foreign keys +(`cataloged_team_id`, `cataloged_corporate_entity_id`) written during +`persist_post_summary`. `fetch_persisted_summary` reads those columns. +It does not join `corporate_entity` by `entity_name`. + +Migration `0019_role_catalog_identity.sql` backfills existing rows from +a post-scoped mention only when the name match is unique on that post. +Two same-named mentions stay unbound rather than guessing. + +## Consequences + +- Open a post whose R&R names an organization that shares a display + name with another catalog row. The button walks the resolved id, not + the homonym. +- Clicking that name still uses `GET /api/corporate-entities/{id}/related` + or `GET /api/teams/{id}/related`. Authz stays person/entity-parity: + a team mentioned only on another corp's private post is 403; an + unknown UUID is 404. + +## References + +Codd, E. F. (1970). A relational model of data for large shared data +banks. *Communications of the ACM, 13*(6), 377–387. +https://doi.org/10.1145/362384.362685 + +Date, C. J. (2019). *Database design and relational theory: Normal forms +and all that jazz* (2nd ed.). Apress. +https://doi.org/10.1007/978-1-4842-5540-7 + +International Organization for Standardization. (2023). *ISO/IEC +11179-1:2023: Information technology—Metadata registries (MDR)—Part 1: +Framework*. + +Reynolds, D. (Ed.). (2014). *The organization ontology*. World Wide Web +Consortium. https://www.w3.org/TR/vocab-org/ diff --git a/docs/doctoring/ANALYSIS_RUN_REGISTRY_REFERENCES.md b/docs/doctoring/ANALYSIS_RUN_REGISTRY_REFERENCES.md new file mode 100644 index 000000000..b41b31c17 --- /dev/null +++ b/docs/doctoring/ANALYSIS_RUN_REGISTRY_REFERENCES.md @@ -0,0 +1,106 @@ +# Analysis-run registry standards and research traceability + +**Status:** Active PR evidence; not protected-main truth until merge. +**Scope:** Migration 0018, ADR 0013, rollback, and real-PostgreSQL contract tests. + +## Standards mapped to implementation + +| Source | Product implication | Implemented evidence | +|---|---|---| +| W3C PROV-DM and PROV-O | Preserve identifiable entities, activities, agents, generation/use, and derivation without flattening provenance into display-only edges. | `analysis_source_snapshot`, `analysis_run`, authenticated requester, append-only status events, immutable digests; later product bindings continue to use the separate `provenance_*` layer from ADR 0011. | +| W3C Time Ontology in OWL | Keep temporal concepts explicit and avoid collapsing distinct clocks. | Evidence availability and snapshot capture remain on `analysis_source_snapshot`; analysis knowledge cutoff and request time remain on `analysis_run`; status occurrence and database record time remain distinct. `GET /api/analysis-runs/{id}` visible posts apply `created_at <= knowledge_cutoff` (ADR 0016). Opening a listed title warns that the live body may have changed after that cutoff. | +| W3C Accessible Name and Description Computation 1.1 | Do not let `aria-label` replace visible text the operator must hear. | Analysis-run digest prefixes live in a labeled group; the prefixes remain the accessible contents and the full digest is on `title` for hover verification. | +| ISO 8601-1:2019 | Use unambiguous timestamp representation and timezone-aware persistence. | PostgreSQL `timestamptz` for availability, capture, cutoff, request, occurrence, and record clocks; tests use explicit `Z` offsets. | +| PostgreSQL 18 constraints and trigger contracts | Put integrity close to durable truth and use constraints for row shape while triggers enforce cross-row state and serialization. | Digest/check constraints, category allowlists, account-scoped uniqueness, shape constraints, immutable-row triggers, shared snapshot-row locking, and serialized status transitions. | +| NIST SP 800-92 | Treat audit records as bounded, protected operational evidence rather than unstructured application logging. | Append-only status events, machine failure codes, actor identity, occurrence/record clocks, fail-closed rollback, and exclusion of raw source/provider payloads. | +| OpenAPI 3.2.0 | Define explicit versioned API schemas rather than exposing database rows or implementation-specific payloads. | API intentionally deferred; ADR 0013 requires a source-redacting run list/detail contract before a product surface is claimed. | + +## Temporal reasoning + +The registry applies a bitemporal discipline without claiming a complete +general-purpose bitemporal database: + +- `maximum_available_time` answers when the newest admitted evidence became + knowable; +- `captured_at` answers when the immutable source snapshot was materialized; +- `knowledge_cutoff` answers what a specific analysis was allowed to know; +- `requested_at` answers when that analysis was requested; +- `occurred_at` and `recorded_at` distinguish lifecycle occurrence from durable + database recording. + +The database requires the aggregate leakage boundary: + +```text +maximum_available_time <= knowledge_cutoff <= requested_at +captured_at <= requested_at +``` + +TEPP remains the authority for finer event/assertion/document/system/available +clocks and temporal psychometrics. The registry does not duplicate TEPP +measurement outputs. + +## Audit and privacy boundary + +The registry may store: + +- opaque product UUIDs; +- authenticated account UUIDs; +- SHA-256 digests; +- bounded configuration/version identifiers; +- aggregate counts; +- bounded status/failure codes; +- timezone-aware clocks. + +The registry must not store: + +- source SQL or source-table names; +- DSNs, credentials, or provider secrets; +- raw posts, HTML, images, base64 data, or attachments; +- model prompts/responses or raw exceptions; +- another service's application tables; +- organization-specific source identifiers in public fixtures or documentation. + +Necessary PII remains available in its purpose-bound authorized product/source +context. Auditability is achieved with actor identity, access control, +provenance, retention, and immutable evidence rather than blanket masking. + +## Verification matrix + +| Claim | Falsifiable test | +|---|---| +| One snapshot supports multiple analyses | Insert two runs over one snapshot with different valid cutoffs. | +| Future evidence is excluded | Reject a run whose cutoff precedes the snapshot's maximum availability time. A late own-corp post stays out of `visible_posts`. | +| Evidence cannot change after derivation | Reject snapshot/count updates and count insert/delete after the first run. | +| Count/run race is serialized | Both paths acquire the snapshot row first; a later concurrency test must prove one legal winner and no lost freeze. | +| Request identity is stable | Reject analysis-run updates; scope and lifecycle live in their own relations. | +| Idempotency is actor-scoped | Permit identical opaque keys for two accounts and reject reuse by the same account. | +| Lifecycle is ordered | Require pending first, contiguous ordinals, monotonic time, legal transitions, terminal finality, and append-only rows. | +| Rollback does not erase audit data silently | Reject rollback with any registry rows and allow replay after explicit cleanup. | + +## APA 7th references + +International Organization for Standardization. (2019). *ISO 8601-1:2019: Date +and time—Representations for information interchange—Part 1: Basic rules* +(confirmed 2024; Amendment 1:2022). + +Kent, K., & Souppaya, M. (2006). *Guide to computer security log management* +(NIST Special Publication 800-92). National Institute of Standards and +Technology. https://doi.org/10.6028/NIST.SP.800-92 + +Moreau, L., & Missier, P. (Eds.). (2013). *PROV-DM: The PROV data model*. +World Wide Web Consortium. https://www.w3.org/TR/prov-dm/ + +OpenAPI Initiative. (2025). *OpenAPI specification, version 3.2.0*. +https://spec.openapis.org/oas/v3.2.0.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: +5.5. Constraints*. https://www.postgresql.org/docs/current/ddl-constraints.html + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology* (W3C +Recommendation). https://www.w3.org/TR/prov-o/ + +World Wide Web Consortium. (2018). *Accessible name and description +computation 1.1* (W3C Recommendation). https://www.w3.org/TR/accname-1.1/ + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C Recommendation). +https://www.w3.org/TR/owl-time/ diff --git a/docs/doctoring/IMAGE_CONTENT_REFERENCES.md b/docs/doctoring/IMAGE_CONTENT_REFERENCES.md new file mode 100644 index 000000000..3aff83a9a --- /dev/null +++ b/docs/doctoring/IMAGE_CONTENT_REFERENCES.md @@ -0,0 +1,30 @@ +# Embedded image content — doctoring + +These are the standards and papers that ground +`docs/image-content-schema.md`, `lineageweave/image_content.py`, and the +product popup in `frontend/src/PostBody.tsx`. Cite them in APA 7th when +you extend OCR, captioning, tagging, or position-preserving storage. + +Li, M., Lv, T., Chen, J., Cui, L., Lu, Y., Florencio, D., Zhang, C., Li, +Z., & Wei, F. (2023). TrOCR: Transformer-based optical character +recognition with pre-trained models. *Proceedings of the AAAI Conference +on Artificial Intelligence, 37*(11), 13094–13102. +https://doi.org/10.1609/aaai.v37i11.26538 + +Radford, A., Kim, J. W., Hallacy, C., Ramesh, A., Goh, G., Agarwal, S., +Sastry, G., Askell, A., Mishkin, P., Clark, J., Krueger, G., & Sutskever, +I. (2021). Learning transferable visual models from natural language +supervision. In M. Meila & T. Zhang (Eds.), *Proceedings of the 38th +International Conference on Machine Learning* (pp. 8748–8763). PMLR. +https://proceedings.mlr.press/v139/radford21a.html + +Masinter, L. (1998). *The "data" URL scheme* (RFC 2397). Internet +Engineering Task Force. https://doi.org/10.17487/RFC2397 + +Crockford, D. (2008). *The application/json media type for JavaScript +Object Notation (JSON)* (RFC 4627; see also RFC 8259). Internet +Engineering Task Force. https://doi.org/10.17487/RFC8259 + +World Wide Web Consortium. (2014). *HTML5: A vocabulary and associated +APIs for HTML and XHTML* (W3C Recommendation). +https://www.w3.org/TR/html5/ diff --git a/docs/doctoring/PROV_O_REFERENCES.md b/docs/doctoring/PROV_O_REFERENCES.md new file mode 100644 index 000000000..ed3167a0e --- /dev/null +++ b/docs/doctoring/PROV_O_REFERENCES.md @@ -0,0 +1,22 @@ +# PROV-O references + +## Normative source — APA 7th + +Lebo, T., Sahoo, S., McGuinness, D., Belhajjame, K., Cheney, J., Corsar, D., Garijo, D., Soiland-Reyes, S., Zednik, S., & Zhao, J. (2013). *PROV-O: The PROV ontology* (W3C Recommendation). World Wide Web Consortium. http://www.w3.org/TR/2013/REC-prov-o-20130430/ + +## Related PROV family documents + +Moreau, L., & Missier, P. (Eds.). (2013). *PROV-DM: The PROV data model* (W3C Recommendation). World Wide Web Consortium. https://www.w3.org/TR/prov-dm/ + +Cheney, J., Missier, P., & Moreau, L. (Eds.). (2013). *Constraints of the PROV data model* (W3C Recommendation). World Wide Web Consortium. https://www.w3.org/TR/prov-constraints/ + +## Implementation traceability + +| Source section | LineageWeave artifact | +|---|---| +| Section 4 term index | `PROV_CLASSES`, `PROV_RELATIONS` | +| Tables 2 and 3 | `PROV_QUALIFICATIONS`, `provenance_qualification_definition` | +| Class/property cross-reference | domain, range, hierarchy registries and normalized tables | +| Appendix B | `PROV_RECOMMENDED_INVERSES`, `provenance_inverse_definition` | +| Qualified form guidance | `ProvGraph.materialized_assertions()` | +| OWL profile union domains | strict union-domain validation in Python and PostgreSQL | diff --git a/docs/doctoring/RELATED_NODE_TEAM_ORG_REFERENCES.md b/docs/doctoring/RELATED_NODE_TEAM_ORG_REFERENCES.md new file mode 100644 index 000000000..fc7ded98f --- /dev/null +++ b/docs/doctoring/RELATED_NODE_TEAM_ORG_REFERENCES.md @@ -0,0 +1,31 @@ +# Related-node team and organization walk — doctoring + +These are the standards and papers that ground ADR 0018. Cite them in +APA 7th when you extend the walk or the catalog identity layer. + +Reynolds, D. (Ed.). (2014). *The organization ontology*. World Wide Web +Consortium. https://www.w3.org/TR/vocab-org/ + +Tong, H., Faloutsos, C., & Pan, J.-Y. (2006). Fast random walk with +restart and its applications. *Proceedings of the Sixth International +Conference on Data Mining (ICDM'06)*, 613–622. +https://doi.org/10.1109/ICDM.2006.70 + +International Organization for Standardization. (2019). *ISO 8601-1:2019: +Date and time—Representations for information interchange—Part 1: Basic +rules* (confirmed 2024; Amendment 1:2022). + +World Wide Web Consortium. (2022). *Time ontology in OWL* (W3C +Recommendation). https://www.w3.org/TR/owl-time/ + +Codd, E. F. (1970). A relational model of data for large shared data +banks. *Communications of the ACM, 13*(6), 377–387. +https://doi.org/10.1145/362384.362685 + +Date, C. J. (2019). *Database design and relational theory: Normal forms +and all that jazz* (2nd ed.). Apress. +https://doi.org/10.1007/978-1-4842-5540-7 + +International Organization for Standardization. (2023). *ISO/IEC +11179-1:2023: Information technology—Metadata registries (MDR)—Part 1: +Framework*. diff --git a/docs/image-content-schema.md b/docs/image-content-schema.md index f023d56a0..57ac3e5a1 100644 --- a/docs/image-content-schema.md +++ b/docs/image-content-schema.md @@ -82,6 +82,23 @@ picture sat relative to the surrounding paragraphs." | `chunk_position` | `integer not null` | 0-based index among ALL of this document's chunks (text and image together) -- matches `Chunk.index` from `chunk_by_dom` | | primary key | `(source_document_id, chunk_position)` | one image slot per position per document | +## Viewer contract (before persistence exists) + +The demo popup does not yet read these tables. It splits the live +`post_body` the same way `extract_base64_images` does: each +`data:image/...;base64,...` payload becomes an `` at its original +character offset, and the surrounding HTML is shown as text. Quoted or +unquoted `src` and optional data-URI parameters such as `charset=utf-8` +are accepted so a real export still shows the picture. A buyer who +opens the post sees the picture that sat between the paragraphs, not the +base64 wall. Remote `src="https://..."` tags are stripped, never fetched; +a remote-only body tells the operator to re-export with the picture +embedded. Undecodable payloads say the same. This screen does not read +text inside the picture. OCR, caption, and tag search still require the +vision client on extract / Ask (Li et al., 2023; Radford et al., 2021) +and, in a real deployment, the tables below. See +`docs/doctoring/IMAGE_CONTENT_REFERENCES.md`. + ## Query shapes this supports - **"Find images whose extracted text or tags match a search query, then @@ -105,3 +122,17 @@ picture sat relative to the surrounding paragraphs." ON CONFLICT DO NOTHING` before the provider call, or a short-lived lease row) to close that race; this schema documents the storage guarantee, not that concurrency control. + +## References + +Li, M., Lv, T., Chen, J., Cui, L., Lu, Y., Florencio, D., Zhang, C., Li, Z., +& Wei, F. (2023). TrOCR: Transformer-based optical character recognition +with pre-trained models. *Proceedings of the AAAI Conference on Artificial +Intelligence, 37*(11), 13094–13102. https://doi.org/10.1609/aaai.v37i11.26538 + +Radford, A., Kim, J. W., Hallacy, C., Ramesh, A., Goh, G., Agarwal, S., +Sastry, G., Askell, A., Mishkin, P., Clark, J., Krueger, G., & Sutskever, I. +(2021). Learning transferable visual models from natural language +supervision. In M. Meila & T. Zhang (Eds.), *Proceedings of the 38th +International Conference on Machine Learning* (pp. 8748–8763). PMLR. +https://proceedings.mlr.press/v139/radford21a.html diff --git a/docs/ontology/lineageweave-kg.ttl b/docs/ontology/lineageweave-kg.ttl index c892d609f..04e8d1b61 100644 --- a/docs/ontology/lineageweave-kg.ttl +++ b/docs/ontology/lineageweave-kg.ttl @@ -4,23 +4,27 @@ @prefix rdfs: . @prefix skos: . @prefix xsd: . +@prefix prov: . +@prefix org: . ################################################################# # LineageWeave Knowledge Graph Ontology # # The formal OWL 2 / RDFS / SKOS vocabulary for the -# `knowledge_graph_edge` table's node/edge types and the +# `knowledge_graph_edge` table's node/edge types, the # `entity_relationship_type` / `person_side` / `corporate_entity_level` -# controlled vocabularies in migrations/0001_initial_schema.sql. +# controlled vocabularies in migrations/0001_initial_schema.sql, and +# `post_summary_role.actor_type_code` (migrations/0012). # # `knowledge_graph_edge` (source_node_type_code, source_node_id) -- # [edge_type_code] --> (target_node_type_code, target_node_id) is # already an RDF triple in shape (Cyganiak, Wood, & Lanthaler, 2014); # this file is the formal semantic layer over it -- PostgreSQL stays # the source of record. See docs/adr/0004-knowledge-graph-ontology.md -# for the full design rationale, and tests/test_ontology.py for the -# round-trip check that every code below actually exists as a -# common_lookup_value row, and vice versa. +# for the KG design rationale, docs/adr/0006-role-responsibility-agent-ontology.md +# for the R&R actor-type rationale (grounded in W3C PROV-O), and +# tests/test_ontology.py for the round-trip check that every code below +# actually exists as a common_lookup_value row, and vice versa. # # Every custom term carries a :lookupCode annotation naming the exact # `common_lookup_value.lookup_code` it corresponds to -- that literal @@ -29,7 +33,7 @@ a owl:Ontology ; rdfs:label "LineageWeave Knowledge Graph Ontology" ; - rdfs:comment "Formal OWL 2 / RDFS / SKOS vocabulary for LineageWeave's knowledge_graph_edge node and edge types, entity_relationship_type, person_side, and corporate_entity_level controlled vocabularies." . + rdfs:comment "Formal OWL 2 / RDFS / SKOS vocabulary for LineageWeave's knowledge_graph_edge node and edge types, entity_relationship_type, person_side, corporate_entity_level, and post_summary_role.actor_type_code controlled vocabularies." . :lookupCode a owl:AnnotationProperty ; rdfs:label "lookup code" ; @@ -65,6 +69,12 @@ rdfs:comment "A corporate_entity row. Also a skos:Concept so the self-referencing parent_entity_id hierarchy (e.g. Group -> Company -> Plant) is expressible with skos:broader/skos:narrower on instances." ; :lookupCode "node_corporate_entity" . +:Team a owl:Class ; + rdfs:subClassOf org:OrganizationalUnit ; + rdfs:label "Team" ; + rdfs:comment "A cataloged_team row: a named company sub-unit (ADR 0009) with a stable team_id, distinct from :RoleActorTeam (ADR 0007's per-row actor_type_code classification) the same way :Person is distinct from :RoleActorPerson." ; + :lookupCode "node_team" . + ################################################################# # Object properties -- edge_type (knowledge_graph_edge.edge_type_code) ################################################################# @@ -90,6 +100,36 @@ rdfs:comment "Two people named in the same post -- symmetric by construction." ; :lookupCode "edge_co_mention" . +################################################################# +# Object properties -- ADR 0009 cross-post identity resolution edges. +# Kept distinct from :mentions/:affiliatedWith (not reused with a +# broadened domain/range) so an edge_type_code alone always tells you +# which node types it connects -- stating rdfs:domain for the same +# property twice (once :Person, once :Team) would make RDFS entail +# every :mentions subject is BOTH a :Person and a :Team, which is false. +################################################################# + +:mentionsTeam a owl:ObjectProperty ; + rdfs:domain :Team ; + rdfs:range :Post ; + rdfs:label "mentioned in post" ; + rdfs:comment "A cataloged team is named by a post (post_team_mention)." ; + :lookupCode "edge_mention_team" . + +:teamAffiliatedWith a owl:ObjectProperty ; + rdfs:domain :Team ; + rdfs:range :CorporateEntity ; + rdfs:label "team affiliated with" ; + rdfs:comment "The company a cataloged team belongs to (cataloged_team.affiliated_corporate_entity_id)." ; + :lookupCode "edge_team_affiliation" . + +:mentionsOrganization a owl:ObjectProperty ; + rdfs:domain :CorporateEntity ; + rdfs:range :Post ; + rdfs:label "mentioned in post" ; + rdfs:comment "A resolved organization is named by a post (post_organization_mention)." ; + :lookupCode "edge_mention_organization" . + ################################################################# # Object properties -- entity_relationship_type # (post_counterparty_entity.relationship_type_code) @@ -152,3 +192,61 @@ :GroupLevel skos:narrower :CompanyLevel . :CompanyLevel skos:narrower :PlantLevel . + +################################################################# +# Classes -- prov_agent_type (post_summary_role.actor_type_code) +# +# A post's R&R (roles & responsibilities) actor is not always a person +# -- business correspondence routinely names an organization acting +# in its own name ("당사" [our company], "Demo Corp"). Grounded +# directly in W3C PROV-O (Lebo, Sahoo, & McGuinness, +# 2013): prov:Agent is the general acting-party class, with prov:Person +# and prov:Organization its two recognized subclasses. These are +# distinct from :Person / :OurSidePerson / :CounterpartyPerson above: +# node_type's :Person is a cataloged_person row with a stable person_id +# a Keyman panel links to; an R&R actor is a free-text name with no +# cataloged identity of its own (it may not even resolve to a Keyman). +# +# A third, meso-level case real data surfaced: a named sub-unit of a +# company ("설계팀" [design team]) is neither prov:Person nor the +# prov:Organization itself -- it is the company's own internal +# structure. PROV-O has no such class; the W3C Organization Ontology +# (Reynolds, 2014) does: org:OrganizationalUnit, "used to represent +# division of a particular organization into sub-organizational units," +# linked to its parent via org:unitOf. See docs/adr/0007-team-actor-type.md. +################################################################# + +:RoleActorPerson a owl:Class ; + rdfs:subClassOf prov:Person ; + rdfs:label "Role actor (person)" ; + rdfs:comment "An R&R actor that is a named individual, per prov:Person." ; + :lookupCode "prov_person" . + +:RoleActorOrganization a owl:Class ; + rdfs:subClassOf prov:Organization ; + rdfs:label "Role actor (organization)" ; + rdfs:comment "An R&R actor that is an organization acting in its own name, per prov:Organization." ; + :lookupCode "prov_organization" . + +:RoleActorTeam a owl:Class ; + rdfs:subClassOf org:OrganizationalUnit ; + rdfs:label "Role actor (team)" ; + rdfs:comment "An R&R actor that is a named sub-unit of a company (e.g. 설계팀), per org:OrganizationalUnit -- not the company itself." ; + :lookupCode "prov_team" . + +################################################################# +# organization_name_resolution (raw/canonical organization-name pairs) +# +# ADR 0008: an abbreviated/slang organization mention (e.g. "AGP") +# is resolved to its full canonical name ("Aurora Grid Power") and +# cross-verified via external search before being trusted. This is not +# a new KG node/edge type -- no new :lookupCode term is declared here, +# since organization_name_resolution's columns are not a +# common_lookup_value category (there is nothing for +# tests/test_ontology.py's round-trip check to enforce). Documented +# here for the Ontology/Semantic-Layer grounding itself: +# `organization_name_resolution.raw_organization_name` corresponds to +# SKOS `skos:altLabel` (an alternative label -- an abbreviation is +# exactly this) and `resolved_organization_name` to `skos:prefLabel` +# (the single preferred/canonical label), per Miles & Bechhofer (2009). +################################################################# diff --git a/docs/ontology/prov-o-support-profile.ttl b/docs/ontology/prov-o-support-profile.ttl new file mode 100644 index 000000000..0175dd70b --- /dev/null +++ b/docs/ontology/prov-o-support-profile.ttl @@ -0,0 +1,18 @@ +@prefix : . +@prefix dcterms: . +@prefix org: . +@prefix owl: . +@prefix prov: . +@prefix rdfs: . + + + a owl:Ontology ; + dcterms:title "LineageWeave PROV-O support profile"@en ; + dcterms:conformsTo ; + owl:imports ; + rdfs:comment "The runtime supports all 30 PROV-O classes, all 50 normative properties, both qualification tables, and Appendix B inverse names without redefining the W3C vocabulary."@en . + +:Post rdfs:subClassOf prov:Entity . +:Person rdfs:subClassOf prov:Person . +:CorporateEntity rdfs:subClassOf prov:Organization . +:Team rdfs:subClassOf prov:Organization, org:OrganizationalUnit . diff --git a/docs/superpowers/plans/2026-08-14-prov-o-standard-relations.md b/docs/superpowers/plans/2026-08-14-prov-o-standard-relations.md new file mode 100644 index 000000000..d3a112453 --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-prov-o-standard-relations.md @@ -0,0 +1,23 @@ +# PROV-O standard relations implementation plan + +## Completed TDD sequence + +1. Write failing tests for exact class/property inventories, datatype-property set, qualification tables, inverse-name table, graph validation, inference, RDF serialization, SQL seed coverage, naming rules, and support-profile mappings. +2. Confirm test collection fails before `lineageweave.prov_o` exists. +3. Implement the complete immutable registry and validated graph API. +4. Implement deterministic fixed-point materialization. +5. Generate the normalized PostgreSQL migration from the same registry and verify every IRI/code is present. +6. Add the ontology support profile and product-class mappings. +7. Add ADR, implementation architecture, complete matrix, and APA 7th doctoring references. +8. Run focused tests, branch coverage, compile checks, exact-head CI/security review, then return the PR to Ready. + +## Merge gates + +- 30/30 classes and 50/50 properties present. +- 14/14 qualification implications pass. +- 44/44 object-property inverse names present. +- Focused production statement and branch coverage 100%. +- Public callable docstrings 100%. +- Migration executes on PostgreSQL 16 in CI and rejects wrong object kinds/domains/ranges. +- Exact-head Tests, Security Scan, and SAST succeed. +- No valid unresolved review thread. diff --git a/docs/superpowers/plans/2026-08-15-analysis-run-registry.md b/docs/superpowers/plans/2026-08-15-analysis-run-registry.md new file mode 100644 index 000000000..a3ed77d27 --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-analysis-run-registry.md @@ -0,0 +1,83 @@ +# Analysis-run registry implementation plan + +> Execute test-first. Preserve the reviewed LineageWeave product and keep +> private actual-data evidence outside public source control. + +**Goal:** Establish one normalized, temporally truthful, actor-scoped registry +for Milestone 2 analysis requests and lifecycle evidence. + +## Task 1 — RED: database contract + +**File:** `tests/test_analysis_run_registry_schema.py` + +1. Require the five normalized relations, current-status view, rollback, and + fresh-install wiring. +2. Reject the retained experiment's denormalized table and JSON metadata. +3. Require evidence-owned availability/capture clocks and a run-owned cutoff. +4. Require non-null requester identity and account-scoped idempotency. +5. Require immutable snapshot, count, and run request rows. +6. Require shared row locking between count mutation and first run creation. +7. Require pending-first, contiguous, monotonic, legal status transitions and + append-only status rows. +8. Require fail-closed rollback and descriptive database-object names. + +## Task 2 — GREEN: normalized migration and rollback + +**Files:** + +- `migrations/0018_analysis_run_registry.sql` +- `migrations/rollback/0018_analysis_run_registry.sql` +- `docker/postgres-init/Dockerfile` + +1. Insert category-checked lookup values idempotently. +2. Add snapshot, count, run, scope, and status-event relations in 3NF. +3. Keep `maximum_available_time` on the snapshot and `knowledge_cutoff` on the + run. +4. Serialize count freeze and run creation through the same snapshot row lock. +5. Reject mutation of immutable evidence and request configuration. +6. Implement the lifecycle state machine as a serialized insert trigger. +7. Add the current-status read view. +8. Refuse rollback while any audit evidence exists. +9. Apply migration 0018 after the PROV-O migration on fresh PostgreSQL images. + +## Task 3 — Documentation and evidence + +**Files:** + +- `docs/adr/0013-normalized-analysis-run-registry.md` +- `docs/doctoring/ANALYSIS_RUN_REGISTRY_REFERENCES.md` +- `CHANGELOG.d/milestone2-analysis-run-registry.md` + +1. Record product/service ownership and deferred API/UI claims. +2. Trace temporal, provenance, audit, privacy, concurrency, and rollback + decisions to current authoritative sources in APA 7th form. +3. Mark active-PR decisions as non-main truth. +4. Keep public fixtures synthetic and exclude private source identifiers. + +## Task 4 — Exact-head verification + +1. Run the static test without PostgreSQL and prove it fails before migration. +2. Run all registry cases against real PostgreSQL after implementation. +3. Replay the migration and rollback. +4. Run the complete Python product suite against PostgreSQL. +5. Run frontend lint, complete tests, and production build. +6. Run `compileall`, security, SAST, documentation hygiene, and public-content + scans. +7. Inspect the exact final diff for temporary workflows/scripts. +8. Obtain independent exact-head review and merge only after the parent PR is on + protected `main` and base-sensitive evidence is regenerated. + +## Task 5 — Next bounded vertical slice + +After this registry reaches protected main: + +1. Write failing repository tests for atomic run + scope + pending-event + creation and idempotent request comparison. +2. Implement the async PostgreSQL repository with no cross-service SQL. +3. Add RBAC/ABAC-protected source-redacting list/detail endpoints. +4. Add the DB-grounded read-only administrator surface and Storybook states. +5. Add normalized outbox + Valkey delivery. +6. Integrate TEPP and contextual-orchestrator only through reviewed versioned + contracts. +7. Execute private actual-data analysis and retain signed aggregate acceptance + artifacts outside public Git history. diff --git a/docs/superpowers/specs/2026-08-14-prov-o-standard-relations-design.md b/docs/superpowers/specs/2026-08-14-prov-o-standard-relations-design.md new file mode 100644 index 000000000..9406d7114 --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-prov-o-standard-relations-design.md @@ -0,0 +1,30 @@ +# PROV-O standard relations design + +## Goal + +Extend PR #74 from three actor categories to complete, interoperable W3C PROV-O relation support while preserving the current product navigation graph. + +## Considered approaches + +### A. Widen `knowledge_graph_edge` + +Rejected. It has one UUID object and cannot represent RDF literals, qualified influence resources, or multiple classes per resource. + +### B. Store opaque RDF only + +Rejected. It would support interchange but provide no fail-closed domain/range, datatype, or relational-integrity contract. + +### C. Standards layer plus explicit product projection — selected + +A complete PROV-O registry, validator, inference engine, normalized relational store, RDF serializer, and support profile sit beside the compact product graph. Existing nodes may be bound to standard resources, and only an explicit projector creates navigation edges. + +## Invariants + +1. Exact W3C namespace and local names are preserved. +2. The registry count is exactly 30 classes and 50 properties for this Recommendation version. +3. A property is object or datatype, never both. +4. Qualified forms imply unqualified forms. +5. Reserved inverse names never create ad hoc vocabulary. +6. Existing product data is not silently retyped or projected. +7. SQL and Python reject invalid assertion shape/domain/range. +8. Definitions and observations remain normalized and independently versionable. diff --git a/frontend/package.json b/frontend/package.json index 9c84795d9..b5209226e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "0.71.0", + "version": "0.86.3", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/App.css b/frontend/src/App.css index 76cf3665c..b3fab25d1 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -85,15 +85,59 @@ cursor: pointer; } +:root { + --lw-opacity-meta: 0.7; + --lw-font-size-meta: 0.85rem; +} + .post-meta { - opacity: 0.7; - font-size: 0.85rem; + opacity: var(--lw-opacity-meta); + font-size: var(--lw-font-size-meta); +} + +.visually-hidden { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip-path: inset(50%); + white-space: nowrap; + border: 0; } .post-body { + display: flex; + flex-direction: column; + gap: var(--post-body-gap); +} + +.post-body-text { + margin: 0; white-space: pre-wrap; } +.post-embedded-image { + margin: 0; + padding: var(--post-image-padding); + border: 1px solid var(--post-image-border); + border-radius: var(--post-image-radius); + background: var(--post-image-bg); +} + +.post-embedded-image img { + display: block; + max-width: 100%; + height: auto; +} + +.post-embedded-image figcaption { + margin-top: 0.4rem; + font-size: 0.85rem; + color: var(--text); +} + .popup-placeholder { margin-top: 1.5rem; padding: 1rem; @@ -239,12 +283,46 @@ font-size: 0.85rem; } +.keyman-role-title { + opacity: 0.6; + font-size: 0.8rem; + font-style: italic; +} + .verification-badge { font-size: 0.8rem; padding: 0.1rem 0.5rem; border-radius: 1rem; } +.actor-type-badge { + font-size: 0.7rem; + padding: 0.05rem 0.4rem; + border-radius: 0.3rem; + text-transform: uppercase; + letter-spacing: 0.02em; +} + +.actor-type-prov_person { + background: #e8eaf6; + color: #303f9f; +} + +.actor-type-prov_organization { + background: #fff3e0; + color: #9a3412; +} + +.actor-type-prov_team { + background: #e0f2f1; + color: #00695c; +} + +.rr-affiliation { + opacity: 0.7; + font-size: 0.9rem; +} + .verification-verify_pending { background: #e0e0e0; color: #444; diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 415e1419f..a6d98a885 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -59,6 +59,12 @@ describe("App, authenticated", () => { chatUnavailable?: boolean; searchUnavailable?: boolean; verificationEvidenceUrl?: string | null; + failedLineageRun?: boolean; + failedReportRun?: boolean; + succeededTeppRun?: boolean; + pendingTeppRun?: boolean; + postBody?: string; + evidencePostBody?: string; }) { const statusLabel: Record = { open: "Open", @@ -169,6 +175,267 @@ describe("App, authenticated", () => { jsonResponse({ post_id: "post-1", has_commitment: true, ticket }), ); } + if (url.endsWith("/api/analysis-runs/run-demo-report")) { + return Promise.resolve( + jsonResponse({ + analysis_run_id: "run-demo-report", + run_kind_code: "analysis_run_report", + run_kind_label: "Period report", + scope_kind_code: "analysis_scope_corporate_entity", + scope_kind_label: "Corporate entity", + scope_entity_name: "Demo Corp", + status_code: "analysis_status_failed", + status_label: "Failed", + knowledge_cutoff: "2026-01-12T12:00:00Z", + requested_at: "2026-01-12T12:38:00Z", + source_counts: [ + { + count_type_code: "analysis_count_document", + count_type_label: "Documents", + count_value: 3, + }, + ], + visible_posts: [], + status_history: [ + { + status_ordinal: 1, + status_code: "analysis_status_pending", + status_label: "Pending", + occurred_at: "2026-01-12T12:39:00Z", + }, + { + status_ordinal: 2, + status_code: "analysis_status_failed", + status_label: "Failed", + occurred_at: "2026-01-12T12:40:00Z", + failure_code: "period_report_rebuild_failed", + }, + ], + }), + ); + } + if (url.endsWith("/api/analysis-runs/run-demo-tepp")) { + const teppStatus = options?.succeededTeppRun + ? "analysis_status_succeeded" + : options?.pendingTeppRun + ? "analysis_status_pending" + : "analysis_status_failed"; + const teppLabel = options?.succeededTeppRun + ? "Succeeded" + : options?.pendingTeppRun + ? "Pending" + : "Failed"; + return Promise.resolve( + jsonResponse({ + analysis_run_id: "run-demo-tepp", + run_kind_code: "analysis_run_tepp", + run_kind_label: "TEPP measurement", + scope_kind_code: "analysis_scope_corporate_entity", + scope_kind_label: "Corporate entity", + scope_entity_name: "Demo Corp", + status_code: teppStatus, + status_label: teppLabel, + knowledge_cutoff: "2026-01-12T12:00:00Z", + requested_at: "2026-01-12T12:34:00Z", + source_counts: [ + { + count_type_code: "analysis_count_document", + count_type_label: "Documents", + count_value: 3, + }, + ], + visible_posts: [{ post_id: "post-1", post_title: "Public post" }], + status_history: options?.pendingTeppRun + ? [ + { + status_ordinal: 1, + status_code: "analysis_status_pending", + status_label: "Pending", + occurred_at: "2026-01-12T12:35:00Z", + }, + ] + : [ + { + status_ordinal: 1, + status_code: "analysis_status_pending", + status_label: "Pending", + occurred_at: "2026-01-12T12:35:00Z", + }, + { + status_ordinal: 2, + status_code: "analysis_status_running", + status_label: "Running", + occurred_at: "2026-01-12T12:36:00Z", + }, + { + status_ordinal: 3, + status_code: options?.succeededTeppRun + ? "analysis_status_succeeded" + : "analysis_status_failed", + status_label: options?.succeededTeppRun ? "Succeeded" : "Failed", + occurred_at: "2026-01-12T12:37:00Z", + ...(options?.succeededTeppRun + ? {} + : { failure_code: "tepp_not_available" }), + }, + ], + }), + ); + } + if (url.endsWith("/api/analysis-runs/run-demo-lineage")) { + return Promise.resolve( + jsonResponse({ + analysis_run_id: "run-demo-lineage", + run_kind_code: "analysis_run_lineage", + run_kind_label: "Lineage reconstruction", + scope_kind_code: "analysis_scope_corporate_entity", + scope_kind_label: "Corporate entity", + scope_entity_name: "Demo Corp", + status_code: "analysis_status_succeeded", + status_label: "Succeeded", + knowledge_cutoff: "2026-01-12T12:00:00Z", + requested_at: "2026-01-12T12:30:00Z", + source_counts: [ + { + count_type_code: "analysis_count_document", + count_type_label: "Documents", + count_value: 3, + }, + ], + visible_posts: [{ post_id: "post-1", post_title: "Public post" }], + code_revision_sha: "abcdef0123456789deadbeefcafebabe", + configuration_sha256: + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + status_history: [ + { + status_ordinal: 1, + status_code: "analysis_status_pending", + status_label: "Pending", + occurred_at: "2026-01-12T12:31:00Z", + }, + { + status_ordinal: 2, + status_code: "analysis_status_running", + status_label: "Running", + occurred_at: "2026-01-12T12:32:00Z", + }, + { + status_ordinal: 3, + status_code: "analysis_status_succeeded", + status_label: "Succeeded", + occurred_at: "2026-01-12T12:33:00Z", + }, + ], + }), + ); + } + if (url.endsWith("/api/analysis-runs") && method === "POST") { + const created = { + analysis_run_id: "run-demo-lineage-pending", + run_kind_code: "analysis_run_lineage", + run_kind_label: "Lineage reconstruction", + scope_kind_code: "analysis_scope_corporate_entity", + scope_kind_label: "Corporate entity", + scope_entity_name: "Demo Corp", + status_code: "analysis_status_pending", + status_label: "Pending", + knowledge_cutoff: "2026-01-12T12:00:00Z", + requested_at: "2026-01-12T12:35:00Z", + source_counts: [], + visible_posts: [{ post_id: "post-1", post_title: "Public post" }], + status_history: [ + { + status_ordinal: 1, + status_code: "analysis_status_pending", + status_label: "Pending", + occurred_at: "2026-01-12T12:35:00Z", + }, + ], + }; + return Promise.resolve(new Response(JSON.stringify(created), { status: 201 })); + } + if (url.endsWith("/api/analysis-runs")) { + return Promise.resolve( + jsonResponse({ + analysis_runs: [ + { + analysis_run_id: "run-demo-lineage", + run_kind_code: "analysis_run_lineage", + run_kind_label: "Lineage reconstruction", + scope_kind_code: "analysis_scope_corporate_entity", + scope_kind_label: "Corporate entity", + scope_entity_name: "Demo Corp", + status_code: options?.failedLineageRun + ? "analysis_status_failed" + : "analysis_status_succeeded", + status_label: options?.failedLineageRun ? "Failed" : "Succeeded", + knowledge_cutoff: "2026-01-12T12:00:00Z", + requested_at: "2026-01-12T12:30:00Z", + source_counts: [ + { + count_type_code: "analysis_count_document", + count_type_label: "Documents", + count_value: 3, + }, + ], + code_revision_sha: "abcdef0123456789deadbeefcafebabe", + configuration_sha256: + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + }, + { + analysis_run_id: "run-demo-tepp", + run_kind_code: "analysis_run_tepp", + run_kind_label: "TEPP measurement", + scope_kind_code: "analysis_scope_corporate_entity", + scope_kind_label: "Corporate entity", + scope_entity_name: "Demo Corp", + status_code: options?.succeededTeppRun + ? "analysis_status_succeeded" + : options?.pendingTeppRun + ? "analysis_status_pending" + : "analysis_status_failed", + status_label: options?.succeededTeppRun + ? "Succeeded" + : options?.pendingTeppRun + ? "Pending" + : "Failed", + knowledge_cutoff: "2026-01-12T12:00:00Z", + requested_at: "2026-01-12T12:34:00Z", + source_counts: [ + { + count_type_code: "analysis_count_document", + count_type_label: "Documents", + count_value: 3, + }, + ], + }, + ...(options?.failedReportRun + ? [ + { + analysis_run_id: "run-demo-report", + run_kind_code: "analysis_run_report" as const, + run_kind_label: "Period report", + scope_kind_code: "analysis_scope_corporate_entity", + scope_kind_label: "Corporate entity", + scope_entity_name: "Demo Corp", + status_code: "analysis_status_failed" as const, + status_label: "Failed", + knowledge_cutoff: "2026-01-12T12:00:00Z", + requested_at: "2026-01-12T12:38:00Z", + source_counts: [ + { + count_type_code: "analysis_count_document", + count_type_label: "Documents", + count_value: 3, + }, + ], + }, + ] + : []), + ], + }), + ); + } if (url.endsWith("/api/calendar")) { return Promise.resolve( jsonResponse({ @@ -403,7 +670,7 @@ describe("App, authenticated", () => { jsonResponse({ post_id: "post-1", post_title: "Public post", - post_body: "The full body text.", + post_body: options?.postBody ?? "The full body text.", voc_type_code: "voc", voc_type_label: "Voice of Customer", visibility_code: "public", @@ -417,7 +684,7 @@ describe("App, authenticated", () => { jsonResponse({ post_id: "post-2", post_title: "Linked post", - post_body: "The evidence panel should show exactly this text.", + post_body: options?.evidencePostBody ?? "The evidence panel should show exactly this text.", voc_type_code: "voc", visibility_code: "public", created_at: "2026-01-02T00:00:00Z", @@ -453,8 +720,32 @@ describe("App, authenticated", () => { korean_summary: "이것은 요약입니다.", key_events: ["첫 번째 이벤트"], roles_and_responsibilities: [ - { person_name: "Ada West", responsibility: "우리 측 후속" }, - { person_name: "Priya Nair", responsibility: "고객 측 수신" }, + { + actor_name: "Ada West", + responsibility: "우리 측 후속", + actor_type_code: "prov_person", + affiliated_organization_name: "Demo Corp", + }, + { + actor_name: "Priya Nair", + responsibility: "고객 측 수신", + actor_type_code: "prov_person", + affiliated_organization_name: "Northridge Grid", + }, + { + actor_name: "당사", + responsibility: "출하 일정 확정", + actor_type_code: "prov_organization", + affiliated_organization_name: null, + }, + { + actor_name: "설계팀", + responsibility: "도면 검토", + actor_type_code: "prov_team", + affiliated_organization_name: "Demo Corp", + catalog_node_id: "team-1", + catalog_node_type_code: "node_team", + }, ], }), ); @@ -468,6 +759,7 @@ describe("App, authenticated", () => { person_name: "Ada West", person_side_code: "our_side", person_side_label: "Our side", + last_known_job_title: "Account manager", mention_context: null, affiliations: [{ organization_name: "Demo Corp", corporate_entity_id: "corp-1", role_title: null }], }, @@ -514,6 +806,8 @@ describe("App, authenticated", () => { ontology_iri: "https://contextualwisdomlab.github.io/lineageweave/ontology#Person", ontology_label: "Person", label: "Ada West", + person_side_code: "our_side", + person_side_label: "Our side", relevance: 0.4, }, ], @@ -533,6 +827,8 @@ describe("App, authenticated", () => { ontology_iri: "https://contextualwisdomlab.github.io/lineageweave/ontology#Person", ontology_label: "Person", label: "Priya Nair", + person_side_code: "counterparty", + person_side_label: "Counterparty", relevance: 0.4, }, { @@ -551,6 +847,32 @@ describe("App, authenticated", () => { label: "Demo Corp", relevance: 0.2, }, + { + node_id: "team-1", + node_type_code: "node_team", + ontology_iri: "https://contextualwisdomlab.github.io/lineageweave/ontology#Team", + ontology_label: "Team", + label: "설계팀", + relevance: 0.15, + }, + ], + }), + ); + } + if (url.endsWith("/api/teams/team-1/related")) { + return Promise.resolve( + jsonResponse({ + team_id: "team-1", + team_name: "설계팀", + related: [ + { + node_id: "post-2", + node_type_code: "node_post", + ontology_iri: "https://contextualwisdomlab.github.io/lineageweave/ontology#Post", + ontology_label: "Post", + label: "Linked post", + relevance: 0.6, + }, ], }), ); @@ -567,6 +889,8 @@ describe("App, authenticated", () => { ontology_iri: "https://contextualwisdomlab.github.io/lineageweave/ontology#Person", ontology_label: "Person", label: "Ada West", + person_side_code: "our_side", + person_side_label: "Our side", relevance: 0.5, }, ], @@ -766,6 +1090,48 @@ describe("App, authenticated", () => { await waitFor(() => expect(screen.getByText("The full body text.")).toBeInTheDocument()); }); + it("shows an embedded invoice image instead of the raw base64 string", async () => { + const tinyPng = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + stubBackend({ + postBody: `

Quote attached.

Please confirm.

`, + }); + render(); + await userEvent.click(await screen.findByRole("button", { name: "View post: Public post" })); + + const image = await screen.findByRole("img", { name: /embedded image at character offset/i }); + expect(image).toHaveAttribute("src", `data:image/png;base64,${tinyPng}`); + expect(screen.getByText("Quote attached.")).toBeInTheDocument(); + expect(screen.getByText("Please confirm.")).toBeInTheDocument(); + expect(screen.getByText(/Text inside the picture is not read on this screen/)).toBeInTheDocument(); + expect(screen.queryByText(/Extract Keyman or ask a question/)).not.toBeInTheDocument(); + expect(screen.queryByText(new RegExp(tinyPng))).not.toBeInTheDocument(); + }); + + it("shows an embedded image in the evidence panel without dumping base64", async () => { + const tinyPng = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + stubBackend({ + evidencePostBody: `

Source quote.

`, + }); + render(); + + await userEvent.click(await screen.findByRole("button", { name: "View post: Public post" })); + await waitFor(() => expect(screen.getByPlaceholderText(/what happened/i)).toBeInTheDocument()); + await userEvent.type(screen.getByPlaceholderText(/what happened/i), "What happened?"); + await userEvent.click(screen.getByRole("button", { name: /^ask$/i })); + await waitFor(() => + expect(screen.getByText("Here is what happened, drawing on the linked post.")).toBeInTheDocument(), + ); + const evidenceChips = screen.getAllByRole("button", { name: "Open evidence: Linked post" }); + await userEvent.click(evidenceChips[evidenceChips.length - 1]); + + const image = await screen.findByRole("img", { name: /embedded image at character offset/i }); + expect(image).toHaveAttribute("src", `data:image/png;base64,${tinyPng}`); + expect(screen.getByText("Source quote.")).toBeInTheDocument(); + expect(screen.queryByText(new RegExp(tinyPng))).not.toBeInTheDocument(); + }); + it("fetches and renders the post list, then opens a detail popup on click", async () => { const fetchMock = stubBackend(); @@ -810,6 +1176,8 @@ describe("App, authenticated", () => { expect(screen.getByText("첫 번째 이벤트")).toBeInTheDocument(); expect(screen.getByText(/우리 측 후속/)).toBeInTheDocument(); expect(screen.getByRole("button", { name: "R&R Keyman: Ada West" })).toBeInTheDocument(); + expect(screen.getByText("당사").closest("li")).toHaveTextContent("Organization"); + expect(screen.queryByRole("button", { name: "R&R Keyman: 당사" })).not.toBeInTheDocument(); await waitFor(() => expect(screen.getByText("간접")).toBeInTheDocument()); expect(screen.getByText("간접").closest("li")).toHaveTextContent("Linked post"); // The popup Event Lineage is the same A-100 reconstruct DAG as the home @@ -956,6 +1324,7 @@ describe("App, authenticated", () => { expect(screen.getByRole("button", { name: "Keyman affiliation: Demo Corp" })).toBeInTheDocument(); expect(screen.getByText("(Company)")).toBeInTheDocument(); expect(screen.getAllByText(/Ada West \(Our side\)/).length).toBeGreaterThanOrEqual(1); + expect(screen.getByText("Account manager")).toBeInTheDocument(); expect(screen.queryByText(/our_side/)).not.toBeInTheDocument(); expect(screen.getByText("unresolved")).toBeInTheDocument(); expect(screen.getByText(/Voice of Customer\s*\(voc\)/)).toBeInTheDocument(); @@ -969,7 +1338,17 @@ describe("App, authenticated", () => { await userEvent.click(screen.getByRole("button", { name: "Related nodes for Ada West" })); await waitFor(() => expect(screen.getByText("Related to Ada West")).toBeInTheDocument()); - expect(screen.getByText("Priya Nair (Person)")).toBeInTheDocument(); + expect(screen.getByText("Related to Ada West").closest(".related-keymen")).toHaveTextContent( + "Priya Nair (Counterparty)", + ); + expect(screen.getByText("Related to Ada West").closest(".related-keymen")).not.toHaveTextContent( + "Priya Nair (Person)", + ); + expect( + screen.getByRole("button", { + name: "Related nodes for Priya Nair (Counterparty)", + }), + ).toBeInTheDocument(); await userEvent.click(screen.getByRole("button", { name: "Open related post: Linked post" })); await waitFor(() => expect(screen.getByText("The evidence panel should show exactly this text.")).toBeInTheDocument(), @@ -982,7 +1361,33 @@ describe("App, authenticated", () => { await userEvent.click(await screen.findByRole("button", { name: "View post: Public post" })); await userEvent.click(await screen.findByRole("button", { name: "R&R Keyman: Ada West" })); await waitFor(() => expect(screen.getByText("Related to Ada West")).toBeInTheDocument()); - expect(screen.getByText("Priya Nair (Person)")).toBeInTheDocument(); + expect(screen.getByText("Related to Ada West").closest(".related-keymen")).toHaveTextContent( + "Priya Nair (Counterparty)", + ); + }); + + it("opens related nodes from an R&R team", async () => { + stubBackend(); + render(); + await userEvent.click(await screen.findByRole("button", { name: "View post: Public post" })); + await userEvent.click(await screen.findByRole("button", { name: "R&R team: 설계팀" })); + await waitFor(() => expect(screen.getByText("Related to 설계팀")).toBeInTheDocument()); + expect(screen.getByText("Related to 설계팀").closest(".related-keymen")).toHaveTextContent( + "Linked post", + ); + }); + + it("opens related nodes from a related team chip", async () => { + stubBackend(); + render(); + await userEvent.click(await screen.findByRole("button", { name: "View post: Public post" })); + await userEvent.click(screen.getByRole("button", { name: "Related nodes for Ada West" })); + await waitFor(() => expect(screen.getByText("Related to Ada West")).toBeInTheDocument()); + await userEvent.click(screen.getByRole("button", { name: "Related nodes for 설계팀" })); + await waitFor(() => expect(screen.getByText("Related to 설계팀")).toBeInTheDocument()); + expect(screen.getByText("Related to 설계팀").closest(".related-keymen")).toHaveTextContent( + "Linked post", + ); }); it("opens related nodes from a related corporate entity", async () => { @@ -993,7 +1398,9 @@ describe("App, authenticated", () => { await waitFor(() => expect(screen.getByText("Related to Ada West")).toBeInTheDocument()); await userEvent.click(screen.getByRole("button", { name: "Related nodes for Demo Corp" })); await waitFor(() => expect(screen.getByText("Related to Demo Corp")).toBeInTheDocument()); - expect(screen.getByText("Ada West (Person)")).toBeInTheDocument(); + expect(screen.getByText("Related to Demo Corp").closest(".related-keymen")).toHaveTextContent( + "Ada West (Our side)", + ); }); it("shows the VOC excerpt under its counterparty, not a detached list", async () => { @@ -1022,7 +1429,9 @@ describe("App, authenticated", () => { await userEvent.click(await screen.findByRole("button", { name: "View post: Public post" })); await userEvent.click(await screen.findByRole("button", { name: "VOC Keyman: Northridge Grid" })); await waitFor(() => expect(screen.getByText("Related to Priya Nair")).toBeInTheDocument()); - expect(screen.getByText("Ada West (Person)")).toBeInTheDocument(); + expect(screen.getByText("Related to Priya Nair").closest(".related-keymen")).toHaveTextContent( + "Ada West (Our side)", + ); }); it("opens related Keyman nodes from an affiliate-tree person", async () => { @@ -1031,7 +1440,9 @@ describe("App, authenticated", () => { await userEvent.click(await screen.findByRole("button", { name: "View post: Public post" })); await userEvent.click(await screen.findByRole("button", { name: "Affiliate Keyman: Priya Nair" })); await waitFor(() => expect(screen.getByText("Related to Priya Nair")).toBeInTheDocument()); - expect(screen.getByText("Ada West (Person)")).toBeInTheDocument(); + expect(screen.getByText("Related to Priya Nair").closest(".related-keymen")).toHaveTextContent( + "Ada West (Our side)", + ); }); it("opens related nodes from a Keyman affiliation organization", async () => { @@ -1040,7 +1451,9 @@ describe("App, authenticated", () => { await userEvent.click(await screen.findByRole("button", { name: "View post: Public post" })); await userEvent.click(await screen.findByRole("button", { name: "Keyman affiliation: Demo Corp" })); await waitFor(() => expect(screen.getByText("Related to Demo Corp")).toBeInTheDocument()); - expect(screen.getByText("Ada West (Person)")).toBeInTheDocument(); + expect(screen.getByText("Related to Demo Corp").closest(".related-keymen")).toHaveTextContent( + "Ada West (Our side)", + ); }); it("opens related nodes from an affiliate-tree organization", async () => { @@ -1049,7 +1462,9 @@ describe("App, authenticated", () => { await userEvent.click(await screen.findByRole("button", { name: "View post: Public post" })); await userEvent.click(await screen.findByRole("button", { name: "Affiliate org: Demo Corp" })); await waitFor(() => expect(screen.getByText("Related to Demo Corp")).toBeInTheDocument()); - expect(screen.getByText("Ada West (Person)")).toBeInTheDocument(); + expect(screen.getByText("Related to Demo Corp").closest(".related-keymen")).toHaveTextContent( + "Ada West (Our side)", + ); expect(screen.queryByRole("button", { name: "Affiliate org: Northridge Grid" })).not.toBeInTheDocument(); }); @@ -1059,7 +1474,9 @@ describe("App, authenticated", () => { await userEvent.click(await screen.findByRole("button", { name: "View post: Public post" })); await userEvent.click(await screen.findByRole("button", { name: "Counterparty org: Demo Corp" })); await waitFor(() => expect(screen.getByText("Related to Demo Corp")).toBeInTheDocument()); - expect(screen.getByText("Ada West (Person)")).toBeInTheDocument(); + expect(screen.getByText("Related to Demo Corp").closest(".related-keymen")).toHaveTextContent( + "Ada West (Our side)", + ); expect(screen.queryByRole("button", { name: "Counterparty org: Northridge Grid" })).not.toBeInTheDocument(); }); @@ -1246,6 +1663,185 @@ describe("App, authenticated", () => { await waitFor(() => expect(screen.getByText("The full body text.")).toBeInTheDocument()); }); + it("shows the seeded analysis run on the home page", async () => { + stubBackend(); + render(); + + expect(await screen.findByRole("heading", { name: "Analysis runs" })).toBeInTheDocument(); + const list = screen.getByRole("list", { name: "Analysis runs" }); + expect(list).toHaveTextContent("Lineage reconstruction · Succeeded · Demo Corp"); + expect(list).toHaveTextContent("TEPP measurement · Failed · Demo Corp"); + expect(list).toHaveTextContent( + "Open this run to see why it failed, then connect the measurement service and re-run.", + ); + expect(list).toHaveTextContent("3 documents"); + expect(list).not.toHaveTextContent("postgresql://"); + expect(list).not.toHaveTextContent("select "); + expect(list).not.toHaveTextContent("Code abcdef012345"); + expect(list).not.toHaveTextContent("Config 0123456789ab"); + expect(list).not.toHaveTextContent("abcdef0123456789deadbeefcafebabe"); + expect(list).not.toHaveTextContent( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + ); + + await userEvent.click( + screen.getByRole("button", { + name: "Open analysis run: Lineage reconstruction · Succeeded · Demo Corp", + }), + ); + expect(await screen.findByRole("heading", { name: "Lineage reconstruction · Succeeded · Demo Corp" })).toBeInTheDocument(); + expect(screen.getByText(/Cutoff 2026-01-12/)).toBeInTheDocument(); + expect(screen.getByText(/Requested 2026-01-12/)).toBeInTheDocument(); + const digests = screen.getByLabelText("Analysis run reproducibility digests"); + expect(digests).toHaveTextContent("Hover a prefix to read the full digest for verification."); + expect(digests).toHaveTextContent("Code abcdef012345"); + expect(digests).toHaveTextContent("Config 0123456789ab"); + expect(digests).not.toHaveTextContent("abcdef0123456789deadbeefcafebabe"); + expect(digests).not.toHaveTextContent( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + ); + expect(screen.getByTitle("abcdef0123456789deadbeefcafebabe")).toHaveTextContent("Code abcdef012345"); + expect( + screen.getByTitle("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), + ).toHaveTextContent("Config 0123456789ab"); + const history = screen.getByRole("list", { name: "Analysis run status history" }); + expect(history).toHaveTextContent("Pending 2026-01-12 12:31"); + expect(history).toHaveTextContent("Running 2026-01-12 12:32"); + expect(history).toHaveTextContent("Succeeded 2026-01-12 12:33"); + expect(screen.getByRole("list", { name: "Posts known at this run cutoff" })).toBeInTheDocument(); + expect( + screen.getByText( + "Opening a title shows the live post. Compare it with cutoff 2026-01-12 before you treat the body as reconstructed evidence — it may have changed after this run.", + ), + ).toBeInTheDocument(); + expect( + screen.getByRole("button", { + name: "Open live post (may have changed after cutoff): Public post", + }), + ).toBeInTheDocument(); + expect(screen.queryByText(/postgresql:\/\//)).not.toBeInTheDocument(); + + await userEvent.click( + screen.getByRole("button", { + name: "Open live post (may have changed after cutoff): Public post", + }), + ); + await waitFor(() => expect(screen.getByText("The full body text.")).toBeInTheDocument()); + + await userEvent.click( + screen.getByRole("button", { + name: "Open analysis run: TEPP measurement · Failed · Demo Corp", + }), + ); + expect( + await screen.findByRole("heading", { name: "TEPP measurement · Failed · Demo Corp" }), + ).toBeInTheDocument(); + const teppHistory = screen.getByRole("list", { name: "Analysis run status history" }); + expect(teppHistory).toHaveTextContent("Failed 2026-01-12 12:37 · tepp_not_available"); + expect(screen.getByText(/cutoff corpus TEPP would measure/i)).toBeInTheDocument(); + expect(teppHistory).not.toHaveTextContent("Succeeded"); + }); + + it("does not tell a failed lineage run to connect the measurement service", async () => { + stubBackend({ failedLineageRun: true }); + render(); + + await screen.findByRole("list", { name: "Analysis runs" }); + const lineageButton = screen.getByRole("button", { + name: "Open analysis run: Lineage reconstruction · Failed · Demo Corp", + }); + const teppButton = screen.getByRole("button", { + name: "Open analysis run: TEPP measurement · Failed · Demo Corp", + }); + expect(lineageButton).toHaveTextContent( + "Open this run to see why it failed, then retry reconstruction from a current snapshot.", + ); + expect(lineageButton).not.toHaveTextContent("measurement service"); + expect(teppButton).toHaveTextContent( + "Open this run to see why it failed, then connect the measurement service and re-run.", + ); + expect(teppButton).not.toHaveTextContent("reconstruction"); + }); + + it("does not tell a failed period report to connect the measurement service", async () => { + stubBackend({ failedReportRun: true }); + render(); + + const reportButton = await screen.findByRole("button", { + name: "Open analysis run: Period report · Failed · Demo Corp", + }); + expect(reportButton).toHaveTextContent( + "Open this run to see why it failed, then rebuild the period report from a current snapshot.", + ); + expect(reportButton).not.toHaveTextContent("measurement service"); + expect(reportButton).not.toHaveTextContent("reconstruction"); + + await userEvent.click(reportButton); + expect( + await screen.findByText( + "No posts were available at this cutoff for the period report. Open a later run, or ask an administrator to capture a newer snapshot.", + ), + ).toBeInTheDocument(); + }); + + it("does not tell a pending TEPP run that it already measured", async () => { + stubBackend({ pendingTeppRun: true }); + render(); + + await userEvent.click( + await screen.findByRole("button", { + name: "Open analysis run: TEPP measurement · Pending · Demo Corp", + }), + ); + expect( + await screen.findByText("These posts are the cutoff corpus TEPP will measure once this run finishes."), + ).toBeInTheDocument(); + expect(screen.queryByText(/replace Failed/i)).not.toBeInTheDocument(); + expect(screen.queryByText(/this TEPP run measured/i)).not.toBeInTheDocument(); + expect(screen.queryByText(/Reconstruction has not started yet/)).not.toBeInTheDocument(); + }); + + it("does not tell a succeeded TEPP run to replace Failed", async () => { + stubBackend({ succeededTeppRun: true }); + render(); + + await userEvent.click( + await screen.findByRole("button", { + name: "Open analysis run: TEPP measurement · Succeeded · Demo Corp", + }), + ); + expect( + await screen.findByText("These posts are the cutoff corpus this TEPP run measured."), + ).toBeInTheDocument(); + expect(screen.queryByText(/replace Failed/i)).not.toBeInTheDocument(); + }); + + it("records a pending lineage run and opens the authorized detail", async () => { + const fetchMock = stubBackend(); + render(); + + await userEvent.click( + await screen.findByRole("button", { name: "Request a lineage reconstruction" }), + ); + expect( + await screen.findByRole("heading", { name: "Lineage reconstruction · Pending · Demo Corp" }), + ).toBeInTheDocument(); + expect( + screen.getByText( + "Open this run to confirm which posts it will use. Reconstruction has not started yet.", + ), + ).toBeInTheDocument(); + const postCall = fetchMock.mock.calls.find( + (call) => String(call[0]).endsWith("/api/analysis-runs") && call[1]?.method === "POST", + ); + expect(postCall).toBeDefined(); + const body = JSON.parse(String(postCall?.[1]?.body)); + expect(body.run_kind_code).toBe("analysis_run_lineage"); + expect(body.idempotency_key).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i, + ); + }); + it("shows the calibrated period-report mean theta on the home page", async () => { stubBackend(); render(); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 1e39a9253..d589a3644 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,12 +1,15 @@ -import { useEffect, useRef, useState } from "react"; +import { useEffect, useRef, useState, type ReactNode } from "react"; import { useAuth } from "react-oidc-context"; import { askPostChat, BackendError, + createAnalysisRun, createPostTicket, deriveCommitment, evaluatePost, extractPostKeymen, + fetchAnalysisRun, + fetchAnalysisRuns, fetchCalendar, fetchLineageGraph, fetchMe, @@ -27,12 +30,14 @@ import { fetchPosts, fetchRelatedEntity, fetchRelatedKeymen, + fetchRelatedTeam, rebuildLineage, rebuildPeriodReports, updateTicketStatus, verifyPostRelations, type ActivityEvent, type AffiliateNode, + type AnalysisRun, type CalendarEntry, type ChatAnswer, type ChatExchange, @@ -50,9 +55,11 @@ import { type PostLineage, type PostSummary, type RelatedNode, + type RelatedNodeType, type VocEvidence, } from "./api"; import { LineageDag } from "./LineageDag"; +import { PostBody } from "./PostBody"; import { subgraphForPost } from "./lineageLayout"; import "./App.css"; @@ -113,7 +120,7 @@ function EvidencePanel({ {post && ( <>

{post.post_title}

-

{post.post_body}

+ )} @@ -468,6 +475,24 @@ function VocEvidenceSection({ const NODE_PERSON = "node_person"; const NODE_POST = "node_post"; const NODE_CORPORATE_ENTITY = "node_corporate_entity"; +const NODE_TEAM = "node_team"; + +const KNOWN_RELATED_NODE_TYPES = [NODE_PERSON, NODE_POST, NODE_CORPORATE_ENTITY, NODE_TEAM] as const; + +function isKnownRelatedNodeType(code: string): code is RelatedNodeType { + return (KNOWN_RELATED_NODE_TYPES as readonly string[]).includes(code); +} + +function relatedNodeCaption(node: RelatedNode): string { + const name = node.label ?? node.node_id; + if (node.node_type_code === NODE_PERSON) { + const side = node.person_side_label ?? node.person_side_code; + if (side) { + return `${name} (${side})`; + } + } + return `${name} (${node.ontology_label ?? node.node_type_code})`; +} const VERIFICATION_BADGE: Record = { verify_pending: "Not yet checked", @@ -523,6 +548,7 @@ function KeymanPanel({ onSelectPost, focusPerson, focusEntity, + focusTeam, }: { postId: string; accessToken: string; @@ -532,6 +558,7 @@ function KeymanPanel({ onSelectPost?: (postId: string) => void; focusPerson?: { personId: string; personName: string } | null; focusEntity?: { entityId: string; entityName: string } | null; + focusTeam?: { teamId: string; teamName: string } | null; }) { const [related, setRelated] = useState(null); const [selectedName, setSelectedName] = useState(null); @@ -569,6 +596,18 @@ function KeymanPanel({ } } + async function handleSelectTeam(teamId: string, teamName: string) { + const requestId = ++relatedRequest.current; + setSelectedName(teamName); + setRelated(null); + try { + const result = await fetchRelatedTeam(accessToken, teamId); + if (requestId === relatedRequest.current) setRelated(result.related); + } catch { + if (requestId === relatedRequest.current) setRelated([]); + } + } + useEffect(() => { if (!focusPerson) return; const requestId = ++relatedRequest.current; @@ -597,6 +636,20 @@ function KeymanPanel({ }); }, [accessToken, focusEntity]); + useEffect(() => { + if (!focusTeam) return; + const requestId = ++relatedRequest.current; + setSelectedName(focusTeam.teamName); + setRelated(null); + fetchRelatedTeam(accessToken, focusTeam.teamId) + .then((result) => { + if (requestId === relatedRequest.current) setRelated(result.related); + }) + .catch(() => { + if (requestId === relatedRequest.current) setRelated([]); + }); + }, [accessToken, focusTeam]); + async function handleExtract() { setExtracting(true); setError(null); @@ -635,6 +688,9 @@ function KeymanPanel({ > {person.person_name} ({person.person_side_label ?? person.person_side_code}) + {person.last_known_job_title && ( + {person.last_known_job_title} + )} {person.affiliations.length > 0 && ( {" -- "} @@ -657,6 +713,9 @@ function KeymanPanel({ ) : ( affiliation.organization_name )} + {affiliation.role_title && ( + ({affiliation.role_title}) + )} ))} @@ -677,49 +736,68 @@ function KeymanPanel({ ) : (
    {related.map((node) => { - const caption = `${node.label ?? node.node_id} (${node.ontology_label ?? node.node_type_code})`; - if (node.node_type_code === NODE_POST && onSelectPost) { - return ( -
  • - -
  • - ); + const caption = relatedNodeCaption(node); + const key = `${node.node_type_code}:${node.node_id}`; + if (!isKnownRelatedNodeType(node.node_type_code)) { + return
  • {caption}
  • ; } - if (node.node_type_code === NODE_PERSON) { - return ( -
  • - -
  • - ); - } - if (node.node_type_code === NODE_CORPORATE_ENTITY) { - return ( -
  • - -
  • - ); + switch (node.node_type_code) { + case NODE_POST: + if (!onSelectPost) { + return
  • {caption}
  • ; + } + return ( +
  • + +
  • + ); + case NODE_PERSON: + return ( +
  • + +
  • + ); + case NODE_CORPORATE_ENTITY: + return ( +
  • + +
  • + ); + case NODE_TEAM: + return ( +
  • + +
  • + ); + default: { + const _exhaustive: never = node.node_type_code; + return
  • {_exhaustive}
  • ; + } } - return ( -
  • {caption}
  • - ); })}
)} @@ -1106,6 +1184,7 @@ function PostDetailPopup({ const [evaluation, setEvaluation] = useState(null); const [focusPerson, setFocusPerson] = useState<{ personId: string; personName: string } | null>(null); const [focusEntity, setFocusEntity] = useState<{ entityId: string; entityName: string } | null>(null); + const [focusTeam, setFocusTeam] = useState<{ teamId: string; teamName: string } | null>(null); function reloadKeymen() { fetchPostKeymen(accessToken, postId).then((r) => setKeymen(r.keymen)).catch(() => setKeymen([])); @@ -1134,6 +1213,7 @@ function PostDetailPopup({ setEvaluation(null); setFocusPerson(null); setFocusEntity(null); + setFocusTeam(null); fetchPost(accessToken, postId).then(setPost).catch((err) => setError(String(err))); fetchPostEvaluation(accessToken, postId) .then((r) => setEvaluation(r.responses)) @@ -1166,7 +1246,7 @@ function PostDetailPopup({ {post.visibility_label ?? post.visibility_code} ·{" "} {new Date(post.created_at).toLocaleString()}

-

{post.post_body}

+

요약 (Summary)

@@ -1188,25 +1268,73 @@ function PostDetailPopup({

R&R

    {summary.roles_and_responsibilities.map((rr, i) => { - const person = keymen?.find((row) => row.person_name === rr.person_name); + const isPerson = rr.actor_type_code === "prov_person"; + const actorTypeLabel = + rr.actor_type_code === "prov_team" + ? "Team" + : isPerson + ? "Person" + : "Organization"; + const person = isPerson + ? keymen?.find((row) => row.person_name === rr.actor_name) + : undefined; + const catalogId = rr.catalog_node_id; + const catalogType = rr.catalog_node_type_code; + let actorName: ReactNode = {rr.actor_name}; + if (person) { + actorName = ( + + ); + } else if (catalogType === NODE_TEAM && catalogId) { + actorName = ( + + ); + } else if (catalogType === NODE_CORPORATE_ENTITY && catalogId) { + actorName = ( + + ); + } return (
  • - {person ? ( - - ) : ( - {rr.person_name} + + {actorTypeLabel} + {" "} + {actorName} + {rr.affiliated_organization_name && ( + ({rr.affiliated_organization_name}) )} : {rr.responsibility}
  • @@ -1234,6 +1362,7 @@ function PostDetailPopup({ affiliateTrees={affiliateTrees} onSelectPerson={(personId, personName) => { setFocusEntity(null); + setFocusTeam(null); setFocusPerson({ personId, personName }); }} /> @@ -1262,10 +1391,12 @@ function PostDetailPopup({ node={node} onSelectPerson={(personId, personName) => { setFocusEntity(null); + setFocusTeam(null); setFocusPerson({ personId, personName }); }} onSelectEntity={(entityId, entityName) => { setFocusPerson(null); + setFocusTeam(null); setFocusEntity({ entityId, entityName }); }} /> @@ -1283,6 +1414,7 @@ function PostDetailPopup({ onSelectPost={onSelectPost} focusPerson={focusPerson} focusEntity={focusEntity} + focusTeam={focusTeam} /> {counterparties && counterparties.length > 0 && ( @@ -1294,6 +1426,7 @@ function PostDetailPopup({ onVerified={reloadCounterparties} onSelectEntity={(entityId, entityName) => { setFocusPerson(null); + setFocusTeam(null); setFocusEntity({ entityId, entityName }); }} /> @@ -1311,6 +1444,336 @@ function PostDetailPopup({ ); } +function analysisRunCaption(run: AnalysisRun): string { + return [run.run_kind_label, run.status_label, run.scope_entity_name ?? run.scope_kind_label] + .filter(Boolean) + .join(" · "); +} + +/** + * Next action for a pending or failed run on the home list and detail. + * + * The machine `failure_code` stays on detail history (ADR 0014). Copy + * is pinned to registered kinds so a pending TEPP row is not mistaken + * for reconstruction, and a failed lineage row is not mistaken for a + * missing TEPP transport. + */ +function analysisRunNextAction(run: AnalysisRun): string | null { + switch (run.status_code) { + case "analysis_status_pending": + switch (run.run_kind_code) { + case "analysis_run_lineage": + return "Open this run to confirm which posts it will use. Reconstruction has not started yet."; + case "analysis_run_tepp": + return "Open this run to confirm which posts TEPP will measure. Measurement has not started yet — this is not a calibrated result."; + case "analysis_run_report": + return "Open this run to confirm which posts the period report will use. The report has not been built yet."; + default: { + const unexpected: never = run.run_kind_code; + return unexpected; + } + } + case "analysis_status_failed": + switch (run.run_kind_code) { + case "analysis_run_tepp": + return "Open this run to see why it failed, then connect the measurement service and re-run."; + case "analysis_run_lineage": + return "Open this run to see why it failed, then retry reconstruction from a current snapshot."; + case "analysis_run_report": + return "Open this run to see why it failed, then rebuild the period report from a current snapshot."; + default: { + const unexpected: never = run.run_kind_code; + return unexpected; + } + } + case "analysis_status_running": + case "analysis_status_succeeded": + case "analysis_status_cancelled": + case null: + return null; + default: { + const unexpected: never = run.status_code; + return unexpected; + } + } +} + +/** + * Empty-corpus copy that tells the operator what to do next. + */ +function analysisRunEmptyPostsHint(run: AnalysisRun): string { + switch (run.run_kind_code) { + case "analysis_run_tepp": + return ( + "No posts were available at this cutoff for TEPP to measure. " + + "Open a later run, or ask an administrator to capture a newer snapshot." + ); + case "analysis_run_lineage": + return ( + "No posts were available at this cutoff for reconstruction. " + + "Open a later run, or ask an administrator to capture a newer snapshot." + ); + case "analysis_run_report": + return ( + "No posts were available at this cutoff for the period report. " + + "Open a later run, or ask an administrator to capture a newer snapshot." + ); + default: { + const unexpected: never = run.run_kind_code; + return unexpected; + } + } +} + +/** + * Corpus copy for a TEPP run that already has cutoff posts. + * + * Those titles are the measurement bag, not a reconstruction result. + * Pending or running must not claim a calibrated measurement. + */ +function analysisRunCorpusHint(run: AnalysisRun): string | null { + if (run.run_kind_code !== "analysis_run_tepp") return null; + switch (run.status_code) { + case "analysis_status_failed": + return ( + "These posts are the cutoff corpus TEPP would measure. Connect a TEPP " + + "transport, then re-run, to replace Failed with a calibrated result." + ); + case "analysis_status_succeeded": + return "These posts are the cutoff corpus this TEPP run measured."; + case "analysis_status_pending": + case "analysis_status_running": + return "These posts are the cutoff corpus TEPP will measure once this run finishes."; + case "analysis_status_cancelled": + return ( + "These posts are the cutoff corpus this TEPP run would have measured. " + + "The run was cancelled before a calibrated result." + ); + case null: + return "These posts are the cutoff corpus attached to this TEPP run."; + default: { + const unexpected: never = run.status_code; + return unexpected; + } + } +} + +/** Git-style prefix. The full digest stays on `title` for verification. */ +const ANALYSIS_RUN_DIGEST_PREFIX_LENGTH = 12; + +function analysisRunDigestPrefix(digest: string): string { + return digest.slice(0, ANALYSIS_RUN_DIGEST_PREFIX_LENGTH); +} + +/** + * Next action when a cutoff title opens the live post (ADR 0016). + * + * Post-body versioning is a later slice. Until then the operator must + * compare the opened body with this run's cutoff instead of treating + * today's text as reconstructed evidence. + */ +function analysisRunLivePostWarning(cutoffIso: string): string { + const cutoffDate = cutoffIso.slice(0, 10); + return ( + `Opening a title shows the live post. Compare it with cutoff ${cutoffDate} ` + + "before you treat the body as reconstructed evidence — it may have changed after this run." + ); +} + +function analysisRunLivePostButtonLabel(postTitle: string): string { + return `Open live post (may have changed after cutoff): ${postTitle}`; +} + +function AnalysisRunReproducibilityDigests({ + codeRevisionSha, + configurationSha256, +}: { + codeRevisionSha?: string; + configurationSha256?: string; +}) { + if (!codeRevisionSha && !configurationSha256) { + return null; + } + return ( +
    +

    + + Hover a prefix to read the full digest for verification.{" "} + + {codeRevisionSha ? ( + {`Code ${analysisRunDigestPrefix(codeRevisionSha)}`} + ) : null} + {codeRevisionSha && configurationSha256 ? " · " : null} + {configurationSha256 ? ( + + {`Config ${analysisRunDigestPrefix(configurationSha256)}`} + + ) : null} +

    +
    + ); +} + +function AnalysisRunsPanel({ + accessToken, + onSelectPost, +}: { + accessToken: string; + onSelectPost: (postId: string) => void; +}) { + const [runs, setRuns] = useState(null); + const [selected, setSelected] = useState(null); + const [error, setError] = useState(null); + const [requesting, setRequesting] = useState(false); + + useEffect(() => { + fetchAnalysisRuns(accessToken) + .then((payload) => setRuns(payload.analysis_runs)) + .catch((err) => setError(String(err))); + }, [accessToken]); + + async function handleRequestLineage() { + setError(null); + setRequesting(true); + try { + const created = await createAnalysisRun(accessToken, { + run_kind_code: "analysis_run_lineage", + idempotency_key: crypto.randomUUID(), + }); + const listed = await fetchAnalysisRuns(accessToken); + setRuns(listed.analysis_runs); + setSelected(created); + } catch (err) { + setError(err instanceof BackendError ? err.message : String(err)); + } finally { + setRequesting(false); + } + } + + async function handleOpen(runId: string) { + setError(null); + try { + setSelected(await fetchAnalysisRun(accessToken, runId)); + } catch (err) { + setSelected(null); + if (err instanceof BackendError && err.status === 404) { + setError("This analysis run is not visible."); + return; + } + setError(String(err)); + } + } + + if (error && runs === null) return

    {error}

    ; + if (runs === null) return

    Loading analysis runs...

    ; + + const corpusHint = selected ? analysisRunCorpusHint(selected) : null; + const selectedNextAction = selected ? analysisRunNextAction(selected) : null; + + return ( +
    +
    +

    Analysis runs

    + +
    + {error &&

    {error}

    } + {runs.length === 0 ? ( +

    + No analysis runs visible to this account yet. Request a lineage + reconstruction, or ask an administrator to run make seed. +

    + ) : ( +
      + {runs.map((run) => { + const documentCount = run.source_counts.find( + (count) => count.count_type_code === "analysis_count_document", + ); + const caption = analysisRunCaption(run); + const nextAction = analysisRunNextAction(run); + return ( +
    • + +
    • + ); + })} +
    + )} + {selected && ( +
    +

    {analysisRunCaption(selected)}

    + {selectedNextAction &&

    {selectedNextAction}

    } +

    + Cutoff {selected.knowledge_cutoff.slice(0, 10)} + {" · "} + Requested {selected.requested_at.slice(0, 10)} +

    + +
      + {selected.source_counts.map((count) => ( +
    • + {count.count_value} {count.count_type_label.toLowerCase()} +
    • + ))} +
    + {selected.status_history && selected.status_history.length > 0 && ( +
      + {selected.status_history.map((event) => ( +
    1. + {event.status_label} {event.occurred_at.slice(0, 16).replace("T", " ")} + {event.failure_code ? ` · ${event.failure_code}` : ""} +
    2. + ))} +
    + )} + {selected.visible_posts && selected.visible_posts.length > 0 ? ( + <> + {corpusHint &&

    {corpusHint}

    } +

    {analysisRunLivePostWarning(selected.knowledge_cutoff)}

    +
      + {selected.visible_posts.map((post) => ( +
    • + +
    • + ))} +
    + + ) : ( +

    {analysisRunEmptyPostsHint(selected)}

    + )} +
    + )} +
    + ); +} + function CalendarPanel({ accessToken, onSelectPost, @@ -1599,6 +2062,7 @@ function PostList({ accessToken }: { accessToken: string }) { return ( <> +
    diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx new file mode 100644 index 000000000..3c7781670 --- /dev/null +++ b/frontend/src/PostBody.test.tsx @@ -0,0 +1,18 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { PostBody } from "./PostBody"; +import { IMAGE_NOT_READ_HERE, UNDECODEABLE_IMAGE } from "./postBodyDisplay"; + +const TINY_PNG_B64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +describe("PostBody", () => { + it("replaces a picture the browser cannot paint with the re-export next action", () => { + render(`} />); + const image = screen.getByRole("img", { name: /embedded image at character offset/i }); + expect(screen.getByText(IMAGE_NOT_READ_HERE)).toBeInTheDocument(); + fireEvent.error(image); + expect(screen.getByText(UNDECODEABLE_IMAGE)).toBeInTheDocument(); + expect(screen.queryByRole("img")).not.toBeInTheDocument(); + }); +}); diff --git a/frontend/src/PostBody.tsx b/frontend/src/PostBody.tsx new file mode 100644 index 000000000..9b87512c3 --- /dev/null +++ b/frontend/src/PostBody.tsx @@ -0,0 +1,51 @@ +import { useState } from "react"; +import { + IMAGE_NOT_READ_HERE, + UNDECODEABLE_IMAGE, + splitPostBody, + type PostBodySegment, +} from "./postBodyDisplay"; + +function EmbeddedPostImage({ src, position }: { src: string; position: number }) { + const [failed, setFailed] = useState(false); + if (failed) { + return

    {UNDECODEABLE_IMAGE}

    ; + } + return ( +
    + {`Embedded setFailed(true)} + /> +
    {IMAGE_NOT_READ_HERE}
    +
    + ); +} + +function renderSegment(segment: PostBodySegment, index: number) { + switch (segment.kind) { + case "text": + return ( +

    + {segment.text} +

    + ); + case "image": + return ( + + ); + default: { + const _exhaustive: never = segment; + throw new Error(`unexpected post body segment: ${JSON.stringify(_exhaustive)}`); + } + } +} + +export function PostBody({ body }: { body: string }) { + return
    {splitPostBody(body).map(renderSegment)}
    ; +} diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 2b6692776..3385d5179 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -26,6 +26,7 @@ export interface Keyman { person_side_code: string; person_side_label?: string; mention_context: string | null; + last_known_job_title: string | null; affiliations: Affiliation[]; } @@ -72,19 +73,30 @@ export interface VocEvidence { counterparties: VocEvidenceCounterparty[]; } +export type RelatedNodeType = + | "node_person" + | "node_post" + | "node_corporate_entity" + | "node_team"; + export interface RelatedNode { node_id: string; - node_type_code: string; + node_type_code: RelatedNodeType | string; relevance: number; label?: string; person_side_code?: string; + person_side_label?: string; ontology_iri?: string; ontology_label?: string; } export interface PostRoleResponsibility { - person_name: string; + actor_name: string; responsibility: string; + actor_type_code: string; + affiliated_organization_name: string | null; + catalog_node_id?: string | null; + catalog_node_type_code?: string | null; } export interface PostAiSummary { @@ -280,6 +292,13 @@ export function fetchRelatedEntity( return backendFetch(`/api/corporate-entities/${entityId}/related`, accessToken); } +export function fetchRelatedTeam( + accessToken: string, + teamId: string, +): Promise<{ team_id: string; team_name: string; related: RelatedNode[] }> { + return backendFetch(`/api/teams/${teamId}/related`, accessToken); +} + export function extractPostKeymen( accessToken: string, postId: string, @@ -488,3 +507,75 @@ export function deriveCommitment(accessToken: string, postId: string): Promise { return backendFetch("/api/calendar", accessToken); } + +export interface AnalysisRunCount { + count_type_code: string; + count_type_label: string; + count_value: number; +} + +/** Registry kinds from `analysis_run.run_kind_code` (migration 0018). */ +export type AnalysisRunKindCode = + | "analysis_run_lineage" + | "analysis_run_report" + | "analysis_run_tepp"; + +/** Registry statuses from `analysis_run_status_event.status_code`. */ +export type AnalysisRunStatusCode = + | "analysis_status_pending" + | "analysis_status_running" + | "analysis_status_succeeded" + | "analysis_status_failed" + | "analysis_status_cancelled"; + +export interface AnalysisRunStatusEvent { + status_ordinal: number; + status_code: AnalysisRunStatusCode; + status_label: string; + occurred_at: string; + failure_code?: string; +} + +export interface AnalysisRun { + analysis_run_id: string; + run_kind_code: AnalysisRunKindCode; + run_kind_label: string; + scope_kind_code: string; + scope_kind_label: string; + scope_entity_name?: string; + status_code: AnalysisRunStatusCode | null; + status_label: string | null; + knowledge_cutoff: string; + requested_at: string; + source_counts: AnalysisRunCount[]; + status_history?: AnalysisRunStatusEvent[]; + visible_posts?: { post_id: string; post_title: string }[]; + code_revision_sha?: string; + configuration_sha256?: string; +} + +export function fetchAnalysisRuns(accessToken: string): Promise<{ analysis_runs: AnalysisRun[] }> { + return backendFetch("/api/analysis-runs", accessToken); +} + +export function fetchAnalysisRun(accessToken: string, analysisRunId: string): Promise { + return backendFetch(`/api/analysis-runs/${analysisRunId}`, accessToken); +} + +export interface CreateAnalysisRunRequest { + run_kind_code?: string; + scope_kind_code?: string; + corporate_entity_id?: string; + knowledge_cutoff?: string; + idempotency_key: string; +} + +export function createAnalysisRun( + accessToken: string, + request: CreateAnalysisRunRequest, +): Promise { + return backendFetch("/api/analysis-runs", accessToken, { + method: "POST", + body: JSON.stringify(request), + }); +} diff --git a/frontend/src/index.css b/frontend/src/index.css index 5fb331302..53f4db2ac 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -8,6 +8,11 @@ --accent-bg: rgba(170, 59, 255, 0.1); --accent-border: rgba(170, 59, 255, 0.5); --social-bg: rgba(244, 243, 236, 0.5); + --post-body-gap: 0.75rem; + --post-image-padding: 0.75rem; + --post-image-radius: 8px; + --post-image-border: var(--border); + --post-image-bg: var(--code-bg); --shadow: rgba(0, 0, 0, 0.1) 0 10px 15px -3px, rgba(0, 0, 0, 0.05) 0 4px 6px -2px; diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts new file mode 100644 index 000000000..c3cd32730 --- /dev/null +++ b/frontend/src/postBodyDisplay.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "vitest"; +import { + REMOTE_IMAGE_SKIPPED, + UNDECODEABLE_IMAGE, + splitPostBody, +} from "./postBodyDisplay"; + +/** 1x1 transparent PNG — the same synthetic fixture the Python vision tests use. */ +const TINY_PNG_B64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +describe("splitPostBody", () => { + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { + expect(splitPostBody("The full body text.")).toEqual([ + { kind: "text", text: "The full body text." }, + ]); + }); + + it("keeps comparison operators that look like broken HTML", () => { + expect(splitPostBody("qty < 50 and price > 10")).toEqual([ + { kind: "text", text: "qty < 50 and price > 10" }, + ]); + }); + + it("renders a data-URI image as its own segment and never leaks the raw base64 into text", () => { + const html = + `

    Quote attached.

    Please confirm.

    `; + const segments = splitPostBody(html); + + expect(segments).toEqual([ + { kind: "text", text: "Quote attached." }, + { + kind: "image", + src: `data:image/png;base64,${TINY_PNG_B64}`, + mimeType: "image/png", + position: html.indexOf(" { + const html = + `

    between

    ` + + ``; + const segments = splitPostBody(html); + expect(segments.map((segment) => segment.kind)).toEqual(["image", "text", "image"]); + expect(segments[1]).toEqual({ kind: "text", text: "between" }); + expect(segments[0]?.kind === "image" && segments[0].position).toBe(0); + expect(segments[2]?.kind === "image" && segments[2].position).toBeGreaterThan(0); + }); + + it("accepts charset parameters and unquoted or single-quoted src", () => { + const charset = ``; + const unquoted = ``; + const single = ``; + + for (const html of [charset, unquoted, single]) { + const segments = splitPostBody(html); + expect(segments).toEqual([ + { + kind: "image", + src: `data:image/png;base64,${TINY_PNG_B64}`, + mimeType: "image/png", + position: 0, + }, + ]); + } + }); + + it("tells the operator to re-export when the base64 payload is not decodable", () => { + const html = ''; + expect(splitPostBody(html)).toEqual([ + { + kind: "text", + text: UNDECODEABLE_IMAGE, + }, + ]); + }); + + it("does not treat valid-base64 non-image bytes as a picture", () => { + expect(splitPostBody('')).toEqual([ + { kind: "text", text: UNDECODEABLE_IMAGE }, + ]); + }); + + it("does not re-dump an invalid alphabet payload", () => { + const html = ''; + expect(splitPostBody(html)).toEqual([{ kind: "text", text: UNDECODEABLE_IMAGE }]); + expect(JSON.stringify(splitPostBody(html))).not.toContain("not-valid-base64"); + }); + + it("does not turn a remote http img into a loaded image", () => { + const html = '

    See

    end

    '; + const segments = splitPostBody(html); + expect(segments.every((segment) => segment.kind === "text")).toBe(true); + expect(segments.map((segment) => (segment.kind === "text" ? segment.text : "")).join(" ")).toContain( + "See", + ); + expect(JSON.stringify(segments)).not.toContain("https://example.test"); + }); + + it("tells the operator to re-export a remote-only image instead of leaking the URL", () => { + const html = ''; + expect(splitPostBody(html)).toEqual([{ kind: "text", text: REMOTE_IMAGE_SKIPPED }]); + expect(JSON.stringify(splitPostBody(html))).not.toContain("https://example.test"); + }); +}); diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts new file mode 100644 index 000000000..d91674965 --- /dev/null +++ b/frontend/src/postBodyDisplay.ts @@ -0,0 +1,157 @@ +/** + * Split a raw `post_body` into text and in-place data-URI images. + * + * The popup used to dump the source string, so a buyer who opened a post + * with an embedded invoice saw a base64 wall instead of the picture. + * Only `data:image/...;base64,...` payloads are turned into images — + * remote `http(s)` img tags are stripped, never fetched. A tag-only body + * never falls back to the raw source string. + */ + +export type PostBodySegment = + | { kind: "text"; text: string } + | { kind: "image"; src: string; mimeType: string; position: number }; + +export const UNDECODEABLE_IMAGE = + "Embedded image could not be decoded. Re-export the source post and open it again."; + +export const REMOTE_IMAGE_SKIPPED = + "This post linked a remote image that was not loaded. Re-export the source with the picture embedded and open it again."; + +export const IMAGE_NOT_READ_HERE = + "Image from this post. Text inside the picture is not read on this screen."; + +const IMG_TAG = /]*>/gi; + +const SRC_ATTR = /\bsrc\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i; + +const DATA_URI = + /^data:(image\/[a-zA-Z0-9.+-]+)(?:;[\w.+-]+=[^;,]*)*;base64,([A-Za-z0-9+/=\s]+)$/i; + +const HTML_TAG = /<\/?[a-zA-Z][^>]*>/g; + +const REMOTE_SRC = /^https?:\/\//i; + +function stripHtmlTags(text: string): string { + return text.replace(HTML_TAG, " ").replace(/\s+/g, " ").trim(); +} + +function decodeBase64(raw: string): Uint8Array | null { + if (raw.length === 0 || raw.length % 4 !== 0 || !/^[A-Za-z0-9+/]+=*$/.test(raw)) { + return null; + } + try { + const binary = atob(raw); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) { + bytes[index] = binary.charCodeAt(index); + } + return bytes; + } catch { + return null; + } +} + +function looksLikeImage(mimeType: string, bytes: Uint8Array): boolean { + const mime = mimeType.toLowerCase(); + if (mime === "image/png") { + return ( + bytes.length >= 8 && + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47 + ); + } + if (mime === "image/jpeg" || mime === "image/jpg") { + return bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff; + } + if (mime === "image/gif") { + return bytes.length >= 6 && bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46; + } + if (mime === "image/webp") { + return ( + bytes.length >= 12 && + bytes[0] === 0x52 && + bytes[1] === 0x49 && + bytes[2] === 0x46 && + bytes[3] === 0x46 && + bytes[8] === 0x57 && + bytes[9] === 0x45 && + bytes[10] === 0x42 && + bytes[11] === 0x50 + ); + } + if (mime === "image/svg+xml") { + const text = new TextDecoder().decode(bytes).trimStart().toLowerCase(); + return text.startsWith(" 0; +} + +function pushText(segments: PostBodySegment[], raw: string): void { + const text = stripHtmlTags(raw); + if (text) { + segments.push({ kind: "text", text }); + } +} + +function srcFromImgTag(tag: string): string | null { + const match = SRC_ATTR.exec(tag); + if (!match) { + return null; + } + return match[1] ?? match[2] ?? match[3] ?? null; +} + +export function splitPostBody(body: string): PostBodySegment[] { + const segments: PostBodySegment[] = []; + const pattern = new RegExp(IMG_TAG.source, "gi"); + let lastIndex = 0; + let match = pattern.exec(body); + let sawRemoteImage = false; + let sawUndecodableImage = false; + + while (match !== null) { + pushText(segments, body.slice(lastIndex, match.index)); + const src = srcFromImgTag(match[0]); + if (src && REMOTE_SRC.test(src)) { + sawRemoteImage = true; + } else if (src) { + const data = DATA_URI.exec(src.trim()); + if (data) { + const mimeType = data[1]; + const rawB64 = data[2].replace(/\s+/g, ""); + const bytes = decodeBase64(rawB64); + if (bytes && looksLikeImage(mimeType, bytes)) { + segments.push({ + kind: "image", + src: `data:${mimeType};base64,${rawB64}`, + mimeType, + position: match.index, + }); + } else { + sawUndecodableImage = true; + segments.push({ kind: "text", text: UNDECODEABLE_IMAGE }); + } + } else if (/^data:image\//i.test(src)) { + sawUndecodableImage = true; + segments.push({ kind: "text", text: UNDECODEABLE_IMAGE }); + } + } + lastIndex = match.index + match[0].length; + match = pattern.exec(body); + } + pushText(segments, body.slice(lastIndex)); + + if (segments.length > 0) { + return segments; + } + if (sawRemoteImage) { + return [{ kind: "text", text: REMOTE_IMAGE_SKIPPED }]; + } + if (sawUndecodableImage) { + return [{ kind: "text", text: UNDECODEABLE_IMAGE }]; + } + return [{ kind: "text", text: body }]; +} diff --git a/lineageweave/__init__.py b/lineageweave/__init__.py index 0ac8e50fe..b84afb519 100644 --- a/lineageweave/__init__.py +++ b/lineageweave/__init__.py @@ -15,6 +15,17 @@ from .models import Edge, Record, Tree from .post_chat import ChatAnswer, cited_post_summaries from .post_summary import PostSummary +from .prov_o import ( + PROV, + PROV_CLASSES, + PROV_QUALIFICATIONS, + PROV_RELATIONS, + PROV_RECOMMENDED_INVERSES, + ProvAssertion, + ProvGraph, + ProvLiteral, + ProvValidationError, +) from .reconstruct import reconstruct from .voc_evidence import sentence_excerpts @@ -22,7 +33,16 @@ "ChatAnswer", "Edge", "OrganizationRelationship", + "PROV", + "PROV_CLASSES", + "PROV_QUALIFICATIONS", + "PROV_RELATIONS", + "PROV_RECOMMENDED_INVERSES", "PostSummary", + "ProvAssertion", + "ProvGraph", + "ProvLiteral", + "ProvValidationError", "Record", "Tree", "build_affiliate_forest", @@ -35,4 +55,4 @@ "sentence_excerpts", ] -__version__ = "0.71.0" +__version__ = "0.86.3" diff --git a/lineageweave/corporate_hierarchy_inference.py b/lineageweave/corporate_hierarchy_inference.py new file mode 100644 index 000000000..174ef4503 --- /dev/null +++ b/lineageweave/corporate_hierarchy_inference.py @@ -0,0 +1,173 @@ +"""Infers where a newly-mentioned organization sits in a Group -> Company +-> Plant style hierarchy (e.g. "Acme Electronics South Plant" -> parent "Acme Electronics +한국" -> parent "Acme Group") when it does not already match an existing +``corporate_entity`` row -- the standing "통합 고객사 계열 tree AI" +(integrated customer affiliate tree) requirement this product has +always named, closing the gap that +:mod:`lineageweave.corporate_hierarchy_resolution`'s similarity +matching leaves open: matching only ever finds an ALREADY-cataloged +entity, it never creates one, so a unseen dataset's first mention of any +new counterparty organization stays permanently unresolved. + +Grounded in the same collective-entity-resolution framing +(Bhattacharya & Getoor, 2007) already cited for +``corporate_hierarchy_resolution`` -- this module is the natural +extension of that same resolution pipeline to entity *creation* when no +existing candidate matches, not a separate technique. The hierarchy +itself is the same SKOS ``skos:broader``/``skos:narrower`` structure +``corporate_entity_level`` (ADR 0004) already uses on top of the +``parent_entity_id`` self-reference. + +Same pluggable-client, never-fake-a-missing-channel, never-trust-an- +unverified-guess discipline as every other channel in this package: a +proposed new entity is only ever created after +:mod:`lineageweave.relation_verification`'s external-search +corroboration, the same reused verification client +:mod:`lineageweave.organization_name_resolution` already established +this pattern for. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from functools import lru_cache +from typing import Protocol + +from .http_client import post_json + +LEVEL_GROUP = "group" +LEVEL_COMPANY = "company" +LEVEL_PLANT = "plant" +_VALID_LEVEL_CODES = frozenset({LEVEL_GROUP, LEVEL_COMPANY, LEVEL_PLANT}) + +@lru_cache(maxsize=1) +def required_corporate_level_codes() -> frozenset[str]: + """Return the level codes every migrated database registers.""" + return _VALID_LEVEL_CODES + + +@dataclass(frozen=True) +class HierarchyProposal: + """One organization's proposed place in the hierarchy. + + Attributes: + level_code: ``corporate_entity_level`` lookup code -- one of + ``group`` / ``company`` / ``plant``. + parent_name: the immediate parent organization's name the text + supports, or ``None`` when this organization has no parent + in the hierarchy the text gives evidence for (a standalone + group-level entity, or the text simply does not say). + """ + + level_code: str + parent_name: str | None + + +class CorporateHierarchyInferenceClient(Protocol): + """Proposes a hierarchy placement for a newly-seen organization name.""" + + available: bool + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal | None: + """Return a proposed placement, or ``None`` when the model + cannot determine one from the given context with real + confidence. + + Implementations must raise if the call itself fails -- a failed + call is not the same outcome as "the model looked and proposed + nothing." Protocol stubs raise ``NotImplementedError`` so a + no-op body is never treated as a successful empty result. + """ + raise NotImplementedError + + +class NullCorporateHierarchyInferenceClient: + """No LLM orchestrator configured -- hierarchy inference is unavailable.""" + + available = False + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal | None: + raise RuntimeError( + "NullCorporateHierarchyInferenceClient cannot infer; check .available first" + ) + + +_INFERENCE_PROMPT_TEMPLATE = """\ +The text below names an organization, "{organization_name}", that is +not yet in our corporate hierarchy catalog. Using ONLY what the text +itself supports (never invent a hierarchy the text gives no evidence +for), determine: + +1. Its level: exactly one of "group" (a top-level conglomerate/group + with no parent), "company" (a company, possibly part of a group), + or "plant" (a specific plant/site/branch/subsidiary of a company). +2. Its immediate parent organization's name, if the text names or + clearly implies one (e.g. "Acme Electronics South Plant" implies its parent is + "Acme Electronics"). Use null when the text gives no parent to infer, or + when this organization is itself a top-level group. + +Reply with ONLY a JSON object (no markdown fences, no prose): + "level": exactly "group", "company", or "plant" + "parent_name": string, or null + +If you cannot determine even the level with real confidence from the +text, reply with exactly: UNKNOWN + +Text: {context} +""" + + +def parse_inference_response(content: str) -> HierarchyProposal | None: + """Parses the LLM's JSON reply into a `HierarchyProposal`. + + Returns `None` for `UNKNOWN`, malformed JSON, or a level outside the + three valid codes -- a model that did not follow the contract gets + treated as "no proposal," never a guessed default. + """ + stripped = content.strip() + if not stripped or stripped.upper() == "UNKNOWN": + return None + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + return None + if not isinstance(parsed, dict): + return None + level = parsed.get("level") + if level not in _VALID_LEVEL_CODES: + return None + parent_raw = parsed.get("parent_name") + parent_name = parent_raw.strip() if isinstance(parent_raw, str) and parent_raw.strip() else None + return HierarchyProposal(level_code=level, parent_name=parent_name) + + +class ContextualOrchestratorHierarchyInferenceClient: + """Calls ``POST {base_url}/v1/chat/completions`` with ``mode="route"``.""" + + available = True + + def __init__( + self, base_url: str, api_key: str, *, reasoning_effort: str = "medium", timeout: float = 30.0 + ) -> None: + self._base_url = base_url.rstrip("/") + self._api_key = api_key + self._reasoning_effort = reasoning_effort + self._timeout = timeout + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal | None: + prompt = _INFERENCE_PROMPT_TEMPLATE.format( + organization_name=organization_name, context=context_text + ) + body = post_json( + f"{self._base_url}/v1/chat/completions", + { + "messages": [{"role": "user", "content": prompt}], + "mode": "route", + "reasoning_effort": self._reasoning_effort, + }, + headers={"authorization": f"Bearer {self._api_key}"}, + timeout=self._timeout, + ) + content = body["choices"][0]["message"]["content"] + return parse_inference_response(content) diff --git a/lineageweave/image_content.py b/lineageweave/image_content.py index 2f6839fcd..e1e0808cd 100644 --- a/lineageweave/image_content.py +++ b/lineageweave/image_content.py @@ -33,8 +33,13 @@ from .http_client import post_json -_DATA_URI_IMG = re.compile( - r']*\bsrc\s*=\s*["\']data:(image/[a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=\s]+)["\']', +_IMG_TAG = re.compile(r"]*>", re.IGNORECASE) +_SRC_ATTR = re.compile( + r"""\bsrc\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))""", + re.IGNORECASE, +) +_DATA_URI = re.compile( + r"^data:(image/[a-zA-Z0-9.+-]+)(?:;[\w.+-]+=[^;,]*)*;base64,([A-Za-z0-9+/=\s]+)$", re.IGNORECASE, ) @@ -63,14 +68,25 @@ class EmbeddedImage: def extract_base64_images(html: str) -> list[EmbeddedImage]: """Find every ```` in document order. - Malformed base64 in a matched tag is skipped rather than raising -- - one corrupt embedded image must not fail extraction of the rest of the - document. + Accepts quoted or unquoted ``src`` and optional data-URI parameters + such as ``charset=utf-8`` so the same picture the popup renders is + also available to the vision channel. Malformed base64 in a matched + tag is skipped rather than raising -- one corrupt embedded image must + not fail extraction of the rest of the document. """ images: list[EmbeddedImage] = [] - for match in _DATA_URI_IMG.finditer(html): - mime_type = match.group(1) - raw_b64 = re.sub(r"\s+", "", match.group(2)) + for match in _IMG_TAG.finditer(html): + src_match = _SRC_ATTR.search(match.group(0)) + if src_match is None: + continue + src = next((group for group in src_match.groups() if group), None) + if src is None: + continue + data_match = _DATA_URI.match(src.strip()) + if data_match is None: + continue + mime_type = data_match.group(1) + raw_b64 = re.sub(r"\s+", "", data_match.group(2)) try: data = base64.b64decode(raw_b64, validate=True) except (binascii.Error, ValueError): @@ -126,35 +142,79 @@ def describe(self, image_bytes: bytes, mime_type: str) -> ImageDescription: # p "CAPTION: \n" "TAGS: " ) -# DOTALL + non-greedy so TEXT: can legitimately span multiple lines (real -# OCR output is often multi-line) without losing everything after the -# first newline, while still stopping at the next expected label. -_DESCRIPTION_PATTERN = re.compile( - r"TEXT:\s*(?P.*?)\s*CAPTION:\s*(?P.*?)\s*TAGS:\s*(?P.*)", - re.DOTALL, +# TEXT may legitimately span multiple lines because OCR output is often +# multi-line. CAPTION and TAGS are explicitly single-line fields. Synthetic +# format-variation fixtures cover common provider drift such as bolded or +# reordered labels without allowing trailing commentary to contaminate the +# searchable caption or tag values. +_LABEL_LINE = re.compile( + r"^\s*(?:[*_`>#\-]\s*)*(TEXT|CAPTION|TAGS)(?:\s*[*_`]+)?\s*:\s*" + r"(?:(?:[*_`]+)(?=\s|$)\s*)?(.*)$", + re.IGNORECASE, ) +_MARKDOWN_EMPHASIS_MARKERS = ("**", "__", "`", "*", "_") class ImageDescriptionParseError(ValueError): - """The vision provider's response didn't match the required - TEXT/CAPTION/TAGS format -- raised instead of silently returning an - empty ImageDescription, so a provider response-format change is - surfaced immediately rather than quietly losing searchable content. + """Neither TEXT nor CAPTION could be found in the vision provider's + response -- raised instead of silently returning an empty + ImageDescription, so a provider response genuinely unusable end to + end is surfaced, not confused with "described nothing." """ +def _strip_outer_markdown_emphasis(value: str) -> str: + """Remove balanced outer Markdown emphasis without changing inner text.""" + cleaned = value.strip() + changed = True + while changed: + changed = False + for marker in _MARKDOWN_EMPHASIS_MARKERS: + if ( + cleaned.startswith(marker) + and cleaned.endswith(marker) + and len(cleaned) > 2 * len(marker) + ): + cleaned = cleaned[len(marker) : -len(marker)].strip() + changed = True + break + return cleaned + + def _parse_description(content: str) -> ImageDescription: - match = _DESCRIPTION_PATTERN.search(content) - if match is None: + fields: dict[str, list[str]] = {"TEXT": [], "CAPTION": [], "TAGS": []} + multiline_field: str | None = None + for line in content.splitlines(): + match = _LABEL_LINE.match(line) + if match: + label = match.group(1).upper() + remainder = _strip_outer_markdown_emphasis(match.group(2)) + if remainder: + fields[label].append(remainder) + multiline_field = "TEXT" if label == "TEXT" else None + continue + + if re.match(r"^\s*[*_`>#\-\s]*[A-Za-z][A-Za-z0-9 _-]*\s*:", line): + multiline_field = None + continue + if multiline_field == "TEXT" and line.strip(): + fields["TEXT"].append(_strip_outer_markdown_emphasis(line)) + + if not fields["TEXT"] and not fields["CAPTION"]: raise ImageDescriptionParseError( - f"vision response did not match the required TEXT/CAPTION/TAGS format: {content!r}" + f"vision response had neither TEXT nor CAPTION content: {content!r}" ) - extracted_text = match.group("text").strip() + + extracted_text = "\n".join(fields["TEXT"]).strip() if extracted_text.upper() == "NONE": extracted_text = "" - caption = match.group("caption").strip() - tags_raw = match.group("tags").strip() - tags = tuple(tag.strip() for tag in tags_raw.split(",") if tag.strip()) + caption = "\n".join(fields["CAPTION"]).strip() + tags_raw = " ".join(fields["TAGS"]).strip() + tags = tuple( + cleaned + for tag in tags_raw.split(",") + if (cleaned := _strip_outer_markdown_emphasis(tag)) + ) return ImageDescription(extracted_text=extracted_text, caption=caption, tags=tags) diff --git a/lineageweave/keyman_extraction.py b/lineageweave/keyman_extraction.py index 5775c0f20..7c3d79397 100644 --- a/lineageweave/keyman_extraction.py +++ b/lineageweave/keyman_extraction.py @@ -35,14 +35,26 @@ class PersonMention: """One person the extractor found in a post's text. - ``affiliated_organization_names`` may be empty (mentioned without a - stated affiliation) or contain more than one name (the N:N case the - product requirement describes). + Attributes: + affiliated_organization_names: may be empty (mentioned without a + stated affiliation) or contain more than one name (the N:N + case the product requirement describes). + job_title: the person's title/position as the text states it + (e.g. "영업팀장," "구매담당"), or ``None`` when the text does + not say. Two different real people can share a name -- a + name alone is not a reliable identity key, and dropping a + stated title would throw away the one signal the text + offers to tell them apart. Persisted onto + ``person_affiliation.role_title`` (a schema column that + already existed, previously never populated) and used by + ``_upsert_person`` as a same-name disambiguation signal: + see ``backend/app/keyman_ingestion.py``. """ person_name: str person_side_code: str affiliated_organization_names: tuple[str, ...] = field(default_factory=tuple) + job_title: str | None = None class KeymanExtractionClient(Protocol): @@ -71,9 +83,13 @@ def extract(self, post_title: str, post_body: str) -> list[PersonMention]: _EXTRACTION_PROMPT_TEMPLATE = """\ Read the post below and list every named person it mentions. For each -person, classify which side they are on and list every organization they +person, classify which side they are on, list every organization they are affiliated with according to the text (a person may belong to more -than one organization, or none if the text does not say). +than one organization, or none if the text does not say), and give their +job title or position if the text states one. Two different real people +can share the same name -- a stated title/position (e.g. "sales +manager," "purchasing lead") is real evidence for telling them apart, so +report it whenever the text gives one rather than leaving it out. Reply with ONLY a JSON array (no markdown fences, no prose), where each element has exactly these fields: @@ -82,6 +98,8 @@ def extract(self, post_title: str, post_body: str) -> list[PersonMention]: "counterparty" (an external customer, partner, competitor, or other outside organization) "affiliations": a JSON array of organization name strings (can be empty) + "job_title": the person's stated title/position as a string, or null + when the text does not give one If no people are named, reply with an empty JSON array: [] @@ -126,8 +144,15 @@ def parse_keyman_response(content: str) -> list[PersonMention]: if not isinstance(affiliations_raw, list): affiliations_raw = [] affiliations = tuple(a.strip() for a in affiliations_raw if isinstance(a, str) and a.strip()) + job_title_raw = entry.get("job_title") + job_title = job_title_raw.strip() if isinstance(job_title_raw, str) and job_title_raw.strip() else None mentions.append( - PersonMention(person_name=name.strip(), person_side_code=side, affiliated_organization_names=affiliations) + PersonMention( + person_name=name.strip(), + person_side_code=side, + affiliated_organization_names=affiliations, + job_title=job_title, + ) ) return mentions diff --git a/lineageweave/knowledge_graph.py b/lineageweave/knowledge_graph.py index cf750e978..6af4fb4a2 100644 --- a/lineageweave/knowledge_graph.py +++ b/lineageweave/knowledge_graph.py @@ -129,9 +129,19 @@ def select_related_nodes( NODE_PERSON = "node_person" NODE_CORPORATE_ENTITY = "node_corporate_entity" NODE_POST = "node_post" +NODE_TEAM = "node_team" EDGE_MENTION = "edge_mention" EDGE_AFFILIATION = "edge_affiliation" EDGE_CO_MENTION = "edge_co_mention" +# ADR 0009: cross-post identity resolution for R&R team/organization +# actors -- a team is meso-level (ADR 0007), so it gets its own mention +# edge distinct from a person's, plus its own affiliation edge to the +# company it belongs to (parallel to edge_affiliation for persons, kept +# distinct rather than reused so an edge_type_code alone always tells +# you which node types it connects, without inspecting the row). +EDGE_MENTION_TEAM = "edge_mention_team" +EDGE_TEAM_AFFILIATION = "edge_team_affiliation" +EDGE_MENTION_ORGANIZATION = "edge_mention_organization" @dataclass(frozen=True) @@ -166,23 +176,36 @@ def knowledge_graph_edges_for_post( post_id: str, person_ids: Sequence[str], person_corporate_entity_ids: Sequence[tuple[str, str]] = (), + team_ids: Sequence[str] = (), + team_corporate_entity_ids: Sequence[tuple[str, str]] = (), + organization_corporate_entity_ids: Sequence[str] = (), ) -> list[KnowledgeGraphEdgeSpec]: - """Populate the three Phase 2 edge kinds for one post. + """Populate this post's Phase 2 + ADR 0009 edge kinds. - person <-> post (``edge_mention``) for every mentioned person - person <-> corporate_entity (``edge_affiliation``) for every affiliation that resolved to a real ``corporate_entity`` row - person <-> person (``edge_co_mention``) for every unordered pair of people named in the same post - - Affiliation names that did not resolve to a ``corporate_entity`` are - stored on ``person_affiliation`` but do not become graph edges -- a - free-text org with no node id cannot be a knowledge_graph_edge - endpoint. Directed storage is canonical (person -> post/org, and - lexicographic person-id order for co-mentions); loaders treat the - graph as undirected. + - team <-> post (``edge_mention_team``) for every mentioned, + cataloged team (ADR 0009 -- cross-post team identity) + - team <-> corporate_entity (``edge_team_affiliation``) for every + team whose parent organization resolved to a real + ``corporate_entity`` row + - corporate_entity <-> post (``edge_mention_organization``) for + every R&R organization actor that resolved to a real + ``corporate_entity`` row (ADR 0009) + + Affiliation/organization names that did not resolve to a + ``corporate_entity`` are stored on the relevant table but do not + become graph edges -- a free-text org with no node id cannot be a + knowledge_graph_edge endpoint. Directed storage is canonical + (person/team/org -> post/org, and lexicographic person-id order for + co-mentions); loaders treat the graph as undirected. """ unique_person_ids = list(dict.fromkeys(person_ids)) + unique_team_ids = list(dict.fromkeys(team_ids)) + unique_organization_ids = list(dict.fromkeys(organization_corporate_entity_ids)) edges: list[KnowledgeGraphEdgeSpec] = [] for person_id in unique_person_ids: @@ -224,6 +247,44 @@ def knowledge_graph_edges_for_post( ) ) + for team_id in unique_team_ids: + edges.append( + KnowledgeGraphEdgeSpec( + source_node_type_code=NODE_TEAM, + source_node_id=team_id, + target_node_type_code=NODE_POST, + target_node_id=post_id, + edge_type_code=EDGE_MENTION_TEAM, + ) + ) + + seen_team_affiliations: set[tuple[str, str]] = set() + for team_id, corporate_entity_id in team_corporate_entity_ids: + pair = (team_id, corporate_entity_id) + if pair in seen_team_affiliations: + continue + seen_team_affiliations.add(pair) + edges.append( + KnowledgeGraphEdgeSpec( + source_node_type_code=NODE_TEAM, + source_node_id=team_id, + target_node_type_code=NODE_CORPORATE_ENTITY, + target_node_id=corporate_entity_id, + edge_type_code=EDGE_TEAM_AFFILIATION, + ) + ) + + for corporate_entity_id in unique_organization_ids: + edges.append( + KnowledgeGraphEdgeSpec( + source_node_type_code=NODE_CORPORATE_ENTITY, + source_node_id=corporate_entity_id, + target_node_type_code=NODE_POST, + target_node_id=post_id, + edge_type_code=EDGE_MENTION_ORGANIZATION, + ) + ) + return edges diff --git a/lineageweave/organization_name_resolution.py b/lineageweave/organization_name_resolution.py new file mode 100644 index 000000000..69b64d156 --- /dev/null +++ b/lineageweave/organization_name_resolution.py @@ -0,0 +1,196 @@ +"""Resolves an abbreviated or slang organization name (e.g. "AGP") to +its full canonical name using LLM context, then cross-verifies the +proposed name against external web search before it is trusted -- +:mod:`lineageweave.corporate_hierarchy_resolution`'s character-similarity +matching cannot bridge this gap on its own: an initialism/acronym shares +almost no substring with its expansion, so no similarity threshold +recovers it. This module runs first, so its output feeds +``resolve_corporate_entity`` a name with a real chance of matching, not +instead of it. + +Grounded in SKOS (Miles & Bechhofer, 2009): ``skos:prefLabel`` (a +resource's single preferred/canonical label) and ``skos:altLabel`` (an +alternative label -- exactly the abbreviation/synonym case) are the +standard vocabulary for this raw-name/canonical-name pair. See +docs/adr/0008-organization-abbreviation-resolution.md. + +Same pluggable-client, never-fake-a-missing-channel discipline as every +other channel in this package -- and, specifically, an LLM's proposed +canonical name is never trusted on its own: it is only usable once +:mod:`lineageweave.relation_verification`'s external-search check +corroborates it, reusing that module's client rather than duplicating a +second web-search integration. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +from .http_client import post_json +from .relation_verification import ( + STATUS_PENDING, + RelationVerificationClient, +) + + +@dataclass(frozen=True) +class OrganizationNameResolution: + """One raw name's resolution outcome, ready to persist to + ``organization_name_resolution``. + + Attributes: + raw_organization_name: the abbreviated/slang name as mentioned + in the source text (``skos:altLabel``). + resolved_organization_name: the LLM's proposed full/canonical + name (``skos:prefLabel``). + verification_status_code: ``relation_verification_status`` + lookup code -- whether external search corroborated the + resolved name, reusing the same category and semantics + :mod:`lineageweave.relation_verification` already defines. + verification_evidence_url: the corroborating search result's + URL, or ``None`` when uncorroborated or verification itself + was unavailable. + """ + + raw_organization_name: str + resolved_organization_name: str + verification_status_code: str + verification_evidence_url: str | None + + +class OrganizationNameResolutionClient(Protocol): + """Proposes a full/canonical name for an abbreviated organization mention.""" + + available: bool + + def resolve(self, raw_name: str, context_text: str) -> str | None: + """Return the proposed canonical name, or ``None`` when the + model cannot determine one from the given context. + + Implementations must raise if the call itself fails (network + error, malformed response) -- a failed call is not the same + outcome as "the model looked and found nothing to propose." + Protocol stubs raise ``NotImplementedError`` so a no-op body is + never treated as a successful empty result. + """ + raise NotImplementedError + + +class NullOrganizationNameResolutionClient: + """No LLM orchestrator configured -- name resolution is unavailable.""" + + available = False + + def resolve(self, raw_name: str, context_text: str) -> str | None: + raise RuntimeError( + "NullOrganizationNameResolutionClient cannot resolve; check .available first" + ) + + +_RESOLUTION_PROMPT_TEMPLATE = """\ +The text below mentions an organization by the short/abbreviated name +"{raw_name}" (this may be a Korean-style contraction, an initialism, or +another kind of shorthand -- e.g. "AGP" is a synthetic contraction +for "Aurora Grid Power"). + +Using ONLY what the text itself supports (do not guess from the +abbreviation's letters/syllables alone if the text gives no supporting +context), determine the organization's full, real-world name. + +Reply with ONLY the full organization name on a single line, in its +most natural real-world form. If the text gives you no way to determine +the full name with real confidence, reply with exactly: UNKNOWN + +Text: {context} +""" + + +def parse_resolution_response(content: str) -> str | None: + """Parses the LLM's reply into a proposed canonical name, or `None` + when it declined (``UNKNOWN``) or replied with nothing usable. + + A one-line reply is the contract; only the first line is trusted -- + a multi-line reply means the model did not follow instructions, and + trusting the wrong line would risk persisting prose as a name. + """ + stripped = content.strip() + if not stripped or stripped.upper() == "UNKNOWN": + return None + first_line = stripped.splitlines()[0].strip() + if not first_line or first_line.upper() == "UNKNOWN": + return None + return first_line + + +class ContextualOrchestratorOrganizationNameResolutionClient: + """Calls ``POST {base_url}/v1/chat/completions`` with ``mode="route"``.""" + + available = True + + def __init__( + self, base_url: str, api_key: str, *, reasoning_effort: str = "medium", timeout: float = 30.0 + ) -> None: + self._base_url = base_url.rstrip("/") + self._api_key = api_key + self._reasoning_effort = reasoning_effort + self._timeout = timeout + + def resolve(self, raw_name: str, context_text: str) -> str | None: + prompt = _RESOLUTION_PROMPT_TEMPLATE.format(raw_name=raw_name, context=context_text) + body = post_json( + f"{self._base_url}/v1/chat/completions", + { + "messages": [{"role": "user", "content": prompt}], + "mode": "route", + "reasoning_effort": self._reasoning_effort, + }, + headers={"authorization": f"Bearer {self._api_key}"}, + timeout=self._timeout, + ) + content = body["choices"][0]["message"]["content"] + return parse_resolution_response(content) + + +def resolve_and_verify_organization_name( + raw_name: str, + context_text: str, + resolution_client: OrganizationNameResolutionClient, + verification_client: RelationVerificationClient, +) -> OrganizationNameResolution | None: + """Runs the full resolve-then-verify pipeline for one raw name. + + Returns ``None`` when resolution is unavailable, the model proposed + nothing, or it proposed back the same string it was given (not a + real resolution) -- the caller keeps using the raw name as-is in + every one of these cases, the same missing-vs-negative discipline + every other channel in this package follows. A verified result's + ``verification_status_code`` is only ever ``verify_corroborated`` / + ``verify_uncorroborated`` (real search ran) or ``verify_pending`` + (search itself is unavailable, not that it ran and found nothing) -- + never fabricated. + """ + if not resolution_client.available: + return None + candidate = resolution_client.resolve(raw_name, context_text) + if candidate is None: + return None + resolved_name = candidate.strip() + if not resolved_name or resolved_name == raw_name.strip(): + return None + + if not verification_client.available: + return OrganizationNameResolution( + raw_organization_name=raw_name, + resolved_organization_name=resolved_name, + verification_status_code=STATUS_PENDING, + verification_evidence_url=None, + ) + + result = verification_client.verify(resolved_name, raw_name) + return OrganizationNameResolution( + raw_organization_name=raw_name, + resolved_organization_name=resolved_name, + verification_status_code=result.status_code, + verification_evidence_url=result.evidence_url, + ) diff --git a/lineageweave/post_summary.py b/lineageweave/post_summary.py index eac051fda..6b3207a38 100644 --- a/lineageweave/post_summary.py +++ b/lineageweave/post_summary.py @@ -14,7 +14,20 @@ bullet, not a summary sentence. - **R&R (roles & responsibilities)**: semantic role labeling (Gildea & Jurafsky, 2002) -- who did what, framed as an agent/action/responsibility - triple per person named in the post, not prose. + triple per named actor in the post, not prose. The actor is not always + a person: business correspondence routinely names an organization + as the acting party ("당사" [our company], "Demo Corp"), not an + individual. Modeling every actor as a person loses this distinction + and makes an organization's affiliation-less name look like an + unresolved person. Grounded in W3C PROV-O (Lebo, Sahoo, & McGuinness, + 2013): ``prov:Agent`` is the general acting-party class, with + ``prov:Person`` and ``prov:Organization`` as its two recognized + subclasses -- the same distinction ``keyman_extraction``'s two-sided + (our-side/counterparty) person model already keeps for *people*, one + level up. A person actor also gets an inferred + ``affiliated_organization_name`` where the text supports it: a bare + person name without who they work for is hard to place in the same + way an unresolved organization name is. Same pluggable-client, never-fake-a-missing-channel discipline as every other Phase 2/3 channel: :class:`NullPostSummaryClient` makes the channel @@ -30,13 +43,51 @@ from .http_client import post_json +# common_lookup_value category "prov_agent_type" -- PROV-O's prov:Person / +# prov:Organization for the micro/macro cases, plus a meso-level third +# case this repo's own real data needed: a named sub-unit of a company +# ("설계팀" / "design team"), which is neither a person nor the company +# itself. Grounded in the W3C Organization Ontology's org:OrganizationalUnit +# (Reynolds, 2014), not invented -- see docs/adr/0007-team-actor-type.md. +ACTOR_TYPE_PERSON = "prov_person" +ACTOR_TYPE_ORGANIZATION = "prov_organization" +ACTOR_TYPE_TEAM = "prov_team" +_VALID_ACTOR_TYPE_CODES = frozenset({ACTOR_TYPE_PERSON, ACTOR_TYPE_ORGANIZATION, ACTOR_TYPE_TEAM}) + @dataclass(frozen=True) class RoleResponsibility: - """One person's role/responsibility as derived from the post text.""" + """One actor's role/responsibility as derived from the post text. + + Attributes: + actor_name: the person's or organization's name as named in the + text. + responsibility: what they are responsible for or did. + actor_type_code: ``ACTOR_TYPE_PERSON``, ``ACTOR_TYPE_ORGANIZATION``, + or ``ACTOR_TYPE_TEAM`` (PROV-O ``prov:Person`` / + ``prov:Organization``, or the meso-level + ``org:OrganizationalUnit`` for a named sub-unit like "설계팀") + -- which this actor actually is, not assumed to be a person. + affiliated_organization_name: for a person OR team actor, the + organization the text says or implies they belong to, when + the text supports it; ``None`` when the text gives no + affiliation to infer, or for an organization actor (its own + name already answers "which organization"). A team actor + without this is an unplaced team -- the text should usually + support it since a team is always someone's team. + """ - person_name: str + actor_name: str responsibility: str + actor_type_code: str = ACTOR_TYPE_PERSON + affiliated_organization_name: str | None = None + + def __post_init__(self) -> None: + if self.actor_type_code not in _VALID_ACTOR_TYPE_CODES: + raise ValueError( + f"actor_type_code must be one of {sorted(_VALID_ACTOR_TYPE_CODES)}, " + f"got {self.actor_type_code!r}" + ) @dataclass(frozen=True) @@ -81,16 +132,34 @@ def summarize(self, post_title: str, post_body: str) -> PostSummary: 2. A list of key events: discrete, datable occurrences mentioned in the post (e.g. "a bid was submitted", "a delivery date was confirmed"), each as a short phrase. -3. A list of roles & responsibilities: for each named person in the post, - one short phrase describing what they are responsible for or did, - according to the text. +3. A list of roles & responsibilities: for each named actor in the post + -- a person, an organization acting in its own name (e.g. "당사" + [our company], "Demo Corp"), OR a named team/department inside an + organization (e.g. "설계팀" [design team], "Sales Team") -- one short + phrase describing what they are responsible for or did, according to + the text. Do not force an organization's name into a person slot, and + do not force a team's name into an organization slot: a team is a + sub-unit of a company, not the company itself -- decide which of the + three each actor is, and say which. + When the actor is a person and the text names or clearly implies who + they work for, also give that organization's name -- a bare person + name without their employer is hard to place. When the actor is a + team, also give the organization it belongs to (a team is always part + of some company, even when the text only names the team, e.g. a + Korean company's internal 설계팀 -- infer the parent company from + context when the text supports it). Reply with ONLY a JSON object (no markdown fences, no prose) with exactly these fields: "korean_summary": string "key_events": array of strings - "roles_and_responsibilities": array of objects, each with - "person_name" and "responsibility" string fields + "roles_and_responsibilities": array of objects, each with: + "actor_name": string + "responsibility": string + "actor_type": exactly "person", "organization", or "team" + "affiliated_organization_name": string, or null when the actor is an + organization, or when the text gives no affiliation to infer for a + person or team actor Post title: {title} Post body: {body} @@ -134,15 +203,35 @@ def parse_summary_response(content: str) -> PostSummary | None: for entry in rr_raw: if not isinstance(entry, dict): continue - name = entry.get("person_name") + name = entry.get("actor_name") responsibility = entry.get("responsibility") + actor_type_raw = entry.get("actor_type") + if actor_type_raw == "organization": + actor_type_code = ACTOR_TYPE_ORGANIZATION + elif actor_type_raw == "team": + actor_type_code = ACTOR_TYPE_TEAM + else: + actor_type_code = ACTOR_TYPE_PERSON + affiliation_raw = entry.get("affiliated_organization_name") + affiliated_organization_name = ( + affiliation_raw.strip() + if isinstance(affiliation_raw, str) and affiliation_raw.strip() + else None + ) if ( isinstance(name, str) and name.strip() and isinstance(responsibility, str) and responsibility.strip() ): - roles.append(RoleResponsibility(person_name=name.strip(), responsibility=responsibility.strip())) + roles.append( + RoleResponsibility( + actor_name=name.strip(), + responsibility=responsibility.strip(), + actor_type_code=actor_type_code, + affiliated_organization_name=affiliated_organization_name, + ) + ) return PostSummary( korean_summary=korean_summary.strip(), diff --git a/lineageweave/prov_o.py b/lineageweave/prov_o.py new file mode 100644 index 000000000..57d7472f6 --- /dev/null +++ b/lineageweave/prov_o.py @@ -0,0 +1,862 @@ +"""Standards-complete W3C PROV-O relation registry and graph runtime. + +The module implements every class and every object/datatype property in the +PROV-O Recommendation's normative cross-reference. It deliberately keeps +LineageWeave's product-specific knowledge graph separate: PROV-O needs +literal-valued properties and qualified influence resources, neither of +which can be represented faithfully by the existing binary UUID edge table. + +Consumers may assert the compact, unqualified form, the qualified form, or +both. :class:`ProvGraph` materializes the Recommendation's property +hierarchy, declared inverses, symmetry, and the rule that a qualified form +implies its corresponding unqualified relation. Appendix B inverse names +are accepted as import aliases by reversing the assertion into the preferred +PROV-O direction. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import datetime +from typing import Final, Iterable, Literal as TypingLiteral, Mapping, cast + +from rdflib import Graph, Literal, Namespace, URIRef +from rdflib.namespace import RDF, XSD + +PROV: Final = Namespace("http://www.w3.org/ns/prov#") +_PROPERTY_KIND = TypingLiteral["object", "datatype"] + + +class ProvValidationError(ValueError): + """Raised when an assertion violates a PROV-O domain, range, or shape.""" + + +def _snake_case(local_name: str) -> str: + """Convert a PROV-O camel-case local name to stable lower snake case.""" + first_pass = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", local_name) + return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", first_pass).lower() + + +def class_code(local_name: str) -> str: + """Relational code for one PROV-O class, e.g. ``prov_entity``.""" + return f"prov_{_snake_case(local_name)}" + + +def relation_code(local_name: str) -> str: + """Relational code for one PROV-O property, e.g. ``prov_used``.""" + return f"prov_{_snake_case(local_name)}" + + +@dataclass(frozen=True) +class ProvClassSpec: + """One normative PROV-O class and its direct superclass names.""" + + local_name: str + superclasses: tuple[str, ...] = () + + @property + def iri(self) -> str: + """Absolute W3C IRI for the class.""" + return str(PROV[self.local_name]) + + @property + def code(self) -> str: + """Stable multiword snake-case relational code.""" + return class_code(self.local_name) + + +@dataclass(frozen=True) +class ProvRelationSpec: + """One normative PROV-O object or datatype property.""" + + local_name: str + property_kind: _PROPERTY_KIND + domains: tuple[str, ...] + ranges: tuple[str, ...] = () + datatype_iri: str | None = None + superproperties: tuple[str, ...] = () + defined_inverse: str | None = None + symmetric: bool = False + + @property + def iri(self) -> str: + """Absolute W3C IRI for the property.""" + return str(PROV[self.local_name]) + + @property + def code(self) -> str: + """Stable multiword snake-case relational code.""" + return relation_code(self.local_name) + + +@dataclass(frozen=True) +class ProvQualificationSpec: + """Normative mapping from a binary relation to its qualified pattern.""" + + unqualified_relation: str + qualification_relation: str + influence_class: str + influencer_relation: str + + +@dataclass(frozen=True) +class ProvInverseSpec: + """Appendix B recommended inverse name for one object property. + + ``defined_relation`` names a normative PROV-O property when the inverse + is itself part of the 50-term relation registry. Otherwise the name is + reserved for interoperable import/export but is not asserted as a new + ontology property by this implementation. + """ + + relation: str + inverse_local_name: str + defined_relation: str | None = None + + @property + def inverse_iri(self) -> str: + """Absolute reserved inverse IRI in the PROV namespace.""" + return str(PROV[self.inverse_local_name]) + + +# --------------------------------------------------------------------------- +# Normative class registry (30 terms) +# --------------------------------------------------------------------------- + + +def _class(local_name: str, *superclasses: str) -> ProvClassSpec: + return ProvClassSpec(local_name, tuple(superclasses)) + + +PROV_CLASSES: Final[Mapping[str, ProvClassSpec]] = { + spec.local_name: spec + for spec in ( + _class("Entity"), + _class("Activity"), + _class("Agent"), + _class("Collection", "Entity"), + _class("EmptyCollection", "Collection"), + _class("Bundle", "Entity"), + _class("Person", "Agent"), + _class("SoftwareAgent", "Agent"), + _class("Organization", "Agent"), + _class("Location"), + _class("Influence"), + _class("EntityInfluence", "Influence"), + _class("Usage", "InstantaneousEvent", "EntityInfluence"), + _class("Start", "InstantaneousEvent", "EntityInfluence"), + _class("End", "InstantaneousEvent", "EntityInfluence"), + _class("Derivation", "EntityInfluence"), + _class("PrimarySource", "Derivation"), + _class("Quotation", "Derivation"), + _class("Revision", "Derivation"), + _class("ActivityInfluence", "Influence"), + _class("Generation", "InstantaneousEvent", "ActivityInfluence"), + _class("Communication", "ActivityInfluence"), + _class("Invalidation", "InstantaneousEvent", "ActivityInfluence"), + _class("AgentInfluence", "Influence"), + _class("Attribution", "AgentInfluence"), + _class("Association", "AgentInfluence"), + _class("Plan", "Entity"), + _class("Delegation", "AgentInfluence"), + _class("InstantaneousEvent"), + _class("Role"), + ) +} + + +# --------------------------------------------------------------------------- +# Normative property registry (50 terms) +# --------------------------------------------------------------------------- + + +def _object( + local_name: str, + domains: tuple[str, ...], + ranges: tuple[str, ...], + *, + superproperties: tuple[str, ...] = (), + defined_inverse: str | None = None, + symmetric: bool = False, +) -> ProvRelationSpec: + return ProvRelationSpec( + local_name=local_name, + property_kind="object", + domains=domains, + ranges=ranges, + superproperties=superproperties, + defined_inverse=defined_inverse, + symmetric=symmetric, + ) + + +def _datatype( + local_name: str, + domains: tuple[str, ...], + *, + datatype_iri: str | None, +) -> ProvRelationSpec: + return ProvRelationSpec( + local_name=local_name, + property_kind="datatype", + domains=domains, + datatype_iri=datatype_iri, + ) + + +_RESOURCE_UNION = ("Entity", "Activity", "Agent") + +PROV_RELATIONS: Final[Mapping[str, ProvRelationSpec]] = { + spec.local_name: spec + for spec in ( + # Starting-point properties. + _object( + "wasGeneratedBy", + ("Entity",), + ("Activity",), + superproperties=("wasInfluencedBy",), + defined_inverse="generated", + ), + _object( + "wasDerivedFrom", + ("Entity",), + ("Entity",), + superproperties=("wasInfluencedBy",), + ), + _object( + "wasAttributedTo", + ("Entity",), + ("Agent",), + superproperties=("wasInfluencedBy",), + ), + _datatype("startedAtTime", ("Activity",), datatype_iri=str(XSD.dateTime)), + _object("used", ("Activity",), ("Entity",), superproperties=("wasInfluencedBy",)), + _object( + "wasInformedBy", + ("Activity",), + ("Activity",), + superproperties=("wasInfluencedBy",), + ), + _datatype("endedAtTime", ("Activity",), datatype_iri=str(XSD.dateTime)), + _object( + "wasAssociatedWith", + ("Activity",), + ("Agent",), + superproperties=("wasInfluencedBy",), + ), + _object( + "actedOnBehalfOf", + ("Agent",), + ("Agent",), + superproperties=("wasInfluencedBy",), + ), + # Expanded properties. + _object( + "alternateOf", + ("Entity",), + ("Entity",), + defined_inverse="alternateOf", + symmetric=True, + ), + _object( + "specializationOf", + ("Entity",), + ("Entity",), + superproperties=("alternateOf",), + ), + _datatype("generatedAtTime", ("Entity",), datatype_iri=str(XSD.dateTime)), + _object( + "hadPrimarySource", + ("Entity",), + ("Entity",), + superproperties=("wasDerivedFrom",), + ), + _datatype("value", ("Entity",), datatype_iri=None), + _object( + "wasQuotedFrom", + ("Entity",), + ("Entity",), + superproperties=("wasDerivedFrom",), + ), + _object( + "wasRevisionOf", + ("Entity",), + ("Entity",), + superproperties=("wasDerivedFrom",), + ), + _datatype("invalidatedAtTime", ("Entity",), datatype_iri=str(XSD.dateTime)), + _object( + "wasInvalidatedBy", + ("Entity",), + ("Activity",), + superproperties=("wasInfluencedBy",), + defined_inverse="invalidated", + ), + _object( + "hadMember", + ("Collection",), + ("Entity",), + superproperties=("wasInfluencedBy",), + ), + _object( + "wasStartedBy", + ("Activity",), + ("Entity",), + superproperties=("wasInfluencedBy",), + ), + _object( + "wasEndedBy", + ("Activity",), + ("Entity",), + superproperties=("wasInfluencedBy",), + ), + _object( + "invalidated", + ("Activity",), + ("Entity",), + superproperties=("influenced",), + defined_inverse="wasInvalidatedBy", + ), + _object( + "influenced", + _RESOURCE_UNION, + _RESOURCE_UNION, + defined_inverse="wasInfluencedBy", + ), + _object( + "atLocation", + ("Activity", "Agent", "Entity", "InstantaneousEvent"), + ("Location",), + ), + _object( + "generated", + ("Activity",), + ("Entity",), + superproperties=("influenced",), + defined_inverse="wasGeneratedBy", + ), + # Qualified properties. + _object( + "wasInfluencedBy", + _RESOURCE_UNION, + _RESOURCE_UNION, + defined_inverse="influenced", + ), + _object("qualifiedInfluence", _RESOURCE_UNION, ("Influence",)), + _object( + "qualifiedGeneration", + ("Entity",), + ("Generation",), + superproperties=("qualifiedInfluence",), + ), + _object( + "qualifiedDerivation", + ("Entity",), + ("Derivation",), + superproperties=("qualifiedInfluence",), + ), + _object( + "qualifiedPrimarySource", + ("Entity",), + ("PrimarySource",), + superproperties=("qualifiedInfluence",), + ), + _object( + "qualifiedQuotation", + ("Entity",), + ("Quotation",), + superproperties=("qualifiedInfluence",), + ), + _object( + "qualifiedRevision", + ("Entity",), + ("Revision",), + superproperties=("qualifiedInfluence",), + ), + _object( + "qualifiedAttribution", + ("Entity",), + ("Attribution",), + superproperties=("qualifiedInfluence",), + ), + _object( + "qualifiedInvalidation", + ("Entity",), + ("Invalidation",), + superproperties=("qualifiedInfluence",), + ), + _object( + "qualifiedStart", + ("Activity",), + ("Start",), + superproperties=("qualifiedInfluence",), + ), + _object( + "qualifiedUsage", + ("Activity",), + ("Usage",), + superproperties=("qualifiedInfluence",), + ), + _object( + "qualifiedCommunication", + ("Activity",), + ("Communication",), + superproperties=("qualifiedInfluence",), + ), + _object( + "qualifiedAssociation", + ("Activity",), + ("Association",), + superproperties=("qualifiedInfluence",), + ), + _object( + "qualifiedEnd", + ("Activity",), + ("End",), + superproperties=("qualifiedInfluence",), + ), + _object( + "qualifiedDelegation", + ("Agent",), + ("Delegation",), + superproperties=("qualifiedInfluence",), + ), + _object("influencer", ("Influence",), _RESOURCE_UNION), + _object( + "entity", + ("EntityInfluence",), + ("Entity",), + superproperties=("influencer",), + ), + _object("hadUsage", ("Derivation",), ("Usage",)), + _object("hadGeneration", ("Derivation",), ("Generation",)), + _object( + "activity", + ("ActivityInfluence",), + ("Activity",), + superproperties=("influencer",), + ), + _object( + "agent", + ("AgentInfluence",), + ("Agent",), + superproperties=("influencer",), + ), + _object("hadPlan", ("Association",), ("Plan",)), + _object("hadActivity", ("Delegation", "Derivation", "End", "Start"), ("Activity",)), + _datatype("atTime", ("InstantaneousEvent",), datatype_iri=str(XSD.dateTime)), + _object("hadRole", ("Association", "InstantaneousEvent"), ("Role",)), + ) +} + + +# --------------------------------------------------------------------------- +# Normative qualification tables (Tables 2 and 3) +# --------------------------------------------------------------------------- + +PROV_QUALIFICATIONS: Final[tuple[ProvQualificationSpec, ...]] = ( + ProvQualificationSpec("wasGeneratedBy", "qualifiedGeneration", "Generation", "activity"), + ProvQualificationSpec("wasDerivedFrom", "qualifiedDerivation", "Derivation", "entity"), + ProvQualificationSpec("wasAttributedTo", "qualifiedAttribution", "Attribution", "agent"), + ProvQualificationSpec("used", "qualifiedUsage", "Usage", "entity"), + ProvQualificationSpec("wasInformedBy", "qualifiedCommunication", "Communication", "activity"), + ProvQualificationSpec("wasAssociatedWith", "qualifiedAssociation", "Association", "agent"), + ProvQualificationSpec("actedOnBehalfOf", "qualifiedDelegation", "Delegation", "agent"), + ProvQualificationSpec("wasInfluencedBy", "qualifiedInfluence", "Influence", "influencer"), + ProvQualificationSpec("hadPrimarySource", "qualifiedPrimarySource", "PrimarySource", "entity"), + ProvQualificationSpec("wasQuotedFrom", "qualifiedQuotation", "Quotation", "entity"), + ProvQualificationSpec("wasRevisionOf", "qualifiedRevision", "Revision", "entity"), + ProvQualificationSpec("wasInvalidatedBy", "qualifiedInvalidation", "Invalidation", "activity"), + ProvQualificationSpec("wasStartedBy", "qualifiedStart", "Start", "entity"), + ProvQualificationSpec("wasEndedBy", "qualifiedEnd", "End", "entity"), +) + + +# --------------------------------------------------------------------------- +# Appendix B inverse-name registry (all 44 object properties) +# --------------------------------------------------------------------------- + +_INVERSE_NAME_ROWS = { + "actedOnBehalfOf": "hadDelegate", + "activity": "activityOfInfluence", + "agent": "agentOfInfluence", + "alternateOf": "alternateOf", + "atLocation": "locationOf", + "entity": "entityOfInfluence", + "generated": "wasGeneratedBy", + "hadActivity": "wasActivityOfInfluence", + "hadGeneration": "generatedAsDerivation", + "hadMember": "wasMemberOf", + "hadPlan": "wasPlanOf", + "hadPrimarySource": "wasPrimarySourceOf", + "hadRole": "wasRoleIn", + "hadUsage": "wasUsedInDerivation", + "influenced": "wasInfluencedBy", + "influencer": "hadInfluence", + "invalidated": "wasInvalidatedBy", + "qualifiedAssociation": "qualifiedAssociationOf", + "qualifiedAttribution": "qualifiedAttributionOf", + "qualifiedCommunication": "qualifiedCommunicationOf", + "qualifiedDelegation": "qualifiedDelegationOf", + "qualifiedDerivation": "qualifiedDerivationOf", + "qualifiedEnd": "qualifiedEndOf", + "qualifiedGeneration": "qualifiedGenerationOf", + "qualifiedInfluence": "qualifiedInfluenceOf", + "qualifiedInvalidation": "qualifiedInvalidationOf", + "qualifiedPrimarySource": "qualifiedSourceOf", + "qualifiedQuotation": "qualifiedQuotationOf", + "qualifiedRevision": "revisedEntity", + "qualifiedStart": "qualifiedStartOf", + "qualifiedUsage": "qualifiedUsingActivity", + "specializationOf": "generalizationOf", + "used": "wasUsedBy", + "wasAssociatedWith": "wasAssociateFor", + "wasAttributedTo": "contributed", + "wasDerivedFrom": "hadDerivation", + "wasEndedBy": "ended", + "wasGeneratedBy": "generated", + "wasInfluencedBy": "influenced", + "wasInformedBy": "informed", + "wasInvalidatedBy": "invalidated", + "wasQuotedFrom": "quotedAs", + "wasRevisionOf": "hadRevision", + "wasStartedBy": "started", +} + +PROV_RECOMMENDED_INVERSES: Final[Mapping[str, ProvInverseSpec]] = { + relation: ProvInverseSpec( + relation=relation, + inverse_local_name=inverse_name, + defined_relation=inverse_name if inverse_name in PROV_RELATIONS else None, + ) + for relation, inverse_name in _INVERSE_NAME_ROWS.items() +} + +# Non-standard-but-reserved aliases are safe to normalize because canonical +# PROV-O names always win when the same local name is itself a real property. +_INVERSE_ALIAS_TO_CANONICAL: Final[Mapping[str, str]] = { + spec.inverse_local_name: relation + for relation, spec in PROV_RECOMMENDED_INVERSES.items() + if spec.inverse_local_name not in PROV_RELATIONS +} + + +@dataclass(frozen=True) +class ProvLiteral: + """RDF literal used as the object of a PROV-O datatype property.""" + + lexical_value: str + datatype_iri: str | None = None + language_tag: str | None = None + + def __post_init__(self) -> None: + if self.datatype_iri and self.language_tag: + raise ProvValidationError("a literal cannot have both datatype_iri and language_tag") + if self.language_tag and not re.fullmatch(r"[A-Za-z]+(?:-[A-Za-z0-9]+)*", self.language_tag): + raise ProvValidationError("language_tag must be a valid BCP 47-style tag") + + @classmethod + def datetime(cls, value: datetime) -> "ProvLiteral": + """Create a timezone-aware ``xsd:dateTime`` literal.""" + if value.tzinfo is None or value.utcoffset() is None: + raise ProvValidationError("PROV-O dateTime values must be timezone-aware") + return cls(value.isoformat(), datatype_iri=str(XSD.dateTime)) + + def to_rdflib(self) -> Literal: + """Convert to an rdflib literal without changing lexical form.""" + return Literal( + self.lexical_value, + datatype=URIRef(self.datatype_iri) if self.datatype_iri else None, + lang=self.language_tag, + ) + + +@dataclass(frozen=True) +class ProvAssertion: + """One canonical PROV-O assertion with exactly one object kind.""" + + subject_iri: str + relation: str + object_resource_iri: str | None = None + object_literal: ProvLiteral | None = None + + def __post_init__(self) -> None: + if (self.object_resource_iri is None) == (self.object_literal is None): + raise ProvValidationError( + "a provenance assertion must have exactly one resource or literal object" + ) + + @classmethod + def resource(cls, subject_iri: str, relation: str, object_iri: str) -> "ProvAssertion": + """Construct a resource-to-resource assertion.""" + return cls(subject_iri, relation, object_resource_iri=object_iri) + + @classmethod + def literal( + cls, subject_iri: str, relation: str, object_literal: ProvLiteral + ) -> "ProvAssertion": + """Construct a resource-to-literal assertion.""" + return cls(subject_iri, relation, object_literal=object_literal) + + +class ProvGraph: + """Validated in-memory PROV-O graph with deterministic entailment. + + Resource IRIs are explicitly typed. Assertions may use a local PROV + name, ``prov:`` compact name, full PROV IRI, or an Appendix B reserved + inverse name. Reserved inverse names are rewritten into the preferred + PROV-O direction at insertion time. + """ + + def __init__(self) -> None: + self._resource_types: dict[str, set[str]] = {} + self._explicit_assertions: set[ProvAssertion] = set() + + @property + def resource_types(self) -> Mapping[str, frozenset[str]]: + """Read-only snapshot of explicitly assigned resource types.""" + return {iri: frozenset(types) for iri, types in self._resource_types.items()} + + @property + def explicit_assertions(self) -> frozenset[ProvAssertion]: + """Assertions supplied by callers after inverse-alias normalization.""" + return frozenset(self._explicit_assertions) + + def add_resource(self, resource_iri: str, *class_names: str) -> None: + """Declare one resource and one or more normative PROV-O types.""" + if not resource_iri: + raise ProvValidationError("resource_iri is required") + if not class_names: + raise ProvValidationError("at least one PROV-O class is required") + normalized = {self._normalize_class_name(name) for name in class_names} + self._resource_types.setdefault(resource_iri, set()).update(normalized) + + def add_assertion( + self, + subject_iri: str, + relation: str, + object_value: str | ProvLiteral, + ) -> ProvAssertion: + """Validate, canonicalize, and store one PROV-O assertion.""" + relation_name, reverse = self._normalize_relation_name(relation) + if reverse: + if isinstance(object_value, ProvLiteral): + raise ProvValidationError("an inverse object-property alias cannot reverse a literal") + subject_iri, object_value = object_value, subject_iri + + spec = PROV_RELATIONS[relation_name] + self._validate_subject(subject_iri, spec) + if spec.property_kind == "object": + if isinstance(object_value, ProvLiteral): + raise ProvValidationError(f"{relation_name} requires a resource object") + self._validate_resource_object(object_value, spec) + assertion = ProvAssertion.resource(subject_iri, relation_name, object_value) + else: + if isinstance(object_value, str): + raise ProvValidationError(f"{relation_name} requires a literal object") + self._validate_literal_object(object_value, spec) + assertion = ProvAssertion.literal(subject_iri, relation_name, object_value) + self._explicit_assertions.add(assertion) + return assertion + + def materialized_assertions(self) -> frozenset[ProvAssertion]: + """Return explicit assertions plus deterministic PROV-O entailments. + + Materialization includes transitive superproperty closure, declared + standard inverses, ``alternateOf`` symmetry, all fourteen + qualified-to-unqualified mappings, and the four direct time + shortcuts defined by qualified Generation/Invalidation/Start/End. + """ + assertions = set(self._explicit_assertions) + changed = True + while changed: + changed = False + additions: set[ProvAssertion] = set() + + for assertion in assertions: + relation_spec = PROV_RELATIONS[assertion.relation] + for superproperty in relation_spec.superproperties: + additions.add(self._same_object(assertion, superproperty)) + if assertion.object_resource_iri is not None: + if relation_spec.defined_inverse is not None: + additions.add( + ProvAssertion.resource( + assertion.object_resource_iri, + relation_spec.defined_inverse, + assertion.subject_iri, + ) + ) + if relation_spec.symmetric: + additions.add( + ProvAssertion.resource( + assertion.object_resource_iri, + assertion.relation, + assertion.subject_iri, + ) + ) + + by_relation: dict[str, list[ProvAssertion]] = {} + for assertion in assertions | additions: + by_relation.setdefault(assertion.relation, []).append(assertion) + + for qualification in PROV_QUALIFICATIONS: + qualified_edges = by_relation.get(qualification.qualification_relation, []) + influencer_edges = by_relation.get(qualification.influencer_relation, []) + influencers_by_node: dict[str, list[str]] = {} + for edge in influencer_edges: + influencer_iri = cast(str, edge.object_resource_iri) + influencers_by_node.setdefault(edge.subject_iri, []).append(influencer_iri) + for edge in qualified_edges: + qualified_node = cast(str, edge.object_resource_iri) + for influencer_iri in influencers_by_node.get(qualified_node, []): + additions.add( + ProvAssertion.resource( + edge.subject_iri, + qualification.unqualified_relation, + influencer_iri, + ) + ) + + # Direct time properties are shorthand for atTime on the + # corresponding qualified instantaneous event. + for qualified_relation, direct_time_relation in ( + ("qualifiedGeneration", "generatedAtTime"), + ("qualifiedInvalidation", "invalidatedAtTime"), + ("qualifiedStart", "startedAtTime"), + ("qualifiedEnd", "endedAtTime"), + ): + event_times: dict[str, list[ProvLiteral]] = {} + for at_time in by_relation.get("atTime", []): + literal = cast(ProvLiteral, at_time.object_literal) + event_times.setdefault(at_time.subject_iri, []).append(literal) + for edge in by_relation.get(qualified_relation, []): + event_iri = cast(str, edge.object_resource_iri) + for literal in event_times.get(event_iri, []): + additions.add( + ProvAssertion.literal(edge.subject_iri, direct_time_relation, literal) + ) + + new_assertions = additions - assertions + if new_assertions: + assertions.update(new_assertions) + changed = True + + return frozenset(assertions) + + def to_rdflib(self, *, materialize: bool = False) -> Graph: + """Serialize explicit or materialized content to an rdflib graph.""" + graph = Graph() + graph.bind("prov", PROV) + for resource_iri, types in self._resource_types.items(): + for class_name in sorted(types): + graph.add((URIRef(resource_iri), RDF.type, PROV[class_name])) + assertions: Iterable[ProvAssertion] + assertions = self.materialized_assertions() if materialize else self.explicit_assertions + for assertion in assertions: + subject = URIRef(assertion.subject_iri) + predicate = PROV[assertion.relation] + if assertion.object_resource_iri is not None: + object_node = URIRef(assertion.object_resource_iri) + else: + assert assertion.object_literal is not None + object_node = assertion.object_literal.to_rdflib() + graph.add((subject, predicate, object_node)) + return graph + + @staticmethod + def _same_object(assertion: ProvAssertion, relation: str) -> ProvAssertion: + """Copy an object-property assertion under one of its superproperties.""" + return ProvAssertion.resource( + assertion.subject_iri, relation, cast(str, assertion.object_resource_iri) + ) + + @staticmethod + def _normalize_class_name(class_name: str) -> str: + local_name = _local_name(class_name) + if local_name not in PROV_CLASSES: + raise ProvValidationError(f"unknown PROV-O class {class_name!r}") + return local_name + + @staticmethod + def _normalize_relation_name(relation: str) -> tuple[str, bool]: + local_name = _local_name(relation) + if local_name in PROV_RELATIONS: + return local_name, False + canonical = _INVERSE_ALIAS_TO_CANONICAL.get(local_name) + if canonical is None: + raise ProvValidationError(f"unknown PROV-O relation {relation!r}") + return canonical, True + + def _validate_subject(self, subject_iri: str, spec: ProvRelationSpec) -> None: + actual_types = self._resource_types.get(subject_iri) + if actual_types is None: + raise ProvValidationError(f"subject resource {subject_iri!r} has not been declared") + if not _matches_any_class(actual_types, spec.domains): + expected = " or ".join(spec.domains) + raise ProvValidationError( + f"subject {subject_iri!r} of {spec.local_name} must be {expected}" + ) + + def _validate_resource_object(self, object_iri: str, spec: ProvRelationSpec) -> None: + actual_types = self._resource_types.get(object_iri) + if actual_types is None: + raise ProvValidationError(f"object resource {object_iri!r} has not been declared") + if not _matches_any_class(actual_types, spec.ranges): + expected = " or ".join(spec.ranges) + raise ProvValidationError( + f"object {object_iri!r} of {spec.local_name} must be {expected}" + ) + + @staticmethod + def _validate_literal_object(literal: ProvLiteral, spec: ProvRelationSpec) -> None: + if spec.datatype_iri is not None and literal.datatype_iri != spec.datatype_iri: + raise ProvValidationError( + f"{spec.local_name} requires datatype {spec.datatype_iri}, " + f"got {literal.datatype_iri!r}" + ) + + +def _local_name(value: str) -> str: + """Return the local name from a local, compact, or absolute PROV IRI.""" + if value.startswith(str(PROV)): + return value[len(str(PROV)) :] + if value.startswith("prov:"): + return value[5:] + return value + + +def _class_ancestors(class_name: str) -> frozenset[str]: + """Return a class and every transitive superclass without duplicates.""" + ancestors = {class_name} + pending = [class_name] + while pending: + current = pending.pop() + unseen = set(PROV_CLASSES[current].superclasses) - ancestors + ancestors.update(unseen) + pending.extend(unseen) + return frozenset(ancestors) + + +def _matches_any_class(actual_types: Iterable[str], expected_types: Iterable[str]) -> bool: + expected = set(expected_types) + return any(bool(_class_ancestors(actual) & expected) for actual in actual_types) + + +__all__ = [ + "PROV", + "PROV_CLASSES", + "PROV_QUALIFICATIONS", + "PROV_RELATIONS", + "PROV_RECOMMENDED_INVERSES", + "ProvAssertion", + "ProvClassSpec", + "ProvGraph", + "ProvInverseSpec", + "ProvLiteral", + "ProvQualificationSpec", + "ProvRelationSpec", + "ProvValidationError", + "class_code", + "relation_code", +] diff --git a/migrations/0001_initial_schema.sql b/migrations/0001_initial_schema.sql index 6a2676a4c..a94b05a0c 100644 --- a/migrations/0001_initial_schema.sql +++ b/migrations/0001_initial_schema.sql @@ -43,7 +43,8 @@ comment on table common_lookup_value is 'Every ENUM-like value in this schema (voc_type, post_visibility, ' 'entity_relationship_type, person_side, edge_type, node_type, ' 'ticket_status, permission, corporate_entity_level, ' - 'relation_verification_status, evaluation_criterion) lives here once. ' + 'relation_verification_status, evaluation_criterion, prov_agent_type) ' + 'lives here once. ' 'lookup_code is unique across all categories -- see the unique(lookup_code) comment.'; -- --------------------------------------------------------------------- @@ -211,11 +212,17 @@ create table post_summary_event ( primary key (post_id, event_ordinal) ); +-- actor_type_code: R&R Ontology, see migrations/0012_role_responsibility_agent_type.sql +-- and ADR 0006 -- a named actor is not always a person (an organization +-- can act in its own name, e.g. "당사," "Demo Corp"), so this is not folded +-- into person_name's own meaning. create table post_summary_role ( post_id uuid not null references post_summary_result (post_id) on delete cascade, - person_name text not null, + actor_name text not null, responsibility text not null, - primary key (post_id, person_name) + actor_type_code text not null default 'prov_person' references common_lookup_value (lookup_code), + affiliated_organization_name text, + primary key (post_id, actor_name) ); -- Persisted in-popup Q&A. Seed writes a synthetic exchange so @@ -326,10 +333,16 @@ create table report_item_information ( -- --------------------------------------------------------------------- -- Cataloged people mentioned in posts (Keyman). Named cataloged_person, -- not person, so every table name is two or more snake_case words. +-- last_known_job_title: the disambiguation signal migrations/0013 adds. +-- Lives here, not only on person_affiliation.role_title, because a +-- stated title ("our legal counsel, Sam Okonkwo") is real same-name +-- evidence even when the text names no specific organization to attach +-- a person_affiliation row to. create table cataloged_person ( person_id uuid primary key default uuid_generate_v4(), person_name text not null, person_side_code text not null references common_lookup_value (lookup_code), + last_known_job_title text, created_at timestamptz not null default now() ); @@ -343,6 +356,9 @@ create table person_affiliation ( ); create index person_affiliation_person_idx on person_affiliation (person_id); +create index person_affiliation_corporate_entity_idx + on person_affiliation (affiliated_corporate_entity_id) + where affiliated_corporate_entity_id is not null; create table post_person_mention ( post_id uuid not null references source_post (post_id), @@ -351,6 +367,59 @@ create table post_person_mention ( primary key (post_id, person_id) ); +create table post_summary_person_mention ( + post_id uuid not null references source_post (post_id) on delete cascade, + person_id uuid not null references cataloged_person (person_id), + primary key (post_id, person_id) +); + +-- Read-side union only. The two writable tables retain the evidence source: +-- post_person_mention is Keyman extraction; post_summary_person_mention is R&R. +create view combined_post_person_mention as + select post_id, person_id from post_person_mention + union + select post_id, person_id from post_summary_person_mention; + +-- --------------------------------------------------------------------- +-- Cross-post identity resolution for R&R actors (ADR 0009/0007): a +-- team named across two posts (e.g. 설계팀) must resolve to the same +-- row, the same way cataloged_person/corporate_entity already give +-- persons/organizations a shared identity across posts. +-- --------------------------------------------------------------------- +create table cataloged_team ( + team_id uuid primary key default uuid_generate_v4(), + team_name text not null, + affiliated_organization_name text, + affiliated_corporate_entity_id uuid references corporate_entity (corporate_entity_id), + created_at timestamptz not null default now(), + unique nulls not distinct (team_name, affiliated_organization_name) +); + +create index cataloged_team_corporate_entity_idx + on cataloged_team (affiliated_corporate_entity_id) + where affiliated_corporate_entity_id is not null; + +create table post_team_mention ( + post_id uuid not null references source_post (post_id), + team_id uuid not null references cataloged_team (team_id), + primary key (post_id, team_id) +); + +create table post_organization_mention ( + post_id uuid not null references source_post (post_id), + corporate_entity_id uuid not null references corporate_entity (corporate_entity_id), + primary key (post_id, corporate_entity_id) +); + +-- ADR 0019: store the resolved catalog id on the role row itself. +-- corporate_entity.entity_name is not unique, and mention tables are +-- post-scoped, so reconstructing identity by name is not 3NF. +alter table post_summary_role + add column cataloged_team_id uuid references cataloged_team (team_id); +alter table post_summary_role + add column cataloged_corporate_entity_id uuid + references corporate_entity (corporate_entity_id); + -- --------------------------------------------------------------------- -- Knowledge graph: person/company/post nodes, typed edges. The type -- codes (which kind of node, which kind of edge) are real enums and DO @@ -369,12 +438,81 @@ create table knowledge_graph_edge ( target_node_id uuid not null, edge_type_code text not null references common_lookup_value (lookup_code), edge_weight numeric not null default 1.0, - created_at timestamptz not null default now() + created_at timestamptz not null default now(), + constraint knowledge_graph_edge_identity_uq unique ( + source_node_type_code, source_node_id, + target_node_type_code, target_node_id, + edge_type_code + ) ); create index knowledge_graph_edge_source_idx on knowledge_graph_edge (source_node_type_code, source_node_id); create index knowledge_graph_edge_target_idx on knowledge_graph_edge (target_node_type_code, target_node_id); +create table if not exists knowledge_graph_edge_evidence ( + knowledge_graph_edge_id uuid not null + references knowledge_graph_edge (knowledge_graph_edge_id) on delete cascade, + evidence_post_id uuid not null references source_post (post_id) on delete cascade, + primary key (knowledge_graph_edge_id, evidence_post_id) +); + +create index if not exists knowledge_graph_edge_evidence_post_idx + on knowledge_graph_edge_evidence (evidence_post_id, knowledge_graph_edge_id); + +create or replace function register_knowledge_graph_edge_evidence() +returns trigger +language plpgsql +as $$ +begin + if new.edge_type_code in ( + 'edge_mention', + 'edge_mention_team', + 'edge_mention_organization' + ) and new.target_node_type_code = 'node_post' then + insert into knowledge_graph_edge_evidence + (knowledge_graph_edge_id, evidence_post_id) + values (new.knowledge_graph_edge_id, new.target_node_id) + on conflict do nothing; + elsif new.edge_type_code = 'edge_co_mention' then + insert into knowledge_graph_edge_evidence + (knowledge_graph_edge_id, evidence_post_id) + select distinct new.knowledge_graph_edge_id, left_mention.post_id + from combined_post_person_mention left_mention + join combined_post_person_mention right_mention + on right_mention.post_id = left_mention.post_id + where left_mention.person_id = new.source_node_id + and right_mention.person_id = new.target_node_id + on conflict do nothing; + elsif new.edge_type_code = 'edge_affiliation' then + insert into knowledge_graph_edge_evidence + (knowledge_graph_edge_id, evidence_post_id) + select distinct new.knowledge_graph_edge_id, mention.post_id + from combined_post_person_mention mention + join person_affiliation affiliation + on affiliation.person_id = mention.person_id + where mention.person_id = new.source_node_id + and affiliation.affiliated_corporate_entity_id = new.target_node_id + on conflict do nothing; + elsif new.edge_type_code = 'edge_team_affiliation' then + insert into knowledge_graph_edge_evidence + (knowledge_graph_edge_id, evidence_post_id) + select distinct new.knowledge_graph_edge_id, mention.post_id + from post_team_mention mention + join cataloged_team team on team.team_id = mention.team_id + where mention.team_id = new.source_node_id + and team.affiliated_corporate_entity_id = new.target_node_id + on conflict do nothing; + end if; + return new; +end +$$; + +drop trigger if exists knowledge_graph_edge_evidence_register + on knowledge_graph_edge; +create trigger knowledge_graph_edge_evidence_register +after insert or update on knowledge_graph_edge +for each row execute function register_knowledge_graph_edge_evidence(); + -- --------------------------------------------------------------------- -- Issue tickets tied to a post. -- --------------------------------------------------------------------- @@ -412,4 +550,26 @@ create table post_lineage_edge ( primary key (parent_post_id, child_post_id) ); +-- --------------------------------------------------------------------- +-- Caches an abbreviated/slang organization name's LLM-inferred +-- canonical name plus external search cross-verification (ADR 0008), +-- e.g. "AGP" -> "Aurora Grid Power" -- keyed by the raw name so the same +-- abbreviation across many posts is resolved once, not re-queried +-- every mention. Grounded in SKOS skos:altLabel/skos:prefLabel (see +-- docs/ontology/lineageweave-kg.ttl); verification_status_code reuses +-- relation_verification_status (migration 0004) rather than a +-- near-duplicate category -- a resolved name is corroborated/ +-- uncorroborated the same way a classified relationship is. +-- --------------------------------------------------------------------- +create table organization_name_resolution ( + raw_organization_name text primary key, + resolved_organization_name text not null, + verification_status_code text not null references common_lookup_value (lookup_code), + verification_evidence_url text, + resolved_at timestamptz not null default now() +); + +comment on table organization_name_resolution is + 'Caches LLM-proposed canonical names for abbreviated/slang organization mentions (e.g. AGP -> Aurora Grid Power), cross-verified via external search before being trusted.'; + commit; diff --git a/migrations/0012_role_responsibility_agent_type.sql b/migrations/0012_role_responsibility_agent_type.sql new file mode 100644 index 000000000..3bcea3499 --- /dev/null +++ b/migrations/0012_role_responsibility_agent_type.sql @@ -0,0 +1,33 @@ +-- Roles & responsibilities' named actor is not always a person -- +-- business correspondence routinely names an organization acting in its +-- own name ("당사" [our company], "Demo Corp"), not an +-- individual. Adds a PROV-O-grounded person/organization distinction +-- (see ADR 0006) plus an inferred affiliated-organization name for +-- person actors. The rename below (person_name -> actor_name) preserves +-- every existing row's data -- a plain RENAME COLUMN, not a drop/recreate +-- -- since a volume that already ran the pre-0006 0001 has real rows +-- under the old name. + +insert into common_lookup_value (lookup_category, lookup_code, lookup_label, display_order) values + ('prov_agent_type', 'prov_person', 'Person', 0), + ('prov_agent_type', 'prov_organization', 'Organization', 1) +on conflict (lookup_code) do nothing; + +do $$ +begin + if exists ( + select 1 from information_schema.columns + where table_schema = 'public' + and table_name = 'post_summary_role' + and column_name = 'person_name' + ) then + alter table post_summary_role rename column person_name to actor_name; + end if; +end $$; + +alter table post_summary_role + add column if not exists actor_type_code text not null default 'prov_person' + references common_lookup_value (lookup_code); + +alter table post_summary_role + add column if not exists affiliated_organization_name text; diff --git a/migrations/0013_person_job_title.sql b/migrations/0013_person_job_title.sql new file mode 100644 index 000000000..5904a6e0b --- /dev/null +++ b/migrations/0013_person_job_title.sql @@ -0,0 +1,11 @@ +-- Same-name-people disambiguation signal: a stated job title/position is +-- real evidence a same person_name+person_side_code match is NOT the +-- same real individual. Lives on cataloged_person itself, not only +-- person_affiliation.role_title, because a title is real disambiguation +-- evidence even when the text names no specific organization to attach +-- an affiliation row to (e.g. "our legal counsel, Sam Okonkwo"). +-- ADD COLUMN IF NOT EXISTS so a volume that already ran 0001 still +-- upgrades. + +alter table cataloged_person + add column if not exists last_known_job_title text; diff --git a/migrations/0014_role_responsibility_team_actor_type.sql b/migrations/0014_role_responsibility_team_actor_type.sql new file mode 100644 index 000000000..e701aef3f --- /dev/null +++ b/migrations/0014_role_responsibility_team_actor_type.sql @@ -0,0 +1,11 @@ +-- A third roles-and-responsibilities actor case real data surfaced: +-- a named sub-unit of a company ("설계팀" [design team]) is meso-level -- +-- neither a person nor the company itself. Adds `prov_team` alongside +-- `prov_person`/`prov_organization` (migration 0012); grounded in the +-- W3C Organization Ontology's org:OrganizationalUnit (see ADR 0007), +-- not PROV-O, which has no sub-organization concept. Purely additive: +-- no existing row's actor_type_code changes. + +insert into common_lookup_value (lookup_category, lookup_code, lookup_label, display_order) values + ('prov_agent_type', 'prov_team', 'Team', 2) +on conflict (lookup_code) do nothing; diff --git a/migrations/0015_organization_name_resolution.sql b/migrations/0015_organization_name_resolution.sql new file mode 100644 index 000000000..7d18320ff --- /dev/null +++ b/migrations/0015_organization_name_resolution.sql @@ -0,0 +1,18 @@ +-- Caches an abbreviated/slang organization name's LLM-inferred +-- canonical name plus external search cross-verification (ADR 0008), +-- e.g. "AGP" -> "Aurora Grid Power". corporate_hierarchy_resolution's +-- character-similarity matching cannot bridge this gap (an initialism +-- shares almost no substring with its expansion), so a genuine +-- LLM-context + web-evidence step is needed instead. Keyed by the raw +-- name so the same abbreviation across many posts is resolved once. + +create table if not exists organization_name_resolution ( + raw_organization_name text primary key, + resolved_organization_name text not null, + verification_status_code text not null references common_lookup_value (lookup_code), + verification_evidence_url text, + resolved_at timestamptz not null default now() +); + +comment on table organization_name_resolution is + 'Caches LLM-proposed canonical names for abbreviated/slang organization mentions (e.g. AGP -> Aurora Grid Power), cross-verified via external search before being trusted.'; diff --git a/migrations/0016_cross_post_actor_identity.sql b/migrations/0016_cross_post_actor_identity.sql new file mode 100644 index 000000000..a20266994 --- /dev/null +++ b/migrations/0016_cross_post_actor_identity.sql @@ -0,0 +1,179 @@ +-- Cross-post identity resolution for R&R actors (ADR 0009). Extraction +-- runs per-post, but the same team, person, or organization named +-- across two different posts must resolve to the same catalog row -- +-- otherwise every extraction is an island and can never become a +-- cross-post Knowledge Graph clue. +-- +-- Teams (prov_team, ADR 0007) had no catalog at all until now, unlike +-- persons (cataloged_person, already Keyman's identity catalog) and +-- organizations (corporate_entity, already the corporate hierarchy +-- catalog). This migration adds the missing team catalog and two +-- mention join tables (post_team_mention, post_organization_mention) +-- so knowledge_graph_edge writers can derive Team/Organization mention +-- edges the same way they already derive Person mention edges from +-- post_person_mention. + +create table if not exists cataloged_team ( + team_id uuid primary key default uuid_generate_v4(), + team_name text not null, + affiliated_organization_name text, + affiliated_corporate_entity_id uuid references corporate_entity (corporate_entity_id), + created_at timestamptz not null default now(), + -- A team name alone rarely uniquely identifies it across a whole + -- product's real-world scope ("설계팀" exists at many companies); + -- the (name, org) pair almost always does. NULLS NOT DISTINCT makes + -- a missing affiliation participate in the same identity key, so + -- concurrent upserts of the same unplaced team return one row. + unique nulls not distinct (team_name, affiliated_organization_name) +); + +create index if not exists cataloged_team_corporate_entity_idx + on cataloged_team (affiliated_corporate_entity_id) + where affiliated_corporate_entity_id is not null; + +create table if not exists post_team_mention ( + post_id uuid not null references source_post (post_id), + team_id uuid not null references cataloged_team (team_id), + primary key (post_id, team_id) +); + +create table if not exists post_organization_mention ( + post_id uuid not null references source_post (post_id), + corporate_entity_id uuid not null references corporate_entity (corporate_entity_id), + primary key (post_id, corporate_entity_id) +); + +insert into common_lookup_value (lookup_category, lookup_code, lookup_label, display_order) values + ('corporate_entity_level', 'group', 'Group', 0), + ('corporate_entity_level', 'company', 'Company', 1), + ('corporate_entity_level', 'plant', 'Plant', 2), + ('node_type', 'node_team', 'Team', 3), + ('edge_type', 'edge_mention_team', 'Team mentioned in', 3), + ('edge_type', 'edge_team_affiliation', 'Team affiliated with', 4), + ('edge_type', 'edge_mention_organization', 'Organization mentioned in', 5) +on conflict (lookup_code) do nothing; +-- Keyman and R&R person mentions are independent replaceable evidence +-- channels. The upgrade copies matching R&R actor names into +-- post_summary_person_mention and leaves post_person_mention (including +-- mention_context) untouched. combined_post_person_mention already unions +-- both sources; deleting Keyman rows would drop mention_context and let a +-- later persist_post_summary erase the only remaining person evidence. +create table if not exists post_summary_person_mention ( + post_id uuid not null references source_post (post_id) on delete cascade, + person_id uuid not null references cataloged_person (person_id), + primary key (post_id, person_id) + ); + + create or replace view combined_post_person_mention as + select post_id, person_id from post_person_mention + union + select post_id, person_id from post_summary_person_mention; + + insert into post_summary_person_mention (post_id, person_id) + select distinct role.post_id, matched_person.person_id + from post_summary_role role + join lateral ( + select person.person_id + from cataloged_person person + where person.person_name = role.actor_name + order by person.created_at, person.person_id + limit 1 + ) matched_person on true + where role.actor_type_code = 'prov_person' + on conflict do nothing; + + with ranked_edge as ( + select knowledge_graph_edge_id, + row_number() over ( + partition by source_node_type_code, source_node_id, + target_node_type_code, target_node_id, + edge_type_code + order by created_at, knowledge_graph_edge_id + ) as duplicate_rank + from knowledge_graph_edge + ) + delete from knowledge_graph_edge edge_row + using ranked_edge duplicate + where edge_row.knowledge_graph_edge_id = duplicate.knowledge_graph_edge_id + and duplicate.duplicate_rank > 1; + + create unique index if not exists knowledge_graph_edge_identity_uq + on knowledge_graph_edge ( + source_node_type_code, source_node_id, + target_node_type_code, target_node_id, + edge_type_code + ); + + create table if not exists knowledge_graph_edge_evidence ( + knowledge_graph_edge_id uuid not null + references knowledge_graph_edge (knowledge_graph_edge_id) on delete cascade, + evidence_post_id uuid not null references source_post (post_id) on delete cascade, + primary key (knowledge_graph_edge_id, evidence_post_id) +); + +create index if not exists knowledge_graph_edge_evidence_post_idx + on knowledge_graph_edge_evidence (evidence_post_id, knowledge_graph_edge_id); + +create or replace function register_knowledge_graph_edge_evidence() +returns trigger +language plpgsql +as $$ +begin + if new.edge_type_code in ( + 'edge_mention', + 'edge_mention_team', + 'edge_mention_organization' + ) and new.target_node_type_code = 'node_post' then + insert into knowledge_graph_edge_evidence + (knowledge_graph_edge_id, evidence_post_id) + values (new.knowledge_graph_edge_id, new.target_node_id) + on conflict do nothing; + elsif new.edge_type_code = 'edge_co_mention' then + insert into knowledge_graph_edge_evidence + (knowledge_graph_edge_id, evidence_post_id) + select distinct new.knowledge_graph_edge_id, left_mention.post_id + from combined_post_person_mention left_mention + join combined_post_person_mention right_mention + on right_mention.post_id = left_mention.post_id + where left_mention.person_id = new.source_node_id + and right_mention.person_id = new.target_node_id + on conflict do nothing; + elsif new.edge_type_code = 'edge_affiliation' then + insert into knowledge_graph_edge_evidence + (knowledge_graph_edge_id, evidence_post_id) + select distinct new.knowledge_graph_edge_id, mention.post_id + from combined_post_person_mention mention + join person_affiliation affiliation + on affiliation.person_id = mention.person_id + where mention.person_id = new.source_node_id + and affiliation.affiliated_corporate_entity_id = new.target_node_id + on conflict do nothing; + elsif new.edge_type_code = 'edge_team_affiliation' then + insert into knowledge_graph_edge_evidence + (knowledge_graph_edge_id, evidence_post_id) + select distinct new.knowledge_graph_edge_id, mention.post_id + from post_team_mention mention + join cataloged_team team on team.team_id = mention.team_id + where mention.team_id = new.source_node_id + and team.affiliated_corporate_entity_id = new.target_node_id + on conflict do nothing; + end if; + return new; +end +$$; + +drop trigger if exists knowledge_graph_edge_evidence_register + on knowledge_graph_edge; +create trigger knowledge_graph_edge_evidence_register +after insert or update on knowledge_graph_edge +for each row execute function register_knowledge_graph_edge_evidence(); + + -- Re-run the support trigger for every surviving legacy edge, then prune + -- rows that cannot be tied to current post evidence. + update knowledge_graph_edge set edge_weight = edge_weight; + delete from knowledge_graph_edge edge_row + where not exists ( + select 1 + from knowledge_graph_edge_evidence evidence + where evidence.knowledge_graph_edge_id = edge_row.knowledge_graph_edge_id + ); diff --git a/migrations/0017_prov_o_standard_relations.sql b/migrations/0017_prov_o_standard_relations.sql new file mode 100644 index 000000000..46867acc1 --- /dev/null +++ b/migrations/0017_prov_o_standard_relations.sql @@ -0,0 +1,636 @@ +-- W3C PROV-O standards-complete provenance layer (ADR 0011). +-- +-- Implements every one of the Recommendation's 30 classes and 50 +-- normative object/datatype properties, both qualification tables, the +-- property/class hierarchies, and every Appendix B recommended inverse +-- name. Runtime provenance data is stored separately from the product's +-- compact knowledge_graph_edge table because PROV-O must represent +-- literals and qualified Influence resources without flattening them. +-- +-- All database objects use two-or-more-word snake_case and the catalog is +-- normalized: class/property definitions, hierarchies, domains, ranges, +-- qualification mappings, inverse names, resources, types, literals, and +-- assertions each have one authoritative table. + +begin; + +create table if not exists provenance_class_definition ( + class_code text primary key, + class_iri text not null unique, + class_local_name text not null unique, + class_label text not null +); + +create table if not exists provenance_class_hierarchy ( + child_class_code text not null references provenance_class_definition (class_code), + parent_class_code text not null references provenance_class_definition (class_code), + primary key (child_class_code, parent_class_code), + check (child_class_code <> parent_class_code) +); + +create table if not exists provenance_relation_definition ( + relation_code text primary key, + relation_iri text not null unique, + relation_local_name text not null unique, + relation_label text not null, + property_kind_code text not null check (property_kind_code in ('object', 'datatype')), + datatype_iri text, + symmetric_flag boolean not null default false, + check (property_kind_code = 'datatype' or datatype_iri is null) +); + +create table if not exists provenance_relation_hierarchy ( + child_relation_code text not null references provenance_relation_definition (relation_code), + parent_relation_code text not null references provenance_relation_definition (relation_code), + primary key (child_relation_code, parent_relation_code), + check (child_relation_code <> parent_relation_code) +); + +create table if not exists provenance_relation_domain ( + relation_code text not null references provenance_relation_definition (relation_code), + domain_class_code text not null references provenance_class_definition (class_code), + primary key (relation_code, domain_class_code) +); + +create table if not exists provenance_relation_resource_range ( + relation_code text not null references provenance_relation_definition (relation_code), + range_class_code text not null references provenance_class_definition (class_code), + primary key (relation_code, range_class_code) +); + +create table if not exists provenance_qualification_definition ( + unqualified_relation_code text primary key references provenance_relation_definition (relation_code), + qualification_relation_code text not null unique references provenance_relation_definition (relation_code), + influence_class_code text not null references provenance_class_definition (class_code), + influencer_relation_code text not null references provenance_relation_definition (relation_code) +); + +create table if not exists provenance_inverse_definition ( + relation_code text primary key references provenance_relation_definition (relation_code), + inverse_local_name text not null, + inverse_iri text not null, + inverse_relation_code text references provenance_relation_definition (relation_code), + inverse_kind_code text not null check (inverse_kind_code in ('defined', 'recommended')), + check ( + (inverse_kind_code = 'defined' and inverse_relation_code is not null) + or (inverse_kind_code = 'recommended' and inverse_relation_code is null) + ) +); + +create table if not exists provenance_resource ( + resource_id uuid primary key default uuid_generate_v4(), + resource_iri text not null unique, + resource_label text, + created_at timestamptz not null default now() +); + +create table if not exists provenance_resource_type ( + resource_id uuid not null references provenance_resource (resource_id) on delete cascade, + class_code text not null references provenance_class_definition (class_code), + primary key (resource_id, class_code) +); + +create table if not exists provenance_literal_value ( + literal_id uuid primary key default uuid_generate_v4(), + lexical_value text not null, + datatype_iri text, + language_tag text, + created_at timestamptz not null default now(), + check (datatype_iri is null or language_tag is null) +); + +create table if not exists provenance_resource_binding ( + resource_id uuid not null references provenance_resource (resource_id) on delete cascade, + node_type_code text not null references common_lookup_value (lookup_code), + node_id uuid not null, + primary key (resource_id, node_type_code, node_id), + unique (node_type_code, node_id) +); + +create table if not exists provenance_assertion ( + assertion_id uuid primary key default uuid_generate_v4(), + subject_resource_id uuid not null references provenance_resource (resource_id), + relation_code text not null references provenance_relation_definition (relation_code), + object_resource_id uuid references provenance_resource (resource_id), + object_literal_id uuid references provenance_literal_value (literal_id), + bundle_resource_id uuid references provenance_resource (resource_id), + created_at timestamptz not null default now(), + check (num_nonnulls(object_resource_id, object_literal_id) = 1) +); + +create unique index if not exists provenance_assertion_resource_unique_idx + on provenance_assertion (subject_resource_id, relation_code, object_resource_id, bundle_resource_id) + nulls not distinct + where object_resource_id is not null; + +create unique index if not exists provenance_assertion_literal_unique_idx + on provenance_assertion (subject_resource_id, relation_code, object_literal_id, bundle_resource_id) + nulls not distinct + where object_literal_id is not null; + +create table if not exists provenance_assertion_derivation ( + derived_assertion_id uuid not null references provenance_assertion (assertion_id) on delete cascade, + source_assertion_id uuid not null references provenance_assertion (assertion_id) on delete cascade, + primary key (derived_assertion_id, source_assertion_id), + check (derived_assertion_id <> source_assertion_id) +); + +create or replace function validate_provenance_assertion_contract() +returns trigger +language plpgsql +as $$ +declare + relation_kind text; + required_datatype text; + literal_datatype text; + literal_lexical text; +begin + select property_kind_code, datatype_iri + into relation_kind, required_datatype + from provenance_relation_definition + where relation_code = new.relation_code; + + if relation_kind = 'object' and new.object_resource_id is null then + raise exception 'PROV-O object property % requires object_resource_id', new.relation_code; + end if; + if relation_kind = 'datatype' and new.object_literal_id is null then + raise exception 'PROV-O datatype property % requires object_literal_id', new.relation_code; + end if; + + if not exists ( + with recursive subject_class (class_code) as ( + select class_code + from provenance_resource_type + where resource_id = new.subject_resource_id + union + select hierarchy.parent_class_code + from subject_class + join provenance_class_hierarchy hierarchy + on hierarchy.child_class_code = subject_class.class_code + ) + select 1 + from subject_class + join provenance_relation_domain domain_rule + on domain_rule.domain_class_code = subject_class.class_code + where domain_rule.relation_code = new.relation_code + ) then + raise exception 'subject resource % violates PROV-O domain for %', + new.subject_resource_id, new.relation_code; + end if; + + if relation_kind = 'object' and not exists ( + with recursive object_class (class_code) as ( + select class_code + from provenance_resource_type + where resource_id = new.object_resource_id + union + select hierarchy.parent_class_code + from object_class + join provenance_class_hierarchy hierarchy + on hierarchy.child_class_code = object_class.class_code + ) + select 1 + from object_class + join provenance_relation_resource_range range_rule + on range_rule.range_class_code = object_class.class_code + where range_rule.relation_code = new.relation_code + ) then + raise exception 'object resource % violates PROV-O range for %', + new.object_resource_id, new.relation_code; + end if; + + if relation_kind = 'datatype' then + select datatype_iri, lexical_value + into literal_datatype, literal_lexical + from provenance_literal_value + where literal_id = new.object_literal_id; + + if required_datatype is not null + and literal_datatype is distinct from required_datatype then + raise exception 'literal % violates datatype % for %', + new.object_literal_id, required_datatype, new.relation_code; + end if; + + if required_datatype = 'http://www.w3.org/2001/XMLSchema#dateTime' then + -- XSD dateTime offsets are Z or ±hh:mm with a maximum of ±14:00. + if literal_lexical !~ ( + '^[0-9]{4}-(0[1-9]|1[0-2])-' + '(0[1-9]|[12][0-9]|3[01])T' + '([01][0-9]|2[0-3]):[0-5][0-9]:' + '[0-5][0-9](\.[0-9]+)?' + '(Z|[+-]((0[0-9]|1[0-3]):[0-5][0-9]|14:00))$' + ) then + raise exception 'literal % violates lexical xsd:dateTime for %', + new.object_literal_id, new.relation_code; + end if; + begin + perform literal_lexical::timestamptz; + exception when others then + raise exception 'literal % violates lexical xsd:dateTime for %', + new.object_literal_id, new.relation_code; + end; + end if; + end if; + + return new; +end; +$$; + +drop trigger if exists provenance_assertion_contract_trigger on provenance_assertion; +create trigger provenance_assertion_contract_trigger +before insert or update on provenance_assertion +for each row execute function validate_provenance_assertion_contract(); + +create or replace function protect_provenance_contract_reference() +returns trigger +language plpgsql +as $$ +begin + if tg_table_name = 'provenance_resource_type' and exists ( + select 1 + from provenance_assertion + where subject_resource_id = (to_jsonb(old)->>'resource_id')::uuid + or object_resource_id = (to_jsonb(old)->>'resource_id')::uuid + ) then + raise exception 'referenced provenance resource types are immutable'; + end if; + + if tg_table_name = 'provenance_literal_value' and exists ( + select 1 + from provenance_assertion + where object_literal_id = (to_jsonb(old)->>'literal_id')::uuid + ) then + raise exception 'referenced provenance literal values are immutable'; + end if; + if tg_op = 'UPDATE' then + return new; + end if; + return old; +end; +$$; + +drop trigger if exists provenance_resource_type_reference_trigger + on provenance_resource_type; +create trigger provenance_resource_type_reference_trigger +before update or delete on provenance_resource_type +for each row execute function protect_provenance_contract_reference(); + +drop trigger if exists provenance_literal_value_reference_trigger + on provenance_literal_value; +create trigger provenance_literal_value_reference_trigger +before update or delete on provenance_literal_value +for each row execute function protect_provenance_contract_reference(); + +insert into provenance_class_definition (class_code, class_iri, class_local_name, class_label) values + ('prov_entity', 'http://www.w3.org/ns/prov#Entity', 'Entity', 'Entity'), + ('prov_activity', 'http://www.w3.org/ns/prov#Activity', 'Activity', 'Activity'), + ('prov_agent', 'http://www.w3.org/ns/prov#Agent', 'Agent', 'Agent'), + ('prov_collection', 'http://www.w3.org/ns/prov#Collection', 'Collection', 'Collection'), + ('prov_empty_collection', 'http://www.w3.org/ns/prov#EmptyCollection', 'EmptyCollection', 'Empty Collection'), + ('prov_bundle', 'http://www.w3.org/ns/prov#Bundle', 'Bundle', 'Bundle'), + ('prov_person', 'http://www.w3.org/ns/prov#Person', 'Person', 'Person'), + ('prov_software_agent', 'http://www.w3.org/ns/prov#SoftwareAgent', 'SoftwareAgent', 'Software Agent'), + ('prov_organization', 'http://www.w3.org/ns/prov#Organization', 'Organization', 'Organization'), + ('prov_location', 'http://www.w3.org/ns/prov#Location', 'Location', 'Location'), + ('prov_influence', 'http://www.w3.org/ns/prov#Influence', 'Influence', 'Influence'), + ('prov_entity_influence', 'http://www.w3.org/ns/prov#EntityInfluence', 'EntityInfluence', 'Entity Influence'), + ('prov_usage', 'http://www.w3.org/ns/prov#Usage', 'Usage', 'Usage'), + ('prov_start', 'http://www.w3.org/ns/prov#Start', 'Start', 'Start'), + ('prov_end', 'http://www.w3.org/ns/prov#End', 'End', 'End'), + ('prov_derivation', 'http://www.w3.org/ns/prov#Derivation', 'Derivation', 'Derivation'), + ('prov_primary_source', 'http://www.w3.org/ns/prov#PrimarySource', 'PrimarySource', 'Primary Source'), + ('prov_quotation', 'http://www.w3.org/ns/prov#Quotation', 'Quotation', 'Quotation'), + ('prov_revision', 'http://www.w3.org/ns/prov#Revision', 'Revision', 'Revision'), + ('prov_activity_influence', 'http://www.w3.org/ns/prov#ActivityInfluence', 'ActivityInfluence', 'Activity Influence'), + ('prov_generation', 'http://www.w3.org/ns/prov#Generation', 'Generation', 'Generation'), + ('prov_communication', 'http://www.w3.org/ns/prov#Communication', 'Communication', 'Communication'), + ('prov_invalidation', 'http://www.w3.org/ns/prov#Invalidation', 'Invalidation', 'Invalidation'), + ('prov_agent_influence', 'http://www.w3.org/ns/prov#AgentInfluence', 'AgentInfluence', 'Agent Influence'), + ('prov_attribution', 'http://www.w3.org/ns/prov#Attribution', 'Attribution', 'Attribution'), + ('prov_association', 'http://www.w3.org/ns/prov#Association', 'Association', 'Association'), + ('prov_plan', 'http://www.w3.org/ns/prov#Plan', 'Plan', 'Plan'), + ('prov_delegation', 'http://www.w3.org/ns/prov#Delegation', 'Delegation', 'Delegation'), + ('prov_instantaneous_event', 'http://www.w3.org/ns/prov#InstantaneousEvent', 'InstantaneousEvent', 'Instantaneous Event'), + ('prov_role', 'http://www.w3.org/ns/prov#Role', 'Role', 'Role') +on conflict (class_code) do update set + class_iri = excluded.class_iri, + class_local_name = excluded.class_local_name, + class_label = excluded.class_label; + +insert into provenance_class_hierarchy (child_class_code, parent_class_code) values + ('prov_collection', 'prov_entity'), + ('prov_empty_collection', 'prov_collection'), + ('prov_bundle', 'prov_entity'), + ('prov_person', 'prov_agent'), + ('prov_software_agent', 'prov_agent'), + ('prov_organization', 'prov_agent'), + ('prov_entity_influence', 'prov_influence'), + ('prov_usage', 'prov_instantaneous_event'), + ('prov_usage', 'prov_entity_influence'), + ('prov_start', 'prov_instantaneous_event'), + ('prov_start', 'prov_entity_influence'), + ('prov_end', 'prov_instantaneous_event'), + ('prov_end', 'prov_entity_influence'), + ('prov_derivation', 'prov_entity_influence'), + ('prov_primary_source', 'prov_derivation'), + ('prov_quotation', 'prov_derivation'), + ('prov_revision', 'prov_derivation'), + ('prov_activity_influence', 'prov_influence'), + ('prov_generation', 'prov_instantaneous_event'), + ('prov_generation', 'prov_activity_influence'), + ('prov_communication', 'prov_activity_influence'), + ('prov_invalidation', 'prov_instantaneous_event'), + ('prov_invalidation', 'prov_activity_influence'), + ('prov_agent_influence', 'prov_influence'), + ('prov_attribution', 'prov_agent_influence'), + ('prov_association', 'prov_agent_influence'), + ('prov_plan', 'prov_entity'), + ('prov_delegation', 'prov_agent_influence') +on conflict do nothing; + +insert into provenance_relation_definition (relation_code, relation_iri, relation_local_name, relation_label, property_kind_code, datatype_iri, symmetric_flag) values + ('prov_was_generated_by', 'http://www.w3.org/ns/prov#wasGeneratedBy', 'wasGeneratedBy', 'Was Generated By', 'object', null, false), + ('prov_was_derived_from', 'http://www.w3.org/ns/prov#wasDerivedFrom', 'wasDerivedFrom', 'Was Derived From', 'object', null, false), + ('prov_was_attributed_to', 'http://www.w3.org/ns/prov#wasAttributedTo', 'wasAttributedTo', 'Was Attributed To', 'object', null, false), + ('prov_started_at_time', 'http://www.w3.org/ns/prov#startedAtTime', 'startedAtTime', 'Started At Time', 'datatype', 'http://www.w3.org/2001/XMLSchema#dateTime', false), + ('prov_used', 'http://www.w3.org/ns/prov#used', 'used', 'Used', 'object', null, false), + ('prov_was_informed_by', 'http://www.w3.org/ns/prov#wasInformedBy', 'wasInformedBy', 'Was Informed By', 'object', null, false), + ('prov_ended_at_time', 'http://www.w3.org/ns/prov#endedAtTime', 'endedAtTime', 'Ended At Time', 'datatype', 'http://www.w3.org/2001/XMLSchema#dateTime', false), + ('prov_was_associated_with', 'http://www.w3.org/ns/prov#wasAssociatedWith', 'wasAssociatedWith', 'Was Associated With', 'object', null, false), + ('prov_acted_on_behalf_of', 'http://www.w3.org/ns/prov#actedOnBehalfOf', 'actedOnBehalfOf', 'Acted On Behalf Of', 'object', null, false), + ('prov_alternate_of', 'http://www.w3.org/ns/prov#alternateOf', 'alternateOf', 'Alternate Of', 'object', null, true), + ('prov_specialization_of', 'http://www.w3.org/ns/prov#specializationOf', 'specializationOf', 'Specialization Of', 'object', null, false), + ('prov_generated_at_time', 'http://www.w3.org/ns/prov#generatedAtTime', 'generatedAtTime', 'Generated At Time', 'datatype', 'http://www.w3.org/2001/XMLSchema#dateTime', false), + ('prov_had_primary_source', 'http://www.w3.org/ns/prov#hadPrimarySource', 'hadPrimarySource', 'Had Primary Source', 'object', null, false), + ('prov_value', 'http://www.w3.org/ns/prov#value', 'value', 'Value', 'datatype', null, false), + ('prov_was_quoted_from', 'http://www.w3.org/ns/prov#wasQuotedFrom', 'wasQuotedFrom', 'Was Quoted From', 'object', null, false), + ('prov_was_revision_of', 'http://www.w3.org/ns/prov#wasRevisionOf', 'wasRevisionOf', 'Was Revision Of', 'object', null, false), + ('prov_invalidated_at_time', 'http://www.w3.org/ns/prov#invalidatedAtTime', 'invalidatedAtTime', 'Invalidated At Time', 'datatype', 'http://www.w3.org/2001/XMLSchema#dateTime', false), + ('prov_was_invalidated_by', 'http://www.w3.org/ns/prov#wasInvalidatedBy', 'wasInvalidatedBy', 'Was Invalidated By', 'object', null, false), + ('prov_had_member', 'http://www.w3.org/ns/prov#hadMember', 'hadMember', 'Had Member', 'object', null, false), + ('prov_was_started_by', 'http://www.w3.org/ns/prov#wasStartedBy', 'wasStartedBy', 'Was Started By', 'object', null, false), + ('prov_was_ended_by', 'http://www.w3.org/ns/prov#wasEndedBy', 'wasEndedBy', 'Was Ended By', 'object', null, false), + ('prov_invalidated', 'http://www.w3.org/ns/prov#invalidated', 'invalidated', 'Invalidated', 'object', null, false), + ('prov_influenced', 'http://www.w3.org/ns/prov#influenced', 'influenced', 'Influenced', 'object', null, false), + ('prov_at_location', 'http://www.w3.org/ns/prov#atLocation', 'atLocation', 'At Location', 'object', null, false), + ('prov_generated', 'http://www.w3.org/ns/prov#generated', 'generated', 'Generated', 'object', null, false), + ('prov_was_influenced_by', 'http://www.w3.org/ns/prov#wasInfluencedBy', 'wasInfluencedBy', 'Was Influenced By', 'object', null, false), + ('prov_qualified_influence', 'http://www.w3.org/ns/prov#qualifiedInfluence', 'qualifiedInfluence', 'Qualified Influence', 'object', null, false), + ('prov_qualified_generation', 'http://www.w3.org/ns/prov#qualifiedGeneration', 'qualifiedGeneration', 'Qualified Generation', 'object', null, false), + ('prov_qualified_derivation', 'http://www.w3.org/ns/prov#qualifiedDerivation', 'qualifiedDerivation', 'Qualified Derivation', 'object', null, false), + ('prov_qualified_primary_source', 'http://www.w3.org/ns/prov#qualifiedPrimarySource', 'qualifiedPrimarySource', 'Qualified Primary Source', 'object', null, false), + ('prov_qualified_quotation', 'http://www.w3.org/ns/prov#qualifiedQuotation', 'qualifiedQuotation', 'Qualified Quotation', 'object', null, false), + ('prov_qualified_revision', 'http://www.w3.org/ns/prov#qualifiedRevision', 'qualifiedRevision', 'Qualified Revision', 'object', null, false), + ('prov_qualified_attribution', 'http://www.w3.org/ns/prov#qualifiedAttribution', 'qualifiedAttribution', 'Qualified Attribution', 'object', null, false), + ('prov_qualified_invalidation', 'http://www.w3.org/ns/prov#qualifiedInvalidation', 'qualifiedInvalidation', 'Qualified Invalidation', 'object', null, false), + ('prov_qualified_start', 'http://www.w3.org/ns/prov#qualifiedStart', 'qualifiedStart', 'Qualified Start', 'object', null, false), + ('prov_qualified_usage', 'http://www.w3.org/ns/prov#qualifiedUsage', 'qualifiedUsage', 'Qualified Usage', 'object', null, false), + ('prov_qualified_communication', 'http://www.w3.org/ns/prov#qualifiedCommunication', 'qualifiedCommunication', 'Qualified Communication', 'object', null, false), + ('prov_qualified_association', 'http://www.w3.org/ns/prov#qualifiedAssociation', 'qualifiedAssociation', 'Qualified Association', 'object', null, false), + ('prov_qualified_end', 'http://www.w3.org/ns/prov#qualifiedEnd', 'qualifiedEnd', 'Qualified End', 'object', null, false), + ('prov_qualified_delegation', 'http://www.w3.org/ns/prov#qualifiedDelegation', 'qualifiedDelegation', 'Qualified Delegation', 'object', null, false), + ('prov_influencer', 'http://www.w3.org/ns/prov#influencer', 'influencer', 'Influencer', 'object', null, false), + ('prov_entity', 'http://www.w3.org/ns/prov#entity', 'entity', 'Entity', 'object', null, false), + ('prov_had_usage', 'http://www.w3.org/ns/prov#hadUsage', 'hadUsage', 'Had Usage', 'object', null, false), + ('prov_had_generation', 'http://www.w3.org/ns/prov#hadGeneration', 'hadGeneration', 'Had Generation', 'object', null, false), + ('prov_activity', 'http://www.w3.org/ns/prov#activity', 'activity', 'Activity', 'object', null, false), + ('prov_agent', 'http://www.w3.org/ns/prov#agent', 'agent', 'Agent', 'object', null, false), + ('prov_had_plan', 'http://www.w3.org/ns/prov#hadPlan', 'hadPlan', 'Had Plan', 'object', null, false), + ('prov_had_activity', 'http://www.w3.org/ns/prov#hadActivity', 'hadActivity', 'Had Activity', 'object', null, false), + ('prov_at_time', 'http://www.w3.org/ns/prov#atTime', 'atTime', 'At Time', 'datatype', 'http://www.w3.org/2001/XMLSchema#dateTime', false), + ('prov_had_role', 'http://www.w3.org/ns/prov#hadRole', 'hadRole', 'Had Role', 'object', null, false) +on conflict (relation_code) do update set + relation_iri = excluded.relation_iri, + relation_local_name = excluded.relation_local_name, + relation_label = excluded.relation_label, + property_kind_code = excluded.property_kind_code, + datatype_iri = excluded.datatype_iri, + symmetric_flag = excluded.symmetric_flag; + +insert into provenance_relation_hierarchy (child_relation_code, parent_relation_code) values + ('prov_was_generated_by', 'prov_was_influenced_by'), + ('prov_was_derived_from', 'prov_was_influenced_by'), + ('prov_was_attributed_to', 'prov_was_influenced_by'), + ('prov_used', 'prov_was_influenced_by'), + ('prov_was_informed_by', 'prov_was_influenced_by'), + ('prov_was_associated_with', 'prov_was_influenced_by'), + ('prov_acted_on_behalf_of', 'prov_was_influenced_by'), + ('prov_specialization_of', 'prov_alternate_of'), + ('prov_had_primary_source', 'prov_was_derived_from'), + ('prov_was_quoted_from', 'prov_was_derived_from'), + ('prov_was_revision_of', 'prov_was_derived_from'), + ('prov_was_invalidated_by', 'prov_was_influenced_by'), + ('prov_had_member', 'prov_was_influenced_by'), + ('prov_was_started_by', 'prov_was_influenced_by'), + ('prov_was_ended_by', 'prov_was_influenced_by'), + ('prov_invalidated', 'prov_influenced'), + ('prov_generated', 'prov_influenced'), + ('prov_qualified_generation', 'prov_qualified_influence'), + ('prov_qualified_derivation', 'prov_qualified_influence'), + ('prov_qualified_primary_source', 'prov_qualified_influence'), + ('prov_qualified_quotation', 'prov_qualified_influence'), + ('prov_qualified_revision', 'prov_qualified_influence'), + ('prov_qualified_attribution', 'prov_qualified_influence'), + ('prov_qualified_invalidation', 'prov_qualified_influence'), + ('prov_qualified_start', 'prov_qualified_influence'), + ('prov_qualified_usage', 'prov_qualified_influence'), + ('prov_qualified_communication', 'prov_qualified_influence'), + ('prov_qualified_association', 'prov_qualified_influence'), + ('prov_qualified_end', 'prov_qualified_influence'), + ('prov_qualified_delegation', 'prov_qualified_influence'), + ('prov_entity', 'prov_influencer'), + ('prov_activity', 'prov_influencer'), + ('prov_agent', 'prov_influencer') +on conflict do nothing; + +insert into provenance_relation_domain (relation_code, domain_class_code) values + ('prov_was_generated_by', 'prov_entity'), + ('prov_was_derived_from', 'prov_entity'), + ('prov_was_attributed_to', 'prov_entity'), + ('prov_started_at_time', 'prov_activity'), + ('prov_used', 'prov_activity'), + ('prov_was_informed_by', 'prov_activity'), + ('prov_ended_at_time', 'prov_activity'), + ('prov_was_associated_with', 'prov_activity'), + ('prov_acted_on_behalf_of', 'prov_agent'), + ('prov_alternate_of', 'prov_entity'), + ('prov_specialization_of', 'prov_entity'), + ('prov_generated_at_time', 'prov_entity'), + ('prov_had_primary_source', 'prov_entity'), + ('prov_value', 'prov_entity'), + ('prov_was_quoted_from', 'prov_entity'), + ('prov_was_revision_of', 'prov_entity'), + ('prov_invalidated_at_time', 'prov_entity'), + ('prov_was_invalidated_by', 'prov_entity'), + ('prov_had_member', 'prov_collection'), + ('prov_was_started_by', 'prov_activity'), + ('prov_was_ended_by', 'prov_activity'), + ('prov_invalidated', 'prov_activity'), + ('prov_influenced', 'prov_entity'), + ('prov_influenced', 'prov_activity'), + ('prov_influenced', 'prov_agent'), + ('prov_at_location', 'prov_activity'), + ('prov_at_location', 'prov_agent'), + ('prov_at_location', 'prov_entity'), + ('prov_at_location', 'prov_instantaneous_event'), + ('prov_generated', 'prov_activity'), + ('prov_was_influenced_by', 'prov_entity'), + ('prov_was_influenced_by', 'prov_activity'), + ('prov_was_influenced_by', 'prov_agent'), + ('prov_qualified_influence', 'prov_entity'), + ('prov_qualified_influence', 'prov_activity'), + ('prov_qualified_influence', 'prov_agent'), + ('prov_qualified_generation', 'prov_entity'), + ('prov_qualified_derivation', 'prov_entity'), + ('prov_qualified_primary_source', 'prov_entity'), + ('prov_qualified_quotation', 'prov_entity'), + ('prov_qualified_revision', 'prov_entity'), + ('prov_qualified_attribution', 'prov_entity'), + ('prov_qualified_invalidation', 'prov_entity'), + ('prov_qualified_start', 'prov_activity'), + ('prov_qualified_usage', 'prov_activity'), + ('prov_qualified_communication', 'prov_activity'), + ('prov_qualified_association', 'prov_activity'), + ('prov_qualified_end', 'prov_activity'), + ('prov_qualified_delegation', 'prov_agent'), + ('prov_influencer', 'prov_influence'), + ('prov_entity', 'prov_entity_influence'), + ('prov_had_usage', 'prov_derivation'), + ('prov_had_generation', 'prov_derivation'), + ('prov_activity', 'prov_activity_influence'), + ('prov_agent', 'prov_agent_influence'), + ('prov_had_plan', 'prov_association'), + ('prov_had_activity', 'prov_delegation'), + ('prov_had_activity', 'prov_derivation'), + ('prov_had_activity', 'prov_end'), + ('prov_had_activity', 'prov_start'), + ('prov_at_time', 'prov_instantaneous_event'), + ('prov_had_role', 'prov_association'), + ('prov_had_role', 'prov_instantaneous_event') +on conflict do nothing; + +insert into provenance_relation_resource_range (relation_code, range_class_code) values + ('prov_was_generated_by', 'prov_activity'), + ('prov_was_derived_from', 'prov_entity'), + ('prov_was_attributed_to', 'prov_agent'), + ('prov_used', 'prov_entity'), + ('prov_was_informed_by', 'prov_activity'), + ('prov_was_associated_with', 'prov_agent'), + ('prov_acted_on_behalf_of', 'prov_agent'), + ('prov_alternate_of', 'prov_entity'), + ('prov_specialization_of', 'prov_entity'), + ('prov_had_primary_source', 'prov_entity'), + ('prov_was_quoted_from', 'prov_entity'), + ('prov_was_revision_of', 'prov_entity'), + ('prov_was_invalidated_by', 'prov_activity'), + ('prov_had_member', 'prov_entity'), + ('prov_was_started_by', 'prov_entity'), + ('prov_was_ended_by', 'prov_entity'), + ('prov_invalidated', 'prov_entity'), + ('prov_influenced', 'prov_entity'), + ('prov_influenced', 'prov_activity'), + ('prov_influenced', 'prov_agent'), + ('prov_at_location', 'prov_location'), + ('prov_generated', 'prov_entity'), + ('prov_was_influenced_by', 'prov_entity'), + ('prov_was_influenced_by', 'prov_activity'), + ('prov_was_influenced_by', 'prov_agent'), + ('prov_qualified_influence', 'prov_influence'), + ('prov_qualified_generation', 'prov_generation'), + ('prov_qualified_derivation', 'prov_derivation'), + ('prov_qualified_primary_source', 'prov_primary_source'), + ('prov_qualified_quotation', 'prov_quotation'), + ('prov_qualified_revision', 'prov_revision'), + ('prov_qualified_attribution', 'prov_attribution'), + ('prov_qualified_invalidation', 'prov_invalidation'), + ('prov_qualified_start', 'prov_start'), + ('prov_qualified_usage', 'prov_usage'), + ('prov_qualified_communication', 'prov_communication'), + ('prov_qualified_association', 'prov_association'), + ('prov_qualified_end', 'prov_end'), + ('prov_qualified_delegation', 'prov_delegation'), + ('prov_influencer', 'prov_entity'), + ('prov_influencer', 'prov_activity'), + ('prov_influencer', 'prov_agent'), + ('prov_entity', 'prov_entity'), + ('prov_had_usage', 'prov_usage'), + ('prov_had_generation', 'prov_generation'), + ('prov_activity', 'prov_activity'), + ('prov_agent', 'prov_agent'), + ('prov_had_plan', 'prov_plan'), + ('prov_had_activity', 'prov_activity'), + ('prov_had_role', 'prov_role') +on conflict do nothing; + +insert into provenance_qualification_definition (unqualified_relation_code, qualification_relation_code, influence_class_code, influencer_relation_code) values + ('prov_was_generated_by', 'prov_qualified_generation', 'prov_generation', 'prov_activity'), + ('prov_was_derived_from', 'prov_qualified_derivation', 'prov_derivation', 'prov_entity'), + ('prov_was_attributed_to', 'prov_qualified_attribution', 'prov_attribution', 'prov_agent'), + ('prov_used', 'prov_qualified_usage', 'prov_usage', 'prov_entity'), + ('prov_was_informed_by', 'prov_qualified_communication', 'prov_communication', 'prov_activity'), + ('prov_was_associated_with', 'prov_qualified_association', 'prov_association', 'prov_agent'), + ('prov_acted_on_behalf_of', 'prov_qualified_delegation', 'prov_delegation', 'prov_agent'), + ('prov_was_influenced_by', 'prov_qualified_influence', 'prov_influence', 'prov_influencer'), + ('prov_had_primary_source', 'prov_qualified_primary_source', 'prov_primary_source', 'prov_entity'), + ('prov_was_quoted_from', 'prov_qualified_quotation', 'prov_quotation', 'prov_entity'), + ('prov_was_revision_of', 'prov_qualified_revision', 'prov_revision', 'prov_entity'), + ('prov_was_invalidated_by', 'prov_qualified_invalidation', 'prov_invalidation', 'prov_activity'), + ('prov_was_started_by', 'prov_qualified_start', 'prov_start', 'prov_entity'), + ('prov_was_ended_by', 'prov_qualified_end', 'prov_end', 'prov_entity') +on conflict (unqualified_relation_code) do update set + qualification_relation_code = excluded.qualification_relation_code, + influence_class_code = excluded.influence_class_code, + influencer_relation_code = excluded.influencer_relation_code; + +insert into provenance_inverse_definition (relation_code, inverse_local_name, inverse_iri, inverse_relation_code, inverse_kind_code) values + ('prov_acted_on_behalf_of', 'hadDelegate', 'http://www.w3.org/ns/prov#hadDelegate', null, 'recommended'), + ('prov_activity', 'activityOfInfluence', 'http://www.w3.org/ns/prov#activityOfInfluence', null, 'recommended'), + ('prov_agent', 'agentOfInfluence', 'http://www.w3.org/ns/prov#agentOfInfluence', null, 'recommended'), + ('prov_alternate_of', 'alternateOf', 'http://www.w3.org/ns/prov#alternateOf', 'prov_alternate_of', 'defined'), + ('prov_at_location', 'locationOf', 'http://www.w3.org/ns/prov#locationOf', null, 'recommended'), + ('prov_entity', 'entityOfInfluence', 'http://www.w3.org/ns/prov#entityOfInfluence', null, 'recommended'), + ('prov_generated', 'wasGeneratedBy', 'http://www.w3.org/ns/prov#wasGeneratedBy', 'prov_was_generated_by', 'defined'), + ('prov_had_activity', 'wasActivityOfInfluence', 'http://www.w3.org/ns/prov#wasActivityOfInfluence', null, 'recommended'), + ('prov_had_generation', 'generatedAsDerivation', 'http://www.w3.org/ns/prov#generatedAsDerivation', null, 'recommended'), + ('prov_had_member', 'wasMemberOf', 'http://www.w3.org/ns/prov#wasMemberOf', null, 'recommended'), + ('prov_had_plan', 'wasPlanOf', 'http://www.w3.org/ns/prov#wasPlanOf', null, 'recommended'), + ('prov_had_primary_source', 'wasPrimarySourceOf', 'http://www.w3.org/ns/prov#wasPrimarySourceOf', null, 'recommended'), + ('prov_had_role', 'wasRoleIn', 'http://www.w3.org/ns/prov#wasRoleIn', null, 'recommended'), + ('prov_had_usage', 'wasUsedInDerivation', 'http://www.w3.org/ns/prov#wasUsedInDerivation', null, 'recommended'), + ('prov_influenced', 'wasInfluencedBy', 'http://www.w3.org/ns/prov#wasInfluencedBy', 'prov_was_influenced_by', 'defined'), + ('prov_influencer', 'hadInfluence', 'http://www.w3.org/ns/prov#hadInfluence', null, 'recommended'), + ('prov_invalidated', 'wasInvalidatedBy', 'http://www.w3.org/ns/prov#wasInvalidatedBy', 'prov_was_invalidated_by', 'defined'), + ('prov_qualified_association', 'qualifiedAssociationOf', 'http://www.w3.org/ns/prov#qualifiedAssociationOf', null, 'recommended'), + ('prov_qualified_attribution', 'qualifiedAttributionOf', 'http://www.w3.org/ns/prov#qualifiedAttributionOf', null, 'recommended'), + ('prov_qualified_communication', 'qualifiedCommunicationOf', 'http://www.w3.org/ns/prov#qualifiedCommunicationOf', null, 'recommended'), + ('prov_qualified_delegation', 'qualifiedDelegationOf', 'http://www.w3.org/ns/prov#qualifiedDelegationOf', null, 'recommended'), + ('prov_qualified_derivation', 'qualifiedDerivationOf', 'http://www.w3.org/ns/prov#qualifiedDerivationOf', null, 'recommended'), + ('prov_qualified_end', 'qualifiedEndOf', 'http://www.w3.org/ns/prov#qualifiedEndOf', null, 'recommended'), + ('prov_qualified_generation', 'qualifiedGenerationOf', 'http://www.w3.org/ns/prov#qualifiedGenerationOf', null, 'recommended'), + ('prov_qualified_influence', 'qualifiedInfluenceOf', 'http://www.w3.org/ns/prov#qualifiedInfluenceOf', null, 'recommended'), + ('prov_qualified_invalidation', 'qualifiedInvalidationOf', 'http://www.w3.org/ns/prov#qualifiedInvalidationOf', null, 'recommended'), + ('prov_qualified_primary_source', 'qualifiedSourceOf', 'http://www.w3.org/ns/prov#qualifiedSourceOf', null, 'recommended'), + ('prov_qualified_quotation', 'qualifiedQuotationOf', 'http://www.w3.org/ns/prov#qualifiedQuotationOf', null, 'recommended'), + ('prov_qualified_revision', 'revisedEntity', 'http://www.w3.org/ns/prov#revisedEntity', null, 'recommended'), + ('prov_qualified_start', 'qualifiedStartOf', 'http://www.w3.org/ns/prov#qualifiedStartOf', null, 'recommended'), + ('prov_qualified_usage', 'qualifiedUsingActivity', 'http://www.w3.org/ns/prov#qualifiedUsingActivity', null, 'recommended'), + ('prov_specialization_of', 'generalizationOf', 'http://www.w3.org/ns/prov#generalizationOf', null, 'recommended'), + ('prov_used', 'wasUsedBy', 'http://www.w3.org/ns/prov#wasUsedBy', null, 'recommended'), + ('prov_was_associated_with', 'wasAssociateFor', 'http://www.w3.org/ns/prov#wasAssociateFor', null, 'recommended'), + ('prov_was_attributed_to', 'contributed', 'http://www.w3.org/ns/prov#contributed', null, 'recommended'), + ('prov_was_derived_from', 'hadDerivation', 'http://www.w3.org/ns/prov#hadDerivation', null, 'recommended'), + ('prov_was_ended_by', 'ended', 'http://www.w3.org/ns/prov#ended', null, 'recommended'), + ('prov_was_generated_by', 'generated', 'http://www.w3.org/ns/prov#generated', 'prov_generated', 'defined'), + ('prov_was_influenced_by', 'influenced', 'http://www.w3.org/ns/prov#influenced', 'prov_influenced', 'defined'), + ('prov_was_informed_by', 'informed', 'http://www.w3.org/ns/prov#informed', null, 'recommended'), + ('prov_was_invalidated_by', 'invalidated', 'http://www.w3.org/ns/prov#invalidated', 'prov_invalidated', 'defined'), + ('prov_was_quoted_from', 'quotedAs', 'http://www.w3.org/ns/prov#quotedAs', null, 'recommended'), + ('prov_was_revision_of', 'hadRevision', 'http://www.w3.org/ns/prov#hadRevision', null, 'recommended'), + ('prov_was_started_by', 'started', 'http://www.w3.org/ns/prov#started', null, 'recommended') +on conflict (relation_code) do update set + inverse_local_name = excluded.inverse_local_name, + inverse_iri = excluded.inverse_iri, + inverse_relation_code = excluded.inverse_relation_code, + inverse_kind_code = excluded.inverse_kind_code; + +commit; diff --git a/migrations/0018_analysis_run_registry.sql b/migrations/0018_analysis_run_registry.sql new file mode 100644 index 000000000..b08d80b3b --- /dev/null +++ b/migrations/0018_analysis_run_registry.sql @@ -0,0 +1,569 @@ +-- Milestone 2 additive runtime bridge: normalized analysis-run registry. +-- +-- This migration records reproducibility, authorization scope, aggregate +-- reconciliation, and lifecycle evidence without storing source SQL, DSNs, +-- raw records, image bytes, provider payloads, credentials, or free-form JSON. +-- Snapshot availability is evidence-owned; the knowledge cutoff is run-owned, +-- so one immutable capture can support multiple historically valid analyses. + +begin; + +insert into common_lookup_value + (lookup_category, lookup_code, lookup_label, display_order) +values + ('analysis_run_kind', 'analysis_run_lineage', 'Lineage reconstruction', 0), + ('analysis_run_kind', 'analysis_run_report', 'Period report', 1), + ('analysis_run_kind', 'analysis_run_tepp', 'TEPP measurement', 2), + ('analysis_run_status', 'analysis_status_pending', 'Pending', 0), + ('analysis_run_status', 'analysis_status_running', 'Running', 1), + ('analysis_run_status', 'analysis_status_succeeded', 'Succeeded', 2), + ('analysis_run_status', 'analysis_status_failed', 'Failed', 3), + ('analysis_run_status', 'analysis_status_cancelled', 'Cancelled', 4), + ('analysis_run_scope', 'analysis_scope_all_visible', 'All authorized records', 0), + ('analysis_run_scope', 'analysis_scope_corporate_entity', 'Corporate entity', 1), + ('analysis_run_scope', 'analysis_scope_process_unit', 'Process unit', 2), + ('analysis_run_scope', 'analysis_scope_thread_group', 'Thread group', 3), + ('analysis_source_count', 'analysis_count_source_row', 'Source rows', 0), + ('analysis_source_count', 'analysis_count_document', 'Documents', 1), + ('analysis_source_count', 'analysis_count_thread', 'Threads', 2), + ('analysis_source_count', 'analysis_count_lineage_node', 'Lineage nodes', 3), + ('analysis_source_count', 'analysis_count_lineage_edge', 'Lineage edges', 4) +on conflict (lookup_code) do nothing; + +-- common_lookup_value deliberately makes lookup_code globally unique. A code +-- that already exists under another category is a migration conflict rather +-- than permission to attach the wrong vocabulary to an analysis column. +do $$ +declare + lookup_mismatch_count integer; +begin + select count(*) + into lookup_mismatch_count + from common_lookup_value as actual + join (values + ('analysis_run_lineage', 'analysis_run_kind'), + ('analysis_run_report', 'analysis_run_kind'), + ('analysis_run_tepp', 'analysis_run_kind'), + ('analysis_status_pending', 'analysis_run_status'), + ('analysis_status_running', 'analysis_run_status'), + ('analysis_status_succeeded', 'analysis_run_status'), + ('analysis_status_failed', 'analysis_run_status'), + ('analysis_status_cancelled', 'analysis_run_status'), + ('analysis_scope_all_visible', 'analysis_run_scope'), + ('analysis_scope_corporate_entity', 'analysis_run_scope'), + ('analysis_scope_process_unit', 'analysis_run_scope'), + ('analysis_scope_thread_group', 'analysis_run_scope'), + ('analysis_count_source_row', 'analysis_source_count'), + ('analysis_count_document', 'analysis_source_count'), + ('analysis_count_thread', 'analysis_source_count'), + ('analysis_count_lineage_node', 'analysis_source_count'), + ('analysis_count_lineage_edge', 'analysis_source_count') + ) as expected(lookup_code, lookup_category) + on expected.lookup_code = actual.lookup_code + where actual.lookup_category <> expected.lookup_category; + + if lookup_mismatch_count <> 0 then + raise exception 'analysis_run_registry_lookup_conflict'; + end if; +end +$$; + +create table if not exists analysis_source_snapshot ( + analysis_source_snapshot_id uuid primary key default uuid_generate_v4(), + snapshot_sha256 text not null unique, + source_contract_version text not null, + maximum_available_time timestamptz not null, + captured_at timestamptz not null, + created_at timestamptz not null default now(), + constraint analysis_source_snapshot_digest_check + check (snapshot_sha256 ~ '^[0-9a-f]{64}$'), + constraint analysis_source_snapshot_contract_check + check (length(btrim(source_contract_version)) between 1 and 128), + constraint analysis_source_snapshot_capture_check + check (maximum_available_time <= captured_at), + constraint analysis_source_snapshot_created_check + check (captured_at <= created_at) +); + +comment on table analysis_source_snapshot is + 'Immutable captured-source identity and latest evidence-availability time; ' + 'knowledge cutoffs belong to analysis_run, not the reusable snapshot.'; + +create table if not exists analysis_source_count ( + analysis_source_snapshot_id uuid not null + references analysis_source_snapshot (analysis_source_snapshot_id) + on delete cascade, + count_type_code text not null + references common_lookup_value (lookup_code), + count_value bigint not null, + primary key (analysis_source_snapshot_id, count_type_code), + constraint analysis_source_count_type_check + check (count_type_code in ( + 'analysis_count_source_row', + 'analysis_count_document', + 'analysis_count_thread', + 'analysis_count_lineage_node', + 'analysis_count_lineage_edge' + )), + constraint analysis_source_count_nonnegative_check + check (count_value >= 0) +); + +comment on table analysis_source_count is + 'One normalized aggregate reconciliation count per immutable snapshot and ' + 'count vocabulary; no source record is stored.'; + +create table if not exists analysis_run ( + analysis_run_id uuid primary key default uuid_generate_v4(), + analysis_source_snapshot_id uuid not null + references analysis_source_snapshot (analysis_source_snapshot_id), + run_kind_code text not null + references common_lookup_value (lookup_code), + requested_by_account_id uuid not null + references user_account (user_account_id), + idempotency_key text not null, + knowledge_cutoff timestamptz not null, + configuration_schema_version text not null, + configuration_sha256 text not null, + model_contract_sha256 text, + prompt_bundle_sha256 text, + code_revision_sha text not null, + requested_at timestamptz not null default now(), + constraint analysis_run_kind_check + check (run_kind_code in ( + 'analysis_run_lineage', + 'analysis_run_report', + 'analysis_run_tepp' + )), + constraint analysis_run_idempotency_key_check + check ( + idempotency_key = btrim(idempotency_key) + and length(idempotency_key) between 1 and 256 + and idempotency_key !~ '[[:cntrl:]]' + ), + constraint analysis_run_configuration_version_check + check ( + configuration_schema_version = btrim(configuration_schema_version) + and length(configuration_schema_version) between 1 and 128 + ), + constraint analysis_run_configuration_digest_check + check (configuration_sha256 ~ '^[0-9a-f]{64}$'), + constraint analysis_run_model_digest_check + check ( + model_contract_sha256 is null + or model_contract_sha256 ~ '^[0-9a-f]{64}$' + ), + constraint analysis_run_prompt_digest_check + check ( + prompt_bundle_sha256 is null + or prompt_bundle_sha256 ~ '^[0-9a-f]{64}$' + ), + constraint analysis_run_code_revision_check + check (code_revision_sha ~ '^(?:[0-9a-f]{40}|[0-9a-f]{64})$'), + constraint analysis_run_request_time_check + check (knowledge_cutoff <= requested_at), + unique (requested_by_account_id, idempotency_key) +); + +create index if not exists analysis_run_snapshot_idx + on analysis_run (analysis_source_snapshot_id); +create index if not exists analysis_run_kind_requested_idx + on analysis_run (run_kind_code, requested_at desc); +create index if not exists analysis_run_requester_idx + on analysis_run (requested_by_account_id, requested_at desc); + +comment on table analysis_run is + 'Immutable account-scoped analysis request bound to one snapshot, one ' + 'knowledge cutoff, and reproducibility digests; lifecycle is event-derived.'; + +create table if not exists analysis_run_scope ( + analysis_run_id uuid primary key + references analysis_run (analysis_run_id), + scope_kind_code text not null + references common_lookup_value (lookup_code), + corporate_entity_id uuid + references corporate_entity (corporate_entity_id), + process_unit_id uuid + references process_unit (process_unit_id), + scope_key text, + constraint analysis_run_scope_kind_check + check (scope_kind_code in ( + 'analysis_scope_all_visible', + 'analysis_scope_corporate_entity', + 'analysis_scope_process_unit', + 'analysis_scope_thread_group' + )), + constraint analysis_run_scope_shape_check + check ( + (scope_kind_code = 'analysis_scope_all_visible' + and corporate_entity_id is null + and process_unit_id is null + and scope_key is null) + or + (scope_kind_code = 'analysis_scope_corporate_entity' + and corporate_entity_id is not null + and process_unit_id is null + and scope_key is null) + or + (scope_kind_code = 'analysis_scope_process_unit' + and corporate_entity_id is null + and process_unit_id is not null + and scope_key is null) + or + (scope_kind_code = 'analysis_scope_thread_group' + and corporate_entity_id is null + and process_unit_id is null + and scope_key is not null + and scope_key = btrim(scope_key) + and length(scope_key) between 1 and 256 + and scope_key !~ '[[:cntrl:]]') + ) +); + +create index if not exists analysis_run_scope_entity_idx + on analysis_run_scope (corporate_entity_id) + where corporate_entity_id is not null; +create index if not exists analysis_run_scope_unit_idx + on analysis_run_scope (process_unit_id) + where process_unit_id is not null; + +comment on table analysis_run_scope is + 'One immutable authorization-relevant scope is required before lifecycle ' + 'evidence; process-unit ownership remains derivable from process_unit.'; + +create table if not exists analysis_run_status_event ( + analysis_run_id uuid not null + references analysis_run (analysis_run_id), + status_ordinal integer not null, + status_code text not null + references common_lookup_value (lookup_code), + occurred_at timestamptz not null, + recorded_at timestamptz not null default clock_timestamp(), + failure_code text, + retryable boolean not null default false, + primary key (analysis_run_id, status_ordinal), + constraint analysis_run_status_code_check + check (status_code in ( + 'analysis_status_pending', + 'analysis_status_running', + 'analysis_status_succeeded', + 'analysis_status_failed', + 'analysis_status_cancelled' + )), + constraint analysis_run_status_ordinal_check + check (status_ordinal >= 1), + constraint analysis_run_status_time_check + check (occurred_at <= recorded_at), + constraint analysis_run_status_failure_shape_check + check ( + (status_code = 'analysis_status_failed' + and failure_code is not null + and failure_code ~ '^[a-z][a-z0-9_]{0,127}$') + or + (status_code <> 'analysis_status_failed' + and failure_code is null + and retryable = false) + ) +); + +create index if not exists analysis_run_status_current_idx + on analysis_run_status_event (analysis_run_id, status_ordinal desc); + +comment on table analysis_run_status_event is + 'Append-only, contiguous, monotonic state-machine evidence; failure_code is ' + 'a bounded machine code and never contains raw provider or source payloads.'; + +create or replace function reject_analysis_source_snapshot_update() +returns trigger +language plpgsql +as $$ +begin + raise exception 'analysis_source_snapshot_is_immutable'; +end +$$; + +comment on function reject_analysis_source_snapshot_update() is + 'Rejects mutation of captured source identity and availability evidence.'; + +drop trigger if exists analysis_source_snapshot_update_reject + on analysis_source_snapshot; +create trigger analysis_source_snapshot_update_reject +before update on analysis_source_snapshot +for each row execute function reject_analysis_source_snapshot_update(); + +create or replace function reject_analysis_source_count_update() +returns trigger +language plpgsql +as $$ +begin + raise exception 'analysis_source_count_is_immutable'; +end +$$; + +comment on function reject_analysis_source_count_update() is + 'Rejects replacement of a snapshot aggregate; delete and reinsert is only ' + 'permitted before the snapshot is attached to a run.'; + +drop trigger if exists analysis_source_count_update_reject + on analysis_source_count; +create trigger analysis_source_count_update_reject +before update on analysis_source_count +for each row execute function reject_analysis_source_count_update(); + +create or replace function enforce_analysis_source_count_freeze() +returns trigger +language plpgsql +as $$ +declare + affected_snapshot_id uuid; +begin + if tg_op = 'DELETE' then + affected_snapshot_id := old.analysis_source_snapshot_id; + else + affected_snapshot_id := new.analysis_source_snapshot_id; + end if; + + -- Both count mutation and run creation lock this row first. That common + -- lock order closes the race between the final count write and first run. + perform 1 + from analysis_source_snapshot + where analysis_source_snapshot_id = affected_snapshot_id + for update; + + if exists ( + select 1 + from analysis_run + where analysis_source_snapshot_id = affected_snapshot_id + ) then + raise exception 'analysis_source_count_frozen_after_run'; + end if; + + if tg_op = 'DELETE' then + return old; + end if; + return new; +end +$$; + +comment on function enforce_analysis_source_count_freeze() is + 'Serializes count insert/delete against first run creation and rejects ' + 'changes after any run references the snapshot.'; + +drop trigger if exists analysis_source_count_freeze_guard + on analysis_source_count; +create trigger analysis_source_count_freeze_guard +before insert or delete on analysis_source_count +for each row execute function enforce_analysis_source_count_freeze(); + +create or replace function enforce_analysis_run_knowledge_cutoff() +returns trigger +language plpgsql +as $$ +declare + snapshot_available_time timestamptz; + snapshot_capture_time timestamptz; +begin + if new.requested_at > clock_timestamp() then + raise exception 'analysis_run_request_time_in_future'; + end if; + + select maximum_available_time, captured_at + into snapshot_available_time, snapshot_capture_time + from analysis_source_snapshot + where analysis_source_snapshot_id = new.analysis_source_snapshot_id + for update; + + if not found then + raise exception 'analysis_source_snapshot_not_found'; + end if; + if snapshot_available_time > new.knowledge_cutoff then + raise exception 'analysis_run_future_information_leakage'; + end if; + if snapshot_capture_time > new.requested_at then + raise exception 'analysis_run_snapshot_captured_after_request'; + end if; + return new; +end +$$; + +comment on function enforce_analysis_run_knowledge_cutoff() is + 'Locks the immutable snapshot and rejects run cutoffs earlier than the ' + 'latest admitted evidence or requests earlier than snapshot capture.'; + +drop trigger if exists analysis_run_knowledge_cutoff_guard + on analysis_run; +create trigger analysis_run_knowledge_cutoff_guard +before insert on analysis_run +for each row execute function enforce_analysis_run_knowledge_cutoff(); + +drop trigger if exists analysis_run_update_reject + on analysis_run; +drop trigger if exists analysis_run_mutation_reject + on analysis_run; +drop function if exists reject_analysis_run_update(); + +create or replace function reject_analysis_run_mutation() +returns trigger +language plpgsql +as $$ +begin + raise exception 'analysis_run_request_is_immutable'; +end +$$; + +comment on function reject_analysis_run_mutation() is + 'Rejects update or delete of actor, cutoff, idempotency, and reproducibility ' + 'evidence; run progress belongs to append-only status events.'; + +create trigger analysis_run_mutation_reject +before update or delete on analysis_run +for each row execute function reject_analysis_run_mutation(); + +create or replace function reject_analysis_run_scope_mutation() +returns trigger +language plpgsql +as $$ +begin + raise exception 'analysis_run_scope_is_immutable'; +end +$$; + +comment on function reject_analysis_run_scope_mutation() is + 'Rejects update or delete of the authorization-relevant scope attached to ' + 'an immutable analysis request.'; + +drop trigger if exists analysis_run_scope_mutation_reject + on analysis_run_scope; +create trigger analysis_run_scope_mutation_reject +before update or delete on analysis_run_scope +for each row execute function reject_analysis_run_scope_mutation(); + +create or replace function reject_analysis_run_status_mutation() +returns trigger +language plpgsql +as $$ +begin + raise exception 'analysis_run_status_event_is_append_only'; +end +$$; + +comment on function reject_analysis_run_status_mutation() is + 'Rejects update or delete of state-machine evidence.'; + +drop trigger if exists analysis_run_status_event_update_reject + on analysis_run_status_event; +create trigger analysis_run_status_event_update_reject +before update on analysis_run_status_event +for each row execute function reject_analysis_run_status_mutation(); + +drop trigger if exists analysis_run_status_event_delete_reject + on analysis_run_status_event; +create trigger analysis_run_status_event_delete_reject +before delete on analysis_run_status_event +for each row execute function reject_analysis_run_status_mutation(); + +create or replace function enforce_analysis_run_status_transition() +returns trigger +language plpgsql +as $$ +declare + previous_ordinal integer; + previous_status_code text; + previous_occurred_at timestamptz; + run_requested_at timestamptz; +begin + -- The immutable parent row is a per-run serialization lock. It prevents + -- concurrent writers from both accepting the same next ordinal. + select requested_at + into run_requested_at + from analysis_run + where analysis_run_id = new.analysis_run_id + for update; + + if not found then + raise exception 'analysis_run_not_found'; + end if; + if not exists ( + select 1 from analysis_run_scope + where analysis_run_id = new.analysis_run_id + ) then + raise exception 'analysis_run_scope_required'; + end if; + if new.occurred_at < run_requested_at then + raise exception 'analysis_run_status_before_request'; + end if; + new.recorded_at := clock_timestamp(); + + select status_ordinal, status_code, occurred_at + into previous_ordinal, previous_status_code, previous_occurred_at + from analysis_run_status_event + where analysis_run_id = new.analysis_run_id + order by status_ordinal desc + limit 1; + + if previous_ordinal is null then + if new.status_ordinal <> 1 + or new.status_code <> 'analysis_status_pending' then + raise exception 'analysis_run_first_status_must_be_pending'; + end if; + return new; + end if; + + if new.status_ordinal <> previous_ordinal + 1 then + raise exception 'analysis_run_status_ordinal_not_contiguous'; + end if; + if new.occurred_at < previous_occurred_at then + raise exception 'analysis_run_status_time_not_monotonic'; + end if; + + if previous_status_code = 'analysis_status_pending' then + if new.status_code not in ( + 'analysis_status_running', + 'analysis_status_cancelled' + ) then + raise exception 'analysis_run_status_transition_invalid'; + end if; + elsif previous_status_code = 'analysis_status_running' then + if new.status_code not in ( + 'analysis_status_succeeded', + 'analysis_status_failed', + 'analysis_status_cancelled' + ) then + raise exception 'analysis_run_status_transition_invalid'; + end if; + else + raise exception 'analysis_run_terminal_status_has_no_successor'; + end if; + + return new; +end +$$; + +comment on function enforce_analysis_run_status_transition() is + 'Serializes status appends and requires immutable scope, request-time ' + 'ordering, database-recorded time, legal transitions, and terminal finality.'; + +drop trigger if exists analysis_run_status_transition_guard + on analysis_run_status_event; +create trigger analysis_run_status_transition_guard +before insert on analysis_run_status_event +for each row execute function enforce_analysis_run_status_transition(); + +create or replace view analysis_run_current_status as +select distinct on (status_event.analysis_run_id) + status_event.analysis_run_id, + status_event.status_code, + status_event.status_ordinal, + status_event.occurred_at, + status_event.recorded_at, + status_event.failure_code, + status_event.retryable + from analysis_run_status_event as status_event + order by status_event.analysis_run_id, + status_event.status_ordinal desc; + +comment on view analysis_run_current_status is + 'Latest append-only status projection for each run; never a second mutable ' + 'lifecycle authority.'; + +commit; diff --git a/migrations/0019_role_catalog_identity.sql b/migrations/0019_role_catalog_identity.sql new file mode 100644 index 000000000..2881be9b3 --- /dev/null +++ b/migrations/0019_role_catalog_identity.sql @@ -0,0 +1,46 @@ +-- ADR 0019: bind each R&R role to the catalog row resolved for that +-- role. corporate_entity.entity_name is not unique, so a fetch join on +-- name can attach a homonym or duplicate the role. Mention tables are +-- post-scoped, not role-scoped, and cannot reconstruct that binding. + +alter table post_summary_role + add column if not exists cataloged_team_id uuid + references cataloged_team (team_id); + +alter table post_summary_role + add column if not exists cataloged_corporate_entity_id uuid + references corporate_entity (corporate_entity_id); + +-- Teams already have a unique (team_name, affiliated_organization_name) +-- key. Backfill only when that pair was mentioned on the same post. +update post_summary_role as role + set cataloged_team_id = team.team_id + from cataloged_team as team + join post_team_mention as mention + on mention.team_id = team.team_id + where role.actor_type_code = 'prov_team' + and role.cataloged_team_id is null + and mention.post_id = role.post_id + and team.team_name = role.actor_name + and team.affiliated_organization_name + is not distinct from role.affiliated_organization_name; + +-- Organizations: copy a mention only when exactly one mentioned org on +-- that post has this role's actor_name. Two same-named mentions stay +-- unbound rather than guessing. +update post_summary_role role + set cataloged_corporate_entity_id = matched.corporate_entity_id + from ( + select mention.post_id, + org.entity_name, + min(org.corporate_entity_id) as corporate_entity_id + from post_organization_mention mention + join corporate_entity org + on org.corporate_entity_id = mention.corporate_entity_id + group by mention.post_id, org.entity_name + having count(*) = 1 + ) matched + where role.actor_type_code = 'prov_organization' + and role.cataloged_corporate_entity_id is null + and role.post_id = matched.post_id + and role.actor_name = matched.entity_name; diff --git a/migrations/rollback/0018_analysis_run_registry.sql b/migrations/rollback/0018_analysis_run_registry.sql new file mode 100644 index 000000000..f91abc47c --- /dev/null +++ b/migrations/rollback/0018_analysis_run_registry.sql @@ -0,0 +1,70 @@ +-- Fail-closed rollback for migration 0018. +-- +-- Registry evidence must be exported or explicitly deleted under an approved +-- retention procedure before these objects can be removed. Re-running this +-- rollback after a successful empty rollback is safe. + +begin; + +do $$ +declare + relation_name text; + relation_has_rows boolean; +begin + foreach relation_name in array array[ + 'analysis_run_status_event', + 'analysis_run_scope', + 'analysis_run', + 'analysis_source_count', + 'analysis_source_snapshot' + ] loop + if to_regclass('public.' || relation_name) is not null then + execute format('select exists (select 1 from %I)', relation_name) + into relation_has_rows; + if relation_has_rows then + raise exception 'analysis_run_registry_not_empty'; + end if; + end if; + end loop; +end +$$; + +drop view if exists analysis_run_current_status; +drop table if exists analysis_run_status_event; +drop table if exists analysis_run_scope; +drop table if exists analysis_run; +drop table if exists analysis_source_count; +drop table if exists analysis_source_snapshot; + +drop function if exists enforce_analysis_run_status_transition(); +drop function if exists reject_analysis_run_status_mutation(); +drop function if exists reject_analysis_run_scope_mutation(); +drop function if exists reject_analysis_run_mutation(); +drop function if exists reject_analysis_run_update(); +drop function if exists enforce_analysis_run_knowledge_cutoff(); +drop function if exists enforce_analysis_source_count_freeze(); +drop function if exists reject_analysis_source_count_update(); +drop function if exists reject_analysis_source_snapshot_update(); + +delete from common_lookup_value + where lookup_code in ( + 'analysis_run_lineage', + 'analysis_run_report', + 'analysis_run_tepp', + 'analysis_status_pending', + 'analysis_status_running', + 'analysis_status_succeeded', + 'analysis_status_failed', + 'analysis_status_cancelled', + 'analysis_scope_all_visible', + 'analysis_scope_corporate_entity', + 'analysis_scope_process_unit', + 'analysis_scope_thread_group', + 'analysis_count_source_row', + 'analysis_count_document', + 'analysis_count_thread', + 'analysis_count_lineage_node', + 'analysis_count_lineage_edge' + ); + +commit; diff --git a/migrations/rollback/0019_role_catalog_identity.sql b/migrations/rollback/0019_role_catalog_identity.sql new file mode 100644 index 000000000..5efafed8b --- /dev/null +++ b/migrations/rollback/0019_role_catalog_identity.sql @@ -0,0 +1,8 @@ +-- Drop role-scoped catalog identity columns added by 0019. +-- Mention tables remain; only the role-row binding is removed. + +alter table post_summary_role + drop column if exists cataloged_team_id; + +alter table post_summary_role + drop column if exists cataloged_corporate_entity_id; diff --git a/pyproject.toml b/pyproject.toml index 9a2272d3a..12222bdad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lineageweave" -version = "0.71.0" +version = "0.86.3" description = "Reconstructs git-branch-style lineage DAGs from scattered short records using multi-channel score fusion and LLM adjudication." readme = "README.md" license = { text = "MIT" } @@ -17,8 +17,8 @@ dependencies = [ # distributions don't reliably inherit the OS trust store. "certifi>=2024.0.0", # The standard Python RDF/OWL library -- parses and validates - # docs/ontology/lineageweave-kg.ttl (ADR 0004). Pure Python, no - # Rust/C toolchain, unlike fast-mlsirm. + # docs/ontology/lineageweave-kg.ttl and the standards-complete PROV-O + # support profile (ADR 0011). Pure Python, no Rust/C toolchain. "rdflib>=7.0.0", ] @@ -26,6 +26,7 @@ dependencies = [ dev = [ "pillow>=12.3.0", "psycopg2-binary>=2.9.12", + "coverage>=7.6", "pyjwt[crypto]>=2.8.0", "pytest>=8.0", "httpx>=0.27.0", diff --git a/scripts/seed_demo_data.py b/scripts/seed_demo_data.py index 72318f33a..9d246445d 100644 --- a/scripts/seed_demo_data.py +++ b/scripts/seed_demo_data.py @@ -13,12 +13,14 @@ HTTP goes through ``lineageweave.http_client`` (http(s) allowlist). -Usage: python3 scripts/seed_demo_data.py [--postgres-dsn ...] [--keycloak-base-url ...] [--valkey-url ...] +Usage: KEYCLOAK_ADMIN_PASSWORD=... python3 scripts/seed_demo_data.py [--postgres-dsn ...] [--keycloak-base-url ...] [--valkey-url ...] """ from __future__ import annotations import argparse +import hashlib +import os import sys from pathlib import Path from urllib.parse import urlencode @@ -29,14 +31,20 @@ import psycopg2 from lineageweave.http_client import get_json_list, post_form +from lineageweave.tepp_client import AnalysisRunRequest, TeppClient, TeppNotAvailable REALM = "lineageweave-demo" DEFAULT_POSTGRES_DSN = "postgresql://lineageweave:lineageweave_dev_only@localhost:15432/lineageweave" DEFAULT_KEYCLOAK_BASE_URL = "http://localhost:18080" -DEFAULT_KEYCLOAK_ADMIN_USER = "admin" -DEFAULT_KEYCLOAK_ADMIN_PASSWORD = "admin_dev_only" # nosec B105 -- throwaway local-dev-only Keycloak seed credential +DEFAULT_KEYCLOAK_ADMIN_USER = os.environ.get("KEYCLOAK_ADMIN", "admin") DEFAULT_VALKEY_URL = "redis://localhost:16379/0" +# ADR 0013: one Demo Corp capture, many runs (lineage + TEPP). +DEMO_SOURCE_SNAPSHOT_MATERIAL = b"lineageweave-synthetic-demo-snapshot-v1" +DEMO_SOURCE_CONTRACT_VERSION = "demo-source-contract-v1" +DEMO_LINEAGE_IDEMPOTENCY_KEY = "demo-lineage-seed-2026-w02" +DEMO_TEPP_IDEMPOTENCY_KEY = "demo-tepp-seed-2026-w02" + # (post_title, ticket_title, due_date) -- Event Lineage fixtures a report # member click opens. Activity seed uses the same titles so Valkey matches. FIXTURE_TICKET_SPECS = ( @@ -106,6 +114,13 @@ def seed( cur.execute((migrations / "0009_shared_metric_bank.sql").read_text()) cur.execute((migrations / "0010_report_item_information.sql").read_text()) cur.execute((migrations / "0011_post_chat_result.sql").read_text()) + cur.execute((migrations / "0012_role_responsibility_agent_type.sql").read_text()) + cur.execute((migrations / "0013_person_job_title.sql").read_text()) + cur.execute((migrations / "0014_role_responsibility_team_actor_type.sql").read_text()) + cur.execute((migrations / "0015_organization_name_resolution.sql").read_text()) + cur.execute((migrations / "0016_cross_post_actor_identity.sql").read_text()) + cur.execute((migrations / "0018_analysis_run_registry.sql").read_text()) + cur.execute((migrations / "0019_role_catalog_identity.sql").read_text()) cur.execute( """ insert into common_lookup_value (lookup_category, lookup_code, lookup_label, display_order) values @@ -217,18 +232,23 @@ def seed( cur.execute("select post_id from source_post where post_title = 'Demo public post'") if cur.fetchone() is None: cur.execute( - "insert into source_post (author_account_id, corporate_entity_id, process_unit_id, post_title, post_body, voc_type_code, visibility_code) " + "insert into source_post (author_account_id, corporate_entity_id, process_unit_id, post_title, post_body, voc_type_code, visibility_code, created_at) " "values (%s, %s, %s, 'Demo public post', " "'Ada West at Demo Corp followed up with Priya Nair at Northridge Grid about the delayed shipment.', " - "'voc', 'public')", + "'voc', 'public', '2026-01-10T12:00:00Z')", (account_ids["demo.analyst"], corporate_entity_id, process_units["DEMO-PU-A"]), ) cur.execute( - "insert into source_post (author_account_id, corporate_entity_id, process_unit_id, post_title, post_body, voc_type_code, visibility_code) " - "values (%s, %s, %s, 'Demo private post', 'A synthetic private post scoped to Demo Corp accounts.', 'vom', 'private')", + "insert into source_post (author_account_id, corporate_entity_id, process_unit_id, post_title, post_body, voc_type_code, visibility_code, created_at) " + "values (%s, %s, %s, 'Demo private post', 'A synthetic private post scoped to Demo Corp accounts.', 'vom', 'private', '2026-01-10T12:00:00Z')", (account_ids["demo.admin"], corporate_entity_id, process_units["DEMO-PU-HQ"]), ) + cur.execute( + "update source_post set created_at = '2026-01-10T12:00:00Z' " + "where post_title in ('Demo public post', 'Demo private post') " + "and created_at > '2026-01-12T12:00:00Z'" + ) cur.execute("select post_id from source_post where post_title = 'Demo public post'") demo_public_post_id = cur.fetchone()[0] cur.execute( @@ -251,8 +271,9 @@ def seed( from lineageweave.knowledge_graph import knowledge_graph_edges_for_post cur.execute( - "insert into cataloged_person (person_name, person_side_code) values " - "('Ada West', 'our_side'), ('Priya Nair', 'counterparty') " + "insert into cataloged_person (person_name, person_side_code, last_known_job_title) values " + "('Ada West', 'our_side', 'Account manager'), " + "('Priya Nair', 'counterparty', 'Procurement lead') " "returning person_name, person_id" ) people = dict(cur.fetchall()) @@ -290,6 +311,8 @@ def seed( ), ) + _seed_demo_public_summary(cur, demo_public_post_id) + _seed_reconstructed_lineage( cur, account_ids["demo.analyst"], @@ -302,10 +325,10 @@ def seed( corporate_entity_id, process_units["DEMO-PU-LINEAGE"], ) + _seed_fixture_keymen_and_voc(cur, corporate_entity_id) _seed_fixture_summaries(cur) _seed_fixture_chats(cur) _seed_fixture_evaluations(cur) - _seed_fixture_keymen_and_voc(cur, corporate_entity_id) _seed_fixture_tickets(cur) _seed_fixture_ticket_activity(cur, account_ids["demo.analyst"], valkey_url) _seed_demo_period_report( @@ -314,6 +337,16 @@ def seed( corporate_entity_id, process_units["DEMO-PU-LINEAGE"], ) + _seed_demo_analysis_run( + cur, + account_ids["demo.analyst"], + corporate_entity_id, + ) + _seed_demo_tepp_run( + cur, + account_ids["demo.analyst"], + corporate_entity_id, + ) conn.commit() finally: @@ -392,6 +425,7 @@ def _seed_reconstructed_lineage(cur, author_account_id, corporate_entity_id, pro def _write_post_summary(cur, post_id, summary) -> None: """Replace the stored summary for ``post_id`` (idempotent re-seed).""" + cur.execute("delete from post_summary_person_mention where post_id = %s", (post_id,)) cur.execute("delete from post_summary_result where post_id = %s", (post_id,)) cur.execute( "insert into post_summary_result (post_id, korean_summary) values (%s, %s)", @@ -404,10 +438,37 @@ def _write_post_summary(cur, post_id, summary) -> None: ) for role in summary.roles_and_responsibilities: cur.execute( - "insert into post_summary_role (post_id, person_name, responsibility) values (%s, %s, %s)", - (post_id, role.person_name, role.responsibility), + "insert into post_summary_role " + "(post_id, actor_name, responsibility, actor_type_code, affiliated_organization_name) " + "values (%s, %s, %s, %s, %s)", + ( + post_id, + role.actor_name, + role.responsibility, + role.actor_type_code, + role.affiliated_organization_name, + ), ) + cur.execute( + """ + insert into post_summary_person_mention (post_id, person_id) + select distinct role.post_id, matched_person.person_id + from post_summary_role role + join lateral ( + select person.person_id + from cataloged_person person + where person.person_name = role.actor_name + order by person.created_at, person.person_id + limit 1 + ) matched_person on true + where role.post_id = %s + and role.actor_type_code = 'prov_person' + on conflict do nothing + """, + (post_id,), + ) + def _write_post_chat(cur, post_id, question: str, chat) -> None: """Replace the stored Ask exchange for ``(post_id, question)``.""" @@ -572,22 +633,27 @@ def _seed_fixture_evaluations(cur) -> None: def _ensure_demo_people(cur, corporate_entity_id) -> dict[str, str]: """Ada West / Priya Nair / Jordan Hale plus their affiliations. Idempotent.""" people: dict[str, str] = {} - for name, side in ( - ("Ada West", "our_side"), - ("Priya Nair", "counterparty"), - ("Jordan Hale", "our_side"), + for name, side, title in ( + ("Ada West", "our_side", "Account manager"), + ("Priya Nair", "counterparty", "Procurement lead"), + ("Jordan Hale", "our_side", "Bid coordinator"), ): cur.execute("select person_id from cataloged_person where person_name = %s", (name,)) row = cur.fetchone() if row is None: cur.execute( - "insert into cataloged_person (person_name, person_side_code) " - "values (%s, %s) returning person_id", - (name, side), + "insert into cataloged_person (person_name, person_side_code, last_known_job_title) " + "values (%s, %s, %s) returning person_id", + (name, side, title), ) people[name] = str(cur.fetchone()[0]) else: people[name] = str(row[0]) + cur.execute( + "update cataloged_person set last_known_job_title = coalesce(last_known_job_title, %s) " + "where person_id = %s", + (title, people[name]), + ) cur.execute( "insert into person_affiliation " "(person_id, affiliated_organization_name, affiliated_corporate_entity_id) " @@ -1151,14 +1217,249 @@ def _seed_demo_period_report(cur, author_account_id, corporate_entity_id, proces _persist_seed_period_report(cur, "process_unit", high_key, w03, week3[high_key]) +def demo_source_snapshot_sha256() -> str: + """Return the reusable Demo Corp snapshot digest (never a source row).""" + return hashlib.sha256(DEMO_SOURCE_SNAPSHOT_MATERIAL).hexdigest() + + +def _ensure_demo_source_snapshot(cur): + """Return the shared Demo Corp capture, inserting it on first seed. + + Lineage and TEPP runs share this snapshot (ADR 0013: one capture, + many runs). The digest is a hash of a fixed demo contract string -- + never a source row or DSN. + """ + digest = demo_source_snapshot_sha256() + cur.execute( + "select analysis_source_snapshot_id from analysis_source_snapshot " + "where snapshot_sha256 = %s", + (digest,), + ) + snapshot_row = cur.fetchone() + if snapshot_row is not None: + return snapshot_row[0] + cur.execute( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, %s, + '2026-01-12T00:00:00Z', '2026-01-12T00:05:00Z') + returning analysis_source_snapshot_id + """, + (digest, DEMO_SOURCE_CONTRACT_VERSION), + ) + return cur.fetchone()[0] + + +def _ensure_demo_source_counts(cur, snapshot_id) -> None: + """Insert demo counts only when the snapshot still has none. + + ``enforce_analysis_source_count_freeze`` runs BEFORE INSERT. After + the first run points at the snapshot, a later ``INSERT ... ON + CONFLICT DO NOTHING`` still raises ``analysis_source_count_frozen_after_run`` + and rolls back the whole ``seed()`` transaction. Skip when counts + already exist so ``make seed`` can be re-run. + """ + cur.execute( + "select 1 from analysis_source_count " + "where analysis_source_snapshot_id = %s limit 1", + (snapshot_id,), + ) + if cur.fetchone() is not None: + return + cur.execute( + """ + insert into analysis_source_count + (analysis_source_snapshot_id, count_type_code, count_value) + values + (%s, 'analysis_count_document', 3), + (%s, 'analysis_count_thread', 1), + (%s, 'analysis_count_lineage_node', 5), + (%s, 'analysis_count_lineage_edge', 4) + """, + (snapshot_id, snapshot_id, snapshot_id, snapshot_id), + ) + + +def _seed_demo_analysis_run(cur, requested_by_account_id, corporate_entity_id) -> None: + """Insert one Demo-Corp lineage run so Analysis runs is not empty. + + Aggregates only: three synthetic documents, one thread. Reuses the + shared Demo Corp snapshot so a later TEPP run can attach to the + same capture. + """ + snapshot_id = _ensure_demo_source_snapshot(cur) + _ensure_demo_source_counts(cur, snapshot_id) + cur.execute( + """ + select analysis_run_id from analysis_run + where requested_by_account_id = %s + and idempotency_key = %s + """, + (requested_by_account_id, DEMO_LINEAGE_IDEMPOTENCY_KEY), + ) + run_row = cur.fetchone() + if run_row is None: + cur.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_lineage', %s, + %s, '2026-01-12T12:00:00Z', 'lineage-run-v1', %s, %s, + '2026-01-12T12:30:00Z') + returning analysis_run_id + """, + ( + snapshot_id, + DEMO_LINEAGE_IDEMPOTENCY_KEY, + requested_by_account_id, + "b" * 64, + "c" * 40, + ), + ) + run_id = cur.fetchone()[0] + else: + run_id = run_row[0] + cur.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values (%s, 'analysis_scope_corporate_entity', %s) + on conflict (analysis_run_id) do nothing + """, + (run_id, corporate_entity_id), + ) + for ordinal, status, occurred in ( + (1, "analysis_status_pending", "2026-01-12T12:31:00Z"), + (2, "analysis_status_running", "2026-01-12T12:32:00Z"), + (3, "analysis_status_succeeded", "2026-01-12T12:33:00Z"), + ): + cur.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values (%s, %s, %s, %s) + on conflict do nothing + """, + (run_id, ordinal, status, occurred), + ) + + +def tepp_seed_request() -> AnalysisRunRequest: + """Build the Demo Corp TEPP request against the shared snapshot digest.""" + return AnalysisRunRequest( + idempotency_key=DEMO_TEPP_IDEMPOTENCY_KEY, + tenant_workspace_id="demo-workspace", + snapshot_id=demo_source_snapshot_sha256(), + knowledge_cutoff="2026-01-12T12:00:00Z", + model_contract_version="tepp-analysis-run-v1", + output_profile="calibrated_event_measurement", + ) + + +def tepp_seed_outcome(client: TeppClient | None = None) -> tuple[str, str | None]: + """Ask TEPP through the published client. A missing transport is Failed. + + Never invents a psychometric score. ``tepp_not_available`` means the + channel was dropped, not a calibrated negative result. A live + envelope is also not a persistable measurement in this seed, so the + run is not stamped Succeeded. + """ + request = tepp_seed_request() + try: + (client or TeppClient()).submit_analysis_run(request) + except TeppNotAvailable: + return "analysis_status_failed", "tepp_not_available" + return "analysis_status_failed", "tepp_result_not_persisted" + + +def _seed_demo_tepp_run(cur, requested_by_account_id, corporate_entity_id) -> None: + """Insert one Demo-Corp TEPP run so the kind is visible without a live TEPP. + + Uses :func:`tepp_seed_outcome` against the shared lineage snapshot. + Default transport is unavailable, so the run ends Failed / + ``tepp_not_available`` -- never a fake theta. + """ + snapshot_id = _ensure_demo_source_snapshot(cur) + _ensure_demo_source_counts(cur, snapshot_id) + cur.execute( + """ + select analysis_run_id from analysis_run + where requested_by_account_id = %s + and idempotency_key = %s + """, + (requested_by_account_id, DEMO_TEPP_IDEMPOTENCY_KEY), + ) + run_row = cur.fetchone() + if run_row is None: + cur.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_tepp', %s, + %s, '2026-01-12T12:00:00Z', 'tepp-run-v1', %s, %s, + '2026-01-12T12:34:00Z') + returning analysis_run_id + """, + ( + snapshot_id, + DEMO_TEPP_IDEMPOTENCY_KEY, + requested_by_account_id, + "d" * 64, + "e" * 40, + ), + ) + run_id = cur.fetchone()[0] + else: + run_id = run_row[0] + cur.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values (%s, 'analysis_scope_corporate_entity', %s) + on conflict (analysis_run_id) do nothing + """, + (run_id, corporate_entity_id), + ) + final_status, failure_code = tepp_seed_outcome() + events = [ + (1, "analysis_status_pending", "2026-01-12T12:35:00Z", None), + (2, "analysis_status_running", "2026-01-12T12:36:00Z", None), + (3, final_status, "2026-01-12T12:37:00Z", failure_code), + ] + for ordinal, status, occurred, fail in events: + cur.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at, failure_code) + values (%s, %s, %s, %s, %s) + on conflict do nothing + """, + (run_id, ordinal, status, occurred, fail), + ) + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--postgres-dsn", default=DEFAULT_POSTGRES_DSN) parser.add_argument("--keycloak-base-url", default=DEFAULT_KEYCLOAK_BASE_URL) parser.add_argument("--keycloak-admin-user", default=DEFAULT_KEYCLOAK_ADMIN_USER) - parser.add_argument("--keycloak-admin-password", default=DEFAULT_KEYCLOAK_ADMIN_PASSWORD) + parser.add_argument( + "--keycloak-admin-password", + default=os.environ.get("KEYCLOAK_ADMIN_PASSWORD"), + help="Keycloak master admin password (or KEYCLOAK_ADMIN_PASSWORD). Required.", + ) parser.add_argument("--valkey-url", default=DEFAULT_VALKEY_URL) args = parser.parse_args() + if not args.keycloak_admin_password: + parser.error("set KEYCLOAK_ADMIN_PASSWORD or pass --keycloak-admin-password") subjects = _fetch_demo_user_subjects(args.keycloak_base_url, args.keycloak_admin_user, args.keycloak_admin_password) seed(args.postgres_dsn, subjects, args.valkey_url) diff --git a/tests/test_analysis_run_authorization.py b/tests/test_analysis_run_authorization.py new file mode 100644 index 000000000..730825c14 --- /dev/null +++ b/tests/test_analysis_run_authorization.py @@ -0,0 +1,254 @@ +"""SQL authorization for the Milestone 2 analysis-run read projection.""" + +from __future__ import annotations + +import os +import uuid +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +import psycopg2 +import pytest +from psycopg2 import sql + +_ROOT = Path(__file__).resolve().parents[1] +_INITIAL_MIGRATION = _ROOT / "migrations" / "0001_initial_schema.sql" +_REGISTRY_MIGRATION = _ROOT / "migrations" / "0018_analysis_run_registry.sql" +_ADMIN_DSN = os.environ.get( + "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" +) + + +def _postgres_available() -> bool: + """Return whether the configured administrator DSN is reachable.""" + try: + psycopg2.connect(_ADMIN_DSN, connect_timeout=2).close() + return True + except psycopg2.OperationalError: + return False + + +def _database_dsn(database_name: str) -> str: + """Replace only the database path while preserving DSN query options.""" + parsed = urlsplit(_ADMIN_DSN) + return urlunsplit(parsed._replace(path=f"/{database_name}")) + + +@pytest.fixture +def authz_db(): + """Yield a throwaway database migrated through the registry schema.""" + if not _postgres_available(): + pytest.skip("a reachable PostgreSQL administrator DSN is required") + database_name = f"lineageweave_authz_{uuid.uuid4().hex[:12]}" + admin_connection = psycopg2.connect(_ADMIN_DSN) + admin_connection.autocommit = True + with admin_connection.cursor() as cursor: + cursor.execute( + sql.SQL("create database {}").format(sql.Identifier(database_name)) + ) + try: + connection = psycopg2.connect(_database_dsn(database_name)) + try: + connection.autocommit = True + with connection.cursor() as cursor: + cursor.execute(_INITIAL_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_REGISTRY_MIGRATION.read_text(encoding="utf-8")) + yield connection + finally: + connection.close() + finally: + with admin_connection.cursor() as cursor: + cursor.execute( + sql.SQL("drop database {}").format(sql.Identifier(database_name)) + ) + admin_connection.close() + + +def _insert_account(cursor, label: str) -> str: + """Insert one synthetic authenticated account and return its UUID.""" + suffix = uuid.uuid4().hex + cursor.execute( + """ + insert into user_account + (external_subject_id, display_name, email_address) + values (%s, %s, %s) + returning user_account_id + """, + (f"{label}-{suffix}", f"{label.title()} User", f"{label}-{suffix}@example.test"), + ) + return str(cursor.fetchone()[0]) + + +def _insert_corp(cursor, code: str, name: str) -> str: + """Insert one synthetic corporate entity.""" + cursor.execute( + """ + insert into common_lookup_value (lookup_category, lookup_code, lookup_label) + values ('corporate_entity_level', 'company', 'Company') + on conflict (lookup_code) do nothing + """ + ) + cursor.execute( + """ + insert into corporate_entity (corporate_entity_code, entity_name, entity_level_code) + values (%s, %s, 'company') + returning corporate_entity_id + """, + (code, name), + ) + return str(cursor.fetchone()[0]) + + +def _complete_run( + cursor, + *, + account_id: str, + digest: str, + idempotency_key: str, + scope_kind: str, + corporate_entity_id: str | None = None, +) -> str: + """Insert one succeeded run with one document-count aggregate.""" + cursor.execute( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', + '2026-01-12T00:00:00Z', '2026-01-12T00:05:00Z') + returning analysis_source_snapshot_id + """, + (digest,), + ) + snapshot_id = cursor.fetchone()[0] + cursor.execute( + """ + insert into analysis_source_count + (analysis_source_snapshot_id, count_type_code, count_value) + values (%s, 'analysis_count_document', 3) + """, + (snapshot_id,), + ) + cursor.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, 'analysis_run_lineage', %s, %s, + '2026-01-12T12:00:00Z', 'lineage-run-v1', %s, %s, + '2026-01-12T12:30:00Z') + returning analysis_run_id + """, + (snapshot_id, idempotency_key, account_id, "b" * 64, "c" * 40), + ) + run_id = str(cursor.fetchone()[0]) + if scope_kind == "analysis_scope_corporate_entity": + cursor.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code, corporate_entity_id) + values (%s, %s, %s) + """, + (run_id, scope_kind, corporate_entity_id), + ) + else: + cursor.execute( + """ + insert into analysis_run_scope + (analysis_run_id, scope_kind_code) + values (%s, %s) + """, + (run_id, scope_kind), + ) + for ordinal, status, occurred in ( + (1, "analysis_status_pending", "2026-01-12T12:31:00Z"), + (2, "analysis_status_running", "2026-01-12T12:32:00Z"), + (3, "analysis_status_succeeded", "2026-01-12T12:33:00Z"), + ): + cursor.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values (%s, %s, %s, %s) + """, + (run_id, ordinal, status, occurred), + ) + return run_id + + +def _visible_ids(cursor, account_id: str, entity_ids: list[str]) -> set[str]: + """Apply the same visibility predicate the product API uses.""" + cursor.execute( + """ + select run.analysis_run_id + from analysis_run run + join analysis_run_scope scope on scope.analysis_run_id = run.analysis_run_id + where + run.requested_by_account_id = %s + or ( + scope.scope_kind_code = 'analysis_scope_corporate_entity' + and scope.corporate_entity_id = any(%s::uuid[]) + ) + or ( + scope.scope_kind_code = 'analysis_scope_process_unit' + and exists ( + select 1 from account_affiliation aff + where aff.user_account_id = %s + and aff.process_unit_id = scope.process_unit_id + ) + ) + """, + (account_id, entity_ids, account_id), + ) + return {str(row[0]) for row in cursor.fetchall()} + + +def test_hidden_scope_does_not_leak_through_all_visible_or_other_corp(authz_db) -> None: + """A Demo-Corp viewer never sees another tenant's run or its aggregates.""" + with authz_db.cursor() as cursor: + viewer = _insert_account(cursor, "viewer") + outsider = _insert_account(cursor, "outsider") + own_corp = _insert_corp(cursor, "DEMO-CORP-AUTHZ", "Demo Corp") + other_corp = _insert_corp(cursor, "OTHER-CORP-AUTHZ", "Other Corp") + cursor.execute( + """ + insert into account_affiliation (user_account_id, corporate_entity_id) + values (%s, %s) + """, + (viewer, own_corp), + ) + own_run = _complete_run( + cursor, + account_id=viewer, + digest="a" * 64, + idempotency_key="own-corp", + scope_kind="analysis_scope_corporate_entity", + corporate_entity_id=own_corp, + ) + hidden_all_visible = _complete_run( + cursor, + account_id=outsider, + digest="d" * 64, + idempotency_key="hidden-all", + scope_kind="analysis_scope_all_visible", + ) + hidden_other_corp = _complete_run( + cursor, + account_id=outsider, + digest="e" * 64, + idempotency_key="hidden-other", + scope_kind="analysis_scope_corporate_entity", + corporate_entity_id=other_corp, + ) + + visible = _visible_ids(cursor, viewer, [own_corp]) + assert own_run in visible + assert hidden_all_visible not in visible + assert hidden_other_corp not in visible + + outsider_visible = _visible_ids(cursor, outsider, [other_corp]) + assert hidden_all_visible in outsider_visible + assert hidden_other_corp in outsider_visible + assert own_run not in outsider_visible diff --git a/tests/test_analysis_run_create.py b/tests/test_analysis_run_create.py new file mode 100644 index 000000000..4e24a4228 --- /dev/null +++ b/tests/test_analysis_run_create.py @@ -0,0 +1,134 @@ +"""Authorized analysis-run create hashes the cutoff bag, never a score.""" + +from datetime import datetime, timezone + +from backend.app.analysis_run_ingestion import ( + AnalysisRunCreateError, + _resolve_corporate_entity_id, + plan_analysis_run_capture, +) +import pytest + + +_CUTOFF = datetime(2026, 1, 12, 12, 0, tzinfo=timezone.utc) +_EARLIER = datetime(2026, 1, 10, 9, 0, tzinfo=timezone.utc) + + +def test_capture_digest_is_stable_for_the_same_authorized_bag() -> None: + first = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + post_ids=["post-b", "post-a"], + thread_keys=["thread-a", "thread-a"], + latest_post_created_at=_EARLIER, + ) + second = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + post_ids=["post-a", "post-b"], + thread_keys=["thread-a", "thread-a"], + latest_post_created_at=_EARLIER, + ) + assert first.snapshot_sha256 == second.snapshot_sha256 + assert first.configuration_sha256 == second.configuration_sha256 + assert first.document_count == 2 + assert first.thread_count == 1 + assert first.maximum_available_time == _EARLIER + assert "theta" not in first.snapshot_sha256 + assert first.configuration_schema_version == "lineage-run-v1" + + +def test_later_cutoff_or_other_kind_does_not_reuse_the_wrong_digest() -> None: + lineage = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + post_ids=["post-a"], + thread_keys=["thread-a"], + latest_post_created_at=_EARLIER, + ) + later = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=datetime(2026, 1, 13, 12, 0, tzinfo=timezone.utc), + idempotency_key="client-key-1", + post_ids=["post-a"], + thread_keys=["thread-a"], + latest_post_created_at=_EARLIER, + ) + tepp = plan_analysis_run_capture( + run_kind_code="analysis_run_tepp", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + post_ids=["post-a"], + thread_keys=["thread-a"], + latest_post_created_at=_EARLIER, + ) + assert lineage.snapshot_sha256 != later.snapshot_sha256 + assert lineage.snapshot_sha256 == tepp.snapshot_sha256 + assert lineage.configuration_sha256 != tepp.configuration_sha256 + assert tepp.configuration_schema_version == "tepp-run-v1" + + +def test_omitted_cutoff_keeps_the_same_client_key_stable() -> None: + first = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + post_ids=["post-a"], + thread_keys=["thread-a"], + latest_post_created_at=_EARLIER, + cutoff_explicit=False, + ) + later_clock = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=datetime(2026, 1, 13, 12, 0, tzinfo=timezone.utc), + idempotency_key="client-key-1", + post_ids=["post-a"], + thread_keys=["thread-a"], + latest_post_created_at=_EARLIER, + cutoff_explicit=False, + ) + assert first.configuration_sha256 == later_clock.configuration_sha256 + assert first.snapshot_sha256 == later_clock.snapshot_sha256 + + +def test_empty_corpus_uses_the_cutoff_as_latest_available_time() -> None: + capture = plan_analysis_run_capture( + run_kind_code="analysis_run_lineage", + scope_kind_code="analysis_scope_corporate_entity", + corporate_entity_id="corp-1", + knowledge_cutoff=_CUTOFF, + idempotency_key="client-key-1", + post_ids=[], + thread_keys=[], + latest_post_created_at=None, + ) + assert capture.document_count == 0 + assert capture.thread_count == 0 + assert capture.maximum_available_time == _CUTOFF + + +def test_create_rejects_an_unaffiliated_or_ambiguous_corporate_entity() -> None: + with pytest.raises(AnalysisRunCreateError) as hidden: + _resolve_corporate_entity_id("corp-other", ["corp-1"]) + assert hidden.value.status_code == 404 + with pytest.raises(AnalysisRunCreateError) as ambiguous: + _resolve_corporate_entity_id(None, ["corp-1", "corp-2"]) + assert ambiguous.value.status_code == 422 + assert _resolve_corporate_entity_id(None, ["corp-1"]) == "corp-1" diff --git a/tests/test_analysis_run_registry_schema.py b/tests/test_analysis_run_registry_schema.py new file mode 100644 index 000000000..f2b38badf --- /dev/null +++ b/tests/test_analysis_run_registry_schema.py @@ -0,0 +1,716 @@ +"""Real-PostgreSQL contracts for the normalized Milestone 2 run registry.""" + +from __future__ import annotations + +import os +import re +import uuid +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +import psycopg2 +import psycopg2.errors +import pytest +from psycopg2 import sql + +_ROOT = Path(__file__).resolve().parents[1] +_INITIAL_MIGRATION = _ROOT / "migrations" / "0001_initial_schema.sql" +_REGISTRY_MIGRATION = _ROOT / "migrations" / "0018_analysis_run_registry.sql" +_REGISTRY_ROLLBACK = _ROOT / "migrations" / "rollback" / "0018_analysis_run_registry.sql" +_POSTGRES_IMAGE = _ROOT / "docker" / "postgres-init" / "Dockerfile" +_ADMIN_DSN = os.environ.get( + "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" +) +_REQUIRED_TABLES = { + "analysis_source_snapshot", + "analysis_source_count", + "analysis_run", + "analysis_run_scope", + "analysis_run_status_event", +} +_REQUIRED_LOOKUP_CODES = { + "analysis_run_lineage", + "analysis_run_report", + "analysis_run_tepp", + "analysis_status_pending", + "analysis_status_running", + "analysis_status_succeeded", + "analysis_status_failed", + "analysis_status_cancelled", + "analysis_scope_all_visible", + "analysis_scope_corporate_entity", + "analysis_scope_process_unit", + "analysis_scope_thread_group", + "analysis_count_source_row", + "analysis_count_document", + "analysis_count_thread", + "analysis_count_lineage_node", + "analysis_count_lineage_edge", +} + + +def _postgres_available() -> bool: + """Return whether the configured administrator DSN is reachable.""" + + try: + psycopg2.connect(_ADMIN_DSN, connect_timeout=2).close() + return True + except psycopg2.OperationalError: + return False + + +def _database_dsn(database_name: str) -> str: + """Replace only the database path while preserving DSN query options.""" + + parsed = urlsplit(_ADMIN_DSN) + return urlunsplit(parsed._replace(path=f"/{database_name}")) + + +def _table_definition(migration: str, table_name: str) -> str: + """Return one table definition from the deterministic migration text.""" + + match = re.search( + rf"create table if not exists {re.escape(table_name)}\s*\((.*?)\n\);", + migration, + re.IGNORECASE | re.DOTALL, + ) + assert match is not None, table_name + return match.group(1) + + +@pytest.fixture +def registry_db(): + """Yield a throwaway database migrated through the registry schema.""" + + if not _postgres_available(): + pytest.skip("a reachable PostgreSQL administrator DSN is required") + database_name = f"lineageweave_registry_{uuid.uuid4().hex[:12]}" + admin_connection = psycopg2.connect(_ADMIN_DSN) + admin_connection.autocommit = True + with admin_connection.cursor() as cursor: + cursor.execute( + sql.SQL("create database {}").format(sql.Identifier(database_name)) + ) + try: + connection = psycopg2.connect(_database_dsn(database_name)) + try: + connection.autocommit = True + with connection.cursor() as cursor: + cursor.execute(_INITIAL_MIGRATION.read_text(encoding="utf-8")) + cursor.execute(_REGISTRY_MIGRATION.read_text(encoding="utf-8")) + yield connection + finally: + connection.close() + finally: + with admin_connection.cursor() as cursor: + cursor.execute( + sql.SQL("drop database {}").format(sql.Identifier(database_name)) + ) + admin_connection.close() + + +def _insert_account(cursor, label: str = "operator") -> str: + """Insert one synthetic authenticated account and return its UUID.""" + + suffix = uuid.uuid4().hex + cursor.execute( + """ + insert into user_account + (external_subject_id, display_name, email_address) + values (%s, %s, %s) + returning user_account_id + """, + (f"{label}-{suffix}", f"{label.title()} User", f"{label}-{suffix}@example.test"), + ) + return str(cursor.fetchone()[0]) + + +def _insert_snapshot( + cursor, + *, + digest: str = "a" * 64, + maximum_available_time: str = "2026-08-15T00:00:00Z", + captured_at: str = "2026-08-15T00:05:00Z", +) -> str: + """Insert one immutable source snapshot and return its UUID.""" + + cursor.execute( + """ + insert into analysis_source_snapshot + (snapshot_sha256, source_contract_version, + maximum_available_time, captured_at) + values (%s, 'source-contract-v1', %s, %s) + returning analysis_source_snapshot_id + """, + (digest, maximum_available_time, captured_at), + ) + return str(cursor.fetchone()[0]) + + +def _insert_run( + cursor, + *, + snapshot_id: str, + account_id: str, + idempotency_key: str, + knowledge_cutoff: str = "2026-08-15T00:30:00Z", + run_kind_code: str = "analysis_run_lineage", + requested_at: str = "2026-08-15T00:45:00Z", +) -> str: + """Insert one immutable account-scoped analysis request.""" + + cursor.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + requested_by_account_id, knowledge_cutoff, + configuration_schema_version, configuration_sha256, + code_revision_sha, requested_at) + values (%s, %s, %s, %s, %s, 'lineage-run-v1', %s, %s, %s) + returning analysis_run_id + """, + ( + snapshot_id, + run_kind_code, + idempotency_key, + account_id, + knowledge_cutoff, + "b" * 64, + "c" * 40, + requested_at, + ), + ) + return str(cursor.fetchone()[0]) + + +def test_registry_contract_is_normalized_and_has_one_temporal_authority() -> None: + """Static contract rejects the parallel prototype and duplicated clocks.""" + + migration = _REGISTRY_MIGRATION.read_text(encoding="utf-8") + rollback = _REGISTRY_ROLLBACK.read_text(encoding="utf-8") + dockerfile = _POSTGRES_IMAGE.read_text(encoding="utf-8") + created_tables = set( + re.findall(r"create table if not exists\s+([a-z0-9_]+)", migration, re.I) + ) + assert _REQUIRED_TABLES <= created_tables + assert "analysis_run_records" not in created_tables + assert "metadata_payload" not in migration + assert "jsonb" not in migration.casefold() + assert _REQUIRED_LOOKUP_CODES <= set( + re.findall(r"'(analysis_[a-z0-9_]+)'", migration) + ) + assert "0018_analysis_run_registry.sql" in dockerfile + assert "analysis_run_registry_not_empty" in rollback + + snapshot_definition = _table_definition(migration, "analysis_source_snapshot") + run_definition = _table_definition(migration, "analysis_run") + assert "maximum_available_time" in snapshot_definition + assert "knowledge_cutoff" not in snapshot_definition + assert "knowledge_cutoff" in run_definition + assert "requested_by_account_id uuid not null" in run_definition + assert "unique (requested_by_account_id, idempotency_key)" in run_definition + assert "enforce_analysis_run_knowledge_cutoff" in migration + assert "reject_analysis_source_snapshot_update" in migration + assert "reject_analysis_run_mutation" in migration + assert "reject_analysis_run_scope_mutation" in migration + assert "analysis_run_scope_required" in migration + assert "enforce_analysis_source_count_freeze" in migration + assert "enforce_analysis_run_status_transition" in migration + assert "analysis_run_current_status" in migration + + object_patterns = ( + r"create table if not exists\s+([a-z0-9_]+)", + r"create(?: unique)? index if not exists\s+([a-z0-9_]+)", + r"create or replace function\s+([a-z0-9_]+)", + r"create trigger\s+([a-z0-9_]+)", + r"create or replace view\s+([a-z0-9_]+)", + ) + for pattern in object_patterns: + for object_name in re.findall(pattern, migration, re.I): + assert len(object_name.split("_")) >= 2, object_name + + +def test_registry_migration_is_idempotent(registry_db) -> None: + """Sequential migration replay preserves one object set.""" + + with registry_db.cursor() as cursor: + cursor.execute(_REGISTRY_MIGRATION.read_text(encoding="utf-8")) + cursor.execute( + "select table_name from information_schema.tables " + "where table_schema = 'public'" + ) + tables = {row[0] for row in cursor.fetchall()} + cursor.execute( + "select table_name from information_schema.views " + "where table_schema = 'public'" + ) + views = {row[0] for row in cursor.fetchall()} + assert _REQUIRED_TABLES <= tables + assert "analysis_run_current_status" in views + + +def test_registry_persists_scope_counts_and_legal_status_history(registry_db) -> None: + """A valid run keeps normalized scope, counts, and current status.""" + + with registry_db.cursor() as cursor: + account_id = _insert_account(cursor) + snapshot_id = _insert_snapshot(cursor) + cursor.execute( + "insert into analysis_source_count values " + "(%s, 'analysis_count_document', 12)", + (snapshot_id,), + ) + run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="synthetic-run-1", + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (run_id,), + ) + cursor.execute( + """ + insert into analysis_run_status_event + (analysis_run_id, status_ordinal, status_code, occurred_at) + values + (%s, 1, 'analysis_status_pending', '2026-08-15T01:00:01Z'), + (%s, 2, 'analysis_status_running', '2026-08-15T01:00:02Z'), + (%s, 3, 'analysis_status_succeeded', '2026-08-15T01:00:03Z') + """, + (run_id, run_id, run_id), + ) + cursor.execute( + "select status_code, status_ordinal from analysis_run_current_status " + "where analysis_run_id = %s", + (run_id,), + ) + assert cursor.fetchone() == ("analysis_status_succeeded", 3) + + +def test_snapshot_supports_multiple_run_owned_cutoffs_and_blocks_future_evidence( + registry_db, +) -> None: + """One capture is reusable, but each run must respect its own cutoff.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + first_account_id = _insert_account(cursor, "first") + second_account_id = _insert_account(cursor, "second") + first_run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=first_account_id, + idempotency_key="cutoff-one", + knowledge_cutoff="2026-08-15T00:30:00Z", + ) + second_run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=second_account_id, + idempotency_key="cutoff-two", + knowledge_cutoff="2026-08-16T00:00:00Z", + requested_at="2026-08-16T00:30:00Z", + ) + assert first_run_id != second_run_id + with pytest.raises(psycopg2.errors.RaiseException): + _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=first_account_id, + idempotency_key="future-leakage", + knowledge_cutoff="2026-08-14T23:59:59Z", + ) + + +def test_snapshot_counts_and_run_request_are_immutable(registry_db) -> None: + """Evidence and request configuration freeze before derivation starts.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + cursor.execute( + "insert into analysis_source_count values " + "(%s, 'analysis_count_document', 12)", + (snapshot_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "update analysis_source_snapshot set source_contract_version = 'x' " + "where analysis_source_snapshot_id = %s", + (snapshot_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "update analysis_source_count set count_value = 13 " + "where analysis_source_snapshot_id = %s", + (snapshot_id,), + ) + account_id = _insert_account(cursor) + run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="freeze-evidence", + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "update analysis_run set knowledge_cutoff = now() " + "where analysis_run_id = %s", + (run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_source_count values " + "(%s, 'analysis_count_thread', 8)", + (snapshot_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "delete from analysis_source_count " + "where analysis_source_snapshot_id = %s", + (snapshot_id,), + ) + + +def test_idempotency_is_scoped_to_the_authenticated_account(registry_db) -> None: + """Two actors may use one opaque key; one actor may not reuse it.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + first_account_id = _insert_account(cursor, "first") + second_account_id = _insert_account(cursor, "second") + _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=first_account_id, + idempotency_key="shared-key", + ) + _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=second_account_id, + idempotency_key="shared-key", + ) + with pytest.raises(psycopg2.errors.UniqueViolation): + _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=first_account_id, + idempotency_key="shared-key", + ) + + +def test_registry_rejects_invalid_evidence_and_missing_actor(registry_db) -> None: + """Database constraints reject malformed audit evidence before persistence.""" + + with registry_db.cursor() as cursor: + with pytest.raises(psycopg2.errors.CheckViolation): + cursor.execute( + "insert into analysis_source_snapshot " + "(snapshot_sha256, source_contract_version, " + "maximum_available_time, captured_at) " + "values ('bad', 'source-contract-v1', now(), now())" + ) + snapshot_id = _insert_snapshot(cursor) + with pytest.raises(psycopg2.errors.CheckViolation): + cursor.execute( + "insert into analysis_source_count values " + "(%s, 'analysis_count_source_row', -1)", + (snapshot_id,), + ) + with pytest.raises(psycopg2.errors.NotNullViolation): + cursor.execute( + """ + insert into analysis_run + (analysis_source_snapshot_id, run_kind_code, idempotency_key, + knowledge_cutoff, configuration_schema_version, + configuration_sha256, code_revision_sha) + values (%s, 'analysis_run_report', 'missing-actor', now(), + 'report-run-v1', %s, %s) + """, + (snapshot_id, "d" * 64, "e" * 40), + ) + + +def test_status_history_enforces_shape_order_time_and_legal_transitions( + registry_db, +) -> None: + """Append-only status evidence is a serialized state machine.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + account_id = _insert_account(cursor) + first_run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="first-status", + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (first_run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 1, 'analysis_status_running', now())", + (first_run_id,), + ) + + second_run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="second-status", + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (second_run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 1, 'analysis_status_pending', " + "'2026-08-15T01:00:00Z')", + (second_run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 3, 'analysis_status_running', " + "'2026-08-15T01:00:01Z')", + (second_run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 2, 'analysis_status_succeeded', " + "'2026-08-15T01:00:01Z')", + (second_run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 2, 'analysis_status_running', " + "'2026-08-15T01:00:02Z')", + (second_run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 3, 'analysis_status_succeeded', " + "'2026-08-15T01:00:01Z')", + (second_run_id,), + ) + with pytest.raises(psycopg2.errors.CheckViolation): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 3, 'analysis_status_failed', " + "'2026-08-15T01:00:03Z')", + (second_run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 3, 'analysis_status_succeeded', " + "'2026-08-15T01:00:03Z')", + (second_run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 4, 'analysis_status_running', " + "'2026-08-15T01:00:04Z')", + (second_run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "update analysis_run_status_event set retryable = true " + "where analysis_run_id = %s and status_ordinal = 3", + (second_run_id,), + ) + + + +def test_run_scope_and_request_evidence_are_immutable(registry_db) -> None: + """Authorization scope and request identity cannot be rewritten or erased.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + account_id = _insert_account(cursor) + run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="immutable-run", + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "update analysis_run_scope set scope_kind_code = scope_kind_code " + "where analysis_run_id = %s", + (run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "delete from analysis_run_scope where analysis_run_id = %s", + (run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "delete from analysis_run where analysis_run_id = %s", + (run_id,), + ) + + +def test_status_requires_scope_and_cannot_predate_request(registry_db) -> None: + """Lifecycle evidence starts only after an immutable authorized request.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + account_id = _insert_account(cursor) + run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="scoped-status", + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 1, 'analysis_status_pending', " + "'2026-08-15T01:00:00Z')", + (run_id,), + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (run_id,), + ) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 1, 'analysis_status_pending', " + "'2026-08-15T00:44:59Z')", + (run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at, recorded_at) " + "values (%s, 1, 'analysis_status_pending', " + "'2026-08-15T01:00:00Z', '2099-01-01T00:00:00Z') " + "returning recorded_at", + (run_id,), + ) + recorded_at = cursor.fetchone()[0] + assert recorded_at.year < 2099 + + +def test_machine_codes_and_canonical_idempotency_are_fail_closed(registry_db) -> None: + """Audit identifiers are canonical and failure details stay machine-safe.""" + + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + account_id = _insert_account(cursor) + with pytest.raises(psycopg2.errors.RaiseException): + _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="future-request", + requested_at="2099-01-01T00:00:00Z", + ) + with pytest.raises(psycopg2.errors.CheckViolation): + _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key=" padded-key ", + ) + run_id = _insert_run( + cursor, + snapshot_id=snapshot_id, + account_id=account_id, + idempotency_key="machine-safe", + ) + cursor.execute( + "insert into analysis_run_scope " + "(analysis_run_id, scope_kind_code) " + "values (%s, 'analysis_scope_all_visible')", + (run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 1, 'analysis_status_pending', " + "'2026-08-15T01:00:00Z')", + (run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at) " + "values (%s, 2, 'analysis_status_running', " + "'2026-08-15T01:00:00Z')", + (run_id,), + ) + with pytest.raises(psycopg2.errors.CheckViolation): + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at, " + "failure_code, retryable) " + "values (%s, 3, 'analysis_status_failed', " + "'2026-08-15T01:00:00Z', 'provider timeout', true)", + (run_id,), + ) + cursor.execute( + "insert into analysis_run_status_event " + "(analysis_run_id, status_ordinal, status_code, occurred_at, " + "failure_code, retryable) " + "values (%s, 3, 'analysis_status_failed', " + "'2026-08-15T01:00:00Z', 'provider_timeout', true)", + (run_id,), + ) + +def test_rollback_refuses_data_loss_then_removes_an_empty_registry(registry_db) -> None: + """Downgrade fails closed until audit evidence is explicitly removed.""" + + rollback_sql = _REGISTRY_ROLLBACK.read_text(encoding="utf-8") + with registry_db.cursor() as cursor: + snapshot_id = _insert_snapshot(cursor) + with pytest.raises(psycopg2.errors.RaiseException): + cursor.execute(rollback_sql) + # The rollback script opens an explicit transaction on this + # autocommit connection. A RAISE leaves that transaction aborted, and + # connection.rollback() is a no-op while autocommit is true. + cursor.execute("rollback") + with registry_db.cursor() as cursor: + cursor.execute( + "delete from analysis_source_snapshot " + "where analysis_source_snapshot_id = %s", + (snapshot_id,), + ) + cursor.execute(rollback_sql) + cursor.execute("select to_regclass('public.analysis_run')") + assert cursor.fetchone()[0] is None + cursor.execute(rollback_sql) diff --git a/tests/test_corporate_hierarchy_inference.py b/tests/test_corporate_hierarchy_inference.py new file mode 100644 index 000000000..b8795bb0a --- /dev/null +++ b/tests/test_corporate_hierarchy_inference.py @@ -0,0 +1,71 @@ +"""Tests for lineageweave.corporate_hierarchy_inference (ADR 0010). + +Pure parse-function tests, same style as test_organization_name_resolution.py +-- the HTTP mechanics are already covered by test_http_client.py. +""" + +from __future__ import annotations + +from lineageweave.corporate_hierarchy_inference import ( + LEVEL_COMPANY, + LEVEL_GROUP, + LEVEL_PLANT, + HierarchyProposal, + parse_inference_response, +) + + +def test_parses_a_plant_with_a_parent() -> None: + content = '{"level": "plant", "parent_name": "Acme Electronics"}' + assert parse_inference_response(content) == HierarchyProposal( + level_code=LEVEL_PLANT, parent_name="Acme Electronics" + ) + + +def test_parses_a_group_with_no_parent() -> None: + content = '{"level": "group", "parent_name": null}' + assert parse_inference_response(content) == HierarchyProposal(level_code=LEVEL_GROUP, parent_name=None) + + +def test_company_level_recognized() -> None: + content = '{"level": "company", "parent_name": "Some Group"}' + result = parse_inference_response(content) + assert result is not None + assert result.level_code == LEVEL_COMPANY + + +def test_unknown_response_returns_none() -> None: + assert parse_inference_response("UNKNOWN") is None + assert parse_inference_response("unknown\n") is None + + +def test_empty_response_returns_none() -> None: + assert parse_inference_response("") is None + + +def test_malformed_json_returns_none() -> None: + assert parse_inference_response("not json at all") is None + + +def test_invalid_level_code_returns_none() -> None: + """A level outside the three valid codes must not be silently + accepted as if it were a real classification.""" + content = '{"level": "division", "parent_name": null}' + assert parse_inference_response(content) is None + + +def test_blank_parent_name_becomes_none() -> None: + content = '{"level": "company", "parent_name": " "}' + result = parse_inference_response(content) + assert result is not None + assert result.parent_name is None + + +def test_markdown_fenced_json_is_rejected_not_stripped() -> None: + """Unlike post_summary's parser, this one does not strip code + fences -- the prompt asks for raw JSON only; a fenced response + means the model did not follow instructions and should not be + silently repaired into a trusted hierarchy claim. + """ + content = '```json\n{"level": "company", "parent_name": null}\n```' + assert parse_inference_response(content) is None diff --git a/tests/test_documentation_hygiene.py b/tests/test_documentation_hygiene.py new file mode 100644 index 000000000..6dc89dff8 --- /dev/null +++ b/tests/test_documentation_hygiene.py @@ -0,0 +1,39 @@ +"""Permanent hygiene checks for committed architecture-decision records.""" + +from __future__ import annotations + +import re +from collections import Counter +from pathlib import Path + +_ROOT = Path(__file__).resolve().parents[1] +_ADR_DIRECTORY = _ROOT / "docs" / "adr" +_ADR_NAME = re.compile(r"^(?P[0-9]{4})-.+\.md$") +_FORBIDDEN_MARKERS = ( + "PLACEHOLDER_DO_NOT_WRITE", + "TODO_WRITE_ADR", +) + + +def test_adr_numbers_are_unique_and_documents_are_not_placeholders() -> None: + """Every committed ADR number identifies one substantive UTF-8 document.""" + paths = sorted(_ADR_DIRECTORY.glob("*.md")) + assert paths, "the repository must contain architecture-decision records" + + numbered_paths: list[tuple[str, Path]] = [] + for path in paths: + match = _ADR_NAME.fullmatch(path.name) + assert match is not None, f"ADR filename is not numbered: {path.name}" + numbered_paths.append((match.group("number"), path)) + + content = path.read_text(encoding="utf-8") + assert content.strip(), f"ADR is empty: {path.relative_to(_ROOT)}" + for marker in _FORBIDDEN_MARKERS: + assert marker not in content, ( + f"ADR contains forbidden placeholder {marker!r}: " + f"{path.relative_to(_ROOT)}" + ) + + counts = Counter(number for number, _ in numbered_paths) + duplicates = sorted(number for number, count in counts.items() if count > 1) + assert duplicates == [], f"duplicate ADR numbers: {duplicates}" diff --git a/tests/test_image_content.py b/tests/test_image_content.py index 572909d6f..333b3fb0b 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -46,6 +46,18 @@ def test_extract_base64_images_skips_malformed_base64() -> None: assert extract_base64_images(html) == [] +def test_extract_base64_images_accepts_charset_and_unquoted_src() -> None: + quoted_charset = f'' + unquoted = f"" + expected = base64.b64decode(_TINY_PNG_B64) + + for html in (quoted_charset, unquoted): + images = extract_base64_images(html) + assert len(images) == 1 + assert images[0].mime_type == "image/png" + assert images[0].data == expected + + def test_extract_base64_images_ignores_non_data_uri_images() -> None: html = '' assert extract_base64_images(html) == [] @@ -82,6 +94,56 @@ def test_parse_description_preserves_multiline_ocr_text() -> None: assert description.caption == "A scanned page." +def test_parse_description_tolerates_markdown_emphasis_on_labels() -> None: + """Synthetic provider drift may bold labels without changing content.""" + content = "**TEXT:** LT7\n**CAPTION:** A close-up of a component.\n**TAGS:** component, close-up" + description = _parse_description(content) + assert description.extracted_text == "LT7" + assert description.caption == "A close-up of a component." + assert description.tags == ("component", "close-up") + + +def test_parse_description_strips_balanced_markdown_emphasis_from_values() -> None: + content = "TEXT: **LT7**\nCAPTION: _A synthetic component._\nTAGS: `component`, close-up" + description = _parse_description(content) + assert description.extracted_text == "LT7" + assert description.caption == "A synthetic component." + assert description.tags == ("component", "close-up") + + +def test_parse_description_tolerates_reordered_labels() -> None: + content = "CAPTION: A blue sky.\nTEXT: NONE\nTAGS: sky" + description = _parse_description(content) + assert description.caption == "A blue sky." + assert description.extracted_text == "" + assert description.tags == ("sky",) + + +def test_parse_description_missing_tags_still_recovers_text_and_caption() -> None: + """Missing optional tags must not discard provided TEXT/CAPTION fields.""" + content = "TEXT: Quarterly Budget Report\nCAPTION: A printed report cover page." + description = _parse_description(content) + assert description.extracted_text == "Quarterly Budget Report" + assert description.caption == "A printed report cover page." + assert description.tags == () + + +def test_parse_description_leading_commentary_before_labels_is_ignored() -> None: + content = "Sure, here is the analysis:\n\nTEXT: LT7\nCAPTION: A component.\nTAGS: component" + description = _parse_description(content) + assert description.extracted_text == "LT7" + + +def test_parse_description_does_not_absorb_trailing_commentary() -> None: + content = ( + "TEXT: NONE\nCAPTION: A turbine diagram.\nTAGS: turbine, diagram\n" + "Let me know if you need more detail." + ) + description = _parse_description(content) + assert description.caption == "A turbine diagram." + assert description.tags == ("turbine", "diagram") + + def test_vision_client_rejects_non_http_url_schemes() -> None: with pytest.raises(ValueError, match="unsupported vision client URL scheme: file"): OpenAiCompatibleVisionClient( @@ -146,3 +208,11 @@ def test_image_content_client_protocol_stub_raises() -> None: """ with pytest.raises(NotImplementedError): ImageContentClient.describe(None, b"", "image/png") # type: ignore[arg-type] + + +def test_parse_description_does_not_absorb_unknown_labels_into_tags() -> None: + parsed = _parse_description( + "TEXT: NONE\nCAPTION: A turbine diagram\n" + "TAGS: turbine, diagram\nNOTE: synthetic" + ) + assert parsed.tags == ("turbine", "diagram") diff --git a/tests/test_ingestion_transaction_contracts.py b/tests/test_ingestion_transaction_contracts.py new file mode 100644 index 000000000..d2994e2c4 --- /dev/null +++ b/tests/test_ingestion_transaction_contracts.py @@ -0,0 +1,509 @@ +"""Regression contracts for ingestion transactions and review documentation.""" + +from __future__ import annotations + +import asyncio +import uuid +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +from backend.app import corporate_entity_ingestion as corporate_ingestion +from backend.app import keyman_ingestion +from backend.app import post_summary_ingestion as summary_ingestion +from lineageweave.corporate_hierarchy_inference import HierarchyProposal +from lineageweave.keyman_extraction import OUR_SIDE, PersonMention +from lineageweave.post_summary import ( + ACTOR_TYPE_ORGANIZATION, + ACTOR_TYPE_TEAM, + PostSummary, + RoleResponsibility, +) +from lineageweave.relation_verification import STATUS_CORROBORATED + + +class _RecordedTransaction: + """Record transaction entry and exit for one fake asyncpg connection.""" + + def __init__(self, events: list[Any], owner: Any | None = None) -> None: + self._events = events + self._owner = owner + + async def __aenter__(self) -> "_RecordedTransaction": + if self._owner is not None: + assert not self._owner.in_transaction + self._owner.in_transaction = True + self._events.append("transaction:enter") + return self + + async def __aexit__(self, exc_type, exc, traceback) -> bool: + self._events.append("transaction:exit") + if self._owner is not None: + self._owner.in_transaction = False + return False + + +class _InferenceClient: + """Return one verified root-company proposal without network access.""" + + available = True + + def __init__(self, events: list[Any]) -> None: + self._events = events + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal: + self._events.append("inference") + return HierarchyProposal(level_code="company", parent_name=None) + + +class _VerificationClient: + """Corroborate the synthetic proposal while recording call order.""" + + available = True + + def __init__(self, events: list[Any]) -> None: + self._events = events + + def verify(self, subject: str, relation: str) -> SimpleNamespace: + self._events.append("verification") + return SimpleNamespace(status_code=STATUS_CORROBORATED) + + +class _CorporateConnection: + """Minimal asyncpg-compatible connection for creation-lock behavior.""" + + def __init__( + self, + events: list[Any], + *, + reloaded_rows: tuple[dict[str, Any], ...] = (), + inserted_id: uuid.UUID | None = None, + allow_insert: bool = True, + ) -> None: + self._events = events + self._reloaded_rows = reloaded_rows + self._inserted_id = inserted_id or uuid.uuid4() + self._allow_insert = allow_insert + + def transaction(self) -> _RecordedTransaction: + self._events.append("transaction:open") + return _RecordedTransaction(self._events) + + async def execute(self, query: str, *args: Any) -> str: + compact = " ".join(query.split()) + assert "pg_advisory_xact_lock" in compact + self._events.append(("creation_lock", args, compact)) + return "SELECT 1" + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + compact = " ".join(query.split()) + assert compact == "select corporate_entity_id, entity_name from corporate_entity" + self._events.append("candidate_reload") + return list(self._reloaded_rows) + + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any]: + assert self._allow_insert, "locked candidate recheck should avoid insertion" + compact = " ".join(query.split()) + assert compact.startswith("insert into corporate_entity") + self._events.append("entity_insert") + return {"corporate_entity_id": self._inserted_id} + + +def test_corporate_entity_creation_locks_rechecks_and_inserts_in_one_transaction() -> None: + """Network verification precedes one transaction-scoped global creation lock.""" + events: list[Any] = [] + inserted_id = uuid.uuid4() + connection = _CorporateConnection(events, inserted_id=inserted_id) + candidates: list[Any] = [] + + result = asyncio.run( + corporate_ingestion.get_or_create_corporate_entity( + connection, + "Synthetic Energy", + "Synthetic context", + _InferenceClient(events), + _VerificationClient(events), + candidates, + ) + ) + + assert result == str(inserted_id) + assert [candidate.entity_name for candidate in candidates] == ["Synthetic Energy"] + assert events[:4] == [ + "inference", + "verification", + "transaction:open", + "transaction:enter", + ] + lock_event = events[4] + assert lock_event[0] == "creation_lock" + assert lock_event[1] == ("lineageweave:corporate_entity_creation",) + assert events[5:] == ["candidate_reload", "entity_insert", "transaction:exit"] + + +def test_locked_candidate_recheck_reuses_concurrently_created_entity() -> None: + """A same-name row committed after inference wins over a duplicate insert.""" + events: list[Any] = [] + existing_id = uuid.uuid4() + connection = _CorporateConnection( + events, + reloaded_rows=( + { + "corporate_entity_id": existing_id, + "entity_name": "Synthetic Energy", + }, + ), + allow_insert=False, + ) + candidates: list[Any] = [] + + result = asyncio.run( + corporate_ingestion.get_or_create_corporate_entity( + connection, + "Synthetic Energy", + "Synthetic context", + _InferenceClient(events), + _VerificationClient(events), + candidates, + ) + ) + + assert result == str(existing_id) + assert [candidate.entity_name for candidate in candidates] == ["Synthetic Energy"] + assert "entity_insert" not in events + assert events[-1] == "transaction:exit" + + +class _SummaryConnection: + """Minimal connection that records every post-summary database operation.""" + + def __init__(self, events: list[Any]) -> None: + self._events = events + self.in_transaction = False + + def transaction(self) -> _RecordedTransaction: + self._events.append("transaction:open") + return _RecordedTransaction(self._events, self) + + async def execute(self, query: str, *args: Any) -> str: + assert self.in_transaction + compact = " ".join(query.split()) + self._events.append(("execute", compact)) + return "OK" + + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any] | None: + compact = " ".join(query.split()) + self._events.append(("fetchrow", compact)) + if compact.startswith("select korean_summary from post_summary_result"): + assert not self.in_transaction + return {"korean_summary": "합성 요약"} + if compact.startswith("select person_id from cataloged_person"): + assert self.in_transaction + return None + raise AssertionError(f"unexpected fetchrow query: {compact}") + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + compact = " ".join(query.split()) + self._events.append(("fetch", compact)) + assert not self.in_transaction + if "from post_summary_event" in compact: + return [{"event_text": "검토 완료"}] + if "from post_summary_role" in compact: + assert "entity_name" not in compact + assert "cataloged_corporate_entity_id" in compact + return [ + { + "actor_name": "Synthetic Design Team", + "responsibility": "도면 검토", + "actor_type_code": ACTOR_TYPE_TEAM, + "affiliated_organization_name": "Synthetic Energy", + "cataloged_team_id": None, + "cataloged_corporate_entity_id": None, + } + ] + raise AssertionError(f"unexpected fetch query: {compact}") + + +def test_post_summary_replacement_mentions_and_edges_share_one_transaction(monkeypatch) -> None: + """Deletion, replacement, mention regeneration, and edges commit atomically.""" + events: list[Any] = [] + connection = _SummaryConnection(events) + team_id = str(uuid.uuid4()) + + async def load_candidates(conn) -> list[Any]: + events.append("candidate_load") + return [] + + async def upsert_team(conn, team_name, organization_name, candidates) -> str: + assert conn.in_transaction + events.append("team_upsert") + return team_id + + async def persist_edges(conn, post_id) -> list[Any]: + assert conn.in_transaction + events.append("edge_persist") + return [] + + monkeypatch.setattr(summary_ingestion, "_load_corporate_entity_candidates", load_candidates) + monkeypatch.setattr(summary_ingestion, "upsert_team", upsert_team) + monkeypatch.setattr(summary_ingestion, "persist_edges_for_post", persist_edges) + + summary = PostSummary( + korean_summary="합성 요약", + key_events=("검토 완료",), + roles_and_responsibilities=( + RoleResponsibility( + actor_name="Synthetic Design Team", + responsibility="도면 검토", + actor_type_code=ACTOR_TYPE_TEAM, + affiliated_organization_name="Synthetic Energy", + ), + ), + ) + + payload = asyncio.run( + summary_ingestion.persist_post_summary( + connection, + str(uuid.uuid4()), + summary, + ) + ) + + enter_index = events.index("transaction:enter") + exit_index = events.index("transaction:exit") + required_sql = ( + "delete from post_summary_person_mention", + "delete from post_team_mention", + "delete from post_organization_mention", + "delete from post_summary_result", + "insert into post_summary_result", + "insert into post_summary_event", + "insert into post_summary_role", + "insert into post_team_mention", + ) + for fragment in required_sql: + operation_index = next( + index + for index, event in enumerate(events) + if isinstance(event, tuple) + and event[0] == "execute" + and fragment in event[1] + ) + assert enter_index < operation_index < exit_index + assert events.index("candidate_load") < enter_index + assert enter_index < events.index("team_upsert") < exit_index + assert enter_index < events.index("edge_persist") < exit_index + assert payload["korean_summary"] == "합성 요약" + + +def test_organization_enrichment_finishes_before_summary_transaction(monkeypatch) -> None: + """LLM verification and the advisory-lock transaction precede summary writes.""" + events: list[Any] = [] + connection = _SummaryConnection(events) + corporate_entity_id = str(uuid.uuid4()) + + async def load_candidates(conn) -> list[Any]: + events.append(("candidate_load", conn.in_transaction)) + return [] + + async def resolve_organization( + conn, + organization_name, + context_text, + inference_client, + verification_client, + candidates, + ) -> str: + events.append(("organization_resolve", conn.in_transaction)) + assert not conn.in_transaction + return corporate_entity_id + + async def persist_edges(conn, post_id) -> list[Any]: + assert conn.in_transaction + events.append("edge_persist") + return [] + + monkeypatch.setattr(summary_ingestion, "_load_corporate_entity_candidates", load_candidates) + monkeypatch.setattr(summary_ingestion, "get_or_create_corporate_entity", resolve_organization) + monkeypatch.setattr(summary_ingestion, "persist_edges_for_post", persist_edges) + + summary = PostSummary( + korean_summary="합성 요약", + roles_and_responsibilities=( + RoleResponsibility( + actor_name="Synthetic Energy", + responsibility="납품 일정 확정", + actor_type_code=ACTOR_TYPE_ORGANIZATION, + ), + ), + ) + + asyncio.run( + summary_ingestion.persist_post_summary( + connection, + str(uuid.uuid4()), + summary, + ) + ) + + assert ("candidate_load", False) in events + assert ("organization_resolve", False) in events + resolve_index = events.index(("organization_resolve", False)) + enter_index = events.index("transaction:enter") + exit_index = events.index("transaction:exit") + mention_index = next( + index + for index, event in enumerate(events) + if isinstance(event, tuple) + and event[0] == "execute" + and "insert into post_organization_mention" in event[1] + ) + role_insert = next( + event[1] + for event in events + if isinstance(event, tuple) + and event[0] == "execute" + and "insert into post_summary_role" in event[1] + ) + assert "cataloged_corporate_entity_id" in role_insert + assert resolve_index < enter_index < mention_index < exit_index + + +class _KeymanConnection: + """Record whether organization enrichment runs outside the write transaction.""" + + def __init__(self, events: list[Any]) -> None: + self._events = events + self.in_transaction = False + + def transaction(self) -> _RecordedTransaction: + self._events.append("transaction:open") + return _RecordedTransaction(self._events, self) + + async def execute(self, query: str, *args: Any) -> str: + assert self.in_transaction + compact = " ".join(query.split()) + self._events.append(("execute", compact)) + return "OK" + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + compact = " ".join(query.split()) + self._events.append(("fetch", compact)) + if compact == "select corporate_entity_id, entity_name from corporate_entity": + assert not self.in_transaction + return [] + if compact.startswith("select person_id, last_known_job_title"): + assert self.in_transaction + return [] + raise AssertionError(f"unexpected fetch query: {compact}") + + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any] | None: + assert self.in_transaction + compact = " ".join(query.split()) + self._events.append(("fetchrow", compact)) + if compact.startswith("select person_id, last_known_job_title"): + return None + if compact.startswith("insert into cataloged_person"): + return {"person_id": uuid.uuid4()} + raise AssertionError(f"unexpected fetchrow query: {compact}") + + +def test_keyman_organization_enrichment_finishes_before_write_transaction(monkeypatch) -> None: + """LLM resolution and hierarchy creation must not hold the Keyman write lock.""" + events: list[Any] = [] + connection = _KeymanConnection(events) + corporate_entity_id = str(uuid.uuid4()) + + async def resolve_name(conn, resolution_client, verification_client, organization_name, post_body) -> str: + events.append(("organization_resolve", conn.in_transaction)) + assert not conn.in_transaction + return "Aurora Grid Power" + + async def resolve_organization( + conn, + organization_name, + context_text, + inference_client, + verification_client, + candidates, + ) -> str: + events.append(("organization_create", conn.in_transaction)) + assert not conn.in_transaction + return corporate_entity_id + + class _Client: + available = True + + def extract(self, post_title: str, post_body: str) -> list[PersonMention]: + return [ + PersonMention( + "Ada West", + OUR_SIDE, + affiliated_organization_names=("AGP",), + ) + ] + + monkeypatch.setattr(keyman_ingestion, "resolve_organization_name", resolve_name) + monkeypatch.setattr(keyman_ingestion, "get_or_create_corporate_entity", resolve_organization) + + asyncio.run( + keyman_ingestion.ingest_post_keymen( + connection, + _Client(), + str(uuid.uuid4()), + "Synthetic post", + "Ada West at AGP followed up.", + persist_graph=False, + ) + ) + + assert ("organization_resolve", False) in events + assert ("organization_create", False) in events + resolve_index = events.index(("organization_resolve", False)) + create_index = events.index(("organization_create", False)) + enter_index = events.index("transaction:enter") + mention_index = next( + index + for index, event in enumerate(events) + if isinstance(event, tuple) + and event[0] == "execute" + and "insert into post_person_mention" in event[1] + ) + assert resolve_index < enter_index + assert create_index < enter_index < mention_index + + +def test_release_notes_describe_balanced_outer_emphasis_stripping() -> None: + """Release notes must match the parser's reviewed normalization contract.""" + content = (Path(__file__).resolve().parents[1] / "CHANGELOG.md").read_text( + encoding="utf-8" + ) + assert "strips balanced outer Markdown emphasis from field values" in content + assert "while still accepting emphasized field labels" in content + assert "preserves Markdown emphasis in field values" not in content + + +def test_role_catalog_identity_is_stored_on_the_role_row() -> None: + """ADR 0019: fetch must not reconstruct organization identity by name.""" + root = Path(__file__).resolve().parents[1] + fetch_source = ( + root / "backend" / "app" / "post_summary_ingestion.py" + ).read_text(encoding="utf-8") + initial = (root / "migrations" / "0001_initial_schema.sql").read_text( + encoding="utf-8" + ) + upgrade = (root / "migrations" / "0019_role_catalog_identity.sql").read_text( + encoding="utf-8" + ) + dockerfile = ( + root / "docker" / "postgres-init" / "Dockerfile" + ).read_text(encoding="utf-8") + changelog = (root / "CHANGELOG.md").read_text(encoding="utf-8") + fetch_sql = fetch_source.split("async def fetch_persisted_summary", 1)[1] + fetch_sql = fetch_sql.split("async def persist_post_summary", 1)[0] + assert "org.entity_name = role.actor_name" not in fetch_sql + assert "cataloged_corporate_entity_id" in fetch_sql + assert "cataloged_team_id" in initial + assert "cataloged_corporate_entity_id" in upgrade + assert "0019_role_catalog_identity.sql" in dockerfile + assert "ADR 0019" in changelog diff --git a/tests/test_keyman_extraction.py b/tests/test_keyman_extraction.py index b1ab49da8..d9de45710 100644 --- a/tests/test_keyman_extraction.py +++ b/tests/test_keyman_extraction.py @@ -47,6 +47,24 @@ def test_parses_a_well_formed_json_array() -> None: assert mentions[1].affiliated_organization_names == ("Acme Corp", "Acme Holdings") +def test_job_title_is_captured_when_present() -> None: + content = '[{"name": "Kim Cheolsu", "side": "counterparty", "affiliations": [], "job_title": "Sales Manager"}]' + mentions = parse_keyman_response(content) + assert mentions[0].job_title == "Sales Manager" + + +def test_job_title_is_none_not_empty_string_when_absent() -> None: + content = '[{"name": "Kim Cheolsu", "side": "counterparty", "affiliations": []}]' + mentions = parse_keyman_response(content) + assert mentions[0].job_title is None + + +def test_null_job_title_is_none_not_the_string_null() -> None: + content = '[{"name": "Kim Cheolsu", "side": "counterparty", "affiliations": [], "job_title": null}]' + mentions = parse_keyman_response(content) + assert mentions[0].job_title is None + + def test_strips_a_markdown_code_fence() -> None: content = '```json\n[{"name": "Jo Park", "side": "our_side", "affiliations": []}]\n```' mentions = parse_keyman_response(content) @@ -110,3 +128,11 @@ def test_contextual_orchestrator_extracts_keymen_from_an_ambiguous_post() -> Non assert jordan.person_side_code == OUR_SIDE assert priya.person_side_code == COUNTERPARTY assert len(priya.affiliated_organization_names) >= 2 + + # Sam Okonkwo is named only by role ("our legal counsel, Sam Okonkwo") -- + # a real assertion that job_title extraction reads the text, not a + # synthetic fixture built just to satisfy this one field. + sam = next((m for name, m in by_name.items() if "Sam" in name or "Okonkwo" in name), None) + assert sam is not None + assert sam.job_title is not None + assert "counsel" in sam.job_title.lower() or "legal" in sam.job_title.lower() diff --git a/tests/test_ontology.py b/tests/test_ontology.py index 41423ada8..0e611bc8e 100644 --- a/tests/test_ontology.py +++ b/tests/test_ontology.py @@ -31,12 +31,31 @@ _SEED_SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "seed_demo_data.py" +# Several covered categories add lookup rows via their own migration +# SQL rather than literally embedded in seed_demo_data.py's own source +# text -- read alongside it below so the round-trip still sees them: +# 0012 (ADR 0006: prov_person/prov_organization), 0014 (ADR 0007: +# prov_team), 0016 (ADR 0009: node_team/edge_mention_team/ +# edge_team_affiliation/edge_mention_organization). +_ADDITIONAL_LOOKUP_MIGRATION_PATHS = ( + Path(__file__).resolve().parents[1] / "migrations" / "0012_role_responsibility_agent_type.sql", + Path(__file__).resolve().parents[1] / "migrations" / "0014_role_responsibility_team_actor_type.sql", + Path(__file__).resolve().parents[1] / "migrations" / "0016_cross_post_actor_identity.sql", +) + # The categories this ontology covers (ADR 0004's scope). seed_demo_data.py # also seeds categories this ontology deliberately does not model yet # (post_visibility, voc_type, permission, ticket_status) -- those are # real, expected gaps, not a test bug. _ONTOLOGY_COVERED_CATEGORIES = frozenset( - {"node_type", "edge_type", "entity_relationship_type", "person_side", "corporate_entity_level"} + { + "node_type", + "edge_type", + "entity_relationship_type", + "person_side", + "corporate_entity_level", + "prov_agent_type", + } ) _INSERT_TUPLE_PATTERN = re.compile(r"\('([a-z_]+)',\s*'([a-z_]+)'") @@ -44,12 +63,14 @@ def _seeded_lookup_codes_for_covered_categories() -> set[str]: """Every `(lookup_category, lookup_code)` pair seed_demo_data.py's own - SQL literally inserts, filtered to the categories this ontology - covers. Parsed from source, not executed -- this is a static - consistency check between two committed files, not a live-database - test. + SQL, plus the additional migrations' SQL, literally inserts, filtered + to the categories this ontology covers. Parsed from source, not + executed -- this is a static consistency check between committed + files, not a live-database test. """ - source = _SEED_SCRIPT_PATH.read_text() + source = _SEED_SCRIPT_PATH.read_text() + "".join( + p.read_text() for p in _ADDITIONAL_LOOKUP_MIGRATION_PATHS + ) return { code for category, code in _INSERT_TUPLE_PATTERN.findall(source) @@ -129,6 +150,38 @@ def test_mentions_property_domain_and_range_match_the_schema() -> None: assert (LW.mentions, RDFS.range, LW.Person) in graph +def test_prov_agent_type_terms_resolve_and_subclass_real_prov_o() -> None: + """Beyond the generic round-trip above: the two prov_agent_type terms + must actually subclass the real external W3C PROV-O classes, not + just carry a matching :lookupCode -- the whole point of grounding + this in a standard ontology is that :RoleActorPerson really is a + prov:Person, not a same-named local invention. + """ + from rdflib import URIRef + from rdflib.namespace import Namespace + + prov = Namespace("http://www.w3.org/ns/prov#") + graph = load_ontology() + assert iri_for_lookup_code("prov_person") == str(LW.RoleActorPerson) + assert iri_for_lookup_code("prov_organization") == str(LW.RoleActorOrganization) + assert (LW.RoleActorPerson, RDFS.subClassOf, URIRef(prov.Person)) in graph + assert (LW.RoleActorOrganization, RDFS.subClassOf, URIRef(prov.Organization)) in graph + + +def test_prov_team_type_resolves_and_subclasses_real_org_ontology() -> None: + """ADR 0007: a team actor is grounded in the real external W3C + Organization Ontology's org:OrganizationalUnit, the meso-level + sub-organization concept PROV-O itself has no equivalent for. + """ + from rdflib import URIRef + from rdflib.namespace import Namespace + + org = Namespace("http://www.w3.org/ns/org#") + graph = load_ontology() + assert iri_for_lookup_code("prov_team") == str(LW.RoleActorTeam) + assert (LW.RoleActorTeam, RDFS.subClassOf, URIRef(org.OrganizationalUnit)) in graph + + def test_corporate_entity_level_hierarchy_is_broadest_first() -> None: """Group is broader than Company is broader than Plant -- the Acme Group -> Acme Electronics Korea -> plant direction the @@ -137,3 +190,12 @@ def test_corporate_entity_level_hierarchy_is_broadest_first() -> None: assert (LW.CompanyLevel, SKOS.broader, LW.GroupLevel) in graph assert (LW.PlantLevel, SKOS.broader, LW.CompanyLevel) in graph assert (LW.GroupLevel, SKOS.broader, LW.CompanyLevel) not in graph + + +def test_actor_mentions_follow_stored_edge_direction() -> None: + """Ontology domain/range matches Team/Organization -> Post storage.""" + graph = load_ontology() + assert (LW.mentionsTeam, RDFS.domain, LW.Team) in graph + assert (LW.mentionsTeam, RDFS.range, LW.Post) in graph + assert (LW.mentionsOrganization, RDFS.domain, LW.CorporateEntity) in graph + assert (LW.mentionsOrganization, RDFS.range, LW.Post) in graph diff --git a/tests/test_organization_name_resolution.py b/tests/test_organization_name_resolution.py new file mode 100644 index 000000000..7eb0a96c3 --- /dev/null +++ b/tests/test_organization_name_resolution.py @@ -0,0 +1,125 @@ +"""Tests for lineageweave.organization_name_resolution (ADR 0008). + +Deterministic fake clients, same style as tests/test_post_summary.py +and tests/test_keyman_extraction.py's pure-parse-function tests -- the +underlying HTTP mechanics (post_json) and SearxngRelationVerificationClient's +own HTTP behavior are already covered in test_http_client.py and +test_relation_verification.py respectively; these tests are for this +module's own resolve-then-verify orchestration logic. +""" + +from __future__ import annotations + +from lineageweave.organization_name_resolution import ( + NullOrganizationNameResolutionClient, + OrganizationNameResolution, + parse_resolution_response, + resolve_and_verify_organization_name, +) +from lineageweave.relation_verification import ( + STATUS_CORROBORATED, + STATUS_PENDING, + STATUS_UNCORROBORATED, + NullRelationVerificationClient, + RelationVerificationResult, +) + + +class _FakeResolutionClient: + available = True + + def __init__(self, candidate: str | None) -> None: + self._candidate = candidate + self.calls: list[tuple[str, str]] = [] + + def resolve(self, raw_name: str, context_text: str) -> str | None: + self.calls.append((raw_name, context_text)) + return self._candidate + + +class _FakeVerificationClient: + available = True + + def __init__(self, result: RelationVerificationResult) -> None: + self._result = result + self.calls: list[tuple[str, str]] = [] + + def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: + self.calls.append((organization_name, relationship_label)) + return self._result + + +def test_parse_resolution_response_extracts_the_first_line() -> None: + assert parse_resolution_response("Aurora Grid Power\n") == "Aurora Grid Power" + + +def test_parse_resolution_response_rejects_unknown() -> None: + assert parse_resolution_response("UNKNOWN") is None + assert parse_resolution_response("unknown\n") is None + + +def test_parse_resolution_response_rejects_empty() -> None: + assert parse_resolution_response("") is None + assert parse_resolution_response(" ") is None + + +def test_no_resolution_when_client_unavailable() -> None: + result = resolve_and_verify_organization_name( + "AGP", "context", NullOrganizationNameResolutionClient(), NullRelationVerificationClient() + ) + assert result is None + + +def test_no_resolution_when_model_proposes_nothing() -> None: + result = resolve_and_verify_organization_name( + "AGP", "context", _FakeResolutionClient(None), NullRelationVerificationClient() + ) + assert result is None + + +def test_no_resolution_when_model_echoes_the_same_name() -> None: + """A "resolution" that just returns the raw name back is not a real + resolution -- must not be persisted as one.""" + result = resolve_and_verify_organization_name( + "AGP", "context", _FakeResolutionClient("AGP"), NullRelationVerificationClient() + ) + assert result is None + + +def test_corroborated_resolution_carries_evidence() -> None: + verification = _FakeVerificationClient( + RelationVerificationResult(status_code=STATUS_CORROBORATED, evidence_url="https://example.org/agp") + ) + resolution_client = _FakeResolutionClient("Aurora Grid Power") + result = resolve_and_verify_organization_name("AGP", "설계팀이 AGP와 회의했다", resolution_client, verification) + assert result == OrganizationNameResolution( + raw_organization_name="AGP", + resolved_organization_name="Aurora Grid Power", + verification_status_code=STATUS_CORROBORATED, + verification_evidence_url="https://example.org/agp", + ) + # The full name and the raw abbreviation are searched together -- + # the specific pairing is what needs corroborating, not just that + # the full name exists as some organization. + assert verification.calls == [("Aurora Grid Power", "AGP")] + + +def test_uncorroborated_resolution_still_returned_with_evidence_none() -> None: + verification = _FakeVerificationClient( + RelationVerificationResult(status_code=STATUS_UNCORROBORATED, evidence_url=None) + ) + result = resolve_and_verify_organization_name( + "AGP", "context", _FakeResolutionClient("Invented Co"), verification + ) + assert result is not None + assert result.verification_status_code == STATUS_UNCORROBORATED + assert result.verification_evidence_url is None + + +def test_verification_unavailable_yields_pending_not_a_fabricated_result() -> None: + result = resolve_and_verify_organization_name( + "AGP", "context", _FakeResolutionClient("Aurora Grid Power"), NullRelationVerificationClient() + ) + assert result is not None + assert result.verification_status_code == STATUS_PENDING + assert result.verification_evidence_url is None diff --git a/tests/test_person_mention_projection.py b/tests/test_person_mention_projection.py new file mode 100644 index 000000000..81e63a75f --- /dev/null +++ b/tests/test_person_mention_projection.py @@ -0,0 +1,544 @@ +"""Real-PostgreSQL regressions for source-aware person and graph projections. + +Keyman extraction and post-summary R&R are independent evidence channels. A +replacement in either channel must remove only that channel's stale person +mentions, then reconcile the buyer-facing Knowledge Graph from the currently +supported union. Orphan graph-registry rows must never become visible. +""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit +import uuid + +import asyncpg +import psycopg2 +from psycopg2 import sql +import pytest + +from backend.app.keyman_ingestion import ingest_post_keymen +from backend.app.knowledge_graph import ( + hydrate_related_nodes, + load_visible_subgraph, + persist_edges_for_post, + related_for_start, + visible_mention_post_ids, +) +from backend.app import post_summary_ingestion as summary_ingestion +from backend.app.post_summary_ingestion import ( + fetch_persisted_summary, + persist_post_summary, +) +from lineageweave.keyman_extraction import OUR_SIDE, PersonMention +from lineageweave.knowledge_graph import ( + EDGE_MENTION, + EDGE_MENTION_TEAM, + NODE_CORPORATE_ENTITY, + NODE_PERSON, + NODE_POST, + NODE_TEAM, +) +from lineageweave.post_summary import ( + ACTOR_TYPE_ORGANIZATION, + PostSummary, + RoleResponsibility, +) + +_ADMIN_DSN = os.environ.get( + "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" +) +_MIGRATION_PATH = Path(__file__).resolve().parents[1] / "migrations" / "0001_initial_schema.sql" + + +def _postgres_available() -> bool: + """Return whether the configured real PostgreSQL test service is reachable.""" + + try: + psycopg2.connect(_ADMIN_DSN, connect_timeout=2).close() + return True + except psycopg2.OperationalError: + return False + + +pytestmark = pytest.mark.skipif( + not _postgres_available(), + reason=f"no reachable PostgreSQL server at {_ADMIN_DSN}", +) + + +class _KeymanClient: + """Mutable deterministic extractor used to model replacement runs.""" + + available = True + + def __init__(self, mentions: list[PersonMention]) -> None: + self.mentions = mentions + + def extract(self, post_title: str, post_body: str) -> list[PersonMention]: + """Return a copy so production code cannot mutate the fixture.""" + + return list(self.mentions) + + +def _database_dsn(database_name: str) -> str: + """Replace only the database path while preserving DSN query parameters.""" + + parsed = urlsplit(_ADMIN_DSN) + return urlunsplit(parsed._replace(path=f"/{database_name}")) + + +@pytest.fixture +def projection_database() -> str: + """Create one freshly migrated PostgreSQL database and seed one post.""" + + database_name = f"lineageweave_projection_{uuid.uuid4().hex[:12]}" + admin = psycopg2.connect(_ADMIN_DSN) + admin.autocommit = True + with admin.cursor() as cursor: + cursor.execute(sql.SQL("create database {}").format(sql.Identifier(database_name))) + try: + database_dsn = _database_dsn(database_name) + connection = psycopg2.connect(database_dsn) + try: + with connection.cursor() as cursor: + cursor.execute(_MIGRATION_PATH.read_text(encoding="utf-8")) + cursor.execute( + """ + insert into common_lookup_value + (lookup_category, lookup_code, lookup_label) + values + ('corporate_entity_level', 'company', 'Company'), + ('post_visibility', 'public', 'Public'), + ('voc_type', 'voc', 'Voice of Customer'), + ('person_side', 'our_side', 'Our side'), + ('person_side', 'counterparty', 'Counterparty'), + ('prov_agent_type', 'prov_person', 'Person'), + ('prov_agent_type', 'prov_organization', 'Organization'), + ('prov_agent_type', 'prov_team', 'Team'), + ('node_type', 'node_person', 'Person node'), + ('node_type', 'node_post', 'Post node'), + ('node_type', 'node_corporate_entity', 'Corporate node'), + ('node_type', 'node_team', 'Team node'), + ('edge_type', 'edge_mention', 'Person mentioned in'), + ('edge_type', 'edge_affiliation', 'Person affiliated with'), + ('edge_type', 'edge_co_mention', 'People co-mentioned'), + ('edge_type', 'edge_mention_team', 'Team mentioned in'), + ('edge_type', 'edge_team_affiliation', 'Team affiliated with'), + ('edge_type', 'edge_mention_organization', 'Organization mentioned in') + """ + ) + cursor.execute( + """ + insert into corporate_entity + (corporate_entity_code, entity_name, entity_level_code) + values ('SYNTH-CORP', 'Synthetic Corp', 'company') + returning corporate_entity_id + """ + ) + corporate_entity_id = cursor.fetchone()[0] + cursor.execute( + """ + insert into user_account + (external_subject_id, display_name, email_address) + values ('projection-subject', 'Projection User', 'projection@example.test') + returning user_account_id + """ + ) + account_id = cursor.fetchone()[0] + cursor.execute( + """ + insert into source_post + (author_account_id, corporate_entity_id, post_title, post_body, + voc_type_code, visibility_code) + values (%s, %s, 'Synthetic post', 'Synthetic body', 'voc', 'public') + returning post_id + """, + (account_id, corporate_entity_id), + ) + post_id = cursor.fetchone()[0] + cursor.execute( + """ + insert into cataloged_person + (person_name, person_side_code, last_known_job_title) + values ('Summary Person', 'counterparty', 'Reviewer') + returning person_id + """ + ) + summary_person_id = cursor.fetchone()[0] + connection.commit() + finally: + connection.close() + yield "|".join((database_dsn, str(post_id), str(summary_person_id))) + finally: + with admin.cursor() as cursor: + cursor.execute(sql.SQL("drop database {}").format(sql.Identifier(database_name))) + admin.close() + + +async def _exercise_projection_contract( + database_dsn: str, + post_id: str, + summary_person_id: str, +) -> None: + """Run Keyman and R&R replacements and prove graph support follows them.""" + + connection = await asyncpg.connect(database_dsn) + try: + keyman = PersonMention("Keyman Person", OUR_SIDE) + client = _KeymanClient([keyman]) + await ingest_post_keymen( + connection, + client, + post_id, + "Synthetic post", + "Synthetic body", + ) + keyman_person_id = str( + await connection.fetchval( + "select person_id from cataloged_person where person_name = 'Keyman Person'" + ) + ) + + await persist_post_summary( + connection, + post_id, + PostSummary( + korean_summary="합성 요약", + roles_and_responsibilities=( + RoleResponsibility( + actor_name="Summary Person", + responsibility="검토", + ), + ), + ), + ) + + keyman_rows = await connection.fetch( + "select person_id from post_person_mention where post_id = $1", + post_id, + ) + summary_rows = await connection.fetch( + "select person_id from post_summary_person_mention where post_id = $1", + post_id, + ) + assert {str(row["person_id"]) for row in keyman_rows} == {keyman_person_id} + assert {str(row["person_id"]) for row in summary_rows} == {summary_person_id} + assert await visible_mention_post_ids( + connection, summary_person_id, lambda row: True + ) == [post_id] + + await persist_post_summary( + connection, + post_id, + PostSummary(korean_summary="역할이 제거된 합성 요약"), + ) + assert await visible_mention_post_ids( + connection, summary_person_id, lambda row: True + ) == [] + assert await visible_mention_post_ids( + connection, keyman_person_id, lambda row: True + ) == [post_id] + visible_edges = await load_visible_subgraph(connection, [post_id]) + visible_person_ids = { + edge.source_node_id + for edge in visible_edges + if edge.source_node_type_code == NODE_PERSON + } | { + edge.target_node_id + for edge in visible_edges + if edge.target_node_type_code == NODE_PERSON + } + assert summary_person_id not in visible_person_ids + assert keyman_person_id in visible_person_ids + + client.mentions = [] + await ingest_post_keymen( + connection, + client, + post_id, + "Synthetic post", + "Synthetic body", + ) + assert await visible_mention_post_ids( + connection, keyman_person_id, lambda row: True + ) == [] + assert await load_visible_subgraph(connection, [post_id]) == [] + + async with connection.transaction(): + await persist_edges_for_post(connection, post_id) + await persist_edges_for_post(connection, post_id) + duplicate_count = await connection.fetchval( + """ + select count(*) + from ( + select source_node_type_code, source_node_id, + target_node_type_code, target_node_id, edge_type_code + from knowledge_graph_edge + group by source_node_type_code, source_node_id, + target_node_type_code, target_node_id, edge_type_code + having count(*) > 1 + ) duplicate_edge + """ + ) + assert duplicate_count == 0 + + orphan_id = await connection.fetchval( + """ + insert into knowledge_graph_edge + (source_node_type_code, source_node_id, target_node_type_code, + target_node_id, edge_type_code, edge_weight) + values ($1, $2::uuid, $3, $4::uuid, $5, 1.0) + on conflict ( + source_node_type_code, source_node_id, + target_node_type_code, target_node_id, edge_type_code + ) do update set edge_weight = excluded.edge_weight + returning knowledge_graph_edge_id + """, + NODE_PERSON, + keyman_person_id, + NODE_POST, + post_id, + EDGE_MENTION, + ) + await connection.execute( + "delete from knowledge_graph_edge_evidence where knowledge_graph_edge_id = $1", + orphan_id, + ) + assert await load_visible_subgraph(connection, [post_id]) == [] + finally: + await connection.close() + + +def test_person_mention_sources_reconcile_without_stale_graph_edges( + projection_database: str, +) -> None: + """Each evidence channel replaces itself and the visible graph follows suit.""" + + database_dsn, post_id, summary_person_id = projection_database.split("|") + asyncio.run( + _exercise_projection_contract(database_dsn, post_id, summary_person_id) + ) + + +def test_cross_post_identity_upgrade_keeps_keyman_mention_context( + projection_database: str, +) -> None: + """Migration 0016 copies R&R names and must not steal Keyman mention_context.""" + + database_dsn, post_id, summary_person_id = projection_database.split("|") + migration = Path(__file__).resolve().parents[1] / "migrations" / "0016_cross_post_actor_identity.sql" + connection = psycopg2.connect(database_dsn) + try: + with connection.cursor() as cursor: + cursor.execute( + """ + insert into post_person_mention (post_id, person_id, mention_context) + values (%s, %s, %s) + """, + ( + post_id, + summary_person_id, + "Keyman extracted this mention from the synthetic body", + ), + ) + cursor.execute( + "insert into post_summary_result (post_id, korean_summary) values (%s, %s)", + (post_id, "합성 요약"), + ) + cursor.execute( + """ + insert into post_summary_role + (post_id, actor_name, responsibility, actor_type_code) + values (%s, 'Summary Person', '검토', 'prov_person') + """, + (post_id,), + ) + cursor.execute(migration.read_text(encoding="utf-8")) + cursor.execute( + """ + select mention_context + from post_person_mention + where post_id = %s and person_id = %s + """, + (post_id, summary_person_id), + ) + keyman_row = cursor.fetchone() + cursor.execute( + """ + select count(*) + from post_summary_person_mention + where post_id = %s and person_id = %s + """, + (post_id, summary_person_id), + ) + summary_count = cursor.fetchone()[0] + connection.commit() + finally: + connection.close() + + assert keyman_row is not None + assert keyman_row[0] == "Keyman extracted this mention from the synthetic body" + assert summary_count == 1 + + +async def _exercise_team_only_related_walk( + database_dsn: str, + first_post_id: str, +) -> None: + """A team mentioned on two posts must walk even when one post has no people.""" + + connection = await asyncpg.connect(database_dsn) + try: + author_id, corporate_entity_id = await connection.fetchrow( + "select author_account_id, corporate_entity_id from source_post where post_id = $1", + first_post_id, + ) + second_post_id = str( + await connection.fetchval( + """ + insert into source_post + (author_account_id, corporate_entity_id, post_title, post_body, + voc_type_code, visibility_code) + values ($1, $2, 'Team-only follow-up', '설계팀이 도면을 재검토했다.', + 'voc', 'public') + returning post_id + """, + author_id, + corporate_entity_id, + ) + ) + team_id = str( + await connection.fetchval( + """ + insert into cataloged_team (team_name, affiliated_organization_name) + values ('설계팀', 'Synthetic Corp') + returning team_id + """ + ) + ) + await connection.execute( + """ + insert into post_team_mention (post_id, team_id) + values ($1, $2), ($3, $2) + """, + first_post_id, + team_id, + second_post_id, + ) + async with connection.transaction(): + await persist_edges_for_post(connection, first_post_id) + await persist_edges_for_post(connection, second_post_id) + + team_only_edges = await load_visible_subgraph(connection, [second_post_id]) + assert any( + edge.edge_type_code == EDGE_MENTION_TEAM + and edge.source_node_id == team_id + and edge.target_node_id == second_post_id + for edge in team_only_edges + ), "a team-only post must still load its mention edge" + + related = await related_for_start( + connection, NODE_TEAM, team_id, [first_post_id, second_post_id] + ) + related_ids = {node["node_id"] for node in related} + assert first_post_id in related_ids + assert second_post_id in related_ids + hydrated = await hydrate_related_nodes( + connection, [(f"{NODE_TEAM}:{team_id}", 1.0)] + ) + assert hydrated[0]["label"] == "설계팀" + assert hydrated[0]["node_type_code"] == NODE_TEAM + finally: + await connection.close() + + +def test_team_only_posts_walk_related_nodes(projection_database: str) -> None: + """ADR 0018: team mention edges must participate in the visible RWR walk.""" + + database_dsn, post_id, _summary_person_id = projection_database.split("|") + asyncio.run(_exercise_team_only_related_walk(database_dsn, post_id)) + + +async def _exercise_homonym_organization_role_binding( + database_dsn: str, + post_id: str, +) -> None: + """A same-named catalog org that this post did not resolve must stay off the role.""" + + connection = await asyncpg.connect(database_dsn) + try: + mentioned_id = str( + await connection.fetchval( + """ + insert into corporate_entity + (corporate_entity_code, entity_name, entity_level_code) + values ('HOMONYM-MENTIONED', 'Homonym Energy', 'company') + returning corporate_entity_id + """ + ) + ) + other_id = str( + await connection.fetchval( + """ + insert into corporate_entity + (corporate_entity_code, entity_name, entity_level_code) + values ('HOMONYM-OTHER', 'Homonym Energy', 'company') + returning corporate_entity_id + """ + ) + ) + + async def resolve_mentioned_organization(*_args, **_kwargs) -> str: + return mentioned_id + + original = summary_ingestion.get_or_create_corporate_entity + summary_ingestion.get_or_create_corporate_entity = resolve_mentioned_organization + try: + payload = await persist_post_summary( + connection, + post_id, + PostSummary( + korean_summary="동명이인 조직이 일정만 확정했다.", + roles_and_responsibilities=( + RoleResponsibility( + actor_name="Homonym Energy", + responsibility="납품 일정 확정", + actor_type_code=ACTOR_TYPE_ORGANIZATION, + ), + ), + ), + ) + finally: + summary_ingestion.get_or_create_corporate_entity = original + + roles = payload["roles_and_responsibilities"] + assert len(roles) == 1 + assert roles[0]["catalog_node_id"] == mentioned_id + assert roles[0]["catalog_node_type_code"] == NODE_CORPORATE_ENTITY + fetched = await fetch_persisted_summary(connection, post_id) + assert fetched is not None + assert fetched["roles_and_responsibilities"][0]["catalog_node_id"] == mentioned_id + assert fetched["roles_and_responsibilities"][0]["catalog_node_id"] != other_id + mention_ids = [ + str(row["corporate_entity_id"]) + for row in await connection.fetch( + "select corporate_entity_id from post_organization_mention " + "where post_id = $1", + post_id, + ) + ] + assert mention_ids == [mentioned_id] + finally: + await connection.close() + + +def test_homonym_organization_role_binds_the_resolved_catalog_id( + projection_database: str, +) -> None: + """ADR 0019: two catalog orgs can share a display name; the role keeps one id.""" + + database_dsn, post_id, _summary_person_id = projection_database.split("|") + asyncio.run(_exercise_homonym_organization_role_binding(database_dsn, post_id)) diff --git a/tests/test_post_summary.py b/tests/test_post_summary.py index 52694bc3b..4f863769a 100644 --- a/tests/test_post_summary.py +++ b/tests/test_post_summary.py @@ -24,6 +24,7 @@ from lineageweave.post_summary import ( ContextualOrchestratorPostSummaryClient, NullPostSummaryClient, + RoleResponsibility, parse_summary_response, ) @@ -39,13 +40,72 @@ def test_parses_a_well_formed_json_object() -> None: content = ( '{"korean_summary": "회의 후속 조치에 대한 요약입니다.", ' '"key_events": ["입찰 워크숍 진행", "검사 일정 확인 요청"], ' - '"roles_and_responsibilities": [{"person_name": "Jordan Hale", "responsibility": "입찰 일정 안내"}]}' + '"roles_and_responsibilities": [{"actor_name": "Jordan Hale", "responsibility": "입찰 일정 안내", ' + '"actor_type": "person", "affiliated_organization_name": "Westfield Power"}]}' ) summary = parse_summary_response(content) assert summary is not None assert summary.korean_summary == "회의 후속 조치에 대한 요약입니다." assert summary.key_events == ("입찰 워크숍 진행", "검사 일정 확인 요청") - assert summary.roles_and_responsibilities[0].person_name == "Jordan Hale" + role = summary.roles_and_responsibilities[0] + assert role.actor_name == "Jordan Hale" + assert role.actor_type_code == "prov_person" + assert role.affiliated_organization_name == "Westfield Power" + + +def test_organization_actor_is_not_forced_into_a_person_slot() -> None: + """A named actor that is genuinely an organization (e.g. our own + company acting in its own name, not a named individual) must parse + as ``prov_organization``, not silently default to person -- the + default only applies when the model omits ``actor_type`` entirely. + """ + content = ( + '{"korean_summary": "당사가 요청 사항을 확인했습니다.", "key_events": [], ' + '"roles_and_responsibilities": [{"actor_name": "당사", "responsibility": "요청 확인", ' + '"actor_type": "organization", "affiliated_organization_name": null}]}' + ) + summary = parse_summary_response(content) + assert summary is not None + role = summary.roles_and_responsibilities[0] + assert role.actor_name == "당사" + assert role.actor_type_code == "prov_organization" + assert role.affiliated_organization_name is None + + +def test_team_actor_is_meso_level_not_organization() -> None: + """A named sub-unit of a company (e.g. 설계팀, "design team") must + parse as ``prov_team``, distinct from both ``prov_person`` and + ``prov_organization`` -- it is part of a company, not the company + itself (ADR 0007), and its parent company's name must still land in + ``affiliated_organization_name``. + """ + content = ( + '{"korean_summary": "설계팀이 도면을 검토했습니다.", "key_events": [], ' + '"roles_and_responsibilities": [{"actor_name": "설계팀", "responsibility": "도면 검토", ' + '"actor_type": "team", "affiliated_organization_name": "Demo Corp"}]}' + ) + summary = parse_summary_response(content) + assert summary is not None + role = summary.roles_and_responsibilities[0] + assert role.actor_name == "설계팀" + assert role.actor_type_code == "prov_team" + assert role.affiliated_organization_name == "Demo Corp" + + +def test_unknown_actor_type_code_is_rejected() -> None: + with pytest.raises(ValueError, match="actor_type_code"): + RoleResponsibility(actor_name="Ada West", responsibility="후속", actor_type_code="person") + + +def test_missing_actor_type_defaults_to_person() -> None: + content = ( + '{"korean_summary": "요약", "key_events": [], ' + '"roles_and_responsibilities": [{"actor_name": "Ada West", "responsibility": "후속"}]}' + ) + summary = parse_summary_response(content) + assert summary is not None + assert summary.roles_and_responsibilities[0].actor_type_code == "prov_person" + assert summary.roles_and_responsibilities[0].affiliated_organization_name is None def test_missing_korean_summary_returns_none() -> None: @@ -75,7 +135,7 @@ def test_every_sample_record_has_a_seeded_korean_summary() -> None: assert summary.korean_summary not in seen seen.add(summary.korean_summary) cast = fixture_thread_cast(rec.label) - names = {role.person_name for role in summary.roles_and_responsibilities} + names = {role.actor_name for role in summary.roles_and_responsibilities} if cast is not None and cast.person_names: assert set(cast.person_names) <= names else: @@ -91,7 +151,7 @@ def test_every_sample_record_has_a_seeded_korean_summary() -> None: def test_malformed_roles_entries_are_skipped_not_crashed_on() -> None: content = ( '{"korean_summary": "요약", "key_events": [], ' - '"roles_and_responsibilities": [{"person_name": "Only Name"}, "not an object"]}' + '"roles_and_responsibilities": [{"actor_name": "Only Name"}, "not an object"]}' ) summary = parse_summary_response(content) assert summary is not None @@ -119,5 +179,5 @@ def test_contextual_orchestrator_summarizes_a_non_trivial_post() -> None: # block -- not just an English sentence handed back unchanged. assert any("가" <= ch <= "힣" for ch in summary.korean_summary) assert len(summary.key_events) >= 1 - people_named = {rr.person_name for rr in summary.roles_and_responsibilities} + people_named = {rr.actor_name for rr in summary.roles_and_responsibilities} assert any("Jordan" in name or "Priya" in name for name in people_named) diff --git a/tests/test_prov_o.py b/tests/test_prov_o.py new file mode 100644 index 000000000..2a33a9b8c --- /dev/null +++ b/tests/test_prov_o.py @@ -0,0 +1,401 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path + +import sys + +import pytest +from rdflib import Graph, Literal, Namespace, URIRef +from rdflib.namespace import RDF, XSD + +from lineageweave.prov_o import ( + PROV, + PROV_CLASSES, + PROV_QUALIFICATIONS, + PROV_RELATIONS, + PROV_RECOMMENDED_INVERSES, + ProvAssertion, + ProvGraph, + ProvLiteral, + ProvValidationError, + class_code, + relation_code, +) + +EXPECTED_CLASS_NAMES = { + "Entity", "Activity", "Agent", "Collection", "EmptyCollection", "Bundle", + "Person", "SoftwareAgent", "Organization", "Location", "Influence", + "EntityInfluence", "Usage", "Start", "End", "Derivation", "PrimarySource", + "Quotation", "Revision", "ActivityInfluence", "Generation", "Communication", + "Invalidation", "AgentInfluence", "Attribution", "Association", "Plan", + "Delegation", "InstantaneousEvent", "Role", +} + +EXPECTED_RELATION_NAMES = { + "wasGeneratedBy", "wasDerivedFrom", "wasAttributedTo", "startedAtTime", "used", + "wasInformedBy", "endedAtTime", "wasAssociatedWith", "actedOnBehalfOf", + "alternateOf", "specializationOf", "generatedAtTime", "hadPrimarySource", "value", + "wasQuotedFrom", "wasRevisionOf", "invalidatedAtTime", "wasInvalidatedBy", + "hadMember", "wasStartedBy", "wasEndedBy", "invalidated", "influenced", + "atLocation", "generated", "wasInfluencedBy", "qualifiedInfluence", + "qualifiedGeneration", "qualifiedDerivation", "qualifiedPrimarySource", + "qualifiedQuotation", "qualifiedRevision", "qualifiedAttribution", + "qualifiedInvalidation", "qualifiedStart", "qualifiedUsage", + "qualifiedCommunication", "qualifiedAssociation", "qualifiedEnd", + "qualifiedDelegation", "influencer", "entity", "hadUsage", "hadGeneration", + "activity", "agent", "hadPlan", "hadActivity", "atTime", "hadRole", +} + +EXPECTED_DATATYPE_RELATIONS = { + "startedAtTime", "endedAtTime", "generatedAtTime", "invalidatedAtTime", "value", "atTime" +} + +EXPECTED_QUALIFICATIONS = { + "wasGeneratedBy": ("qualifiedGeneration", "Generation", "activity"), + "wasDerivedFrom": ("qualifiedDerivation", "Derivation", "entity"), + "wasAttributedTo": ("qualifiedAttribution", "Attribution", "agent"), + "used": ("qualifiedUsage", "Usage", "entity"), + "wasInformedBy": ("qualifiedCommunication", "Communication", "activity"), + "wasAssociatedWith": ("qualifiedAssociation", "Association", "agent"), + "actedOnBehalfOf": ("qualifiedDelegation", "Delegation", "agent"), + "wasInfluencedBy": ("qualifiedInfluence", "Influence", "influencer"), + "hadPrimarySource": ("qualifiedPrimarySource", "PrimarySource", "entity"), + "wasQuotedFrom": ("qualifiedQuotation", "Quotation", "entity"), + "wasRevisionOf": ("qualifiedRevision", "Revision", "entity"), + "wasInvalidatedBy": ("qualifiedInvalidation", "Invalidation", "activity"), + "wasStartedBy": ("qualifiedStart", "Start", "entity"), + "wasEndedBy": ("qualifiedEnd", "End", "entity"), +} + + +def test_registry_contains_every_normative_prov_o_class_and_relation() -> None: + assert set(PROV_CLASSES) == EXPECTED_CLASS_NAMES + assert set(PROV_RELATIONS) == EXPECTED_RELATION_NAMES + assert len(PROV_CLASSES) == 30 + assert len(PROV_RELATIONS) == 50 + + +def test_registry_distinguishes_all_six_datatype_properties() -> None: + actual = {name for name, spec in PROV_RELATIONS.items() if spec.property_kind == "datatype"} + assert actual == EXPECTED_DATATYPE_RELATIONS + assert {name for name, spec in PROV_RELATIONS.items() if spec.property_kind == "object"} == ( + EXPECTED_RELATION_NAMES - EXPECTED_DATATYPE_RELATIONS + ) + + +def test_qualification_table_matches_both_normative_tables() -> None: + actual = { + item.unqualified_relation: ( + item.qualification_relation, + item.influence_class, + item.influencer_relation, + ) + for item in PROV_QUALIFICATIONS + } + assert actual == EXPECTED_QUALIFICATIONS + + +def test_every_object_property_has_the_appendix_b_inverse_name() -> None: + object_properties = { + name for name, spec in PROV_RELATIONS.items() if spec.property_kind == "object" + } + assert set(PROV_RECOMMENDED_INVERSES) == object_properties + assert PROV_RECOMMENDED_INVERSES["actedOnBehalfOf"].inverse_local_name == "hadDelegate" + assert PROV_RECOMMENDED_INVERSES["wasDerivedFrom"].inverse_local_name == "hadDerivation" + assert PROV_RECOMMENDED_INVERSES["specializationOf"].inverse_local_name == "generalizationOf" + assert PROV_RECOMMENDED_INVERSES["wasGeneratedBy"].inverse_local_name == "generated" + assert PROV_RECOMMENDED_INVERSES["alternateOf"].inverse_local_name == "alternateOf" + + +def test_codes_are_stable_two_word_snake_case() -> None: + assert class_code("Entity") == "prov_entity" + assert class_code("InstantaneousEvent") == "prov_instantaneous_event" + assert relation_code("wasGeneratedBy") == "prov_was_generated_by" + assert relation_code("qualifiedPrimarySource") == "prov_qualified_primary_source" + for name in PROV_CLASSES: + assert class_code(name).startswith("prov_") and "_" in class_code(name) + for name in PROV_RELATIONS: + assert relation_code(name).startswith("prov_") and "_" in relation_code(name) + + +def _graph_with_core_resources() -> ProvGraph: + graph = ProvGraph() + graph.add_resource("urn:entity:input", "Entity") + graph.add_resource("urn:entity:output", "Entity") + graph.add_resource("urn:activity:transform", "Activity") + graph.add_resource("urn:agent:operator", "Person") + graph.add_resource("urn:agent:principal", "Organization") + graph.add_resource("urn:location:lab", "Location") + graph.add_resource("urn:plan:procedure", "Plan") + graph.add_resource("urn:role:reviewer", "Role") + return graph + + +def test_graph_rejects_wrong_object_kind_and_wrong_domain() -> None: + graph = _graph_with_core_resources() + with pytest.raises(ProvValidationError, match="requires a resource object"): + graph.add_assertion("urn:activity:transform", "used", ProvLiteral("not-a-resource")) + with pytest.raises(ProvValidationError, match="requires a literal object"): + graph.add_assertion("urn:activity:transform", "startedAtTime", "urn:entity:input") + with pytest.raises(ProvValidationError, match="subject.*Entity"): + graph.add_assertion("urn:agent:operator", "wasDerivedFrom", "urn:entity:input") + + +def test_subclass_membership_satisfies_agent_domain() -> None: + graph = _graph_with_core_resources() + graph.add_assertion("urn:agent:operator", "actedOnBehalfOf", "urn:agent:principal") + assert ProvAssertion.resource( + "urn:agent:operator", "actedOnBehalfOf", "urn:agent:principal" + ) in graph.explicit_assertions + + +@pytest.mark.parametrize( + ("unqualified", "qualified", "influence_class", "influencer_relation", "subject", "object_iri"), + [ + ("wasGeneratedBy", "qualifiedGeneration", "Generation", "activity", "urn:entity:output", "urn:activity:transform"), + ("wasDerivedFrom", "qualifiedDerivation", "Derivation", "entity", "urn:entity:output", "urn:entity:input"), + ("wasAttributedTo", "qualifiedAttribution", "Attribution", "agent", "urn:entity:output", "urn:agent:operator"), + ("used", "qualifiedUsage", "Usage", "entity", "urn:activity:transform", "urn:entity:input"), + ("wasInformedBy", "qualifiedCommunication", "Communication", "activity", "urn:activity:transform", "urn:activity:source"), + ("wasAssociatedWith", "qualifiedAssociation", "Association", "agent", "urn:activity:transform", "urn:agent:operator"), + ("actedOnBehalfOf", "qualifiedDelegation", "Delegation", "agent", "urn:agent:operator", "urn:agent:principal"), + ("wasInfluencedBy", "qualifiedInfluence", "Influence", "influencer", "urn:entity:output", "urn:entity:input"), + ("hadPrimarySource", "qualifiedPrimarySource", "PrimarySource", "entity", "urn:entity:output", "urn:entity:input"), + ("wasQuotedFrom", "qualifiedQuotation", "Quotation", "entity", "urn:entity:output", "urn:entity:input"), + ("wasRevisionOf", "qualifiedRevision", "Revision", "entity", "urn:entity:output", "urn:entity:input"), + ("wasInvalidatedBy", "qualifiedInvalidation", "Invalidation", "activity", "urn:entity:output", "urn:activity:transform"), + ("wasStartedBy", "qualifiedStart", "Start", "entity", "urn:activity:transform", "urn:entity:input"), + ("wasEndedBy", "qualifiedEnd", "End", "entity", "urn:activity:transform", "urn:entity:output"), + ], +) +def test_each_qualified_form_implies_its_unqualified_form( + unqualified: str, + qualified: str, + influence_class: str, + influencer_relation: str, + subject: str, + object_iri: str, +) -> None: + graph = _graph_with_core_resources() + graph.add_resource("urn:activity:source", "Activity") + graph.add_resource("urn:influence:q", influence_class) + graph.add_assertion(subject, qualified, "urn:influence:q") + graph.add_assertion("urn:influence:q", influencer_relation, object_iri) + assert ProvAssertion.resource(subject, unqualified, object_iri) in graph.materialized_assertions() + + +def test_specific_derivation_implies_general_derivation_and_influence() -> None: + graph = _graph_with_core_resources() + graph.add_assertion("urn:entity:output", "wasQuotedFrom", "urn:entity:input") + materialized = graph.materialized_assertions() + assert ProvAssertion.resource("urn:entity:output", "wasDerivedFrom", "urn:entity:input") in materialized + assert ProvAssertion.resource("urn:entity:output", "wasInfluencedBy", "urn:entity:input") in materialized + + +def test_defined_inverse_and_symmetric_properties_are_materialized() -> None: + graph = _graph_with_core_resources() + graph.add_assertion("urn:entity:output", "wasGeneratedBy", "urn:activity:transform") + graph.add_assertion("urn:entity:output", "alternateOf", "urn:entity:input") + materialized = graph.materialized_assertions() + assert ProvAssertion.resource("urn:activity:transform", "generated", "urn:entity:output") in materialized + assert ProvAssertion.resource("urn:entity:input", "alternateOf", "urn:entity:output") in materialized + + +def test_reserved_inverse_alias_is_normalized_by_reversing_endpoints() -> None: + graph = _graph_with_core_resources() + graph.add_assertion("urn:entity:input", "hadDerivation", "urn:entity:output") + assert ProvAssertion.resource( + "urn:entity:output", "wasDerivedFrom", "urn:entity:input" + ) in graph.explicit_assertions + + +def test_qualified_event_time_implies_direct_time_property() -> None: + graph = _graph_with_core_resources() + graph.add_resource("urn:influence:generation", "Generation") + instant = ProvLiteral.datetime(datetime(2026, 8, 14, 4, 0, tzinfo=timezone.utc)) + graph.add_assertion("urn:entity:output", "qualifiedGeneration", "urn:influence:generation") + graph.add_assertion("urn:influence:generation", "activity", "urn:activity:transform") + graph.add_assertion("urn:influence:generation", "atTime", instant) + materialized = graph.materialized_assertions() + assert ProvAssertion.literal("urn:entity:output", "generatedAtTime", instant) in materialized + + +def test_rdf_serialization_uses_exact_prov_namespace_and_xsd_datetime() -> None: + graph = _graph_with_core_resources() + instant = ProvLiteral.datetime(datetime(2026, 8, 14, 4, 0, tzinfo=timezone.utc)) + graph.add_assertion("urn:activity:transform", "startedAtTime", instant) + rdf_graph = graph.to_rdflib(materialize=True) + assert (URIRef("urn:entity:input"), RDF.type, PROV.Entity) in rdf_graph + assert ( + URIRef("urn:activity:transform"), + PROV.startedAtTime, + Literal("2026-08-14T04:00:00+00:00", datatype=XSD.dateTime), + ) in rdf_graph + + +def test_sql_migration_seeds_every_class_relation_and_qualification() -> None: + sql_path = Path(__file__).resolve().parents[1] / "migrations" / "0017_prov_o_standard_relations.sql" + sql = sql_path.read_text() + for name in EXPECTED_CLASS_NAMES: + assert class_code(name) in sql + assert f"http://www.w3.org/ns/prov#{name}" in sql + for name in EXPECTED_RELATION_NAMES: + assert relation_code(name) in sql + assert f"http://www.w3.org/ns/prov#{name}" in sql + for unqualified, (qualified, influence_class, influencer) in EXPECTED_QUALIFICATIONS.items(): + assert relation_code(unqualified) in sql + assert relation_code(qualified) in sql + assert class_code(influence_class) in sql + assert relation_code(influencer) in sql + + +def test_sql_migration_uses_only_multiword_snake_case_table_names() -> None: + import re + + sql = (Path(__file__).resolve().parents[1] / "migrations" / "0017_prov_o_standard_relations.sql").read_text() + names = re.findall(r"create table(?: if not exists)?\s+([a-z_]+)", sql, flags=re.IGNORECASE) + assert names + assert all(len(name.split("_")) >= 2 for name in names) + + +def test_registry_spec_accessors_and_inverse_iri_use_exact_namespace() -> None: + assert PROV_CLASSES["Entity"].iri == "http://www.w3.org/ns/prov#Entity" + assert PROV_CLASSES["Entity"].code == "prov_entity" + assert PROV_RELATIONS["used"].iri == "http://www.w3.org/ns/prov#used" + assert PROV_RELATIONS["used"].code == "prov_used" + assert ( + PROV_RECOMMENDED_INVERSES["actedOnBehalfOf"].inverse_iri + == "http://www.w3.org/ns/prov#hadDelegate" + ) + + +def test_literal_contract_rejects_conflicts_invalid_language_and_naive_time() -> None: + with pytest.raises(ProvValidationError, match="both datatype_iri and language_tag"): + ProvLiteral("x", datatype_iri=str(XSD.string), language_tag="en") + with pytest.raises(ProvValidationError, match="language_tag"): + ProvLiteral("x", language_tag="not_a_tag!") + with pytest.raises(ProvValidationError, match="timezone-aware"): + ProvLiteral.datetime(datetime(2026, 8, 14, 4, 0)) + assert ProvLiteral("bonjour", language_tag="fr").to_rdflib() == Literal("bonjour", lang="fr") + + +def test_assertion_requires_exactly_one_object_kind() -> None: + with pytest.raises(ProvValidationError, match="exactly one"): + ProvAssertion("urn:s", "used") + with pytest.raises(ProvValidationError, match="exactly one"): + ProvAssertion( + "urn:s", + "used", + object_resource_iri="urn:o", + object_literal=ProvLiteral("x"), + ) + + +def test_resource_registration_and_name_normalization_fail_closed() -> None: + graph = ProvGraph() + with pytest.raises(ProvValidationError, match="resource_iri"): + graph.add_resource("", "Entity") + with pytest.raises(ProvValidationError, match="at least one"): + graph.add_resource("urn:empty") + with pytest.raises(ProvValidationError, match="unknown PROV-O class"): + graph.add_resource("urn:bad", "NotAClass") + + graph.add_resource("urn:e", "prov:Entity") + graph.add_resource("urn:a", "http://www.w3.org/ns/prov#Activity") + assert graph.resource_types == { + "urn:e": frozenset({"Entity"}), + "urn:a": frozenset({"Activity"}), + } + + +def test_assertion_name_and_endpoint_validation_fail_closed() -> None: + graph = _graph_with_core_resources() + with pytest.raises(ProvValidationError, match="unknown PROV-O relation"): + graph.add_assertion("urn:entity:input", "notARelation", "urn:entity:output") + with pytest.raises(ProvValidationError, match="subject resource"): + graph.add_assertion("urn:missing", "prov:wasDerivedFrom", "urn:entity:input") + with pytest.raises(ProvValidationError, match="object resource"): + graph.add_assertion( + "urn:entity:output", + "http://www.w3.org/ns/prov#wasDerivedFrom", + "urn:missing", + ) + with pytest.raises(ProvValidationError, match="object.*Entity"): + graph.add_assertion("urn:activity:transform", "used", "urn:role:reviewer") + with pytest.raises(ProvValidationError, match="requires datatype"): + graph.add_assertion( + "urn:activity:transform", + "startedAtTime", + ProvLiteral("2026-08-14T04:00:00Z"), + ) + with pytest.raises(ProvValidationError, match="cannot reverse a literal"): + graph.add_assertion( + "urn:entity:input", + "hadDerivation", + ProvLiteral("invalid"), + ) + + +def test_rdf_serialization_covers_resource_and_literal_objects_without_materialization() -> None: + graph = _graph_with_core_resources() + graph.add_assertion("urn:activity:transform", "used", "urn:entity:input") + graph.add_assertion("urn:entity:input", "value", ProvLiteral("raw value")) + rdf_graph = graph.to_rdflib() + assert ( + URIRef("urn:activity:transform"), + PROV.used, + URIRef("urn:entity:input"), + ) in rdf_graph + assert ( + URIRef("urn:entity:input"), + PROV.value, + Literal("raw value"), + ) in rdf_graph + + +def test_every_public_callable_has_a_docstring() -> None: + import inspect + + module = sys.modules["lineageweave.prov_o"] + + missing: list[str] = [] + for name, value in vars(module).items(): + if name.startswith("_"): + continue + if inspect.isfunction(value) or inspect.isclass(value): + if value.__module__ == module.__name__ and not inspect.getdoc(value): + missing.append(name) + if inspect.isclass(value) and value.__module__ == module.__name__: + for member_name, member in vars(value).items(): + if member_name.startswith("_"): + continue + target = member.fget if isinstance(member, property) else member + if callable(target) and not inspect.getdoc(target): + missing.append(f"{name}.{member_name}") + assert missing == [] + + +def test_support_profile_imports_prov_o_and_maps_product_classes() -> None: + from rdflib.namespace import OWL, RDFS + + profile_path = ( + Path(__file__).resolve().parents[1] + / "docs" + / "ontology" + / "prov-o-support-profile.ttl" + ) + profile = Graph().parse(profile_path, format="turtle") + ontology_iri = URIRef( + "https://contextualwisdomlab.github.io/LineageWeave/prov-o-support" + ) + local = Namespace("https://contextualwisdomlab.github.io/LineageWeave/ontology#") + assert ( + ontology_iri, + OWL.imports, + URIRef("http://www.w3.org/ns/prov-o#"), + ) in profile + assert (local.Post, RDFS.subClassOf, PROV.Entity) in profile + assert (local.Person, RDFS.subClassOf, PROV.Person) in profile + assert (local.CorporateEntity, RDFS.subClassOf, PROV.Organization) in profile + assert (local.Team, RDFS.subClassOf, PROV.Organization) in profile diff --git a/tests/test_prov_o_schema.py b/tests/test_prov_o_schema.py new file mode 100644 index 000000000..733c96076 --- /dev/null +++ b/tests/test_prov_o_schema.py @@ -0,0 +1,261 @@ +"""Real-PostgreSQL contract tests for the PROV-O migration. + +The module applies the actual base and PROV-O migration files to a throwaway +database. It self-skips when no local PostgreSQL is reachable, matching the +repository's existing real-database schema tests. +""" + +from __future__ import annotations + +import os +import uuid +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +import pytest + +psycopg2 = pytest.importorskip("psycopg2") +sql = pytest.importorskip("psycopg2.sql") + +_ADMIN_DSN = os.environ.get( + "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" +) +_ROOT = Path(__file__).resolve().parents[1] +_MIGRATION_PATHS = ( + _ROOT / "migrations" / "0001_initial_schema.sql", + _ROOT / "migrations" / "0017_prov_o_standard_relations.sql", +) + + +def _postgres_available() -> bool: + """Return whether the configured PostgreSQL admin database is reachable.""" + try: + connection = psycopg2.connect(_ADMIN_DSN, connect_timeout=2) + connection.close() + return True + except psycopg2.OperationalError: + return False + + +def _dsn_for_database(admin_dsn: str, database_name: str) -> str: + """Replace only the database path while preserving DSN query options.""" + parsed_admin_dsn = urlsplit(admin_dsn) + return urlunsplit(parsed_admin_dsn._replace(path=f"/{database_name}")) + + +pytestmark = pytest.mark.skipif( + not _postgres_available(), + reason=f"no reachable PostgreSQL server at {_ADMIN_DSN}", +) + + +@pytest.fixture +def prov_schema_db(): + """Yield a freshly migrated database and drop it after the test.""" + database_name = f"lineageweave_prov_{uuid.uuid4().hex[:12]}" + admin_connection = psycopg2.connect(_ADMIN_DSN) + admin_connection.autocommit = True + with admin_connection.cursor() as cursor: + cursor.execute( + sql.SQL("create database {}").format(sql.Identifier(database_name)) + ) + try: + database_dsn = _dsn_for_database(_ADMIN_DSN, database_name) + connection = psycopg2.connect(database_dsn) + try: + with connection.cursor() as cursor: + for migration_path in _MIGRATION_PATHS: + cursor.execute(migration_path.read_text()) + connection.commit() + yield connection + finally: + connection.close() + finally: + with admin_connection.cursor() as cursor: + cursor.execute( + sql.SQL("drop database {}").format(sql.Identifier(database_name)) + ) + admin_connection.close() + + +def _resource(cursor, iri: str, class_code: str) -> str: + """Insert one typed provenance resource and return its UUID.""" + cursor.execute( + "insert into provenance_resource (resource_iri) values (%s) returning resource_id", + (iri,), + ) + resource_id = cursor.fetchone()[0] + cursor.execute( + "insert into provenance_resource_type (resource_id, class_code) values (%s, %s)", + (resource_id, class_code), + ) + return str(resource_id) + + +def test_catalog_has_every_normative_term(prov_schema_db) -> None: + """The database catalog exactly matches the Recommendation inventory.""" + with prov_schema_db.cursor() as cursor: + cursor.execute("select count(*) from provenance_class_definition") + assert cursor.fetchone()[0] == 30 + cursor.execute("select count(*) from provenance_relation_definition") + assert cursor.fetchone()[0] == 50 + cursor.execute("select count(*) from provenance_qualification_definition") + assert cursor.fetchone()[0] == 14 + cursor.execute("select count(*) from provenance_inverse_definition") + assert cursor.fetchone()[0] == 44 + + +def test_database_accepts_valid_generation_and_rejects_wrong_domain(prov_schema_db) -> None: + """Recursive class-domain checks are enforced by PostgreSQL itself.""" + with prov_schema_db.cursor() as cursor: + entity_id = _resource(cursor, "urn:test:entity", "prov_entity") + activity_id = _resource(cursor, "urn:test:activity", "prov_activity") + agent_id = _resource(cursor, "urn:test:agent", "prov_person") + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_resource_id) " + "values (%s, 'prov_was_generated_by', %s)", + (entity_id, activity_id), + ) + with pytest.raises(psycopg2.errors.RaiseException, match="violates PROV-O domain"): + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_resource_id) " + "values (%s, 'prov_was_derived_from', %s)", + (agent_id, entity_id), + ) + prov_schema_db.rollback() + + +def test_database_rejects_literal_for_object_property(prov_schema_db) -> None: + """Object/datatype shape cannot be bypassed by direct SQL writes.""" + with prov_schema_db.cursor() as cursor: + entity_id = _resource(cursor, "urn:test:shape-entity", "prov_entity") + cursor.execute( + "insert into provenance_literal_value (lexical_value) values ('bad') returning literal_id" + ) + literal_id = cursor.fetchone()[0] + with pytest.raises(psycopg2.errors.RaiseException, match="requires object_resource_id"): + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_literal_id) " + "values (%s, 'prov_was_derived_from', %s)", + (entity_id, literal_id), + ) + prov_schema_db.rollback() + + +def test_database_requires_xsd_datetime_for_event_time(prov_schema_db) -> None: + """Date properties reject untyped lexical strings at the storage boundary.""" + with prov_schema_db.cursor() as cursor: + activity_id = _resource(cursor, "urn:test:time-activity", "prov_activity") + cursor.execute( + "insert into provenance_literal_value (lexical_value) " + "values ('2026-08-14T04:00:00Z') returning literal_id" + ) + literal_id = cursor.fetchone()[0] + with pytest.raises(psycopg2.errors.RaiseException, match="violates datatype"): + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_literal_id) " + "values (%s, 'prov_started_at_time', %s)", + (activity_id, literal_id), + ) + prov_schema_db.rollback() + + +def _literal(cursor, lexical_value: str, datatype_iri: str | None) -> str: + """Insert one RDF literal and return its UUID.""" + cursor.execute( + "insert into provenance_literal_value (lexical_value, datatype_iri) " + "values (%s, %s) returning literal_id", + (lexical_value, datatype_iri), + ) + return str(cursor.fetchone()[0]) + + +@pytest.mark.parametrize( + "lexical_value", + ( + "2026-08-14T04:00:00", + "not-a-date", + "2026-02-31T04:00:00Z", + "2026-08-14T04:00:00+14:01", + ), +) +def test_database_rejects_invalid_xsd_datetime(prov_schema_db, lexical_value: str) -> None: + """Malformed and timezone-less xsd:dateTime values fail closed.""" + with prov_schema_db.cursor() as cursor: + activity_id = _resource(cursor, "urn:test:strict-time", "prov_activity") + literal_id = _literal( + cursor, + lexical_value, + "http://www.w3.org/2001/XMLSchema#dateTime", + ) + with pytest.raises(psycopg2.errors.RaiseException, match="lexical xsd:dateTime"): + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_literal_id) " + "values (%s, 'prov_started_at_time', %s)", + (activity_id, literal_id), + ) + prov_schema_db.rollback() + + +def test_database_accepts_timezone_aware_xsd_datetime(prov_schema_db) -> None: + """A valid timezone-aware dateTime reaches the assertion store.""" + with prov_schema_db.cursor() as cursor: + activity_id = _resource(cursor, "urn:test:valid-time", "prov_activity") + literal_id = _literal( + cursor, + "2026-08-14T04:00:00+09:00", + "http://www.w3.org/2001/XMLSchema#dateTime", + ) + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_literal_id) " + "values (%s, 'prov_started_at_time', %s)", + (activity_id, literal_id), + ) + prov_schema_db.rollback() + + +def test_referenced_contract_rows_are_immutable(prov_schema_db) -> None: + """Reference-table mutation cannot invalidate stored assertions.""" + with prov_schema_db.cursor() as cursor: + entity_id = _resource(cursor, "urn:test:immutable-entity", "prov_entity") + activity_id = _resource(cursor, "urn:test:immutable-activity", "prov_activity") + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_resource_id) " + "values (%s, 'prov_was_generated_by', %s)", + (entity_id, activity_id), + ) + with pytest.raises(psycopg2.errors.RaiseException, match="types are immutable"): + cursor.execute( + "delete from provenance_resource_type " + "where resource_id = %s and class_code = 'prov_activity'", + (activity_id,), + ) + prov_schema_db.rollback() + + with prov_schema_db.cursor() as cursor: + activity_id = _resource(cursor, "urn:test:immutable-time", "prov_activity") + literal_id = _literal( + cursor, + "2026-08-14T04:00:00Z", + "http://www.w3.org/2001/XMLSchema#dateTime", + ) + cursor.execute( + "insert into provenance_assertion " + "(subject_resource_id, relation_code, object_literal_id) " + "values (%s, 'prov_started_at_time', %s)", + (activity_id, literal_id), + ) + with pytest.raises(psycopg2.errors.RaiseException, match="literal values are immutable"): + cursor.execute( + "update provenance_literal_value set datatype_iri = null " + "where literal_id = %s", + (literal_id,), + ) + prov_schema_db.rollback() diff --git a/tests/test_schema.py b/tests/test_schema.py index 33e88f705..324661d08 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -17,6 +17,7 @@ import os import uuid from pathlib import Path +from urllib.parse import urlsplit, urlunsplit import psycopg2 import psycopg2.errors @@ -52,7 +53,8 @@ def schema_db(): with admin_conn.cursor() as cur: cur.execute(f'create database "{db_name}"') try: - db_dsn = _ADMIN_DSN.rsplit("/", 1)[0] + f"/{db_name}" + parsed_admin_dsn = urlsplit(_ADMIN_DSN) + db_dsn = urlunsplit(parsed_admin_dsn._replace(path=f"/{db_name}")) conn = psycopg2.connect(db_dsn) try: with conn.cursor() as cur: @@ -88,7 +90,9 @@ def test_migration_applies_cleanly(schema_db) -> None: "cataloged_person", "person_affiliation", "post_person_mention", + "post_summary_person_mention", "knowledge_graph_edge", + "knowledge_graph_edge_evidence", "issue_ticket", "post_lineage_edge", "post_evaluation_response", @@ -187,14 +191,38 @@ def test_lookup_code_is_unique_across_categories(schema_db) -> None: def test_every_created_table_name_has_at_least_two_words() -> None: - """The project naming rule is enforced on the shipped migration, not - only on tables that happen to be created in a live-Postgres run. - """ + """Enforce naming for ordinary and idempotent table declarations.""" import re sql = _MIGRATION_PATH.read_text() - names = re.findall(r"create table (\w+)", sql) + names = re.findall( + r"create\s+table\s+(?:if\s+not\s+exists\s+)?([a-z][a-z0-9_]*)", + sql, + flags=re.IGNORECASE, + ) assert names, "migration must create at least one table" for name in names: words = name.split("_") assert len(words) >= 2, f"table {name!r} must be two or more snake_case words" + + +def test_cataloged_team_null_affiliation_is_unique(schema_db) -> None: + """Repeated NULL-affiliation upserts return one catalog identity.""" + with schema_db.cursor() as cursor: + ids = [] + for _ in range(2): + cursor.execute( + "insert into cataloged_team (team_name, affiliated_organization_name) " + "values ('Synthetic Design Team', null) " + "on conflict (team_name, affiliated_organization_name) do update " + "set team_name = excluded.team_name returning team_id" + ) + ids.append(cursor.fetchone()[0]) + cursor.execute( + "select count(*) from cataloged_team " + "where team_name = 'Synthetic Design Team' " + "and affiliated_organization_name is null" + ) + count = cursor.fetchone()[0] + assert ids[0] == ids[1] + assert count == 1 diff --git a/tests/test_seed_tepp_run.py b/tests/test_seed_tepp_run.py new file mode 100644 index 000000000..b25908cbe --- /dev/null +++ b/tests/test_seed_tepp_run.py @@ -0,0 +1,132 @@ +"""Seeded TEPP analysis runs go through tepp_client, never a local model.""" + +from lineageweave.tepp_client import AnalysisRunRequest, TeppClient, TeppNotAvailable +from scripts.seed_demo_data import ( + _ensure_demo_source_counts, + _seed_demo_tepp_run, + demo_source_snapshot_sha256, + tepp_seed_outcome, + tepp_seed_request, +) + + +class _RecordingUnavailableClient(TeppClient): + """Default-path stand-in that records the request then drops the channel.""" + + def __init__(self) -> None: + super().__init__() + self.submitted: list[AnalysisRunRequest] = [] + + def submit_analysis_run(self, request: AnalysisRunRequest) -> dict[str, object]: + self.submitted.append(request) + raise TeppNotAvailable("TEPP has no live HTTP endpoint yet.") + + +class _AcceptingClient(TeppClient): + """Transport that returns an envelope without a persistable measurement.""" + + def __init__(self) -> None: + super().__init__(transport=lambda _payload: {"status": "accepted"}) + + +class _CountCursor: + """Minimal cursor for proving re-seed skips a frozen count insert.""" + + def __init__(self, existing_counts: bool) -> None: + self.existing_counts = existing_counts + self.statements: list[str] = [] + + def execute(self, sql: str, _params=None) -> None: + self.statements.append(" ".join(sql.split())) + + def fetchone(self): + if self.existing_counts and "from analysis_source_count" in self.statements[-1]: + return (1,) + return None + + +def test_tepp_seed_request_targets_the_shared_demo_snapshot() -> None: + request = tepp_seed_request() + assert request.snapshot_id == demo_source_snapshot_sha256() + assert request.idempotency_key == "demo-tepp-seed-2026-w02" + assert request.model_contract_version == "tepp-analysis-run-v1" + assert request.output_profile == "calibrated_event_measurement" + + +def test_tepp_seed_outcome_calls_client_and_does_not_invent_a_score() -> None: + client = _RecordingUnavailableClient() + status, failure = tepp_seed_outcome(client) + assert status == "analysis_status_failed" + assert failure == "tepp_not_available" + assert client.submitted == [tepp_seed_request()] + + +def test_tepp_seed_outcome_default_client_is_unavailable_not_a_fake_score() -> None: + status, failure = tepp_seed_outcome() + assert status == "analysis_status_failed" + assert failure == "tepp_not_available" + + +def test_tepp_seed_outcome_does_not_treat_an_empty_envelope_as_success() -> None: + status, failure = tepp_seed_outcome(_AcceptingClient()) + assert status == "analysis_status_failed" + assert failure == "tepp_result_not_persisted" + + +def test_ensure_demo_source_counts_skips_insert_when_counts_exist() -> None: + cursor = _CountCursor(existing_counts=True) + _ensure_demo_source_counts(cursor, "snapshot-1") + assert any("from analysis_source_count" in sql for sql in cursor.statements) + assert not any(sql.lstrip().startswith("insert into analysis_source_count") for sql in cursor.statements) + + +def test_ensure_demo_source_counts_inserts_when_the_snapshot_is_empty() -> None: + cursor = _CountCursor(existing_counts=False) + _ensure_demo_source_counts(cursor, "snapshot-1") + assert any(sql.lstrip().startswith("insert into analysis_source_count") for sql in cursor.statements) + + +class _TeppSeedCursor: + """Drive `_seed_demo_tepp_run` without a live database.""" + + def __init__(self) -> None: + self.statements: list[str] = [] + self.params: list[object] = [] + + def execute(self, sql: str, params=None) -> None: + self.statements.append(" ".join(sql.split())) + self.params.append(params) + + def fetchone(self): + last = self.statements[-1] + if last.lstrip().startswith("select") and "from analysis_source_snapshot" in last: + return None + if "insert into analysis_source_snapshot" in last: + return ("snapshot-demo",) + if last.lstrip().startswith("select") and "from analysis_source_count" in last: + return None + if last.lstrip().startswith("select") and "from analysis_run" in last: + return None + if "insert into analysis_run" in last: + return ("run-demo-tepp",) + return None + + +def test_seed_demo_tepp_run_inserts_failed_tepp_not_available() -> None: + cursor = _TeppSeedCursor() + _seed_demo_tepp_run(cursor, "account-1", "corp-1") + run_inserts = [sql for sql in cursor.statements if "insert into analysis_run" in sql] + assert run_inserts, "seed must insert the TEPP analysis_run row" + assert any("analysis_run_tepp" in sql for sql in run_inserts) + status_params = [ + params + for sql, params in zip(cursor.statements, cursor.params, strict=True) + if "insert into analysis_run_status_event" in sql + ] + assert any( + params is not None and "analysis_status_failed" in params and "tepp_not_available" in params + for params in status_params + ) + assert not any( + params is not None and "analysis_status_succeeded" in params for params in status_params + ) diff --git a/uv.lock b/uv.lock index 1964f34b9..179628fdd 100644 --- a/uv.lock +++ b/uv.lock @@ -188,6 +188,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "coverage" +version = "7.15.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/c3/4f2195f512fb172aa425a8803a874b2baa9ba7f80ff7b6080998761fc701/coverage-7.15.4.tar.gz", hash = "sha256:0548198fff07ccf4faf469520bce1c2eceb1ce3e62891921138dec10907f9d00", size = 936952, upload-time = "2026-08-06T13:50:24.442Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/48/bc8d4ba7b37551a767bd863f15b3f80182b271c2f55975356f5f7dbe94c2/coverage-7.15.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4fedd1f7f428f9fe83b1ead5e7cc87a43427be31aadafbac3ac0636dc7abb22", size = 222543, upload-time = "2026-08-06T13:47:37.562Z" }, + { url = "https://files.pythonhosted.org/packages/20/dd/88d6f83f1fffc974a3691a34a97951c5b12df7512a6782c5963883cbc058/coverage-7.15.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:37e2f0cdf58e2e1fed4e4d5a8f8786ae2f7eb80b478016876667dc4a01d60a97", size = 222905, upload-time = "2026-08-06T13:47:38.927Z" }, + { url = "https://files.pythonhosted.org/packages/bd/5c/54ee0d4748585bb0acab9891cd8d92f2d3593165b4e59fc9de113bfb3140/coverage-7.15.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fb55d0e70bb15f2e81477613627286581414693d74ac7963c93a790dd453ca9d", size = 254407, upload-time = "2026-08-06T13:47:40.488Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3f/f0642a372f494bd0d7dad3b497083b910194a5f1c88be2c94fef707c3b59/coverage-7.15.4-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:899b9da30f3c6c336566e3707495bb23e8302d39d862f01fa78c48b99b9437e2", size = 257145, upload-time = "2026-08-06T13:47:41.931Z" }, + { url = "https://files.pythonhosted.org/packages/71/17/8b46d0ed68251016002ec972c8fc0119961a765d0984cafb8bf317c43758/coverage-7.15.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d15715e8c46552827e5e4f30a35575a2dbcad14454cf3284c54483946bd16931", size = 258257, upload-time = "2026-08-06T13:47:43.527Z" }, + { url = "https://files.pythonhosted.org/packages/30/b8/8498a0e72d0adbe15477dd07463d2b3bb2c9f6a4815e8589e50939e2c3ae/coverage-7.15.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:002a438859f7b430bc99afeaf01a6d187dad1d0dc907b64cdeffc632a5db8fd8", size = 260517, upload-time = "2026-08-06T13:47:45.121Z" }, + { url = "https://files.pythonhosted.org/packages/41/e1/7dce19c3bdb1e3dd63e769508216500edad81bd5f69a26d724e32aceaf78/coverage-7.15.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4193a04b518f7968f3099755f5509ee7cccc6dc2b92a6b14841934d22e222c9", size = 254785, upload-time = "2026-08-06T13:47:46.541Z" }, + { url = "https://files.pythonhosted.org/packages/dd/b1/e1494703c675a2561723cd9b89f45c9168782c31280c611b1f767851e57c/coverage-7.15.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e98dcc55d572b38e69d117da7e8e8efb8500f1f5eaf81ecd460a63220790b839", size = 256176, upload-time = "2026-08-06T13:47:48.155Z" }, + { url = "https://files.pythonhosted.org/packages/73/76/a5629d270fb638a43a4b10466f51e2f49d532c1aa4da2913cbbb150bbe0a/coverage-7.15.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:af6c538498ce66c10d3fd541c2a8d5b03da5850355add34e6cba564210cb9e72", size = 254321, upload-time = "2026-08-06T13:47:49.757Z" }, + { url = "https://files.pythonhosted.org/packages/ff/4f/9c44447218435d5766b911534f9d798144a5560f85e9a54ebe5f3f5d19f9/coverage-7.15.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1d10025d96ea89fc2f73714dbc4cbd433fe012c1ac9e23f895d7728b238b6e52", size = 258390, upload-time = "2026-08-06T13:47:51.248Z" }, + { url = "https://files.pythonhosted.org/packages/de/36/c1e127616fb3fa18a9ff71e76c417f2fd7424332a4870015ac224ef4c039/coverage-7.15.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d802e1947603162ded419bff83ac7489820355d2b856dfb09206574e3a37ac0c", size = 253894, upload-time = "2026-08-06T13:47:52.816Z" }, + { url = "https://files.pythonhosted.org/packages/e9/b9/fdb92c8ae7a8bb9b850cc253b7b3b9c8526f68130002048b5671cd510d09/coverage-7.15.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c2de40895718f91951b86712b4c5b694acaf9a0a49be13874896f599a1eed3f4", size = 255763, upload-time = "2026-08-06T13:47:54.296Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c0/a7d51b2587c7bdb76e71b0896d2565bf7d60436b5122fc83e511adb1f7cd/coverage-7.15.4-cp312-cp312-win32.whl", hash = "sha256:5c3431b2161279b7db5c2a1aa58ae02e5cb8c3c42d93a5094be3f5537bd5b11b", size = 224597, upload-time = "2026-08-06T13:47:56.074Z" }, + { url = "https://files.pythonhosted.org/packages/49/b9/5c5f80cc55f5acaaca6dee677626bfcec8c87204a7809b438b08e84f4571/coverage-7.15.4-cp312-cp312-win_amd64.whl", hash = "sha256:6befeab5fb2b51c958ca4ac6c5d141a1e8240f4f76e46350f1911963deda49cd", size = 225135, upload-time = "2026-08-06T13:47:57.52Z" }, + { url = "https://files.pythonhosted.org/packages/47/e4/2a4561f89ff6bf7c925c287d0f2cce8bdf139c3a33735c87e3203401cf94/coverage-7.15.4-cp312-cp312-win_arm64.whl", hash = "sha256:67bc345491ab55b837277d76f5775d057e8c7f1ac44d890d8c2c82adde258c6f", size = 224515, upload-time = "2026-08-06T13:47:58.977Z" }, + { url = "https://files.pythonhosted.org/packages/f1/84/651a9310859673aaa3b3203f1aa1641ca60fcf2494683e1c9474c7172780/coverage-7.15.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c705b28feb2775dc82a25f1d473a370bc37ff93f5177f4e29ce2425f560f6921", size = 222565, upload-time = "2026-08-06T13:48:00.796Z" }, + { url = "https://files.pythonhosted.org/packages/82/f9/4dcf700137e8af550670f4d74d1b63828ce93e1e2b05e5f10710eb2ea987/coverage-7.15.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3ff205ab5e3ecc670f6a4dd19d9cbf12ede53dd41cfc1e15716ec961ea6d314e", size = 222936, upload-time = "2026-08-06T13:48:02.391Z" }, + { url = "https://files.pythonhosted.org/packages/07/4a/612ff1e780b3fbfd637486f542f84adc5503873d8b5d279dec1ffeef9414/coverage-7.15.4-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5172326e861a38b48b48befca15e0f477a26b283337a33a739c8fed229934e36", size = 253926, upload-time = "2026-08-06T13:48:04.382Z" }, + { url = "https://files.pythonhosted.org/packages/b0/04/d1cff1c2ead4708a6a79c01d3736b6a25bd38a36678398f72a8dd33dfad9/coverage-7.15.4-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:12b59c90084e3234fb11184886bf4a40f4f16a8c8f867be2e087b81f8e8868d4", size = 256523, upload-time = "2026-08-06T13:48:05.996Z" }, + { url = "https://files.pythonhosted.org/packages/b9/80/d34e13fb4b293cbdb9665838cf5522077b8ad14ef947550631a4bced36a5/coverage-7.15.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:349062d66f00b40fa2c1c222438bad25fabf755631b5d82937fe985c8008615c", size = 257759, upload-time = "2026-08-06T13:48:08.036Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e7/2c5fe7636fdb0732fe0f09f308a5b066864078b7fc61f6678e8478554f2e/coverage-7.15.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4256ced708e598e05209bc1a8ab4074e04a51dba4c62fb45926a229af675ace7", size = 259890, upload-time = "2026-08-06T13:48:09.834Z" }, + { url = "https://files.pythonhosted.org/packages/92/28/9689f0858dfff59c2ea688938ab9fa2925631235df67126a42b6c5c70ae1/coverage-7.15.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d80f974b20782d9612c8b4c9beeca867074c7cf4079d1419843fa25a26428b25", size = 254121, upload-time = "2026-08-06T13:48:11.459Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e2/785077c230c157243eb5aa9a26c3be260ecd02001bead54a3cada3df8e03/coverage-7.15.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2e179f19bfe1d31f8eeeaa12990194d761c4f62f0759661000bca6cd8729f40b", size = 255891, upload-time = "2026-08-06T13:48:13.209Z" }, + { url = "https://files.pythonhosted.org/packages/d4/90/e20371b17b40f912f21305c2db2f30efa3de306f7320fc916804872c85a4/coverage-7.15.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8bc16bb47b7679670eceff71d78bfb7d6e5b143f6c2cd117487ec7c75e0d4b78", size = 253859, upload-time = "2026-08-06T13:48:14.736Z" }, + { url = "https://files.pythonhosted.org/packages/05/49/25371987ee459a5f67c0427fb75c74f9358e65f2c71fe75bf41c1b6c5fcb/coverage-7.15.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1cd685005cd2c4200adfc14cf39a603b9320efab3f18a8f7f156d20c9cc3345f", size = 258011, upload-time = "2026-08-06T13:48:16.464Z" }, + { url = "https://files.pythonhosted.org/packages/30/6e/32e67467f6154bf4f1c4f63b05acc5097cba4237d45bbeeea446b52e8ac1/coverage-7.15.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:337399ad2c93b3acd2a937627dae8b3e86b66707cd3d3e856347999aadf1ef8d", size = 253676, upload-time = "2026-08-06T13:48:18.493Z" }, + { url = "https://files.pythonhosted.org/packages/03/c1/8b24192e89286399765155251f99ee9f070a9d637109018ac23d99b99f6f/coverage-7.15.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:96e257121228ec5cd2bb919276e94ac11074471bc37d68dbae0e8308cce15fff", size = 255453, upload-time = "2026-08-06T13:48:20.057Z" }, + { url = "https://files.pythonhosted.org/packages/16/6f/8b41ebdf67c87854e17c035336a90f1cfbad0c14c2a584301be6ff148718/coverage-7.15.4-cp313-cp313-win32.whl", hash = "sha256:c65a9e0dfc6143491879da4e13b5e30f8be192055de508d737fb14601edbd22c", size = 224605, upload-time = "2026-08-06T13:48:21.655Z" }, + { url = "https://files.pythonhosted.org/packages/e0/e2/2946c7f0b42b152ecb21ff1bdad72e3d301e790c0c487e4a86e8c9f69347/coverage-7.15.4-cp313-cp313-win_amd64.whl", hash = "sha256:2ff8f5e9b8f7a94f0c11c45631eee103dbcb7d63274edd12c56efe1be690b3b4", size = 225148, upload-time = "2026-08-06T13:48:23.376Z" }, + { url = "https://files.pythonhosted.org/packages/9e/83/3f4a69957f48ae7a0aba76c34743f88963d607b19e03f3f8e66f91cae0f9/coverage-7.15.4-cp313-cp313-win_arm64.whl", hash = "sha256:6e0a8a5083b096487d6cfced94cdd514d8f5db6f113610fb36c0620edb1028cf", size = 224536, upload-time = "2026-08-06T13:48:25.117Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ac/748cf29eeb2d6be34a3176ce26a4f49e38085ee08e8935f05f6f26ed7e0f/coverage-7.15.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:770e9325ab5ea6d56f77e59b29ecfe0ac20b57a82a601876f90494a4dda0386f", size = 222608, upload-time = "2026-08-06T13:48:26.806Z" }, + { url = "https://files.pythonhosted.org/packages/0b/02/1abbf5c984677b0aa439cdacaccbf38d248939d8ef8fe1cc7a50d73edb77/coverage-7.15.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d12b33a3a50a1676b7784dc8d00a0c6d66a9f2add4b85a041c19b6a7e53ef23c", size = 222940, upload-time = "2026-08-06T13:48:28.432Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e1/ff8f9f53d9fcf586125b55d0b1f04ec1c14955fee41e83d5814bee141bb5/coverage-7.15.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5669c8378ebde86f5def7a25d29586631b58acc27ffde04399f678f3dfc6e082", size = 253985, upload-time = "2026-08-06T13:48:29.995Z" }, + { url = "https://files.pythonhosted.org/packages/a1/26/595759762e514e81be1d7d01ed03444303bcd152226a6529998d253f9201/coverage-7.15.4-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ff97a14362eef486483ed44042ca2027ea257df6ff768e62358ee0c9776925ac", size = 256492, upload-time = "2026-08-06T13:48:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/24/68/b79aabac54d482be23b5fcdd4f4662bff24a78edc4ee29201726929936d5/coverage-7.15.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a325e815318638aed1655d9c06e6d7c2d3d46c09231ce988070428a8762d734", size = 257837, upload-time = "2026-08-06T13:48:33.186Z" }, + { url = "https://files.pythonhosted.org/packages/09/0f/bf7f297885a5bf6fd71e5782404e0ff059ca09e8711ceb3a08544abde45a/coverage-7.15.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:474223409d88eb20d2d6a0d37ea60e8647a65a90cc008dc1f0410af5f64f1e0d", size = 260152, upload-time = "2026-08-06T13:48:34.75Z" }, + { url = "https://files.pythonhosted.org/packages/fd/f1/296744e854ff8368542343457414380465e9ceefb9192342feb9d3bc461d/coverage-7.15.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f2f62ae3cd189dd2e13aece758c57b3eecbd27be070dbd4cbd10936049e5dbf", size = 253978, upload-time = "2026-08-06T13:48:36.434Z" }, + { url = "https://files.pythonhosted.org/packages/55/b0/bbdb2e9057493e66220a2e149ca2d301ba0e3a58a83bd6b90de9826d16f3/coverage-7.15.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39ece820e29e0a2ba34b3ecb3be83c27e997eed8926f2ba6fe7ce7a0bda5843b", size = 255846, upload-time = "2026-08-06T13:48:38.317Z" }, + { url = "https://files.pythonhosted.org/packages/96/e4/38015b2b6d21258713bd17e76b59d033b191efb5703589cffd037dfbca20/coverage-7.15.4-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f21b56dcace11dfe013014201f577dcd592b2a9b72182d930361b47cf6f73f25", size = 253808, upload-time = "2026-08-06T13:48:39.993Z" }, + { url = "https://files.pythonhosted.org/packages/0b/64/0d515c1e60ee6fbfd1a0e79c07cd87d388a233b7adc37758735677203808/coverage-7.15.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:93a3a0b662abcc10c73a47cbc72cd60f63618d6989fb2d1286e50eacd974f303", size = 258081, upload-time = "2026-08-06T13:48:41.971Z" }, + { url = "https://files.pythonhosted.org/packages/91/71/04d9e7a3642146c6351338aef4ef85ab11dbbb54744c13245caba1aad1c0/coverage-7.15.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:141fae2cabf5569b782c10afc4c850ce10f618c13f8db54765cba99cc839da1f", size = 253624, upload-time = "2026-08-06T13:48:43.731Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a7/6c28b74c81ebff66987b0e2522ba5cffa3e90b0c33cb6a2eb264d4ee8cf1/coverage-7.15.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:81294c7e6ab30c5f74c0353b11b2fd6320e72d9bee6ac73b357caa8b916323a5", size = 255280, upload-time = "2026-08-06T13:48:45.58Z" }, + { url = "https://files.pythonhosted.org/packages/52/af/bc19996a7014b98d7bbb0f0939453c67074af65784a3aa16a789a07381fa/coverage-7.15.4-cp314-cp314-win32.whl", hash = "sha256:7bbd7d6418e0dab31a206af5203bd43ae36edb8e7fba1940b055d3e9249290d7", size = 224768, upload-time = "2026-08-06T13:48:47.525Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/219484e476d6e101ba0a444852579e05f5b75c37c611a42ed1190f73ef62/coverage-7.15.4-cp314-cp314-win_amd64.whl", hash = "sha256:f0204ed122758782970526057093f448051a39db9d810d4e344bb87a3546f425", size = 225259, upload-time = "2026-08-06T13:48:49.513Z" }, + { url = "https://files.pythonhosted.org/packages/b7/66/fa77daf4e383e5f776dac62c2409b6af81910ae6fe326bd5170dba74cc63/coverage-7.15.4-cp314-cp314-win_arm64.whl", hash = "sha256:9e71e7bc71c686a123347ae47a0de33a175e797a85bb57b791492adf4eec8ed8", size = 224684, upload-time = "2026-08-06T13:48:51.235Z" }, + { url = "https://files.pythonhosted.org/packages/58/5b/f03bf0ce362bbf3f785fa5219620d00778d4ac6fc9e407734828e9c672f6/coverage-7.15.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7c922735321eef3f87c280a3d39afff6b646723a2880b862cda4ac7a093b8aa8", size = 223338, upload-time = "2026-08-06T13:48:52.896Z" }, + { url = "https://files.pythonhosted.org/packages/0f/76/e77d0ae22501831cc9f92193e8a957a5caa1dd177f90a6d1d9b106242d92/coverage-7.15.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f41c17c4668a655ce96d090d8d5ffdc24ef64b5a02f9753884d08483e8a4a41a", size = 223609, upload-time = "2026-08-06T13:48:54.688Z" }, + { url = "https://files.pythonhosted.org/packages/82/1a/b1f089da8d38ac612fa2dd6dc7f4a1a7657d12f3e261d2996edd3a838d0b/coverage-7.15.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:46822e9b6ff1c6a72b518c162c44a8f45a61a1d609c51084bf5b16c023c5037b", size = 264970, upload-time = "2026-08-06T13:48:56.403Z" }, + { url = "https://files.pythonhosted.org/packages/bf/31/e66d98d6e9c7fcc88470f1e234eaf6b1950dc0dfbf797f7282c1c861da24/coverage-7.15.4-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d6f4955b73b5445271379a59e3792b0d978f42d4a01e0cf7a67d9c33a3bb0a5", size = 267088, upload-time = "2026-08-06T13:48:58.41Z" }, + { url = "https://files.pythonhosted.org/packages/59/a1/ae94eb2c541add426378408379f233591e069040b1e2cdb33df9498a0682/coverage-7.15.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3fc9e047706fb4a9abb54f719d3aa643e80e5bb3818182c40aee01ac0f0247ba", size = 269508, upload-time = "2026-08-06T13:49:00.42Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c7/88a10694a1c6a213569766aba9f25847b28155d4ac731b13226db216356d/coverage-7.15.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05e491d4f3165d62d4f5c8fd48dfeabf2ae8f42cbbd484319af33ea851b78982", size = 270629, upload-time = "2026-08-06T13:49:02.234Z" }, + { url = "https://files.pythonhosted.org/packages/b3/34/d8b8232e5e55169933b59aabcef2fedfa4b9d8897361bb80fcbda146505f/coverage-7.15.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:226c66e80ec0598d3b9b4874123df167ccca342aca8714f77cac6829688ee09c", size = 264043, upload-time = "2026-08-06T13:49:04.102Z" }, + { url = "https://files.pythonhosted.org/packages/7e/35/58b009dbf8c471c7224716478b9fed4a7e1af15320e1ed41660978504663/coverage-7.15.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ac41cc14bebda0dbfb0628036b7f75706935c95bcc07fefe9a0f93614aa60a57", size = 266963, upload-time = "2026-08-06T13:49:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/62/aa/57fbda1b42c892968273c56b6ee9dc0f1310850859230a507bc7873b1f65/coverage-7.15.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8af623e5cd92080acddd02b38f2f406a2c3a0893c38950b211890361448fbf26", size = 264569, upload-time = "2026-08-06T13:49:07.706Z" }, + { url = "https://files.pythonhosted.org/packages/98/8a/360e6e7f24d477b7e889703af0afa878d15b6d4d8d2a822b2835c169a879/coverage-7.15.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:07545711d4f0f32852a18f18ad11f76f0109909d09e78b9008b4cfc67e829429", size = 268299, upload-time = "2026-08-06T13:49:09.587Z" }, + { url = "https://files.pythonhosted.org/packages/4e/89/6f701261aee21b6b5fa8f7872229406dc917e125069448292223bf213606/coverage-7.15.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:a0865421cfdc53654b342d515e5a233187590882d20b95752150e53f65460017", size = 263413, upload-time = "2026-08-06T13:49:11.604Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0f/6f04036edc260ed425af83e834f627fad48941ce97b50bfe6edd8b6fa623/coverage-7.15.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:460115e32ee40566476db5048f9bec1e842c127ad8e6f8be745aad3ac9cbc839", size = 265725, upload-time = "2026-08-06T13:49:13.38Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ce/d19b5d4d5c49a7bfb925fd74310fee7d28bc99520ac3367ccbc54e662518/coverage-7.15.4-cp314-cp314t-win32.whl", hash = "sha256:cbde877ef9dd7baf272b9bfef2b8a25edd45d9170fc326951dd20eb480335e85", size = 225079, upload-time = "2026-08-06T13:49:15.265Z" }, + { url = "https://files.pythonhosted.org/packages/26/bb/7aa1b3b173faee0679037ca950bbbe1247273656697994d8d13f80f8d4b4/coverage-7.15.4-cp314-cp314t-win_amd64.whl", hash = "sha256:3da9e92d1c551fd7563833e9ade686efb0c4b7363ab7681a94283958c950bf5e", size = 225911, upload-time = "2026-08-06T13:49:17.279Z" }, + { url = "https://files.pythonhosted.org/packages/81/1c/4ea9e47426d80038d9222db3c4534cb6021a74b237d3ff97ffd33b6600dd/coverage-7.15.4-cp314-cp314t-win_arm64.whl", hash = "sha256:3a54f5a0d85050c73a38f6793090ee83974531e67fe5e57a1da9bee11398aa5e", size = 225219, upload-time = "2026-08-06T13:49:19.293Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c4/dc5d2ac8f9142e7ec7de66e7bf0591db29d78955a040bd915870d9c0e657/coverage-7.15.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:2c9872e4d9dc5d3cf616bf4b382f5a00359305a5be666a3dd0b5cdb4e49597f9", size = 222604, upload-time = "2026-08-06T13:49:21.279Z" }, + { url = "https://files.pythonhosted.org/packages/70/39/33e63df81fe2ee100897451841c821467635923e58e37c6bd4b46dd8106c/coverage-7.15.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:e101dbb4b9b72f0cddd8cdc8c9c5b47f456766f5e0ac82dbfb75e5c55409b78a", size = 222944, upload-time = "2026-08-06T13:49:23.187Z" }, + { url = "https://files.pythonhosted.org/packages/99/1f/ef3ffb5557febc75a0d97aa459d0266d7d741110265121cc6d8539343d44/coverage-7.15.4-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7d1abebdb047729e852b9c77a00497dfbeb11eb3a117e037d7dbc3ac8e5f5c54", size = 254050, upload-time = "2026-08-06T13:49:25.008Z" }, + { url = "https://files.pythonhosted.org/packages/6f/f5/1f0f6f77698c3601ca0ae7431e34b24c62ca2f06fecb23b73ed1f651d2be/coverage-7.15.4-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d28a4a899354d0ea6214cc59b4fa19eefbce1b9ff1688ab579acf49e894bd3fb", size = 256967, upload-time = "2026-08-06T13:49:26.896Z" }, + { url = "https://files.pythonhosted.org/packages/03/7a/2ed9bed79925f4367c83c77f66a89e5ca7229c288d2d19ad5f36d1ca0070/coverage-7.15.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffb3c2aacea411cc7e1d27712490c11108e2de1d39019ae32915493a59a8b9ed", size = 258587, upload-time = "2026-08-06T13:49:28.692Z" }, + { url = "https://files.pythonhosted.org/packages/45/8c/fa34044f71b7cc4ecb6da9c2408770959b0591fa9b5fb6fb6bca38f94298/coverage-7.15.4-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a9447978a92f405d301123cfd39ff49895490efb769a758fe2734c7f631bf8ce", size = 260785, upload-time = "2026-08-06T13:49:30.472Z" }, + { url = "https://files.pythonhosted.org/packages/4f/54/d5727ce36b4524a7394ab9f5f1df378e1f23affcdab01037dc8655185cc7/coverage-7.15.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:050467a7983b8e2fe7dd41a78bb30c3e7f8c0b8cafda14b1c46f8b5e3cf2dd3c", size = 254545, upload-time = "2026-08-06T13:49:32.271Z" }, + { url = "https://files.pythonhosted.org/packages/dc/e6/6e3783e576719590194bdffb6dd6d85490801785b7c331e35a245d8cb8b5/coverage-7.15.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:d003b7a5708ddad5c206c79607a6b92abb6fc13c57d99d8a4468cc03a2941ced", size = 256682, upload-time = "2026-08-06T13:49:34.089Z" }, + { url = "https://files.pythonhosted.org/packages/dc/f2/bacdbde18b69ed2de424fcf64d9fb0a4913753d4f0eca8bae9daad69f4bd/coverage-7.15.4-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c38efe30fd74e5c19e9433f11fb1f5dc9c6522770971b7c6145bbaa413dc8800", size = 254560, upload-time = "2026-08-06T13:49:36.052Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a3/1fb927196e3477c1b48831169ab58ba08f451ba87ae311ff1de68b26a616/coverage-7.15.4-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:1f4f826d70f772ab8b0c052329580d7fe8b8abd191e4ce0c8f81aec6614665d3", size = 258792, upload-time = "2026-08-06T13:49:38.01Z" }, + { url = "https://files.pythonhosted.org/packages/41/58/30d4c149c69053de0edfe325614c1d28d508f62b1783e0e4a234d2e49136/coverage-7.15.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:4a4bf917c9953f57c957be31c1cd504e3bd2f34d4a352b9d391a3025336f6768", size = 253968, upload-time = "2026-08-06T13:49:39.934Z" }, + { url = "https://files.pythonhosted.org/packages/89/e4/77f639371b918aad30dda4051f95404b43578f7f2e2f87ba73e02ed1ff37/coverage-7.15.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:1c9bf40ebef178a45192c75c4964760bb261b0e6ad725da5fc4c93f674f19753", size = 255893, upload-time = "2026-08-06T13:49:41.825Z" }, + { url = "https://files.pythonhosted.org/packages/5c/62/13be29b3ddab35f14c87967a4820a05106d2a3eccb4fa4ff550bf30b75e0/coverage-7.15.4-cp315-cp315-win32.whl", hash = "sha256:43619d04c3671792d2c4706ae8bf45e265dc87bbd4078189ef8b847ea1e74be2", size = 224768, upload-time = "2026-08-06T13:49:44.08Z" }, + { url = "https://files.pythonhosted.org/packages/a1/70/af0c6be0f964af6954f6b74bc109b0dbca02824696d2520fb17fe1ab06e3/coverage-7.15.4-cp315-cp315-win_amd64.whl", hash = "sha256:be619439dbcd31a2eab10b32de9fff62c26ed4bab69dc32b8363fdaaa0882809", size = 225242, upload-time = "2026-08-06T13:49:45.899Z" }, + { url = "https://files.pythonhosted.org/packages/4f/2d/f3bd3aab899fc9efc18b53133ee68f5f98574ef480649b23e12962226387/coverage-7.15.4-cp315-cp315-win_arm64.whl", hash = "sha256:def597967dafc2e8d97c9097ea453c464e0bb8ed38f193a43070f10dc623bb6d", size = 224674, upload-time = "2026-08-06T13:49:48.322Z" }, + { url = "https://files.pythonhosted.org/packages/f5/ca/f69251cd63eabc6438321aea22148754cce758a26bde07dd490e3fe7cfc5/coverage-7.15.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c7dbc748ac8a1e3e59a2b28bea47675e6e778081dbbf081bde0d75def2fcbe1d", size = 223333, upload-time = "2026-08-06T13:49:50.293Z" }, + { url = "https://files.pythonhosted.org/packages/a7/a7/037b53b2885b0d8447064432491a4d5a1014cd9f97a594d53acd0c04541a/coverage-7.15.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:2413074a5ecbb61a01a7888fc72db0ca324d13588c5b38bc0dd8564cdcdfea26", size = 223630, upload-time = "2026-08-06T13:49:52.637Z" }, + { url = "https://files.pythonhosted.org/packages/80/4f/152b8a4779ae90da11bb24f7467df8a59f0be48a5c52acb856325ca48289/coverage-7.15.4-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4e6f6f632b7b2f714bf7a1346e8f97b650ee71f3c298aaad42a2ab60f0f07645", size = 264489, upload-time = "2026-08-06T13:49:54.52Z" }, + { url = "https://files.pythonhosted.org/packages/10/2d/84b4b9e0e1dd6528a51920ff7031f35b789382e467a28ec6a5a578cb8812/coverage-7.15.4-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8df457da2249d3c75ca2e5e835d59c725abfe92d27fdff6cd99eed85b51d5e9a", size = 267567, upload-time = "2026-08-06T13:49:56.721Z" }, + { url = "https://files.pythonhosted.org/packages/53/fc/ba01cc25299f9f8a2c8b02d3b28c53f3543d9fbfbe4e74fa2760b48f163e/coverage-7.15.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:050f66a08805acb5b8a23c6d4a517b1ecf82c08e81ed0e4bd727df065e5c6624", size = 270123, upload-time = "2026-08-06T13:49:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d0/db2647cbf40b14f8c308f94ff7bf89c06d564e59f396906edf50086ec788/coverage-7.15.4-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1587fb771d1ccceef708fdde1e5af8c7ed24b486b61d13a321acb7d8145390aa", size = 271107, upload-time = "2026-08-06T13:50:00.811Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/4d2d17924552c458bb4f77dd631f0e3bc92fbbdf2d2d916cd4b33bbfd5b1/coverage-7.15.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8b4f1c3a69ca580f3fbd6b2046915f536d7f586874f25c1bb23add2a3c88d50f", size = 264955, upload-time = "2026-08-06T13:50:03.023Z" }, + { url = "https://files.pythonhosted.org/packages/ee/de/dc010c7a3691f396d93bbc26bfcafa1c2a3a351cd520470f15faf5795bd5/coverage-7.15.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:ffb58d7eff5b7f6ecc6fa21d6288ab7f968a212cb67d682c269c09b9eba3b66f", size = 267949, upload-time = "2026-08-06T13:50:05.557Z" }, + { url = "https://files.pythonhosted.org/packages/78/ea/dc96a11375e83c045c2f7c61fb6918277cfe9401db7c0f7b1d111a84b2e5/coverage-7.15.4-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:d9df165544774574ee004b953023d1bebada1894a80b1052a43d798b0f676e67", size = 264421, upload-time = "2026-08-06T13:50:07.612Z" }, + { url = "https://files.pythonhosted.org/packages/c8/86/b77131a0f9503ce461cd577076147d7a9040f0c5dda772686f729e2cc9cb/coverage-7.15.4-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:f9de0a24a4079b53e523b5c5e2c5945ec251ab486652659955187cf255a259bc", size = 269121, upload-time = "2026-08-06T13:50:09.58Z" }, + { url = "https://files.pythonhosted.org/packages/24/24/944bc35007862955e7ebf05754e645419dcf5d7526c52735cfa2715e8ebf/coverage-7.15.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:150089274bdc9f940628552cb92844e0223c987f1902ab8efe9f45a2ec758d88", size = 264565, upload-time = "2026-08-06T13:50:11.722Z" }, + { url = "https://files.pythonhosted.org/packages/c7/cc/a3bb9f93e7e740659163e2ea584f8196ddcd2c456a5dbe15f6c50105fec1/coverage-7.15.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:a58a94fed5da6997d258e8f7668c1e195fbd04a691d781b7558f1e468f9e68bc", size = 266522, upload-time = "2026-08-06T13:50:13.786Z" }, + { url = "https://files.pythonhosted.org/packages/49/dd/e0e40f3560d878d888c580698ff5ad1179f5e1c3ac949684ef66b41a3817/coverage-7.15.4-cp315-cp315t-win32.whl", hash = "sha256:ebd5a6d8466ff30836572f3ba2cae8a5e8f85029b1c6d5e2ed338dc472a5166a", size = 225068, upload-time = "2026-08-06T13:50:15.825Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7e/37732ea80eebc30e976e4cdab15c190bc42d96959a42e38ddf6f8c60468f/coverage-7.15.4-cp315-cp315t-win_amd64.whl", hash = "sha256:288bde2a2d7ab6b6c2d7252fcde8b524387f2d970bdba9658fc6f8bbcaef0f9b", size = 225895, upload-time = "2026-08-06T13:50:17.928Z" }, + { url = "https://files.pythonhosted.org/packages/c6/08/1e00f7923eaaba45fb3d51dd794125fc766304b1df264f3a9c6557bfb30e/coverage-7.15.4-cp315-cp315t-win_arm64.whl", hash = "sha256:68be5e1de60ff13c9095bbec0e5a7fa45b33b101752215b91345ea1f61c4a278", size = 225213, upload-time = "2026-08-06T13:50:19.981Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d9/e70c286c979378f061d8266e279b686ab0b0b688e1fe0af864684f23a77d/coverage-7.15.4-py3-none-any.whl", hash = "sha256:964730a1e9de9c0cf11be6a1a3c79ce419c34882842abd256086ba4698705e84", size = 214332, upload-time = "2026-08-06T13:50:22.192Z" }, +] + [[package]] name = "cryptography" version = "50.0.0" @@ -355,7 +454,7 @@ wheels = [ [[package]] name = "lineageweave" -version = "0.71.0" +version = "0.86.3" source = { virtual = "." } dependencies = [ { name = "certifi" }, @@ -374,6 +473,7 @@ backend = [ { name = "uvicorn", extra = ["standard"] }, ] dev = [ + { name = "coverage" }, { name = "httpx" }, { name = "pillow" }, { name = "psycopg2-binary" }, @@ -385,6 +485,7 @@ dev = [ requires-dist = [ { name = "asyncpg", marker = "extra == 'backend'", specifier = ">=0.29.0" }, { name = "certifi", specifier = ">=2024.0.0" }, + { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.6" }, { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=5006c38286a4fa1d81bcf57eeed5ce27ae743f50" }, { name = "fastapi", marker = "extra == 'backend'", specifier = ">=0.115.0" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27.0" },