diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index fe6a94941..ece0509f2 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -864,7 +864,11 @@ still flows through unchanged. Cached in a new (`migrations/0015_organization_name_resolution.sql`) keyed by the raw name, so the same abbreviation across many posts is resolved once. Grounded in SKOS `skos:altLabel`/`skos:prefLabel` (Miles & Bechhofer, -2009). Wired into `backend/app/keyman_ingestion.py`'s affiliation loop +2009). After a pair is search-corroborated, both labels compete as +virtual candidates for the **same** `corporate_entity_id`, so a later +mention of `AGP` or `Aurora Grid Power` reuses one catalog row instead +of inserting a second `AUTO-` identity (ADR 0160). Wired into +`backend/app/keyman_ingestion.py`'s affiliation loop and the offline synthetic-batch script's paced re-implementation of it (the batch script's own copy was also missing `role_title` persistence entirely -- fixed alongside this). diff --git a/CHANGELOG.d/skos-organization-alias-binding.md b/CHANGELOG.d/skos-organization-alias-binding.md new file mode 100644 index 000000000..f2d8766f8 --- /dev/null +++ b/CHANGELOG.d/skos-organization-alias-binding.md @@ -0,0 +1,9 @@ +# Unreleased — SKOS organization alias catalog binding + +## Added + +- Corroborated SKOS `altLabel` / `prefLabel` pairs expand corporate + catalog candidates so a synthetic short form (`AGP`) and full form + (`Aurora Grid Power`) bind one `corporate_entity` row (ADR 0160). + Uncorroborated pairs, short near-matches, and tied scores stay unbound. + Real organization names are not used in fixtures. diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d0005ecd..3bf9f454c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ All notable changes to this project are documented here. Format follows ### Added +- Corroborated SKOS `altLabel` / `prefLabel` pairs now expand corporate + catalog candidates so a synthetic short form (`AGP`) and full form + (`Aurora Grid Power`) bind one `corporate_entity` row instead of + creating a second `AUTO-` identity (ADR 0160). Tied scores still stay + unbound. - ADRs 0150-0153 define the accepted boundaries for Korean relative-time retrieval, multi-thread Event Lineage answers, persisted image-evidence citations, and the focused evidence popup. Their implementations remain @@ -36,6 +41,10 @@ All notable changes to this project are documented here. Format follows - The public ontology now states its OWL 2 Full/RDF-Based semantics for the ADR 0036 RDF-reified project evidence, and the PROV-O support profile uses its canonical lowercase deployed IRI. +- Corporate-entity creation repeats normal similarity classification after + its lock to catch concurrent ties, while excluding the full resolved + ancestor path so no inferred ancestor can absorb its child. The separate + corroborated-alias recheck remains exact-only (ADR 0012 / ADR 0160). - `make smoke` and `make seed` now run through the locked project `uv` environment, so local OIDC and synthetic-data workflows resolve the same pinned dependencies as CI. diff --git a/backend/app/corporate_entity_ingestion.py b/backend/app/corporate_entity_ingestion.py index bfc0e740d..a38e9fb99 100644 --- a/backend/app/corporate_entity_ingestion.py +++ b/backend/app/corporate_entity_ingestion.py @@ -17,6 +17,8 @@ import asyncio import hashlib +from collections.abc import Sequence + import asyncpg from lineageweave.corporate_hierarchy_inference import ( @@ -27,6 +29,8 @@ RESOLUTION_TIE, RESOLUTION_UNIQUE, CorporateEntityCandidate, + OrganizationNameAlias, + expand_candidates_with_skos_aliases, score_corporate_entity, ) from lineageweave.http_client import HttpClientError @@ -35,6 +39,10 @@ RelationVerificationClient, ) +from .organization_name_resolution_ingestion import ( + load_corroborated_organization_name_aliases, +) + _AUTO_CODE_PREFIX = "AUTO-" _MAX_HIERARCHY_DEPTH = 4 _CREATION_LOCK_KEY = "lineageweave:corporate_entity_creation" @@ -111,18 +119,21 @@ async def get_or_create_corporate_entity( verification_client: RelationVerificationClient, candidates: list[CorporateEntityCandidate], *, + aliases: Sequence[OrganizationNameAlias] | None = None, _depth: int = 0, _visited_names: frozenset[str] = frozenset(), + _ancestor_entity_ids: set[str] | None = None, ) -> str | None: """Return a verified catalog id, otherwise ``None``. A unique similarity match is reused. A tied top score stays unbound - and does not create a third same-named row (ADR 0026). Only a genuine - miss -- no candidate at or above ``min_similarity`` -- may enter ADR - 0010 inference. A proposed parent must independently corroborate and - resolve before the child can be inserted. Repeated names in the - recursion path are cycles, including multi-node cycles such as - A -> B -> A. + and does not create a third same-named row (ADR 0026). After a raw + miss, SKOS alt/pref pairs expand the candidate labels so a + synthetic short form and full form bind the same row (ADR 0160). Only + an alias-expanded miss may enter ADR 0010 inference. A proposed parent + must independently corroborate and resolve before the child can be + inserted. Repeated names in the recursion path are cycles, including + multi-node cycles such as A -> B -> A. """ normalized_name = organization_name.strip() if not normalized_name: @@ -130,8 +141,26 @@ async def get_or_create_corporate_entity( visit_key = normalized_name.casefold() if visit_key in _visited_names: return None + ancestor_entity_ids = ( + _ancestor_entity_ids if _ancestor_entity_ids is not None else set() + ) existing = score_corporate_entity(normalized_name, candidates) + if existing.kind == RESOLUTION_UNIQUE and existing.catalog_id is not None: + return existing.catalog_id + if existing.kind == RESOLUTION_TIE: + return None + + resolved_aliases: Sequence[OrganizationNameAlias] + if aliases is None: + resolved_aliases = await load_corroborated_organization_name_aliases(conn) + else: + resolved_aliases = aliases + existing = score_corporate_entity( + normalized_name, + expand_candidates_with_skos_aliases(candidates, resolved_aliases), + min_similarity=1.0, + ) if existing.kind == RESOLUTION_UNIQUE and existing.catalog_id is not None: return existing.catalog_id if existing.kind == RESOLUTION_TIE: @@ -194,24 +223,42 @@ async def get_or_create_corporate_entity( inference_client, verification_client, candidates, + aliases=resolved_aliases, _depth=_depth + 1, _visited_names=visited_names, + _ancestor_entity_ids=ancestor_entity_ids, ) if parent_entity_id is None: return None + ancestor_entity_ids.add(parent_entity_id) async with conn.transaction(): await conn.execute( "select pg_advisory_xact_lock(hashtext($1))", _CREATION_LOCK_KEY, ) - # ponytail: the lock recheck is exact-only; fuzzy matching here can - # mistake an inferred child for the parent just created above. The - # initial lookup remains fuzzy, while this check only prevents a - # concurrent insert of the same normalized name. + # ponytail: exclude the resolved ancestor path before repeating normal + # raw scoring, or an ancestor created by this recursion can absorb its child. + fresh_candidates = await _reload_candidates(conn) + if ancestor_entity_ids: + fresh_candidates = [ + candidate + for candidate in fresh_candidates + if candidate.corporate_entity_id not in ancestor_entity_ids + ] + fresh = score_corporate_entity( + normalized_name, + fresh_candidates, + ) + if fresh.kind == RESOLUTION_UNIQUE and fresh.catalog_id is not None: + _remember_candidate(candidates, fresh.catalog_id, normalized_name) + return fresh.catalog_id + if fresh.kind == RESOLUTION_TIE: + return None + fresh_aliases = await load_corroborated_organization_name_aliases(conn) fresh = score_corporate_entity( normalized_name, - await _reload_candidates(conn), + expand_candidates_with_skos_aliases(fresh_candidates, fresh_aliases), min_similarity=1.0, ) if fresh.kind == RESOLUTION_UNIQUE and fresh.catalog_id is not None: diff --git a/backend/app/keyman_ingestion.py b/backend/app/keyman_ingestion.py index ade1044a1..88a22c2de 100644 --- a/backend/app/keyman_ingestion.py +++ b/backend/app/keyman_ingestion.py @@ -60,6 +60,7 @@ from lineageweave.corporate_hierarchy_resolution import ( RESOLUTION_TIE, CorporateEntityCandidate, + OrganizationNameAlias, score_corporate_entity, ) from lineageweave.keyman_extraction import KeymanExtractionClient, PersonMention @@ -71,7 +72,10 @@ from .corporate_entity_ingestion import get_or_create_corporate_entity from .knowledge_graph import persist_edges_for_post -from .organization_name_resolution_ingestion import resolve_organization_name +from .organization_name_resolution_ingestion import ( + load_corroborated_organization_name_aliases, + resolve_organization_name, +) async def _load_corporate_entity_candidates(conn: asyncpg.Connection) -> list[CorporateEntityCandidate]: @@ -182,8 +186,10 @@ async def _resolve_affiliated_organization( verification_client: RelationVerificationClient, hierarchy_inference_client: CorporateHierarchyInferenceClient, candidates: list[CorporateEntityCandidate], + aliases: list[OrganizationNameAlias] | None = None, ) -> tuple[str, str, str | None]: """Resolve one affiliation without rewriting a known raw-name tie.""" + resolved_aliases = aliases if aliases is not None else [] raw_outcome = score_corporate_entity(organization_name, candidates) if raw_outcome.kind == RESOLUTION_TIE: return organization_name, organization_name, None @@ -202,6 +208,7 @@ async def _resolve_affiliated_organization( hierarchy_inference_client, verification_client, candidates, + aliases=resolved_aliases, ) return organization_name, resolved_name, corporate_entity_id @@ -249,6 +256,7 @@ async def ingest_post_keymen( else: mentions = await asyncio.to_thread(client.extract, post_title, post_body) candidates = await _load_corporate_entity_candidates(conn) + aliases = await load_corroborated_organization_name_aliases(conn) resolved_by_mention: list[tuple[PersonMention, list[tuple[str, str, str | None]]]] = [] for mention in mentions: resolved_orgs: list[tuple[str, str, str | None]] = [] @@ -262,6 +270,7 @@ async def ingest_post_keymen( verification_client, hierarchy_inference_client, candidates, + aliases, ) ) resolved_by_mention.append((mention, resolved_orgs)) diff --git a/backend/app/organization_name_resolution_ingestion.py b/backend/app/organization_name_resolution_ingestion.py index 9300586c4..37109bb1b 100644 --- a/backend/app/organization_name_resolution_ingestion.py +++ b/backend/app/organization_name_resolution_ingestion.py @@ -11,11 +11,43 @@ OrganizationNameResolutionClient, resolve_and_verify_organization_name, ) +from lineageweave.corporate_hierarchy_resolution import OrganizationNameAlias from lineageweave.relation_verification import ( STATUS_CORROBORATED, RelationVerificationClient, ) +_CORROBORATED_ALIAS_SQL = ( + "select raw_organization_name, resolved_organization_name " + "from organization_name_resolution " + "where verification_status_code = $1" +) + + +async def load_corroborated_organization_name_aliases( + conn: asyncpg.Connection, +) -> list[OrganizationNameAlias]: + """Return search-corroborated SKOS alt/pref pairs, or an empty list. + + Callers with a stub connection that has no ``fetch`` (the early-return + tie tests) get no aliases rather than raising. Only + ``verify_corroborated`` rows are returned; pending or uncorroborated + guesses must not bind catalog identities (ADR 0008). + """ + fetch = getattr(conn, "fetch", None) + if not callable(fetch): + return [] + rows = await fetch(_CORROBORATED_ALIAS_SQL, STATUS_CORROBORATED) + aliases: list[OrganizationNameAlias] = [] + for row in rows: + aliases.append( + OrganizationNameAlias( + alt_label=row["raw_organization_name"], + pref_label=row["resolved_organization_name"], + ) + ) + return aliases + async def resolve_organization_name( conn: asyncpg.Connection, diff --git a/backend/app/post_summary_ingestion.py b/backend/app/post_summary_ingestion.py index 7403185ed..022d372c2 100644 --- a/backend/app/post_summary_ingestion.py +++ b/backend/app/post_summary_ingestion.py @@ -60,6 +60,9 @@ from .corporate_entity_ingestion import get_or_create_corporate_entity from .keyman_ingestion import _load_corporate_entity_candidates +from .organization_name_resolution_ingestion import ( + load_corroborated_organization_name_aliases, +) from .knowledge_graph import persist_edges_for_post from .team_ingestion import upsert_team @@ -249,6 +252,11 @@ async def persist_post_summary( verification_client = verification_client or NullRelationVerificationClient() context_text = post_body if post_body is not None else summary.korean_summary + aliases = ( + await load_corroborated_organization_name_aliases(conn) + if summary.roles_and_responsibilities + else [] + ) candidates = ( await _load_corporate_entity_candidates(conn) if summary.roles_and_responsibilities @@ -265,6 +273,7 @@ async def persist_post_summary( hierarchy_inference_client, verification_client, candidates, + aliases=aliases, ) if corporate_entity_id is not None: resolved_organization_ids[role_index] = corporate_entity_id diff --git a/docs/adr/0008-organization-abbreviation-resolution.md b/docs/adr/0008-organization-abbreviation-resolution.md index 72b121253..5704e38f4 100644 --- a/docs/adr/0008-organization-abbreviation-resolution.md +++ b/docs/adr/0008-organization-abbreviation-resolution.md @@ -78,6 +78,11 @@ canonical name is returned to the caller as part of the normalized `PersonMention`, so the same request's relationship classifier uses the canonical form too rather than reintroducing the raw abbreviation. +[ADR 0160](0160-skos-organization-alias-catalog-binding.md) extends this +decision: a corroborated pair also expands catalog candidates so both +labels bind one `corporate_entity` row, including when the catalog was +first stored under the short form. + ## Consequences - The raw abbreviated form is not duplicated onto every row that diff --git a/docs/adr/0012-corporate-entity-creation-lock.md b/docs/adr/0012-corporate-entity-creation-lock.md index f5a01d859..a1cf722f9 100644 --- a/docs/adr/0012-corporate-entity-creation-lock.md +++ b/docs/adr/0012-corporate-entity-creation-lock.md @@ -22,7 +22,12 @@ pg_advisory_xact_lock( The transaction-scoped lock is acquired immediately before persistence and is released automatically by the enclosing transaction's commit or rollback (PostgreSQL Global Development Group, 2024). 1. The lock is acquired only after inference and verification complete. Holding it across network I/O would unnecessarily serialize unrelated workers. -2. Under the lock, candidates are reloaded and similarity matching is repeated. Another transaction may have committed the same entity after the caller's original snapshot was read. +2. Under the lock, candidates and corroborated organization aliases are + reloaded. Raw candidates repeat the normal similarity classification; a + full ancestor path resolved by the current recursion is excluded so an + ancestor cannot absorb its child. Alias-expanded candidates then repeat + ADR 0160's exact-label check. Another transaction may have committed the + same entity or alias after the caller's original snapshot was read. 3. The key is one fixed creation-path key rather than a per-name key. Per-name locking still permits the opposite-order multi-entity deadlock shape `A: [X, Y]` versus `B: [Y, X]`. 4. The already-cataloged resolution path remains lock-free. diff --git a/docs/adr/0160-skos-organization-alias-catalog-binding.md b/docs/adr/0160-skos-organization-alias-catalog-binding.md new file mode 100644 index 000000000..816870773 --- /dev/null +++ b/docs/adr/0160-skos-organization-alias-catalog-binding.md @@ -0,0 +1,76 @@ +# ADR 0160 — Corroborated SKOS aliases bind one corporate catalog row + +**Decision status:** Accepted +**Date:** 2026-08-23 + +## Context + +[ADR 0008](0008-organization-abbreviation-resolution.md) persists a +search-corroborated `skos:altLabel` / `skos:prefLabel` pair (Miles & +Bechhofer, 2009) in `organization_name_resolution`. Keyman then +substitutes the canonical name before +`get_or_create_corporate_entity`. That rewrite is not enough. + +`score_corporate_entity` only compares the mention to +`corporate_entity.entity_name`. An initialism such as the synthetic +fixture `AGP` shares almost no substring with `Aurora Grid Power` +(Bhattacharya & Getoor, 2007 candidate generation). If a catalog row +was created under the short form before corroboration, a later +canonical mention misses and ADR 0010 may insert a second `AUTO-` +row. The reverse is also true: a catalog row stored under the +preferred label does not bind a later abbreviated mention when +resolution is unavailable on that request. + +The product gap baseline described this as missing abbreviation +mapping. Repository artifacts may only use synthetic names +([ADR 0001](0001-demo-identity-and-data-boundary.md)). + +## Decision + +After loading `corporate_entity` candidates, expand them with every +`verify_corroborated` SKOS pair: + +- a row whose `entity_name` matches the preferred label also competes + under the alternative label; +- a row whose `entity_name` matches the alternative label also + competes under the preferred label. + +The expansion is a virtual candidate with the **same** +`corporate_entity_id`. Score the raw catalog first. Alias labels enrich +candidates only after a raw miss; they never override a raw unique result +or tie (ADR 0026). An alias label binds only on exact normalized equality; +short near-matches do not inherit the raw catalog's fuzzy threshold. +Uncorroborated or pending pairs are not loaded. +Identical or empty labels are ignored. + +Under the creation lock, reload both the catalog and corroborated aliases. +First repeat ADR 0026's normal-threshold raw classification, excluding only +the ancestor path resolved by the current recursion; then run the alias-expanded +exact check (`min_similarity=1.0`). A concurrent catalog row stored under the +other label is therefore reused instead of inserting a duplicate. + +Synthetic fixtures only: `AGP` / `Aurora Grid Power` (and similarly +`NRG` / `Northridge Grid` where already present). Real organization +names must not appear in tests, seeds, or docs. + +## Consequences + +- Two corroborated names for the same organization bind one catalog + row in Keyman affiliation and R&R organization-role resolution. +- Live customer abbreviations still require the ADR 0008 + resolve-then-verify pipeline at runtime; this decision does not + ship a real-world alias table. +- Character-similarity matching remains candidate generation, not + proof of identity. + +## Related + +Extends [ADR 0008](0008-organization-abbreviation-resolution.md) and +respects [ADR 0026](0026-tied-organization-similarity.md) and +[ADR 0010](0010-corporate-hierarchy-auto-creation.md). + +## References (APA 7th) + +Bhattacharya, I., & Getoor, L. (2007). Collective entity resolution in relational data. *ACM Transactions on Knowledge Discovery from Data, 1*(1), Article 5. https://doi.org/10.1145/1217299.1217304 + +Miles, A., & Bechhofer, S. (Eds.). (2009). *SKOS simple knowledge organization system reference*. World Wide Web Consortium. https://www.w3.org/TR/skos-reference/ diff --git a/lineageweave/corporate_hierarchy_resolution.py b/lineageweave/corporate_hierarchy_resolution.py index 652f827b2..5cb599f2b 100644 --- a/lineageweave/corporate_hierarchy_resolution.py +++ b/lineageweave/corporate_hierarchy_resolution.py @@ -60,6 +60,20 @@ class CorporateEntityCandidate: entity_name: str +@dataclass(frozen=True) +class OrganizationNameAlias: + """One corroborated SKOS alt/pref pair for the same organization. + + ``alt_label`` is the abbreviated/slang form stored as + ``organization_name_resolution.raw_organization_name``. ``pref_label`` + is the canonical form stored as ``resolved_organization_name``. Only + search-corroborated rows belong here (ADR 0008 / ADR 0160). + """ + + alt_label: str + pref_label: str + + @dataclass(frozen=True) class CorporateEntityResolution: """Candidate-generation outcome for one mentioned organization name. @@ -166,3 +180,58 @@ def resolve_corporate_entity( candidates, min_similarity, ).catalog_id + + +def expand_candidates_with_skos_aliases( + candidates: Sequence[CorporateEntityCandidate], + aliases: Sequence[OrganizationNameAlias], +) -> list[CorporateEntityCandidate]: + """Expose each corroborated SKOS pair as another label on the same row. + + Catalog matching is still string similarity (Bhattacharya & Getoor, + 2007 candidate generation). An initialism shares almost no substring + with its expansion, so a corroborated ``skos:altLabel`` / + ``skos:prefLabel`` pair (Miles & Bechhofer, 2009) is projected onto + the catalog id that already holds either label. Duplicate labels for + the same id stay one candidate. A later ``score_corporate_entity`` + call still fail-closes on a tie. + + Uncorroborated pairs must not be supplied. Empty or identical labels + are ignored so a no-op resolution cannot manufacture a match. + """ + expanded = list(candidates) + seen = { + (candidate.corporate_entity_id, normalize_organization_name(candidate.entity_name)) + for candidate in candidates + if normalize_organization_name(candidate.entity_name) + } + for candidate in candidates: + catalog_key = normalize_organization_name(candidate.entity_name) + if not catalog_key: + continue + for alias in aliases: + alt_key = normalize_organization_name(alias.alt_label) + pref_key = normalize_organization_name(alias.pref_label) + if not alt_key or not pref_key or alt_key == pref_key: + continue + extra_name: str | None = None + if catalog_key == pref_key: + extra_name = alias.alt_label + elif catalog_key == alt_key: + extra_name = alias.pref_label + if extra_name is None: + continue + extra_key = ( + candidate.corporate_entity_id, + normalize_organization_name(extra_name), + ) + if extra_key in seen: + continue + seen.add(extra_key) + expanded.append( + CorporateEntityCandidate( + corporate_entity_id=candidate.corporate_entity_id, + entity_name=extra_name, + ) + ) + return expanded diff --git a/tests/test_corporate_hierarchy_resolution.py b/tests/test_corporate_hierarchy_resolution.py index 72a7927c0..7ebea78a8 100644 --- a/tests/test_corporate_hierarchy_resolution.py +++ b/tests/test_corporate_hierarchy_resolution.py @@ -7,6 +7,8 @@ RESOLUTION_TIE, RESOLUTION_UNIQUE, CorporateEntityCandidate, + OrganizationNameAlias, + expand_candidates_with_skos_aliases, normalize_organization_name, resolve_corporate_entity, score_corporate_entity, @@ -110,3 +112,59 @@ def test_normalize_strips_suffix_punctuation_and_case() -> None: "Acme Electronics Korea, Ltd." ) == "acme electronics korea" assert normalize_organization_name(" ACME Group ") == "acme group" + + +_AGP_ALIAS = OrganizationNameAlias(alt_label="AGP", pref_label="Aurora Grid Power") + + +def test_skos_alias_binds_short_form_to_pref_label_catalog_row() -> None: + """A corroborated AGP altLabel must resolve to the Aurora Grid Power row.""" + catalog = [CorporateEntityCandidate("aurora-id", "Aurora Grid Power")] + expanded = expand_candidates_with_skos_aliases(catalog, [_AGP_ALIAS]) + outcome = score_corporate_entity("AGP", expanded) + assert outcome.kind == RESOLUTION_UNIQUE + assert outcome.catalog_id == "aurora-id" + assert resolve_corporate_entity("AGP", catalog) is None + + +def test_skos_alias_binds_pref_label_to_short_form_catalog_row() -> None: + """A catalog row stored under the altLabel still binds the prefLabel mention.""" + catalog = [CorporateEntityCandidate("aurora-id", "AGP")] + expanded = expand_candidates_with_skos_aliases(catalog, [_AGP_ALIAS]) + outcome = score_corporate_entity("Aurora Grid Power", expanded, min_similarity=1.0) + assert outcome.kind == RESOLUTION_UNIQUE + assert outcome.catalog_id == "aurora-id" + + +def test_skos_alias_does_not_bind_an_unrelated_organization() -> None: + catalog = [CorporateEntityCandidate("northridge-id", "Northridge Grid")] + expanded = expand_candidates_with_skos_aliases(catalog, [_AGP_ALIAS]) + outcome = score_corporate_entity("AGP", expanded) + assert outcome.kind == RESOLUTION_MISS + assert outcome.catalog_id is None + + +def test_skos_alias_tie_across_two_catalog_rows_stays_unbound() -> None: + """Two catalog ids that both own the same prefLabel remain a tie.""" + catalog = [ + CorporateEntityCandidate("aurora-a", "Aurora Grid Power"), + CorporateEntityCandidate("aurora-b", "Aurora Grid Power"), + ] + expanded = expand_candidates_with_skos_aliases(catalog, [_AGP_ALIAS]) + outcome = score_corporate_entity("AGP", expanded) + assert outcome.kind == RESOLUTION_TIE + assert outcome.catalog_id is None + assert set(outcome.top_catalog_ids) == {"aurora-a", "aurora-b"} + + +def test_identical_or_empty_skos_labels_do_not_expand() -> None: + catalog = [CorporateEntityCandidate("aurora-id", "Aurora Grid Power")] + expanded = expand_candidates_with_skos_aliases( + catalog, + [ + OrganizationNameAlias(alt_label="Aurora Grid Power", pref_label="Aurora Grid Power"), + OrganizationNameAlias(alt_label=" ", pref_label="Aurora Grid Power"), + ], + ) + assert expanded == catalog + diff --git a/tests/test_ingestion_transaction_contracts.py b/tests/test_ingestion_transaction_contracts.py index 68441ec0c..a9b8e7557 100644 --- a/tests/test_ingestion_transaction_contracts.py +++ b/tests/test_ingestion_transaction_contracts.py @@ -100,6 +100,8 @@ async def execute(self, query: str, *args: Any) -> str: async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: compact = " ".join(query.split()) + if "organization_name_resolution" in compact: + return [] assert compact == "select corporate_entity_id, entity_name from corporate_entity" self._events.append("candidate_reload") return list(self._reloaded_rows) @@ -232,6 +234,8 @@ async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: "cataloged_person_id": None, } ] + if "organization_name_resolution" in compact: + return [] raise AssertionError(f"unexpected fetch query: {compact}") @@ -324,6 +328,7 @@ async def resolve_organization( inference_client, verification_client, candidates, + **_kwargs, ) -> str: events.append(("organization_resolve", conn.in_transaction)) assert not conn.in_transaction @@ -404,6 +409,9 @@ async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: if compact == "select corporate_entity_id, entity_name from corporate_entity": assert not self.in_transaction return [] + if "organization_name_resolution" in compact: + assert not self.in_transaction + return [] if compact.startswith("select person_id, last_known_job_title"): assert self.in_transaction return [] @@ -438,6 +446,7 @@ async def resolve_organization( inference_client, verification_client, candidates, + **_kwargs, ) -> str: events.append(("organization_create", conn.in_transaction)) assert not conn.in_transaction diff --git a/tests/test_organization_name_resolution_ingestion.py b/tests/test_organization_name_resolution_ingestion.py index afdfa8f32..fef9b3abd 100644 --- a/tests/test_organization_name_resolution_ingestion.py +++ b/tests/test_organization_name_resolution_ingestion.py @@ -7,16 +7,22 @@ import backend.app.organization_name_resolution_ingestion as ingestion from lineageweave.relation_verification import STATUS_CORROBORATED, STATUS_UNCORROBORATED +from lineageweave.corporate_hierarchy_resolution import OrganizationNameAlias class _Connection: def __init__(self, cached: dict[str, str] | None = None) -> None: self.cached = cached self.executed: list[tuple[str, tuple[object, ...]]] = [] + self.alias_rows: list[dict[str, str]] = [] async def fetchrow(self, _query: str, _raw_name: str): return self.cached + async def fetch(self, query: str, *_args: object): + assert "organization_name_resolution" in query + return list(self.alias_rows) + async def execute(self, query: str, *args: object) -> str: self.executed.append((query, args)) return "OK" @@ -73,3 +79,21 @@ def test_new_resolution_is_persisted_but_only_verified_name_is_returned( assert result == expected assert len(conn.executed) == 1 assert "organization_name_resolution" in conn.executed[0][0] + + +def test_load_corroborated_aliases_skips_stub_connections() -> None: + assert asyncio.run(ingestion.load_corroborated_organization_name_aliases(object())) == [] + + +def test_load_corroborated_aliases_returns_search_verified_pairs() -> None: + conn = _Connection() + conn.alias_rows = [ + { + "raw_organization_name": "AGP", + "resolved_organization_name": "Aurora Grid Power", + } + ] + aliases = asyncio.run(ingestion.load_corroborated_organization_name_aliases(conn)) + assert aliases == [ + OrganizationNameAlias(alt_label="AGP", pref_label="Aurora Grid Power") + ] diff --git a/tests/test_skos_organization_alias_binding.py b/tests/test_skos_organization_alias_binding.py new file mode 100644 index 000000000..e0bc7d8e3 --- /dev/null +++ b/tests/test_skos_organization_alias_binding.py @@ -0,0 +1,315 @@ +"""SKOS alt/pref round-trip binds two synthetic names to one catalog row.""" + +from __future__ import annotations + +import asyncio +import uuid +from types import SimpleNamespace +from typing import Any + +from backend.app import corporate_entity_ingestion +from lineageweave.corporate_hierarchy_inference import HierarchyProposal +from lineageweave.corporate_hierarchy_resolution import ( + CorporateEntityCandidate, + OrganizationNameAlias, +) +from lineageweave.relation_verification import STATUS_CORROBORATED + + +_AGP_ALIAS = OrganizationNameAlias(alt_label="AGP", pref_label="Aurora Grid Power") +_SYNTHETIC_CONTEXT = "AGP (Aurora Grid Power) joined the synthetic grid forum." + + +class _RejectInferenceClient: + available = False + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal: + raise AssertionError(f"alias binding must not create {organization_name!r}") + + +class _RejectVerificationClient: + available = False + + def verify(self, subject: str, relation: str) -> SimpleNamespace: + raise AssertionError("alias binding must not call live search") + + +class _CreateIfReachedInferenceClient: + available = True + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal: + return HierarchyProposal(level_code="company", parent_name=None) + + +class _CreateIfReachedVerificationClient: + available = True + + def verify(self, subject: str, relation: str) -> SimpleNamespace: + return SimpleNamespace(status_code=STATUS_CORROBORATED) + + +class _RejectLiveInferenceClient: + available = True + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal: + raise AssertionError("an alias tie must not enter hierarchy inference") + + +class _AliasLockConnection: + """Serve corroborated aliases and a catalog row stored under the altLabel.""" + + def __init__(self, catalog_id: str, entity_name: str) -> None: + self.catalog_id = catalog_id + self.entity_name = entity_name + self.insert_attempted = False + + class _Transaction: + async def __aenter__(self) -> "_AliasLockConnection._Transaction": + return self + + async def __aexit__(self, exc_type: Any, exc: Any, traceback: Any) -> bool: + return False + + def transaction(self) -> "_AliasLockConnection._Transaction": + return self._Transaction() + + async def execute(self, query: str, *args: Any) -> str: + assert "pg_advisory_xact_lock" in query + return "SELECT 1" + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + if "organization_name_resolution" in query: + assert args == (STATUS_CORROBORATED,) + return [ + { + "raw_organization_name": "AGP", + "resolved_organization_name": "Aurora Grid Power", + } + ] + assert "from corporate_entity" in query + return [ + { + "corporate_entity_id": self.catalog_id, + "entity_name": self.entity_name, + } + ] + + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any]: + self.insert_attempted = True + raise AssertionError("alias round-trip must reuse the existing catalog row") + + +class _AliasTieLockConnection(_AliasLockConnection): + """Expose two rows that tie only after their shared alias is expanded.""" + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + if "organization_name_resolution" in query: + return [ + { + "raw_organization_name": "AGP", + "resolved_organization_name": "Aurora Grid Power", + }, + { + "raw_organization_name": "AGP", + "resolved_organization_name": "Alpine Grid Power", + }, + ] + assert "from corporate_entity" in query + return [ + {"corporate_entity_id": "alias-a", "entity_name": "Aurora Grid Power"}, + {"corporate_entity_id": "alias-b", "entity_name": "Alpine Grid Power"}, + ] + + +class _LateAliasLockConnection(_AliasLockConnection): + """Publish one corroborated alias while hierarchy inference is running.""" + + def __init__(self) -> None: + super().__init__("short-row", "AGP") + self.alias_reads = 0 + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + if "organization_name_resolution" in query: + self.alias_reads += 1 + if self.alias_reads == 1: + return [] + return [ + { + "raw_organization_name": "AGP", + "resolved_organization_name": "Aurora Grid Power", + } + ] + assert "from corporate_entity" in query + return [ + {"corporate_entity_id": self.catalog_id, "entity_name": self.entity_name} + ] + + +def test_short_form_mention_reuses_pref_label_catalog_row() -> None: + catalog_id = str(uuid.uuid4()) + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + object(), + "AGP", + _SYNTHETIC_CONTEXT, + _RejectInferenceClient(), + _RejectVerificationClient(), + [CorporateEntityCandidate(catalog_id, "Aurora Grid Power")], + aliases=[_AGP_ALIAS], + ) + ) + assert result == catalog_id + + +def test_raw_unique_match_does_not_load_aliases() -> None: + catalog_id = str(uuid.uuid4()) + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + object(), + "Aurora Grid Power", + _SYNTHETIC_CONTEXT, + _RejectInferenceClient(), + _RejectVerificationClient(), + [CorporateEntityCandidate(catalog_id, "Aurora Grid Power")], + ) + ) + assert result == catalog_id + + +def test_pref_label_mention_reuses_alt_label_catalog_row_under_creation_lock() -> None: + catalog_id = str(uuid.uuid4()) + connection = _AliasLockConnection(catalog_id, "AGP") + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + connection, + "Aurora Grid Power", + _SYNTHETIC_CONTEXT, + _CreateIfReachedInferenceClient(), + _CreateIfReachedVerificationClient(), + [], + ) + ) + assert result == catalog_id + assert connection.insert_attempted is False + + +def test_uncorroborated_alias_does_not_bind() -> None: + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + object(), + "AGP", + _SYNTHETIC_CONTEXT, + _RejectInferenceClient(), + _RejectVerificationClient(), + [CorporateEntityCandidate(str(uuid.uuid4()), "Aurora Grid Power")], + aliases=(), + ) + ) + assert result is None + + +def test_short_near_match_does_not_fuzzy_bind_a_corroborated_alias() -> None: + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + object(), + "AGB", + _SYNTHETIC_CONTEXT, + _RejectInferenceClient(), + _RejectVerificationClient(), + [CorporateEntityCandidate(str(uuid.uuid4()), "Aurora Grid Power")], + aliases=[_AGP_ALIAS], + ) + ) + + assert result is None + + +def test_raw_catalog_tie_precedes_unique_alias_match() -> None: + candidates = [ + CorporateEntityCandidate("tied-north", "Tied Energy North"), + CorporateEntityCandidate("tied-south", "Tied Energy South"), + CorporateEntityCandidate("alias-target", "Aurora Grid Power"), + ] + + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + object(), + "Tied Energy", + _SYNTHETIC_CONTEXT, + _CreateIfReachedInferenceClient(), + _CreateIfReachedVerificationClient(), + candidates, + aliases=[ + OrganizationNameAlias( + alt_label="Tied Energy", + pref_label="Aurora Grid Power", + ) + ], + ) + ) + + assert result is None + + +def test_alias_expansion_tie_does_not_enter_inference() -> None: + candidates = [ + CorporateEntityCandidate("alias-a", "Aurora Grid Power"), + CorporateEntityCandidate("alias-b", "Alpine Grid Power"), + ] + aliases = [ + OrganizationNameAlias("AGP", "Aurora Grid Power"), + OrganizationNameAlias("AGP", "Alpine Grid Power"), + ] + + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + object(), + "AGP", + _SYNTHETIC_CONTEXT, + _RejectLiveInferenceClient(), + _RejectVerificationClient(), + candidates, + aliases=aliases, + ) + ) + + assert result is None + + +def test_post_lock_alias_tie_does_not_insert() -> None: + connection = _AliasTieLockConnection("unused", "unused") + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + connection, + "AGP", + _SYNTHETIC_CONTEXT, + _CreateIfReachedInferenceClient(), + _CreateIfReachedVerificationClient(), + [], + aliases=[ + OrganizationNameAlias("AGP", "Aurora Grid Power"), + OrganizationNameAlias("AGP", "Alpine Grid Power"), + ], + ) + ) + + assert result is None + assert connection.insert_attempted is False + + +def test_post_lock_reloads_aliases_before_inserting() -> None: + connection = _LateAliasLockConnection() + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + connection, + "Aurora Grid Power", + _SYNTHETIC_CONTEXT, + _CreateIfReachedInferenceClient(), + _CreateIfReachedVerificationClient(), + [], + ) + ) + + assert result == "short-row" + assert connection.alias_reads == 2 + assert connection.insert_attempted is False diff --git a/tests/test_tied_organization_no_create.py b/tests/test_tied_organization_no_create.py index 55041715c..1c5cc33b2 100644 --- a/tests/test_tied_organization_no_create.py +++ b/tests/test_tied_organization_no_create.py @@ -9,7 +9,10 @@ from backend.app import corporate_entity_ingestion, keyman_ingestion from lineageweave.corporate_hierarchy_inference import HierarchyProposal -from lineageweave.corporate_hierarchy_resolution import CorporateEntityCandidate +from lineageweave.corporate_hierarchy_resolution import ( + CorporateEntityCandidate, + OrganizationNameAlias, +) from lineageweave.relation_verification import STATUS_CORROBORATED @@ -54,6 +57,29 @@ def infer(self, organization_name: str, context_text: str) -> HierarchyProposal: raise TimeoutError("synthetic orchestrator timeout") +class _ParentProposalInferenceClient: + available = True + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal: + parent_name = ( + "Aurora Grid Power" + if organization_name == "Aurora Grid Power Division" + else None + ) + return HierarchyProposal(level_code="company", parent_name=parent_name) + + +class _GrandparentProposalInferenceClient: + available = True + + def infer(self, organization_name: str, context_text: str) -> HierarchyProposal: + parent_name = { + "Aurora Grid Holdings Division": "Aurora Grid Operations", + "Aurora Grid Operations": "Aurora Grid Holdings", + }.get(organization_name) + return HierarchyProposal(level_code="company", parent_name=parent_name) + + class _Transaction: """Minimal async transaction context manager.""" @@ -78,6 +104,8 @@ async def execute(self, query: str, *args: Any) -> str: return "SELECT 1" async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + if "organization_name_resolution" in query: + return [] assert "from corporate_entity" in query return [ { @@ -95,6 +123,65 @@ async def fetchrow(self, query: str, *args: Any) -> dict[str, Any]: raise AssertionError("a refreshed tie must not insert an AUTO row") +class _ReloadFuzzyTieConnection(_ReloadTieConnection): + """Expose a qualifying raw-name tie only after inference.""" + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + if "organization_name_resolution" in query: + return [] + assert "from corporate_entity" in query + return [ + {"corporate_entity_id": "north", "entity_name": "Tied Energy North"}, + {"corporate_entity_id": "south", "entity_name": "Tied Energy South"}, + ] + + +class _ParentAwareConnection(_ReloadTieConnection): + """Persist a recursively inferred parent before the child lock refresh.""" + + def __init__(self) -> None: + super().__init__() + self.rows: list[dict[str, Any]] = [] + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + if "organization_name_resolution" in query: + return [] + assert "from corporate_entity" in query + return list(self.rows) + + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any]: + organization_name = args[2] + corporate_entity_id = ( + "parent-row" + if organization_name == "Aurora Grid Power" + else "child-row" + ) + self.rows.append( + { + "corporate_entity_id": corporate_entity_id, + "entity_name": organization_name, + } + ) + return {"corporate_entity_id": corporate_entity_id} + + +class _GrandparentAwareConnection(_ParentAwareConnection): + async def fetchrow(self, query: str, *args: Any) -> dict[str, Any]: + organization_name = args[2] + corporate_entity_id = { + "Aurora Grid Holdings": "grandparent-row", + "Aurora Grid Operations": "parent-row", + "Aurora Grid Holdings Division": "child-row", + }[organization_name] + self.rows.append( + { + "corporate_entity_id": corporate_entity_id, + "entity_name": organization_name, + } + ) + return {"corporate_entity_id": corporate_entity_id} + + def test_initial_tie_never_reaches_live_inference_or_creation() -> None: """A known tie returns before any live external client is consulted.""" inference = _LiveInferenceClient() @@ -155,6 +242,70 @@ def test_tie_discovered_under_creation_lock_does_not_insert() -> None: assert connection.insert_attempted is False +def test_fuzzy_tie_discovered_under_creation_lock_does_not_insert() -> None: + """The lock refresh repeats normal raw classification, not exact-only.""" + connection = _ReloadFuzzyTieConnection() + + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + connection, + "Tied Energy", + "Synthetic context", + _LiveInferenceClient(), + _LiveVerificationClient(), + [], + ) + ) + + assert result is None + assert connection.insert_attempted is False + + +def test_post_lock_raw_scoring_excludes_recursively_resolved_parent() -> None: + connection = _ParentAwareConnection() + + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + connection, + "Aurora Grid Power Division", + "Synthetic context", + _ParentProposalInferenceClient(), + _LiveVerificationClient(), + [], + aliases=[], + ) + ) + + assert result == "child-row" + assert [row["corporate_entity_id"] for row in connection.rows] == [ + "parent-row", + "child-row", + ] + + +def test_post_lock_raw_scoring_excludes_entire_resolved_ancestor_path() -> None: + connection = _GrandparentAwareConnection() + + result = asyncio.run( + corporate_entity_ingestion.get_or_create_corporate_entity( + connection, + "Aurora Grid Holdings Division", + "Synthetic context", + _GrandparentProposalInferenceClient(), + _LiveVerificationClient(), + [], + aliases=[], + ) + ) + + assert result == "child-row" + assert [row["corporate_entity_id"] for row in connection.rows] == [ + "grandparent-row", + "parent-row", + "child-row", + ] + + def test_keyman_raw_tie_blocks_abbreviation_rewrite_and_auto_creation() -> None: """Keyman checks the raw tied name before any resolver rewrite.""" result = asyncio.run( @@ -170,3 +321,32 @@ def test_keyman_raw_tie_blocks_abbreviation_rewrite_and_auto_creation() -> None: ) assert result == ("Tied Energy", "Tied Energy", None) + + +def test_keyman_raw_tie_precedes_unique_alias_match() -> None: + """A virtual alias cannot rewrite an already ambiguous raw mention.""" + candidates = [ + CorporateEntityCandidate("tied-north", "Tied Energy North"), + CorporateEntityCandidate("tied-south", "Tied Energy South"), + CorporateEntityCandidate("alias-target", "Aurora Grid Power"), + ] + + result = asyncio.run( + keyman_ingestion._resolve_affiliated_organization( + object(), + "Tied Energy", + "Synthetic context", + object(), + object(), + object(), + candidates, + aliases=[ + OrganizationNameAlias( + alt_label="Tied Energy", + pref_label="Aurora Grid Power", + ) + ], + ) + ) + + assert result == ("Tied Energy", "Tied Energy", None)