diff --git a/backend/app/ontology_neighborhood_ingestion.py b/backend/app/ontology_neighborhood_ingestion.py index f64987ec4..51f56ff2c 100644 --- a/backend/app/ontology_neighborhood_ingestion.py +++ b/backend/app/ontology_neighborhood_ingestion.py @@ -866,6 +866,10 @@ def neighborhood_to_payload(neighborhood: OntologyNeighborhood) -> dict[str, Any "is_primary": assignment.is_primary, "truth_status_code": assignment.truth_status_code, "recorded_at": assignment.recorded_at.isoformat(), + "effective_from": assignment.effective_from.isoformat(), + "effective_to": assignment.effective_to.isoformat() + if assignment.effective_to + else None, "provenance_reference": assignment.provenance_reference, "evidence_post_id": assignment.evidence_post_id, } @@ -890,6 +894,7 @@ async def _load_voice_assignments( """ select voice.post_id, voice.voice_type_code, lookup.lookup_label, voice.is_primary, voice.truth_status_code, voice.recorded_at, + voice.effective_from, voice.effective_to, case when evidence.node_id = any($1::uuid[]) then evidence.node_id end as evidence_post_id from source_post_voice voice @@ -903,10 +908,11 @@ async def _load_voice_assignments( and evidence.node_type_code = 'node_post' where voice.post_id = any($1::uuid[]) and (voice.is_primary or evidence.node_id = any($1::uuid[])) - and (($2::timestamptz is null and voice.effective_to is null) - or ($2::timestamptz is not null - and voice.effective_from <= $2 - and (voice.effective_to is null or $2 < voice.effective_to))) + and voice.effective_from <= coalesce($2::timestamptz, $3::timestamptz) + and ( + voice.effective_to is null + or coalesce($2::timestamptz, $3::timestamptz) < voice.effective_to + ) and voice.recorded_at <= $3::timestamptz order by voice.post_id, voice.is_primary desc, lookup.display_order, voice.voice_type_code @@ -936,6 +942,8 @@ async def _load_voice_assignments( if not row["is_primary"] else "Imported primary voice" ), + effective_from=row["effective_from"], + effective_to=row["effective_to"], evidence_post_id=( str(row["evidence_post_id"]) if row["evidence_post_id"] is not None diff --git a/docs/adr/0252-temporal-primary-voice-history.md b/docs/adr/0252-temporal-primary-voice-history.md new file mode 100644 index 000000000..2cf3f5685 --- /dev/null +++ b/docs/adr/0252-temporal-primary-voice-history.md @@ -0,0 +1,97 @@ +# ADR 0252: Temporal history for imported primary Voice + +## Status + +Accepted (2026-08-27). Extends ADR 0251 and closes issue #748. + +## Context + +ADR 0251 records when a Voice assignment starts, but migration 0237 deletes +the former imported primary when `source_post.voc_type_code` changes. The live +value is honest, yet an authorized knowledge-cutoff read after that update can +no longer recover the primary that was effective at the cutoff. The existing +`(post_id, voice_type_code)` key also cannot represent A → B → A. + +OWL-Time distinguishes instants from intervals and gives an interval explicit +beginning and end bounds. PostgreSQL range types and exclusion constraints are +the native database mechanism for rejecting overlapping periods. Neither +source supplies a missing business-effective instant, so LineageWeave must not +invent one: an imported change becomes effective at the database transaction +instant when no source change instant exists. + +## Decision + +- Keep `source_post_voice` as the normalized assignment relation. Add nullable + `effective_to`; each row is a half-open interval + `[effective_from, effective_to)`. Null means current. +- Change the key to `(post_id, voice_type_code, effective_from)`, allowing the + same atomic Voice to recur in non-overlapping periods. +- Use PostgreSQL GiST exclusion constraints to reject overlapping primary + intervals for one Post. A partial unique index also permits at most one + current row for a `(post_id, voice_type_code)` pair. +- When the imported primary changes, one trigger transaction closes both the + current primary and any current additional assignment for the incoming + Voice, then inserts the new observed primary at one trigger-execution + timestamp. PostgreSQL `clock_timestamp()` is read after the source-row lock + is acquired, so a waiting concurrent update cannot backdate its interval to + the earlier statement start. It never overwrites or fabricates the former + interval. +- Live reads select `effective_to is null`. Cutoff reads select the row whose + interval contains the cutoff. Ontology continuation reads use their frozen + `snapshot_at` when no knowledge cutoff was requested, so a page minted + before a change cannot silently switch to the new primary. +- Existing rows migrate as open intervals. Migration replay changes neither + their starts nor their history. History before ADR 0252 remains unavailable + because the deleted facts cannot be reconstructed honestly. +- This is valid-time history for a source assignment, not psychometric or + mathematical modeling. No weight, confidence, inference, or new Voice code + is introduced. + +## Data model + +```mermaid +classDiagram + class SourcePost { + uuid post_id + text voc_type_code + } + class SourcePostVoice { + uuid post_id + text voice_type_code + boolean is_primary + timestamptz effective_from + timestamptz effective_to + timestamptz recorded_at + } + SourcePost "1" --> "1..*" SourcePostVoice +``` + +```mermaid +sequenceDiagram + participant Import + participant SourcePost + participant VoiceHistory + Import->>SourcePost: update primary A to B + SourcePost->>VoiceHistory: close current A after source-row lock + SourcePost->>VoiceHistory: close current additional B, if present + SourcePost->>VoiceHistory: insert observed primary B at same instant + VoiceHistory-->>Import: one non-overlapping current primary +``` + +## Consequences + +- A → B → A is auditable without copying source content or exposing real + identifiers. +- Half-open bounds assign the exact change instant to the new primary and avoid + double matches. +- The exclusion constraint adds a GiST index and write-time check. This table + is bounded by Voice assignments per Post; partitioning is not warranted + until observed volume or lock evidence shows otherwise. + +## References + +Cox, S. J. D., & Little, C. (2022). *Time ontology in OWL*. World Wide +Web Consortium. https://www.w3.org/TR/owl-time/ + +PostgreSQL Global Development Group. (2025). *PostgreSQL 18 documentation: +Range types*. https://www.postgresql.org/docs/18/rangetypes.html diff --git a/docs/adr/README.md b/docs/adr/README.md index f10df7664..d90943d19 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -9,9 +9,11 @@ decision from them. | Supporting document | Normative ADR | |---|---| -| [`product-requirements.md`](../product-requirements.md) | Product requirements projection across the ADR set; ADRs remain normative | +| [`product-requirements.md`](../product-requirements.md) | Product requirements projection across the ADR set; ADRs remain normative, including [0252](0252-temporal-primary-voice-history.md) | | [`product-technical-gap-baseline.md`](../product-technical-gap-baseline.md) | Product/technical traceability projection across the ADR set; ADRs remain normative | | [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0024](0024-rankweave-fusion-fail-closed.md), [0165](0165-quantity-script-display.md), [0167](0167-rankweave-ranking-channel-evidence.md), [0169](0169-ask-batched-lineage-graph.md), [0172](0172-event-lineage-channel-evidence.md), [0202](0202-ask-event-time-filter.md), [0223](0223-explicit-semantic-content-unit-kinds.md), [0238](0238-source-conversation-turn-import-contract.md) | +| [`voice-combination-technical-requirements.md`](../voice-combination-technical-requirements.md) | [0246](0246-expanded-voice-of-x-post-taxonomy.md), [0251](0251-evidence-bearing-voice-combinations.md), [0252](0252-temporal-primary-voice-history.md) | +| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0024](0024-rankweave-fusion-fail-closed.md), [0165](0165-quantity-script-display.md), [0167](0167-rankweave-ranking-channel-evidence.md), [0169](0169-ask-batched-lineage-graph.md), [0172](0172-event-lineage-channel-evidence.md), [0202](0202-ask-event-time-filter.md), [0223](0223-explicit-semantic-content-unit-kinds.md) | | [`PROV_O_IMPLEMENTATION.md`](../PROV_O_IMPLEMENTATION.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`PROV_O_IMPLEMENTATION_MATRIX.md`](../PROV_O_IMPLEMENTATION_MATRIX.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`ONTOLOGY_NAMESPACE_INVENTORY.md`](../doctoring/ONTOLOGY_NAMESPACE_INVENTORY.md) | [0207](0207-repository-case-ontology-namespace-canonical.md), [0157](0157-public-ontology-namespace-identity.md) | diff --git a/docs/voice-combination-technical-requirements.md b/docs/voice-combination-technical-requirements.md new file mode 100644 index 000000000..f075f7e2e --- /dev/null +++ b/docs/voice-combination-technical-requirements.md @@ -0,0 +1,60 @@ +# Voice-of-X Combination Technical Requirements + +This supporting TRD projects ADR 0246, ADR 0251, and ADR 0252. Those ADRs are +normative when this document and an implementation differ. + +## Scope + +LineageWeave represents a Post's explicitly supplied stakeholder perspectives +without assuming a company, B2B2C chain, or exhaustive industry taxonomy. One +imported primary Voice and zero or more evidence-bearing additional Voices are +atomic assignments; combinations are sets of rows, never compound codes. + +## Requirements + +| ID | Requirement | Verification | +|---|---|---| +| VOC-TR-1 | `source_post.voc_type_code` owns the imported primary; additional assignments cannot demote it | Database trigger and API conflict tests | +| VOC-TR-2 | Every additional Voice references a normalized PROV-O derivation and governed truth status | Foreign keys, category trigger, authenticated write test | +| VOC-TR-3 | Primary assignments use non-overlapping half-open effective intervals and allow A → B → A under serialized concurrent source updates | GiST exclusion constraint and PostgreSQL integration tests | +| VOC-TR-4 | Live reads select current rows; cutoff reads select the containing interval; ontology continuation uses its frozen snapshot when no cutoff exists | Backend SQL-contract tests and authenticated cutoff API test | +| VOC-TR-5 | Post, filter, ontology JSON-LD, exact-value CSV, and UI apply the same RBAC/ABAC and source-eligibility boundary | API, SHACL, frontend interaction, and accessibility tests | +| VOC-TR-6 | Voice stays separate from counterparty relationship, role, topic, channel, lifecycle, and stakeholder salience | ADR/schema review and ontology round-trip tests | +| VOC-TR-7 | Migration replay preserves existing starts and never reconstructs deleted pre-migration history | Migration replay test and non-identifying runtime evidence | + +## Read contract + +```text +reference_time = knowledge_cutoff ?? ontology_snapshot ?? live +live = effective_to IS NULL +historical = effective_from <= reference_time < effective_to +open historical = effective_from <= reference_time AND effective_to IS NULL +``` + +The interval is lower-inclusive and upper-exclusive. The exact primary-change +instant belongs to the new primary, so a read cannot return two primary rows. + +## Component flow + +```mermaid +flowchart LR + Import[Authorized source import] --> SourcePost[(source_post)] + SourcePost --> Trigger[Primary Voice sync trigger] + Trigger --> History[(source_post_voice intervals)] + Admin[post_admin + visible evidence] --> API[Voice assignment API] + API --> Provenance[(PROV-O assertion)] + Provenance --> History + History --> PostRead[Post and filters] + History --> Ontology[Ontology JSON-LD and CSV] + PostRead --> UI[Post and board UI] + Ontology --> Explorer[Ontology explorer] +``` + +## Failure behavior + +- Missing or hidden evidence rejects or omits the additional assignment; it is + never replaced with a placeholder. +- Unknown Voice/truth categories fail with a database check error. +- Overlapping imported-primary intervals fail at the database boundary. +- Cutoffs before retained history return an explicit unavailable state rather + than the current value. diff --git a/lineageweave/ontology_neighborhood.py b/lineageweave/ontology_neighborhood.py index cb07daae0..991b81f10 100644 --- a/lineageweave/ontology_neighborhood.py +++ b/lineageweave/ontology_neighborhood.py @@ -234,7 +234,9 @@ class OntologyVoiceAssignment: is_primary: bool truth_status_code: str recorded_at: datetime + effective_from: datetime provenance_reference: str + effective_to: datetime | None = None evidence_post_id: str | None = None def __post_init__(self) -> None: @@ -260,6 +262,16 @@ def __post_init__(self) -> None: raise OntologyNeighborhoodError( "naive_timestamp", "voice assignment recorded_at must be offset-aware" ) + if self.effective_from.tzinfo is None or ( + self.effective_to is not None and self.effective_to.tzinfo is None + ): + raise OntologyNeighborhoodError( + "naive_timestamp", "voice assignment effective bounds must be offset-aware" + ) + if self.effective_to is not None and self.effective_from >= self.effective_to: + raise OntologyNeighborhoodError( + "invalid_interval", "voice assignment effective interval is empty or inverted" + ) @dataclass(frozen=True) @@ -324,8 +336,10 @@ def exact_value_rows(self) -> tuple[dict[str, str], ...]: "target_type_code": "node_voice_type", "truth_status_code": assignment.truth_status_code, "recorded_at": assignment.recorded_at.isoformat(), - "valid_from": "", - "valid_to": "", + "valid_from": assignment.effective_from.isoformat(), + "valid_to": assignment.effective_to.isoformat() + if assignment.effective_to + else "", "evidence_count": "1" if assignment.evidence_post_id or assignment.is_primary else "0", "evidence_post_id": assignment.evidence_post_id or ( assignment.post_id if assignment.is_primary else "" @@ -414,23 +428,24 @@ def jsonld_document(self) -> dict[str, object]: if evidence_iri is not None else {} ) - graph.append( - { - "@id": assignment_iri, - "@type": str(LW.VoiceAssignment), - str(LW.assignedVoiceType): {"@id": assignment.voice_type_iri}, - str(LW.primaryVoiceAssignment): { - "@value": assignment.is_primary, - "@type": "xsd:boolean", - }, - **provenance, - "lw:truthStatus": assignment.truth_status_code, - "prov:generatedAtTime": { - "@value": assignment.recorded_at.isoformat(), - "@type": "xsd:dateTimeStamp", - }, - } + item: dict[str, object] = { + "@id": assignment_iri, + "@type": str(LW.VoiceAssignment), + str(LW.assignedVoiceType): {"@id": assignment.voice_type_iri}, + str(LW.primaryVoiceAssignment): { + "@value": assignment.is_primary, + "@type": "xsd:boolean", + }, + **provenance, + "lw:truthStatus": assignment.truth_status_code, + } + _add_jsonld_times( + item, + assignment.recorded_at, + assignment.effective_from, + assignment.effective_to, ) + graph.append(item) graph.append( { "@id": assignment.voice_type_iri, diff --git a/migrations/0237_source_post_voice_combination.sql b/migrations/0237_source_post_voice_combination.sql index a010a830c..5011dc1b3 100644 --- a/migrations/0237_source_post_voice_combination.sql +++ b/migrations/0237_source_post_voice_combination.sql @@ -129,7 +129,8 @@ update source_post_voice voice insert into source_post_voice (post_id, voice_type_code, is_primary, truth_status_code, effective_from) -select post.post_id, post.voc_type_code, true, 'truth_observed', post.created_at +select post.post_id, post.voc_type_code, true, 'truth_observed', + least(post.created_at, clock_timestamp()) from source_post post where not exists ( select 1 from source_post_voice voice @@ -159,7 +160,7 @@ begin new.voc_type_code, true, 'truth_observed', - case when tg_op = 'INSERT' then new.created_at else change_at end + case when tg_op = 'INSERT' then least(new.created_at, change_at) else change_at end ) on conflict (post_id, voice_type_code) where effective_to is null do update set is_primary = true, diff --git a/migrations/0243_source_post_voice_history.sql b/migrations/0243_source_post_voice_history.sql new file mode 100644 index 000000000..ad4637754 --- /dev/null +++ b/migrations/0243_source_post_voice_history.sql @@ -0,0 +1,57 @@ +-- ADR 0252: preserve non-overlapping imported primary Voice intervals. + +begin; + +create extension if not exists btree_gist; + +alter table source_post_voice + add column if not exists effective_to timestamptz; + +alter table source_post_voice + drop constraint if exists source_post_voice_effective_interval_check; +alter table source_post_voice + add constraint source_post_voice_effective_interval_check + check (effective_to is null or effective_from < effective_to); + +create unique index if not exists source_post_voice_current_pair_idx + on source_post_voice (post_id, voice_type_code) + where effective_to is null; + +alter table source_post_voice + drop constraint if exists source_post_voice_primary_period_excl; +alter table source_post_voice + add constraint source_post_voice_primary_period_excl + exclude using gist ( + post_id with =, + tstzrange(effective_from, effective_to, '[)') with && + ) where (is_primary); + +create or replace function synchronize_source_post_primary_voice() +returns trigger +language plpgsql +as $$ +declare + change_at timestamptz := clock_timestamp(); +begin + update source_post_voice + set effective_to = change_at + where post_id = new.post_id + and effective_to is null + and (is_primary or voice_type_code = new.voc_type_code); + + insert into source_post_voice + (post_id, voice_type_code, is_primary, truth_status_code, + effective_from, recorded_at) + values ( + new.post_id, + new.voc_type_code, + true, + 'truth_observed', + case when tg_op = 'INSERT' then least(new.created_at, change_at) else change_at end, + change_at + ); + return new; +end; +$$; + +commit; diff --git a/tests/test_ontology_neighborhood.py b/tests/test_ontology_neighborhood.py index 3f6165745..a9421b428 100644 --- a/tests/test_ontology_neighborhood.py +++ b/tests/test_ontology_neighborhood.py @@ -912,6 +912,7 @@ def test_voice_assignments_join_exact_csv_rows_and_jsonld() -> None: is_primary=False, truth_status_code=TRUTH_OBSERVED, recorded_at=T0, + effective_from=T0, provenance_reference="Evidence-backed additional voice", evidence_post_id=POST_ID, ) diff --git a/tests/test_ontology_neighborhood_ingestion.py b/tests/test_ontology_neighborhood_ingestion.py index 333d91df9..546fd28bb 100644 --- a/tests/test_ontology_neighborhood_ingestion.py +++ b/tests/test_ontology_neighborhood_ingestion.py @@ -930,6 +930,8 @@ def test_focus_label_fetch_may_be_empty_when_facts_already_labeled() -> None: "is_primary": True, "truth_status_code": "truth_observed", "recorded_at": T0, + "effective_from": T0, + "effective_to": None, "has_assertion": False, "evidence_post_id": None, } @@ -1003,6 +1005,8 @@ def test_load_voice_assignments_preserves_truth_and_customer_safe_provenance() - "is_primary": True, "truth_status_code": "truth_observed", "recorded_at": T0, + "effective_from": T0, + "effective_to": None, "has_assertion": False, "evidence_post_id": None, }, @@ -1013,6 +1017,8 @@ def test_load_voice_assignments_preserves_truth_and_customer_safe_provenance() - "is_primary": False, "truth_status_code": "truth_observed", "recorded_at": T0, + "effective_from": T0, + "effective_to": None, "has_assertion": True, "evidence_post_id": POST_ID, }, @@ -1030,7 +1036,15 @@ def test_load_voice_assignments_preserves_truth_and_customer_safe_provenance() - assert assignments[1].evidence_post_id == POST_ID assert "evidence.node_id = any($1::uuid[])" in conn.calls[0][0] assert "voice.is_primary or evidence.node_id = any($1::uuid[])" in conn.calls[0][0] - assert "voice.effective_from <= $2" in conn.calls[0][0] + assert ( + "voice.effective_from <= coalesce($2::timestamptz, $3::timestamptz)" + in conn.calls[0][0] + ) + assert ( + "coalesce($2::timestamptz, $3::timestamptz) < voice.effective_to" + in conn.calls[0][0] + ) + assert "$2::timestamptz is null and voice.effective_to is null" not in conn.calls[0][0] assert "voice.recorded_at <= $3" in conn.calls[0][0] assert conn.calls[0][1] == ([POST_ID], T0, T0) diff --git a/tests/test_source_post_voice_history_schema.py b/tests/test_source_post_voice_history_schema.py new file mode 100644 index 000000000..66b7866b9 --- /dev/null +++ b/tests/test_source_post_voice_history_schema.py @@ -0,0 +1,49 @@ +"""Static contract tests for ADR 0252 temporal primary Voice history.""" + +from __future__ import annotations + +from pathlib import Path + +MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0243_source_post_voice_history.sql" +) + + +def test_primary_voice_history_uses_non_overlapping_half_open_intervals() -> None: + """The database preserves recurring Voices and rejects overlapping primaries.""" + sql = MIGRATION.read_text(encoding="utf-8").lower() + + assert "add column if not exists effective_to timestamptz" in sql + assert "drop constraint if exists source_post_voice_pkey" not in sql + assert "primary key (post_id, voice_type_code, effective_from)" not in sql + assert "voice_assignment_id" not in sql + assert "tstzrange(effective_from, effective_to, '[)') with &&" in sql + assert "where (is_primary)" in sql + assert "effective_from < effective_to" in sql + + +def test_primary_voice_change_closes_current_rows_before_insert() -> None: + """One transaction instant closes the prior state and opens the new primary.""" + sql = MIGRATION.read_text(encoding="utf-8").lower() + + assert "change_at timestamptz := clock_timestamp()" in sql + assert "set effective_to = change_at" in sql + assert "and (is_primary or voice_type_code = new.voc_type_code)" in sql + assert sql.index("set effective_to = change_at") < sql.index( + "insert into source_post_voice" + ) + assert "on conflict" not in sql + + +def test_future_source_clock_is_bounded_by_the_recording_clock() -> None: + """A future source timestamp cannot create an interval that closes backwards.""" + assignment_sql = ( + MIGRATION.parent / "0237_source_post_voice_combination.sql" + ).read_text(encoding="utf-8").lower() + history_sql = MIGRATION.read_text(encoding="utf-8").lower() + + assert "least(post.created_at, clock_timestamp())" in assignment_sql + assert "least(new.created_at, change_at)" in assignment_sql + assert "least(new.created_at, change_at)" in history_sql diff --git a/tests/test_source_post_voice_ingestion.py b/tests/test_source_post_voice_ingestion.py index 75a0c48d0..52fdf0b1f 100644 --- a/tests/test_source_post_voice_ingestion.py +++ b/tests/test_source_post_voice_ingestion.py @@ -72,6 +72,7 @@ def test_additional_voice_creates_prov_derivation_and_assignment_atomically() -> assert "prov_was_derived_from" in sql assert "where effective_to is null" in sql assert "where not source_post_voice.is_primary" in sql + assert "where effective_to is null" in sql assert "voice-assignment/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1/vops" in str( conn.calls ) diff --git a/tests/test_source_post_voice_schema.py b/tests/test_source_post_voice_schema.py index 05265bf10..b78135b6b 100644 --- a/tests/test_source_post_voice_schema.py +++ b/tests/test_source_post_voice_schema.py @@ -25,9 +25,10 @@ def test_voice_combination_schema_is_normalized_and_evidence_bearing() -> None: assert "effective_from timestamptz not null" in sql assert "true, 'truth_observed'" in sql assert "where is_primary" in sql - assert "select post.post_id, post.voc_type_code, true, 'truth_observed', post.created_at" in sql + assert "select post.post_id, post.voc_type_code, true, 'truth_observed'" in sql + assert "least(post.created_at, clock_timestamp())" in sql assert "change_at timestamptz := clock_timestamp()" in sql - assert "case when tg_op = 'insert' then new.created_at else change_at end" in sql + assert "case when tg_op = 'insert' then least(new.created_at, change_at) else change_at end" in sql assert "after insert on source_post" in sql assert "after update of voc_type_code on source_post" in sql assert "when (old.voc_type_code is distinct from new.voc_type_code)" in sql