diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2f23dd6d0..9b7f53ed7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -86,12 +86,8 @@ flowchart LR | `commitment_extraction.py` | Pluggable LLM derivation of a customer commitment (promise + deadline) from a post; `Null` default, `ContextualOrchestrator` real impl | | `temporal_expressions.py` | Pure Korean relative-time resolver for Global Ask (ADR 0150) | | `ask_time_axis.py` | Event-time vs ingestion-time clock choice for that window (ADR 0202) | -| `ontology.py` | Loads the governed Turtle source tree (`lineageweave-kg.ttl` plus generated fragments), the formal OWL 2/RDFS/SKOS vocabulary for the Knowledge Graph's node/edge types, source taxonomies, and published O*NET linkages (ADR 0004, ADR 0252, ADR 0255, ADR 0256) | -| `backend/app/occupation_rating_ingestion.py` | Projects authenticated occupation-rating evidence plus persisted source and represented-occupation catalogs (ADR 0258, ADR 0260, ADR 0261) | -| `frontend/src/components/OccupationRatingProfile.tsx` | Selects an imported source, filters stored occupation titles without ranking, and reads exact Dashboard evidence while preserving absence, uncertainty, and warning semantics (ADR 0259–0262) | +| `ontology.py` | Loads `docs/ontology/lineageweave-kg.ttl`, the formal OWL 2/RDFS/SKOS vocabulary for the Knowledge Graph's node/edge types (ADR 0004) | | `ontology_neighborhood.py` | Bounded typed ontology/provenance neighborhood (ADR 0184); PostgreSQL stays authoritative, OWL subclass is not an instance edge | -| `occupational_construct_catalog.py` | Official O*NET 31.0 construct catalog sync (ADR 0250); no ratings or invented IRIs | -| `backend/app/occupational_construct_search.py` | Authorized catalog-label search over assertion-backed constructs (ADR 0257); hidden Posts never mint a hit | | `ontology_source_cursor.py` | Opaque HMAC source-window continuation (ADR 0124); keyset pagination, never OFFSET | | `period_report.py` | Fit GRM/GPCM on persisted IRT rows, FIPC-select, EAP-score a period (ADR 0003 slice 3; Bock & Mislevy, 1982) | | `fixtures.py` | Synthetic demo dataset -- no real data ships in this repo | @@ -379,6 +375,11 @@ governed atomic Voice with an explicit truth state and an ABAC-visible evidence Post. The server creates the normalized PROV-O derivation and assignment in one transaction; clients never submit an internal assertion id, and this route cannot replace the imported primary Voice. +Imported-primary changes are retained in `source_post_voice` as non-overlapping +half-open effective intervals (ADR 0252). The database closes the current row +and opens the new observed primary at one statement instant; live reads select +the open interval, cutoff reads select the containing interval, and an ontology +continuation without an explicit cutoff uses its frozen snapshot instant. The bounded ontology response carries a visible Voice assignment's evidence Post id alongside its exact-value row. The exact-value table therefore offers separate carrying-Post and derivation-evidence actions; hidden evidence removes @@ -683,8 +684,7 @@ vocabulary (`node_type`, `edge_type`, `entity_relationship_type`, `person_side`, `corporate_entity_level`) actually matches what the Ontology/Semantic-Layer claim implies. -`docs/ontology/lineageweave-kg.ttl` and its deterministic governed fragments -are a real OWL 2 / RDFS / SKOS +`docs/ontology/lineageweave-kg.ttl` is a real OWL 2 / RDFS / SKOS ontology in Turtle syntax: classes for `Post`/`Person`/`CorporateEntity` (with `OurSidePerson`/`CounterpartyPerson` subclasses), object properties for each `edge_type_code` and `entity_relationship_type` @@ -698,7 +698,7 @@ specification over it, in the same sense W3C's own stack uses "semantic layer" (RDFS/OWL as the governed conceptual layer over raw data), not a separate BI-metrics product and not a parallel triple store. -`lineageweave/ontology.py` parses the Turtle source tree once with `rdflib` +`lineageweave/ontology.py` parses the Turtle file once with `rdflib` (pure Python, no Rust toolchain, unlike `fast-mlsirm`) and exposes the vocabulary as importable IRI constants, so application code has one canonical name per class/property instead of re-typing lookup codes as @@ -715,19 +715,6 @@ enforcement mechanism: a future PR that adds a new `edge_type` or `entity_relationship_type` code without updating the ontology fails this test, not just a docstring's word. -### Authorized job architecture snapshots - -The public SOC/O*NET vocabulary and an employer's job architecture remain -different graphs. ADR 0263 adds an organization-scoped PostgreSQL source -boundary for private job-family/job-series snapshots: immutable source -metadata owns normalized nodes, source-declared broader/narrower edges, and -optional explicit bindings to a versioned external occupation scheme. An edge -table preserves multiple-family membership; the importer rejects cycles and -never derives a parent or binding from a label or code pattern. The snapshot -is source evidence only. It does not create a person, post, organizational -unit, competency, score, weight, or ontology assertion, and runtime rows never -enter repository artifacts. - ## Phase 6c: post content normalization before any LLM/embedding call The brief's latest revision calls out, explicitly, that a post body mixing diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e57455bb..7953f47b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ All notable changes to this project are documented here. Format follows ### Added - Normalized Voice-of-X composition persistence (ADR 0256): every imported +- Imported primary Voice changes now retain non-overlapping effective intervals + (ADR 0252), including recurring A → B → A values. Live, knowledge-cutoff, + and ontology-snapshot reads select the same period contract; PostgreSQL + rejects overlapping primary intervals instead of relying on application + ordering. +- Normalized Voice-of-X composition persistence (ADR 0251): every imported primary voice is mirrored into `source_post_voice`; each additional voice requires its own PROV-O assertion and truth status. Compound lookup codes, keyword inference, confidence thresholds, and invented weights remain out of diff --git a/backend/app/ontology_neighborhood_ingestion.py b/backend/app/ontology_neighborhood_ingestion.py index f64987ec4..f008ed3fd 100644 --- a/backend/app/ontology_neighborhood_ingestion.py +++ b/backend/app/ontology_neighborhood_ingestion.py @@ -866,6 +866,10 @@ def neighborhood_to_payload(neighborhood: OntologyNeighborhood) -> dict[str, Any "is_primary": assignment.is_primary, "truth_status_code": assignment.truth_status_code, "recorded_at": assignment.recorded_at.isoformat(), + "effective_from": assignment.effective_from.isoformat(), + "effective_to": assignment.effective_to.isoformat() + if assignment.effective_to + else None, "provenance_reference": assignment.provenance_reference, "evidence_post_id": assignment.evidence_post_id, } @@ -890,6 +894,7 @@ async def _load_voice_assignments( """ select voice.post_id, voice.voice_type_code, lookup.lookup_label, voice.is_primary, voice.truth_status_code, voice.recorded_at, + voice.effective_from, voice.effective_to, case when evidence.node_id = any($1::uuid[]) then evidence.node_id end as evidence_post_id from source_post_voice voice @@ -903,10 +908,11 @@ async def _load_voice_assignments( and evidence.node_type_code = 'node_post' where voice.post_id = any($1::uuid[]) and (voice.is_primary or evidence.node_id = any($1::uuid[])) - and (($2::timestamptz is null and voice.effective_to is null) - or ($2::timestamptz is not null - and voice.effective_from <= $2 - and (voice.effective_to is null or $2 < voice.effective_to))) + and voice.effective_from <= coalesce($2::timestamptz, $3::timestamptz) + and ( + voice.effective_to is null + or coalesce($2::timestamptz, $3::timestamptz) < voice.effective_to + ) and voice.recorded_at <= $3::timestamptz order by voice.post_id, voice.is_primary desc, lookup.display_order, voice.voice_type_code @@ -931,6 +937,8 @@ async def _load_voice_assignments( is_primary=row["is_primary"], truth_status_code=row["truth_status_code"], recorded_at=row["recorded_at"], + effective_from=row["effective_from"], + effective_to=row["effective_to"], provenance_reference=( "Evidence-backed additional voice" if not row["is_primary"] diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 4cb798a69..7500bfcc0 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -15,10 +15,11 @@ import asyncio import math import os -import subprocess import uuid +from concurrent.futures import ThreadPoolExecutor from contextlib import closing from pathlib import Path +from threading import Barrier from types import SimpleNamespace import asyncpg @@ -38,6 +39,11 @@ _VALKEY_URL = os.environ.get("LINEAGEWEAVE_TEST_VALKEY_URL", "redis://localhost:16379/0") _REALM = "lineageweave-demo" _MIGRATION_PATH = Path(__file__).resolve().parents[2] / "migrations" / "0001_initial_schema.sql" +_PROVENANCE_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0017_prov_o_standard_relations.sql" +) _REGISTRY_MIGRATION = Path(__file__).resolve().parents[2] / "migrations" / "0018_analysis_run_registry.sql" _RETENTION_MIGRATION = Path(__file__).resolve().parents[2] / "migrations" / "0020_analysis_run_retention_purge.sql" _RECONSTRUCTION_MIGRATION = ( @@ -88,6 +94,9 @@ _SOURCE_ORG_NAMED_HINTS_MIGRATION = ( Path(__file__).resolve().parents[2] / "migrations" / "0039_source_org_named_hints.sql" ) +_VOC_VOCABULARY_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0042_voc_type_vocabulary.sql" +) _MEMBER_LOCALE_MIGRATION = ( Path(__file__).resolve().parents[2] / "migrations" / "0044_member_locale_preference.sql" ) @@ -182,6 +191,11 @@ / "migrations" / "0206_report_leftover_map_reconstruction.sql" ) +_LEFTOVER_MAP_UNEXPLAINED_SHARE_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0233_report_leftover_map_unexplained_share.sql" +) _GLOBAL_ASK_JOB_MIGRATION = ( Path(__file__).resolve().parents[2] / "migrations" @@ -232,6 +246,36 @@ / "migrations" / "0183_source_post_event_occurred_at.sql" ) +_ONTOLOGY_TRUTH_STATUS_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0175_ontology_truth_status.sql" +) +_VOICE_TAXONOMY_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0235_voice_of_x_post_taxonomy.sql" +) +_VOICE_ASSIGNMENT_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0237_source_post_voice_combination.sql" +) +_VOICE_HISTORY_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0243_source_post_voice_history.sql" +) +_OCCUPATIONAL_CONSTRUCT_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0238_occupational_construct_assertion.sql" +) +_OCCUPATIONAL_CATALOG_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0239_occupational_construct_catalog.sql" +) def _postgres_available() -> bool: @@ -317,6 +361,7 @@ def seeded_db(demo_analyst_token): try: with conn.cursor() as cur: cur.execute(_MIGRATION_PATH.read_text()) + cur.execute(_PROVENANCE_MIGRATION.read_text()) cur.execute(_REGISTRY_MIGRATION.read_text()) cur.execute(_RETENTION_MIGRATION.read_text()) cur.execute(_RECONSTRUCTION_MIGRATION.read_text()) @@ -339,6 +384,7 @@ def seeded_db(demo_analyst_token): (Path(__file__).resolve().parents[2] / "migrations" / "0040_post_summary_contract.sql") .read_text() ) + cur.execute(_VOC_VOCABULARY_MIGRATION.read_text()) cur.execute(_MEMBER_LOCALE_MIGRATION.read_text()) cur.execute(_IMAGE_REGION_MIGRATION.read_text()) cur.execute(_POST_CONTENT_STRUCTURE_MIGRATION.read_text()) @@ -402,11 +448,18 @@ def seeded_db(demo_analyst_token): cur.execute(_GLOBAL_ASK_KNOWLEDGE_CUTOFF_MIGRATION.read_text()) cur.execute(_GLOBAL_ASK_PUBLIC_VERIFICATION_MIGRATION.read_text()) cur.execute(_EVENT_OCCURRED_AT_MIGRATION.read_text()) + cur.execute(_ONTOLOGY_TRUTH_STATUS_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_AXIS_MIGRATION.read_text()) cur.execute(_CHANNEL_EVIDENCE_MIGRATION.read_text()) + cur.execute(_OCCUPATIONAL_CONSTRUCT_MIGRATION.read_text()) + cur.execute(_OCCUPATIONAL_CATALOG_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_UNEXPLAINED_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_CROSS_SHARE_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_RECONSTRUCTION_MIGRATION.read_text()) + cur.execute(_VOICE_TAXONOMY_MIGRATION.read_text()) + cur.execute(_VOICE_ASSIGNMENT_MIGRATION.read_text()) + cur.execute(_VOICE_HISTORY_MIGRATION.read_text()) + cur.execute(_LEFTOVER_MAP_UNEXPLAINED_SHARE_MIGRATION.read_text()) cur.execute( "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values " "('corporate_entity_level', 'group', 'Group'), " @@ -414,7 +467,6 @@ def seeded_db(demo_analyst_token): "('corporate_entity_level', 'plant', 'Plant'), " "('post_visibility', 'public', 'Public'), " "('post_visibility', 'private', 'Private'), " - "('voc_type', 'voc', 'Voice of Customer'), " "('permission', 'post_read', 'Read posts'), " "('person_side', 'our_side', 'Our side'), " "('person_side', 'counterparty', 'Counterparty'), " @@ -1864,6 +1916,20 @@ def test_post_list_includes_public_and_own_corp_but_excludes_other_corp(client, assert public["voc_type_label"] == "Voice of Customer" assert public["visibility_label"] == "Public" assert {option["code"] for option in payload["voc_type_options"]} == {"voc"} + assert {option["code"] for option in payload["voice_type_catalog"]} == { + "voc", + "voe", + "vob", + "voi", + "vop", + "vops", + "vos", + "vocc", + "voco", + "vom", + "vor", + "voso", + } assert {option["code"] for option in payload["visibility_options"]} == {"public", "private"} assert next(option for option in payload["visibility_options"] if option["code"] == "public")["label"] == "Public" @@ -1960,6 +2026,100 @@ def test_post_detail_exposes_explicit_and_semantic_project_evidence( assert listed_post["project_evidence"][0]["provenance"] == "post_project_mention.evidence_text" +def test_post_detail_exposes_evidence_bound_occupational_construct( + client, demo_analyst_token, seeded_db +) -> None: + """The authorized detail projection preserves its exact synthetic evidence.""" + conn = psycopg2.connect(seeded_db["dsn"]) + try: + with conn.cursor() as cur: + cur.execute( + """ + insert into post_content_unit + (post_id, unit_index, unit_kind_code, unit_text) + values (%s, 90, 'plain_text', %s) + returning post_content_unit_id + """, + ( + seeded_db["public_post_id"], + "Synthetic record requires oral comprehension.", + ), + ) + unit_id = cur.fetchone()[0] + cur.execute( + """ + insert into occupational_construct_vocabulary + (vocabulary_iri, version_label, license_iri, attribution_text) + values (%s, '31.0', %s, 'Synthetic O*NET attribution') + returning vocabulary_id + """, + ( + "https://www.onetcenter.org/database.html", + "https://creativecommons.org/licenses/by/4.0/", + ), + ) + vocabulary_id = cur.fetchone()[0] + cur.execute( + """ + insert into occupational_construct + (vocabulary_id, construct_iri, construct_family_code, preferred_label) + values (%s, %s, 'cognitive_ability', 'Oral Comprehension') + returning construct_id + """, + (vocabulary_id, "https://data.onetcenter.org/element/1.A.1.a.1"), + ) + construct_id = cur.fetchone()[0] + cur.execute( + """ + insert into post_occupational_construct_assertion + (post_id, post_content_unit_id, construct_id, evidence_text, + truth_status_code, extraction_method, orchestrator_session_id) + values (%s, %s, %s, 'oral comprehension', 'truth_inferred', + 'contextual_orchestrator_structured', 'synthetic-session') + """, + (seeded_db["public_post_id"], unit_id, construct_id), + ) + conn.commit() + finally: + conn.close() + + response = client.get( + f"/api/posts/{seeded_db['public_post_id']}", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200 + assertions = response.json()["occupational_construct_assertions"] + assert assertions[0]["preferred_label"] == "Oral Comprehension" + assert assertions[0]["evidence_text"] == "oral comprehension" + assert assertions[0]["provenance"] == ( + "post_occupational_construct_assertion.evidence_text" + ) + + conn = psycopg2.connect(seeded_db["dsn"]) + try: + with conn.cursor() as cur: + cur.execute( + """ + update post_content_ingestion_job + set source_body_sha256 = %s, + status_code = 'post_content_ingestion_failed' + where post_id = %s + """, + ("f" * 64, seeded_db["public_post_id"]), + ) + conn.commit() + finally: + conn.close() + + stale = client.get( + f"/api/posts/{seeded_db['public_post_id']}", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert stale.status_code == 200 + assert stale.json()["occupational_construct_evidence_status"] == "unavailable" + assert stale.json()["occupational_construct_assertions"] == [] + + def test_post_detail_as_of_returns_the_cutoff_known_body( client, demo_analyst_token, seeded_db ) -> None: @@ -4658,6 +4818,165 @@ def _grant_post_admin(dsn: str) -> None: admin_conn.close() +def test_create_voice_assignment_persists_authorized_prov_o_evidence( + client, demo_analyst_token, seeded_db +) -> None: + """A real OIDC admin write retains its governed Voice and evidence Post.""" + headers = {"Authorization": f"Bearer {demo_analyst_token}"} + endpoint = f"/api/posts/{seeded_db['own_private_post_id']}/voice-assignments" + payload = { + "voice_type_code": "vos", + "truth_status_code": "truth_observed", + "evidence_post_id": seeded_db["public_post_id"], + } + + assert client.post(endpoint, json=payload, headers=headers).status_code == 403 + _grant_post_admin(seeded_db["dsn"]) + with closing(psycopg2.connect(seeded_db["dsn"])) as conn, conn.cursor() as cur: + cur.execute( + "insert into provenance_resource (resource_iri, resource_label) " + "values (%s, 'Synthetic evidence Post') returning resource_id", + (f"https://example.test/posts/{seeded_db['public_post_id']}",), + ) + resource_id = cur.fetchone()[0] + cur.execute( + "insert into provenance_resource_type (resource_id, class_code) " + "values (%s, 'prov_entity')", + (resource_id,), + ) + cur.execute( + "insert into provenance_resource_binding " + "(resource_id, node_type_code, node_id) values (%s, 'node_post', %s)", + (resource_id, seeded_db["public_post_id"]), + ) + conn.commit() + + response = client.post(endpoint, json=payload, headers=headers) + assert response.status_code == 201, response.text + assert response.json() == { + "code": "vos", + "label": "Voice of Supplier", + "is_primary": False, + "truth_status_code": "truth_observed", + "evidence_available": True, + } + + with closing(psycopg2.connect(seeded_db["dsn"])) as conn, conn.cursor() as cur: + cur.execute( + """ + select voice.voice_type_code, voice.is_primary, + assertion.relation_code, binding.node_id + from source_post_voice voice + join provenance_assertion assertion + on assertion.assertion_id = voice.provenance_assertion_id + join provenance_resource_binding binding + on binding.resource_id = assertion.object_resource_id + where voice.post_id = %s and voice.voice_type_code = 'vos' + """, + (seeded_db["own_private_post_id"],), + ) + stored = cur.fetchone() + assert stored == ( + "vos", + False, + "prov_was_derived_from", + seeded_db["public_post_id"], + ) + cur.execute( + "select voice_type_code from source_post_voice " + "where post_id = %s and is_primary", + (seeded_db["own_private_post_id"],), + ) + assert cur.fetchone() == ("voc",) + + +def test_primary_voice_history_survives_concurrent_changes_and_cutoff_reads( + client, demo_analyst_token, seeded_db +) -> None: + """Concurrent A→B→C→A changes retain one non-overlapping primary timeline.""" + post_id = seeded_db["own_private_post_id"] + barrier = Barrier(2) + + with closing(psycopg2.connect(seeded_db["dsn"])) as conn, conn.cursor() as cur: + cur.execute(_VOICE_ASSIGNMENT_MIGRATION.read_text()) + cur.execute(_VOICE_ASSIGNMENT_MIGRATION.read_text()) + + def update_voice(code: str) -> None: + with closing(psycopg2.connect(seeded_db["dsn"])) as conn, conn.cursor() as cur: + barrier.wait() + cur.execute( + "update source_post set voc_type_code = %s where post_id = %s", + (code, post_id), + ) + conn.commit() + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(update_voice, code) for code in ("vop", "voe")] + for future in futures: + future.result() + + with closing(psycopg2.connect(seeded_db["dsn"])) as conn, conn.cursor() as cur: + cur.execute( + "update source_post set voc_type_code = 'voc' where post_id = %s", + (post_id,), + ) + conn.commit() + cur.execute( + "select voice_type_code, effective_from, effective_to " + "from source_post_voice where post_id = %s and is_primary " + "order by effective_from, voice_assignment_id", + (post_id,), + ) + history = cur.fetchall() + + assert len(history) == 4 + assert history[0][0] == "voc" + assert history[-1][0] == "voc" + assert sum(effective_to is None for _, _, effective_to in history) == 1 + assert all( + history[index][2] == history[index + 1][1] + for index in range(len(history) - 1) + ) + for voice_type_code, effective_from, _effective_to in history: + response = client.get( + f"/api/posts/{post_id}", + params={"as_of": effective_from.isoformat()}, + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200, response.text + primaries = [ + voice for voice in response.json()["voice_types"] if voice["is_primary"] + ] + assert [voice["code"] for voice in primaries] == [voice_type_code] + + +def test_voice_migration_family_replays_after_a_primary_change(seeded_db) -> None: + """Every-start replay remains valid after a post accumulates Voice history.""" + post_id = seeded_db["own_private_post_id"] + with closing(psycopg2.connect(seeded_db["dsn"])) as conn, conn.cursor() as cur: + cur.execute( + "update source_post set voc_type_code = 'vob' where post_id = %s", + (post_id,), + ) + conn.commit() + + for _ in range(2): + cur.execute(_VOICE_ASSIGNMENT_MIGRATION.read_text()) + cur.execute(_VOICE_HISTORY_MIGRATION.read_text()) + conn.commit() + + cur.execute( + "select count(*) filter (where effective_to is null), " + "count(*) from source_post_voice " + "where post_id = %s and is_primary", + (post_id,), + ) + current_count, history_count = cur.fetchone() + + assert current_count == 1 + assert history_count >= 2 + + def test_tickets_list_is_empty_before_any_created(client, demo_analyst_token, seeded_db) -> None: response = client.get( f"/api/posts/{seeded_db['own_private_post_id']}/tickets", @@ -5715,10 +6034,15 @@ def test_seed_period_report_surfaces_on_get_reports(client, demo_analyst_token, if share is not None: assert not math.isnan(share) assert not math.isinf(share) + unexplained_share = pair.get("leftover_map_unexplained_share") + assert unexplained_share is None or isinstance(unexplained_share, (int, float)) + if unexplained_share is not None: + assert not math.isnan(unexplained_share) + assert not math.isinf(unexplained_share) + assert unexplained_share >= 0.0 if unexplained is not None and reconstruction is not None: assert unexplained + reconstruction == pytest.approx(pair["leftover_residual"]) assert "leftover_map_explained_share" not in pair - assert "leftover_map_unexplained_share" not in pair leftover_axes = high_report.get("leftover_map_axes", []) assert [axis["axis_index"] for axis in leftover_axes] == [1, 2] assert all(axis["leftover_singular_value"] >= 0 for axis in leftover_axes) @@ -5781,6 +6105,11 @@ def test_seed_period_report_surfaces_on_get_reports(client, demo_analyst_token, or isinstance(pair["leftover_map_reconstruction"], (int, float)) for pair in leftover_thread.get("leftover_pairs", []) ) + assert all( + "leftover_map_unexplained_share" not in pair + and "leftover_map_explained_share" not in pair + for pair in leftover_thread.get("leftover_pairs", []) + ) def test_seed_period_report_includes_fixture_event_lineage_posts( diff --git a/docs/adr/0245-io-occupational-taxonomy-in-the-published-ontology.md b/docs/adr/0245-io-occupational-taxonomy-in-the-published-ontology.md index a8476f33d..e556ba8ae 100644 --- a/docs/adr/0245-io-occupational-taxonomy-in-the-published-ontology.md +++ b/docs/adr/0245-io-occupational-taxonomy-in-the-published-ontology.md @@ -3,10 +3,7 @@ **Status:** Accepted **Date:** 2026-08-26 **Extends:** [ADR 0004](0004-knowledge-graph-ontology.md), [ADR 0145](0145-psychometric-channel-weight-estimation.md), [ADR 0207](0207-repository-case-ontology-namespace-canonical.md), [ADR 0232](0232-worker-function-taxonomy-in-the-published-ontology.md) -<<<<<<< HEAD -======= **Superseded in part by:** [ADR 0252](0252-complete-2018-soc-hierarchy.md), which expands the major-group-only scheme into the complete 2018 SOC hierarchy. ->>>>>>> origin/feat/onet-rating-occupation-filter ## Context diff --git a/docs/adr/0252-temporal-primary-voice-history.md b/docs/adr/0252-temporal-primary-voice-history.md new file mode 100644 index 000000000..2cf3f5685 --- /dev/null +++ b/docs/adr/0252-temporal-primary-voice-history.md @@ -0,0 +1,97 @@ +# ADR 0252: Temporal history for imported primary Voice + +## Status + +Accepted (2026-08-27). Extends ADR 0251 and closes issue #748. + +## Context + +ADR 0251 records when a Voice assignment starts, but migration 0237 deletes +the former imported primary when `source_post.voc_type_code` changes. The live +value is honest, yet an authorized knowledge-cutoff read after that update can +no longer recover the primary that was effective at the cutoff. The existing +`(post_id, voice_type_code)` key also cannot represent A → B → A. + +OWL-Time distinguishes instants from intervals and gives an interval explicit +beginning and end bounds. PostgreSQL range types and exclusion constraints are +the native database mechanism for rejecting overlapping periods. Neither +source supplies a missing business-effective instant, so LineageWeave must not +invent one: an imported change becomes effective at the database transaction +instant when no source change instant exists. + +## Decision + +- Keep `source_post_voice` as the normalized assignment relation. Add nullable + `effective_to`; each row is a half-open interval + `[effective_from, effective_to)`. Null means current. +- Change the key to `(post_id, voice_type_code, effective_from)`, allowing the + same atomic Voice to recur in non-overlapping periods. +- Use PostgreSQL GiST exclusion constraints to reject overlapping primary + intervals for one Post. A partial unique index also permits at most one + current row for a `(post_id, voice_type_code)` pair. +- When the imported primary changes, one trigger transaction closes both the + current primary and any current additional assignment for the incoming + Voice, then inserts the new observed primary at one trigger-execution + timestamp. PostgreSQL `clock_timestamp()` is read after the source-row lock + is acquired, so a waiting concurrent update cannot backdate its interval to + the earlier statement start. It never overwrites or fabricates the former + interval. +- Live reads select `effective_to is null`. Cutoff reads select the row whose + interval contains the cutoff. Ontology continuation reads use their frozen + `snapshot_at` when no knowledge cutoff was requested, so a page minted + before a change cannot silently switch to the new primary. +- Existing rows migrate as open intervals. Migration replay changes neither + their starts nor their history. History before ADR 0252 remains unavailable + because the deleted facts cannot be reconstructed honestly. +- This is valid-time history for a source assignment, not psychometric or + mathematical modeling. No weight, confidence, inference, or new Voice code + is introduced. + +## Data model + +```mermaid +classDiagram + class SourcePost { + uuid post_id + text voc_type_code + } + class SourcePostVoice { + uuid post_id + text voice_type_code + boolean is_primary + timestamptz effective_from + timestamptz effective_to + timestamptz recorded_at + } + SourcePost "1" --> "1..*" SourcePostVoice +``` + +```mermaid +sequenceDiagram + participant Import + participant SourcePost + participant VoiceHistory + Import->>SourcePost: update primary A to B + SourcePost->>VoiceHistory: close current A after source-row lock + SourcePost->>VoiceHistory: close current additional B, if present + SourcePost->>VoiceHistory: insert observed primary B at same instant + VoiceHistory-->>Import: one non-overlapping current primary +``` + +## Consequences + +- A → B → A is auditable without copying source content or exposing real + identifiers. +- Half-open bounds assign the exact change instant to the new primary and avoid + double matches. +- The exclusion constraint adds a GiST index and write-time check. This table + is bounded by Voice assignments per Post; partitioning is not warranted + until observed volume or lock evidence shows otherwise. + +## References + +Cox, S. J. D., & Little, C. (2022). *Time ontology in OWL*. World Wide +Web Consortium. https://www.w3.org/TR/owl-time/ + +PostgreSQL Global Development Group. (2025). *PostgreSQL 18 documentation: +Range types*. https://www.postgresql.org/docs/18/rangetypes.html diff --git a/docs/adr/README.md b/docs/adr/README.md index f10df7664..d90943d19 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -9,9 +9,11 @@ decision from them. | Supporting document | Normative ADR | |---|---| -| [`product-requirements.md`](../product-requirements.md) | Product requirements projection across the ADR set; ADRs remain normative | +| [`product-requirements.md`](../product-requirements.md) | Product requirements projection across the ADR set; ADRs remain normative, including [0252](0252-temporal-primary-voice-history.md) | | [`product-technical-gap-baseline.md`](../product-technical-gap-baseline.md) | Product/technical traceability projection across the ADR set; ADRs remain normative | | [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0024](0024-rankweave-fusion-fail-closed.md), [0165](0165-quantity-script-display.md), [0167](0167-rankweave-ranking-channel-evidence.md), [0169](0169-ask-batched-lineage-graph.md), [0172](0172-event-lineage-channel-evidence.md), [0202](0202-ask-event-time-filter.md), [0223](0223-explicit-semantic-content-unit-kinds.md), [0238](0238-source-conversation-turn-import-contract.md) | +| [`voice-combination-technical-requirements.md`](../voice-combination-technical-requirements.md) | [0246](0246-expanded-voice-of-x-post-taxonomy.md), [0251](0251-evidence-bearing-voice-combinations.md), [0252](0252-temporal-primary-voice-history.md) | +| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0024](0024-rankweave-fusion-fail-closed.md), [0165](0165-quantity-script-display.md), [0167](0167-rankweave-ranking-channel-evidence.md), [0169](0169-ask-batched-lineage-graph.md), [0172](0172-event-lineage-channel-evidence.md), [0202](0202-ask-event-time-filter.md), [0223](0223-explicit-semantic-content-unit-kinds.md) | | [`PROV_O_IMPLEMENTATION.md`](../PROV_O_IMPLEMENTATION.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`PROV_O_IMPLEMENTATION_MATRIX.md`](../PROV_O_IMPLEMENTATION_MATRIX.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`ONTOLOGY_NAMESPACE_INVENTORY.md`](../doctoring/ONTOLOGY_NAMESPACE_INVENTORY.md) | [0207](0207-repository-case-ontology-namespace-canonical.md), [0157](0157-public-ontology-namespace-identity.md) | diff --git a/docs/product-requirements.md b/docs/product-requirements.md index a8b741521..763a116b9 100644 --- a/docs/product-requirements.md +++ b/docs/product-requirements.md @@ -53,6 +53,10 @@ edge exposes the same authorized endpoints and evidence through API and UI. governed atomic Voice only with an ABAC-visible evidence Post and explicit truth state; create the normalized PROV-O derivation server-side and never accept an opaque provenance assertion identifier from the caller. +- Preserve recurring imported-primary history as non-overlapping half-open + intervals; live reads return the current primary, while authorized cutoff + reads return the primary effective at that instant without substituting the + current value. - Let a `post_admin` connect another perspective from the live Post popup by choosing an unassigned atomic Voice and an explicit truth state; use the open authorized Post as evidence and hide the write action on cutoff views. @@ -63,6 +67,8 @@ edge exposes the same authorized endpoints and evidence through API and UI. Acceptance: Turtle, JSON-LD, N-Triples, SHACL, API payloads, persisted IRIs, and rendered labels agree on term kind, direction, namespace, and provenance; an additional Voice cannot demote the imported primary or cite hidden evidence; +an A → B → A imported-primary sequence returns A, B, and A at its three +respective authorized cutoff intervals; the exact-value table opens the carrying Post and its authorized derivation evidence as distinct actions; the authoring form has explicit selections, permission/cutoff gating, retryable @@ -108,10 +114,8 @@ and a digest-bound run record distinguishes a supported empty result from an unavailable provider. ADR 0254 adds the authorized Post-detail evidence review surface and honest complete, processing, and unavailable states. ADR 0255 projects assertion-backed constructs into the existing ABAC-filtered ontology -neighborhood without duplicating graph storage or promoting truth. ADR 0257 -adds authorized catalog-label search: reviewers type an official O*NET label -and open the earliest visible supporting Post. Constructs without visible -evidence stay undisclosed. Occupation ratings remain unavailable. +neighborhood without duplicating graph storage or promoting truth. Catalog +search remains unavailable. ### PRD-FR-2C — FJA I/O-Psychology cognitive, affective & behavioral semantic layer @@ -165,180 +169,6 @@ canonical namespace, and lookup round-trip isolation are enforced by `tests/test_io_taxonomy.py`; `tests/test_ontology.py` continues to pass unchanged. -### PRD-FR-2A — Worker-function taxonomy - -- Publish the DOT/FJA Data/People/Things worker functions (24 concepts, - official definitions verbatim) in the canonical ontology namespace - (ADR 0232), each with its definitional ordinal rank. Do not infer a - DOT-to-O*NET or Fleishman crosswalk that the authorities do not publish. -- Expose the taxonomy through a deterministic application read model with - fail-closed lookups; an absent function is an honest unknown. -- Carry no numeric weight from the taxonomy: ranks are scale positions, - never calibrated weights. - -Acceptance: completeness, full verbatim definitions, deterministic ordering, -and lookup round-trip isolation are enforced by -`tests/test_worker_function_taxonomy.py`; `tests/test_ontology.py` -continues to pass unchanged. - -### PRD-FR-2B — Occupational classification and worker-characteristic taxonomy - -- Publish all four levels of the 2018 Standard Occupational Classification: - 23 major groups, 98 minor groups, 459 broad occupations, and 867 detailed - occupations with exact source parents, titles, and codes (ADR 0252), plus - the four O*NET 31.0 job-zone categories with - published names and source values 2 through 5 (ADR 0245). -- Publish the worker-characteristic families that work-related - cognition, affect, and behavior resolve into: Fleishman's four ability - domains, Holland's six RIASEC interest types with the published - hexagonal adjacency relation, the six explicitly legacy O*NET work-value - clusters, and - the seven higher-order dimensions of the revised O*NET Work Styles - structure. -- Publish all 3,006 O*NET 31.0 Content Model Reference elements with exact - identifiers, names, descriptions, and source-defined outline parents - (ADR 0264). Treat the six roots and 18 second-level branches as navigation - classes, never occupation ratings, person traits, scores, or weights. -- Declare typed derivation properties from classifications to - characteristics but assert no instance binding; binding requires a - versioned released source profile imported with provenance in its own - decision. -- Expose everything through a deterministic application read model with - fail-closed lookups; carry no numeric importance or level rating from - any occupational profile. - -Acceptance: completeness counts, verbatim titles, closed RIASEC -vocabulary, exact published adjacency pairs, deterministic ordering, -canonical namespace, and lookup round-trip isolation are enforced by -`tests/test_io_taxonomy.py`, `tests/test_soc_2018_hierarchy.py`, and -`tests/test_onet_content_model.py`; -`tests/test_ontology.py` continues to pass unchanged. -### PRD-FR-2C — Evidence-bound occupational constructs - -- Keep cognitive abilities, work styles, work activities, affective - reactions, and performance behaviors as non-equivalent construct classes - (ADR 0248). FJA worker functions remain separate. -- Reuse official external identifiers and source-published relationships; - never infer a DPT-to-psychology crosswalk or relabel work style as affect. -- Publish the eight O*NET 31.0 Ability, Essential Skill, Transferable Skill, - and Work Style link tables to Work Activities and Work Context as 1,417 - directed, assertion-level provenance-bearing relations (ADR 0256). Treat - relevance as neither a causal effect nor a numeric weight. -- Bind a construct to record content only through a provenance-bearing, - evidence-cited assertion. Do not promote record evidence to a person trait, - score, causal effect, or job requirement. - -Acceptance: SHACL rejects incomplete record assertions; ontology tests -prohibit FJA equivalence, require exact Post/evidence/PROV statement structure, -and reproduce every pinned O*NET linkage with its exact source table. Runtime -persistence and UI remain unavailable until their separate ADR acceptance. - -### PRD-FR-2D — Occupation-rating source observations - -- Persist released occupation-to-element ratings as source observations, not - ontology weights: release, source table, occupation, element, scale, - optional category, value, sample/error/interval, suppression, relevance, - exact source update month, and domain source remain independently auditable - (ADR 0257); the product must not invent a day for O*NET's `MM/YYYY` field. -- Keep normalized reference identities in third normal form and partition the - observation store by exact release then source table. An unknown partition - fails closed instead of entering a catch-all table. -- Preserve decimals and missingness exactly. No local aggregation, - normalization, person inference, or psychometric estimation is permitted. -- Reject divergent duplicate identities and owner-level truncation. Task - Ratings remain unavailable until their integer Task IDs and statements have - a separate normalized source-target contract. - -Acceptance: the replay-safe migration creates the normalized store; the pinned -CSV importer validates both rating and scale-reference digests and row counts, -reference identity, source scale, uncertainty, flags, and dates before -persistence; PostgreSQL integration proves missing partitions fail closed and -repeated null-category UPSERT is idempotent. -API, UI, and derived modeling remain unavailable until separate accepted -delivery records. - -### PRD-FR-2E — Occupation-rating evidence read - -- Let an authenticated user open one exact release/source/occupation profile - with both rating and scale artifact provenance (ADR 0258). -- Distinguish an unavailable imported source from an available source with no - observation for the occupation. -- Preserve exact decimal text, uncertainty, suppression, relevance, source - month, domain source, and declared bounds; derive no ranking or recommendation. - -Acceptance: invalid identifiers and unbounded pages are rejected; an unavailable -source never appears as a negative profile; pagination is deterministic; and a -suppressed observation retains its value and warning flag together. - -### PRD-FR-2F — Occupation-rating evidence view - -- Let an authenticated user submit an exact O*NET-SOC code, release, and source - from the existing Dashboard without changing the governed GNB (ADR 0259). -- Display published values beside bounds, sample/error/interval evidence, - source time, and text warnings; link both source artifacts. -- Give different next actions for unavailable source, empty occupation, - transport failure, and additional pages. - -Acceptance: keyboard users can operate the form and named horizontally -scrollable table; narrow layouts retain complete values; suppression remains -visible beside its value; and Storybook covers populated, narrow, unavailable, -and empty states using synthetic data. - -### PRD-FR-2G — Imported rating-source catalog - -- Populate the occupation evidence selector only from imported artifacts that - contain observations, preserving release and artifact provenance (ADR 0260). -- Exclude the scale-definition support artifact from the rating-source selector. -- Disable profile submission and state the next action while the catalog is - loading, empty, or unavailable. - -Acceptance: a user never types an internal release/source code; the selector -order follows persisted import time rather than parsed version heuristics; and -the real PostgreSQL integration test proves an imported synthetic artifact is -listed while its supporting scale artifact is not. - -### PRD-FR-2H — Occupations represented in a rating source - -- Populate the occupation selector with exact stored code/title pairs that - have observations in the selected imported source (ADR 0261). -- Clear the current occupation and profile when the source changes, and clear - the profile when the occupation changes; never mix continuation rows across - occupations or sources. -- Keep unavailable source, available-empty source, loading, and transport - failure distinct and actionable. - -Acceptance: a user selects a stored title rather than typing an internal code; -the PostgreSQL integration test proves the source membership predicate; and -component tests prove selector changes clear prior evidence and pagination -stays bound to the loaded profile identifiers. - -### PRD-FR-2I — Occupation catalog title filter - -- Let an authenticated user filter the imported occupation catalog by - published title or retained code without ranking or typed-code fallback - (ADR 0262). -- Reset the filter when the source changes. -- Disable profile submission and state the next action when the filter - matches no catalog occupation. - -Acceptance: submitting still sends only a catalog identity; a non-matching -filter never creates a request; and Storybook covers a no-match state. - -### PRD-FR-2J — Authorized job-family and job-series snapshots - -- Import one authorized, pinned organization-specific source snapshot without - committing runtime rows or creating an organization (ADR 0263). -- Keep job families, job series, standard occupations, organizational units, - positions, people, and psychological constructs as distinct identities. -- Preserve source-declared multiple-family membership and validity dates; infer - no parent or occupation binding from a code, label, similarity, or model. -- Persist a standard-occupation binding only when scheme IRI, version, code, - and source relation are all explicitly supplied. - -Acceptance: synthetic tests reproduce a series with two source-declared family -parents, reject cycles and partial bindings, leave an occupation-looking label -unbound, and prove the normalized snapshot store is immutable. - ### PRD-FR-3 — Bounded ontology exploration - Apply RBAC/ABAC, source eligibility, and knowledge cutoff before graph diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index dccd12f40..4f6a34faf 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,67 @@ # Product & Technical Gap Baseline +> Exact-head overlay: 2026-08-28 00:21 KST. Protected `main` was +> `96109cf06f6611eb0e09c43d33416f8f8cf4242f`; 10 PRs and 10 issues were +> open. This overlay supersedes every older queue count and head snapshot in +> this document. Historical sections remain provenance for earlier decisions, +> not current release evidence. + +## Current protected-delivery and Voice-of-X decision + +ADRs 0246 and 0251 remain distinct authorities: ADR 0246 governs the twelve +atomic, extensible Voice-of-X concepts and ADR 0251 governs the FJA +I/O-psychology semantic layer. ADR 0256 governs evidence-bearing Voice +composition. No fixed compound-code catalog, B2B2C-only cross-product, +confidence weight, or keyword inference is authorized. The accepted literature +and standards cited by those ADRs support an extensible stakeholder vocabulary; +they do not prove that any finite set of combinations is exhaustive. + +Protected `main` implements the twelve atomic concepts, normalized additional +Voice assignments, server-created PROV-O derivation, truth status, authorization +filtering, separate carrying-Post and derivation-evidence actions, and paged +JSON-LD union behavior. Those are implementation facts, not evidence that a +particular runtime contains accepted assignments. Current non-identifying +repository aggregates establish only the open-work inventory above; no +production row, title, organization, or credential was inspected for this +overlay. + +The largest remaining user-visible Voice gap is temporal imported-primary +history. PR #752 exact head `5c2dfed5a32db655264fa62fa46fda626d753954` +adds ADR 0252 and migration 0243 so recurring A to B to A changes retain +non-overlapping half-open intervals. The review repair removes duplicate SQL +clauses and makes ontology continuation evaluate the interval containing the +frozen page snapshot rather than requiring the current open interval. Focused +local Voice/ontology regression evidence passed 93 tests after merging current +`main`; the full frontend suite passed 453 tests. The combined-Voice Storybook +scene was rendered and visually inspected at 1440 by 1000 and 390 by 844 CSS +pixels: carrying-Post and derivation-evidence controls remain distinct, and the +mobile exact-value table preserves its horizontal-scroll access. Hosted +exact-head checks were still pending. Authenticated PostgreSQL API and +authenticated product-UI evidence have not been recollected for this head +because the running canonical Compose project was assembled from multiple +other active worktrees; it was not rebuilt or migrated over their work. Runtime +acceptance therefore remains **unavailable**, not complete. + +| PR | Exact head | Current gate at overlay time | +|---:|---|---| +| #629 | `fcb933ba30a5fe866f01a5a7940c7304cce347f5` | Mergeable with auto-merge retained; exact-head checks/reviews pending and `CHANGES_REQUESTED` not cleared | +| #640 | `bd73e0a43ae139d0091c25696e89786c62a34e74` | Conflicting with `main`; auto-merge retained | +| #643 | `bae04cff19f2e1d54237b7d7168708b9c5df9a3b` | Draft and conflicting; no auto-merge | +| #644 | `f53dd28e50f984ceebd9520f800f84dbea05b7e6` | Conflicting; review required; auto-merge retained | +| #667 | `0c0f4af572a94e63cc8ea4545e48f5eda32a389c` | Conflicting with one prior failed check; review required; auto-merge retained | +| #668 | `234f975ba5b0982e22d918219b505a9cd6a103e2` | Conflicting; review required; auto-merge retained | +| #672 | `a3e87a89185fae03c5f18c79e2d97d12c73e8af9` | Conflicting; review required; auto-merge retained | +| #679 | `135dfe7c4266c7a2098c622b7c9976eaf7304cdd` | Conflicting; review required; auto-merge retained | +| #702 | `ebee9520a5b48c3074135153954c96e067309863` | Conflicting; no auto-merge | +| #752 | `5c2dfed5a32db655264fa62fa46fda626d753954` | Mergeable; exact-head checks and independent review pending; no auto-merge yet | + +Organization ruleset 18156473 currently requires one approval, dismissal of +stale approvals after a push, approval for unattributed changes, resolved +threads, and seven workflows pinned to `.github@main`. Repository ruleset +21065108 and the organization ruleset both prohibit non-fast-forward updates. +No self-approval, admin bypass, force push, stale check transfer, or pre-parent +stack evidence satisfies those gates. + > Voice-of-X delivery snapshot: 2026-08-27 KST. Protected `main` was > `ff7431bd1851c03e737808d22c6a2d43968582f9`; PR #713 was > `850494c3861703862a76cfe564381a41243c6c2d`; stacked PR #717 was @@ -620,6 +682,7 @@ this file per §3.5 of the prior snapshot). | Semantic source rendering | Paragraph, table, list, formula, and indentation work exists across stacks (#394, #427, #448–#450); #515 adds synthetic backend/frontend parity for deterministic rows/cells, footnote boundaries, and encoded scripts | Land the #427 → #515 stack, then gather authenticated browser evidence that list nesting, continuation alignment, and formula units render without authoring-layout artifacts | | Event and project semantics | #663 is the largest current user-visible gap slice: evidence-backed Project nodes, bounded traversal, cutoff/snapshot fencing, exact-value table parity, and localized graph labels. Focus visibility, label-bound, and temporal test-double regressions are repaired. #666's heuristic removal is composed into this parent but is not separately protected-main evidence. #640 separately adds project journeys without claiming authoritative lifecycle status | Combined #663 must pass exact-head checks and independent approval before protected merge. Aggregate authenticated evidence must still prove distinct projects/events and handover intervals without promoting co-occurrence | | Voice primary history | The #717 candidate updates ADR 0256 and migration 0237 with immutable assignment ids and half-open intervals, closing rather than deleting a replaced primary so A → B → A is representable; this is not protected-main evidence | Prove migration replay, concurrent primary changes, non-overlap, and API/ontology cutoff reads against synthetic PostgreSQL at the current exact head, then close #748 only after protected delivery | +| Voice primary history | ADR 0252 and candidate migration 0243 retain half-open imported-primary intervals, allow recurring A → B → A values, and align live, cutoff, and ontology-snapshot reads. Static/unit contracts pass; this branch is not protected-main or authenticated runtime evidence | Prove migration replay and atomic A → B → A cutoff reads against the isolated PostgreSQL/OIDC stack, then pass exact-head hosted checks and protected review before closing #748 | | Knowledge Graph readability | #659 recreates the token-backed node-type repair on current `main`, including regression coverage; it is open and therefore not protected-main evidence | Merge #659 normally, then verify light/dark contrast, keyboard graph navigation, full labels, and evidence tables in the authenticated rendered surface | | Source-code lookup UX | Source state/detail codes remain evidence-bearing machine values and current detail presentation is dense | Catalog-backed display labels with raw-code provenance, compact 5W1H/source-detail hierarchy, keyboard access, and no unsupported customer/project binding | | Calendar / Naruon | #355 delivered the projection contract; v2.17.0 wires operator consumption without forwarding the end-user token. Naruon producer, provider/consumer fixtures, and protected merge remain open (#336) | Verify observed events against the published schema without invented events; keep commitments available when the channel is unwired | @@ -781,23 +844,24 @@ post-merge reruns (not transferable evidence for later heads): | ---: | --- | --- | | #750 | Leftover-map unexplained leftover share persisted (`report_leftover_map_unexplained_share`, share `s = U² / R²`) | ADR 0233 | | #749 | Authorized job-family/job-series import snapshots (`0223_authorized_job_architecture`) | ADR 0263 | -| #759 | ***Promoted** the ONET rating-store stack to `main`: migrations 0222/0223, authenticated rating/rating-sources/rating-occupations endpoints, `OccupationRatingProfile` UI + stories, rating client functions, import scripts, ADR 0252–0263 references. Semgrep SQLi nullified by PL/pgSQL `format(%I/%L)` DDL + documented `nosemgrep`; 1583 Python + 447 frontend tests green | ADR 0257–0263 | | #747 | Current product and MCP manuals (`docs/manuals/*`, contract tests) | ADR 0118-family | | #754 | Customer-actionable copy and ADR 0237 accelerator runtime boundary; share/bookmark/verification call sites reworded and ko/zh/ja/vi translations completed after review | ADR 0237 | -| #700 | Source conversation-turn evidence ingestion (`0233_source_conversation_turn_evidence`, choke/adjacency resilience) | ADR 0238 | -| #658 | Optional Global Ask knowledge cutoff honoring `source_post_revision` cover | ADR 0216 | -| #632 | Graph-fact source provenance preserved through MCP streaming + verified psql-parity migration fixture | ADR 0166 | | #742 | Evidence-bound product-operations relations (stack base) | ADR 0235 | | #743 | Imported occupation-rating source catalog (stack base) | ADR 0260 | | #745 | Occupation catalog title filter (stack base) | ADR 0262 | | #746 | Rating-source occupation selector (stack base) | ADR 0261 | | #740 | Occupation rating evidence view (stack base) | ADR 0259 | +| #732 | O*NET content-model published linkages (stack base) | ADR 0256 | | #720 | Cancel stale test runs on PR close | — | | #716 | Prioritized evidence-bound operations backfill | — | | #711 | Pinned validated structured-workflow runtime | — | | #704 | Current-main external lineage contract publication | — | -The ONET rows stacked into base branches (#743/#745/#746/#740/#732) reached -`main` together through the #759 promotion; their per-base merge records are -historical evidence only. The job-architecture artifact ship originally via -#749 is now re-verified on `main` from the promotion. +Rebased and re-pushed onto current `main` (checks running at this snapshot): +#700 source-conversation-turn contract (ADR 0238), #658 optional Global Ask +knowledge cutoff (ADR 0216). Both unreviewed until exact-head checks pass. + +The ONET stack rows above landed into their stacked base branches rather than +`main`; their content reaches `main` only if the base branch continues into a +`main`-bound PR. Each base branch is recorded in the PR's `baseRefName` and +remains the owner's responsibility to promote. diff --git a/docs/voice-combination-technical-requirements.md b/docs/voice-combination-technical-requirements.md new file mode 100644 index 000000000..f075f7e2e --- /dev/null +++ b/docs/voice-combination-technical-requirements.md @@ -0,0 +1,60 @@ +# Voice-of-X Combination Technical Requirements + +This supporting TRD projects ADR 0246, ADR 0251, and ADR 0252. Those ADRs are +normative when this document and an implementation differ. + +## Scope + +LineageWeave represents a Post's explicitly supplied stakeholder perspectives +without assuming a company, B2B2C chain, or exhaustive industry taxonomy. One +imported primary Voice and zero or more evidence-bearing additional Voices are +atomic assignments; combinations are sets of rows, never compound codes. + +## Requirements + +| ID | Requirement | Verification | +|---|---|---| +| VOC-TR-1 | `source_post.voc_type_code` owns the imported primary; additional assignments cannot demote it | Database trigger and API conflict tests | +| VOC-TR-2 | Every additional Voice references a normalized PROV-O derivation and governed truth status | Foreign keys, category trigger, authenticated write test | +| VOC-TR-3 | Primary assignments use non-overlapping half-open effective intervals and allow A → B → A under serialized concurrent source updates | GiST exclusion constraint and PostgreSQL integration tests | +| VOC-TR-4 | Live reads select current rows; cutoff reads select the containing interval; ontology continuation uses its frozen snapshot when no cutoff exists | Backend SQL-contract tests and authenticated cutoff API test | +| VOC-TR-5 | Post, filter, ontology JSON-LD, exact-value CSV, and UI apply the same RBAC/ABAC and source-eligibility boundary | API, SHACL, frontend interaction, and accessibility tests | +| VOC-TR-6 | Voice stays separate from counterparty relationship, role, topic, channel, lifecycle, and stakeholder salience | ADR/schema review and ontology round-trip tests | +| VOC-TR-7 | Migration replay preserves existing starts and never reconstructs deleted pre-migration history | Migration replay test and non-identifying runtime evidence | + +## Read contract + +```text +reference_time = knowledge_cutoff ?? ontology_snapshot ?? live +live = effective_to IS NULL +historical = effective_from <= reference_time < effective_to +open historical = effective_from <= reference_time AND effective_to IS NULL +``` + +The interval is lower-inclusive and upper-exclusive. The exact primary-change +instant belongs to the new primary, so a read cannot return two primary rows. + +## Component flow + +```mermaid +flowchart LR + Import[Authorized source import] --> SourcePost[(source_post)] + SourcePost --> Trigger[Primary Voice sync trigger] + Trigger --> History[(source_post_voice intervals)] + Admin[post_admin + visible evidence] --> API[Voice assignment API] + API --> Provenance[(PROV-O assertion)] + Provenance --> History + History --> PostRead[Post and filters] + History --> Ontology[Ontology JSON-LD and CSV] + PostRead --> UI[Post and board UI] + Ontology --> Explorer[Ontology explorer] +``` + +## Failure behavior + +- Missing or hidden evidence rejects or omits the additional assignment; it is + never replaced with a placeholder. +- Unknown Voice/truth categories fail with a database check error. +- Overlapping imported-primary intervals fail at the database boundary. +- Cutoffs before retained history return an explicit unavailable state rather + than the current value. diff --git a/lineageweave/ontology_neighborhood.py b/lineageweave/ontology_neighborhood.py index cb07daae0..991b81f10 100644 --- a/lineageweave/ontology_neighborhood.py +++ b/lineageweave/ontology_neighborhood.py @@ -234,7 +234,9 @@ class OntologyVoiceAssignment: is_primary: bool truth_status_code: str recorded_at: datetime + effective_from: datetime provenance_reference: str + effective_to: datetime | None = None evidence_post_id: str | None = None def __post_init__(self) -> None: @@ -260,6 +262,16 @@ def __post_init__(self) -> None: raise OntologyNeighborhoodError( "naive_timestamp", "voice assignment recorded_at must be offset-aware" ) + if self.effective_from.tzinfo is None or ( + self.effective_to is not None and self.effective_to.tzinfo is None + ): + raise OntologyNeighborhoodError( + "naive_timestamp", "voice assignment effective bounds must be offset-aware" + ) + if self.effective_to is not None and self.effective_from >= self.effective_to: + raise OntologyNeighborhoodError( + "invalid_interval", "voice assignment effective interval is empty or inverted" + ) @dataclass(frozen=True) @@ -324,8 +336,10 @@ def exact_value_rows(self) -> tuple[dict[str, str], ...]: "target_type_code": "node_voice_type", "truth_status_code": assignment.truth_status_code, "recorded_at": assignment.recorded_at.isoformat(), - "valid_from": "", - "valid_to": "", + "valid_from": assignment.effective_from.isoformat(), + "valid_to": assignment.effective_to.isoformat() + if assignment.effective_to + else "", "evidence_count": "1" if assignment.evidence_post_id or assignment.is_primary else "0", "evidence_post_id": assignment.evidence_post_id or ( assignment.post_id if assignment.is_primary else "" @@ -414,23 +428,24 @@ def jsonld_document(self) -> dict[str, object]: if evidence_iri is not None else {} ) - graph.append( - { - "@id": assignment_iri, - "@type": str(LW.VoiceAssignment), - str(LW.assignedVoiceType): {"@id": assignment.voice_type_iri}, - str(LW.primaryVoiceAssignment): { - "@value": assignment.is_primary, - "@type": "xsd:boolean", - }, - **provenance, - "lw:truthStatus": assignment.truth_status_code, - "prov:generatedAtTime": { - "@value": assignment.recorded_at.isoformat(), - "@type": "xsd:dateTimeStamp", - }, - } + item: dict[str, object] = { + "@id": assignment_iri, + "@type": str(LW.VoiceAssignment), + str(LW.assignedVoiceType): {"@id": assignment.voice_type_iri}, + str(LW.primaryVoiceAssignment): { + "@value": assignment.is_primary, + "@type": "xsd:boolean", + }, + **provenance, + "lw:truthStatus": assignment.truth_status_code, + } + _add_jsonld_times( + item, + assignment.recorded_at, + assignment.effective_from, + assignment.effective_to, ) + graph.append(item) graph.append( { "@id": assignment.voice_type_iri, diff --git a/migrations/0237_source_post_voice_combination.sql b/migrations/0237_source_post_voice_combination.sql index a010a830c..5011dc1b3 100644 --- a/migrations/0237_source_post_voice_combination.sql +++ b/migrations/0237_source_post_voice_combination.sql @@ -129,7 +129,8 @@ update source_post_voice voice insert into source_post_voice (post_id, voice_type_code, is_primary, truth_status_code, effective_from) -select post.post_id, post.voc_type_code, true, 'truth_observed', post.created_at +select post.post_id, post.voc_type_code, true, 'truth_observed', + least(post.created_at, clock_timestamp()) from source_post post where not exists ( select 1 from source_post_voice voice @@ -159,7 +160,7 @@ begin new.voc_type_code, true, 'truth_observed', - case when tg_op = 'INSERT' then new.created_at else change_at end + case when tg_op = 'INSERT' then least(new.created_at, change_at) else change_at end ) on conflict (post_id, voice_type_code) where effective_to is null do update set is_primary = true, diff --git a/migrations/0243_source_post_voice_history.sql b/migrations/0243_source_post_voice_history.sql new file mode 100644 index 000000000..ad4637754 --- /dev/null +++ b/migrations/0243_source_post_voice_history.sql @@ -0,0 +1,57 @@ +-- ADR 0252: preserve non-overlapping imported primary Voice intervals. + +begin; + +create extension if not exists btree_gist; + +alter table source_post_voice + add column if not exists effective_to timestamptz; + +alter table source_post_voice + drop constraint if exists source_post_voice_effective_interval_check; +alter table source_post_voice + add constraint source_post_voice_effective_interval_check + check (effective_to is null or effective_from < effective_to); + +create unique index if not exists source_post_voice_current_pair_idx + on source_post_voice (post_id, voice_type_code) + where effective_to is null; + +alter table source_post_voice + drop constraint if exists source_post_voice_primary_period_excl; +alter table source_post_voice + add constraint source_post_voice_primary_period_excl + exclude using gist ( + post_id with =, + tstzrange(effective_from, effective_to, '[)') with && + ) where (is_primary); + +create or replace function synchronize_source_post_primary_voice() +returns trigger +language plpgsql +as $$ +declare + change_at timestamptz := clock_timestamp(); +begin + update source_post_voice + set effective_to = change_at + where post_id = new.post_id + and effective_to is null + and (is_primary or voice_type_code = new.voc_type_code); + + insert into source_post_voice + (post_id, voice_type_code, is_primary, truth_status_code, + effective_from, recorded_at) + values ( + new.post_id, + new.voc_type_code, + true, + 'truth_observed', + case when tg_op = 'INSERT' then least(new.created_at, change_at) else change_at end, + change_at + ); + return new; +end; +$$; + +commit; diff --git a/tests/test_ontology_neighborhood.py b/tests/test_ontology_neighborhood.py index 3f6165745..764bceaec 100644 --- a/tests/test_ontology_neighborhood.py +++ b/tests/test_ontology_neighborhood.py @@ -912,6 +912,7 @@ def test_voice_assignments_join_exact_csv_rows_and_jsonld() -> None: is_primary=False, truth_status_code=TRUTH_OBSERVED, recorded_at=T0, + effective_from=T0, provenance_reference="Evidence-backed additional voice", evidence_post_id=POST_ID, ) @@ -922,6 +923,7 @@ def test_voice_assignments_join_exact_csv_rows_and_jsonld() -> None: assert row["target_label"] == "Voice of Process" assert row["evidence_post_id"] == POST_ID assert row["evidence_count"] == "1" + assert row["valid_from"] == T0.isoformat() graph = neighborhood.jsonld_document()["@graph"] assignment_iri = str(LW[f"voice-assignment/{POST_ID}/vops"]) projected = next(item for item in graph if item.get("@id") == assignment_iri) @@ -939,6 +941,10 @@ def test_voice_assignments_join_exact_csv_rows_and_jsonld() -> None: assert projected["prov:wasDerivedFrom"] == { "@id": ontology_node_iri(NODE_POST, POST_ID) } + assert ( + projected["time:hasBeginning"]["time:inXSDDateTimeStamp"]["@value"] + == T0.isoformat() + ) hidden_evidence = replace(assignment, evidence_post_id=None) hidden_row = replace( @@ -957,6 +963,8 @@ def test_voice_assignments_join_exact_csv_rows_and_jsonld() -> None: with pytest.raises(OntologyNeighborhoodError, match="offset-aware"): replace(assignment, recorded_at=T0.replace(tzinfo=None)) + with pytest.raises(OntologyNeighborhoodError, match="offset-aware"): + replace(assignment, effective_from=T0.replace(tzinfo=None)) def test_node_bound_truncation_keeps_nearer_hop_over_farther_alphabetically_earlier_type() -> None: diff --git a/tests/test_ontology_neighborhood_ingestion.py b/tests/test_ontology_neighborhood_ingestion.py index 333d91df9..546fd28bb 100644 --- a/tests/test_ontology_neighborhood_ingestion.py +++ b/tests/test_ontology_neighborhood_ingestion.py @@ -930,6 +930,8 @@ def test_focus_label_fetch_may_be_empty_when_facts_already_labeled() -> None: "is_primary": True, "truth_status_code": "truth_observed", "recorded_at": T0, + "effective_from": T0, + "effective_to": None, "has_assertion": False, "evidence_post_id": None, } @@ -1003,6 +1005,8 @@ def test_load_voice_assignments_preserves_truth_and_customer_safe_provenance() - "is_primary": True, "truth_status_code": "truth_observed", "recorded_at": T0, + "effective_from": T0, + "effective_to": None, "has_assertion": False, "evidence_post_id": None, }, @@ -1013,6 +1017,8 @@ def test_load_voice_assignments_preserves_truth_and_customer_safe_provenance() - "is_primary": False, "truth_status_code": "truth_observed", "recorded_at": T0, + "effective_from": T0, + "effective_to": None, "has_assertion": True, "evidence_post_id": POST_ID, }, @@ -1030,7 +1036,15 @@ def test_load_voice_assignments_preserves_truth_and_customer_safe_provenance() - assert assignments[1].evidence_post_id == POST_ID assert "evidence.node_id = any($1::uuid[])" in conn.calls[0][0] assert "voice.is_primary or evidence.node_id = any($1::uuid[])" in conn.calls[0][0] - assert "voice.effective_from <= $2" in conn.calls[0][0] + assert ( + "voice.effective_from <= coalesce($2::timestamptz, $3::timestamptz)" + in conn.calls[0][0] + ) + assert ( + "coalesce($2::timestamptz, $3::timestamptz) < voice.effective_to" + in conn.calls[0][0] + ) + assert "$2::timestamptz is null and voice.effective_to is null" not in conn.calls[0][0] assert "voice.recorded_at <= $3" in conn.calls[0][0] assert conn.calls[0][1] == ([POST_ID], T0, T0) diff --git a/tests/test_post_filter_options.py b/tests/test_post_filter_options.py index 6ac4dc570..607a9f1bd 100644 --- a/tests/test_post_filter_options.py +++ b/tests/test_post_filter_options.py @@ -57,6 +57,7 @@ def test_post_filter_options_use_one_authorized_source_scan() -> None: assert "cross join lateral" in query assert "('post_visibility', post.visibility_code)" in query assert "left join source_post_voice voice" in query + assert query.count("voice.effective_to is null") == 1 assert "('voc_type', coalesce(voice.voice_type_code, post.voc_type_code))" in query assert "post.corporate_entity_id::text = any($1::text[])" in query assert "post.process_unit_id::text = any($2::text[])" in query diff --git a/tests/test_source_post_voice_history_schema.py b/tests/test_source_post_voice_history_schema.py new file mode 100644 index 000000000..66b7866b9 --- /dev/null +++ b/tests/test_source_post_voice_history_schema.py @@ -0,0 +1,49 @@ +"""Static contract tests for ADR 0252 temporal primary Voice history.""" + +from __future__ import annotations + +from pathlib import Path + +MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0243_source_post_voice_history.sql" +) + + +def test_primary_voice_history_uses_non_overlapping_half_open_intervals() -> None: + """The database preserves recurring Voices and rejects overlapping primaries.""" + sql = MIGRATION.read_text(encoding="utf-8").lower() + + assert "add column if not exists effective_to timestamptz" in sql + assert "drop constraint if exists source_post_voice_pkey" not in sql + assert "primary key (post_id, voice_type_code, effective_from)" not in sql + assert "voice_assignment_id" not in sql + assert "tstzrange(effective_from, effective_to, '[)') with &&" in sql + assert "where (is_primary)" in sql + assert "effective_from < effective_to" in sql + + +def test_primary_voice_change_closes_current_rows_before_insert() -> None: + """One transaction instant closes the prior state and opens the new primary.""" + sql = MIGRATION.read_text(encoding="utf-8").lower() + + assert "change_at timestamptz := clock_timestamp()" in sql + assert "set effective_to = change_at" in sql + assert "and (is_primary or voice_type_code = new.voc_type_code)" in sql + assert sql.index("set effective_to = change_at") < sql.index( + "insert into source_post_voice" + ) + assert "on conflict" not in sql + + +def test_future_source_clock_is_bounded_by_the_recording_clock() -> None: + """A future source timestamp cannot create an interval that closes backwards.""" + assignment_sql = ( + MIGRATION.parent / "0237_source_post_voice_combination.sql" + ).read_text(encoding="utf-8").lower() + history_sql = MIGRATION.read_text(encoding="utf-8").lower() + + assert "least(post.created_at, clock_timestamp())" in assignment_sql + assert "least(new.created_at, change_at)" in assignment_sql + assert "least(new.created_at, change_at)" in history_sql diff --git a/tests/test_source_post_voice_ingestion.py b/tests/test_source_post_voice_ingestion.py index 75a0c48d0..52fdf0b1f 100644 --- a/tests/test_source_post_voice_ingestion.py +++ b/tests/test_source_post_voice_ingestion.py @@ -72,6 +72,7 @@ def test_additional_voice_creates_prov_derivation_and_assignment_atomically() -> assert "prov_was_derived_from" in sql assert "where effective_to is null" in sql assert "where not source_post_voice.is_primary" in sql + assert "where effective_to is null" in sql assert "voice-assignment/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1/vops" in str( conn.calls ) diff --git a/tests/test_source_post_voice_schema.py b/tests/test_source_post_voice_schema.py index 05265bf10..b78135b6b 100644 --- a/tests/test_source_post_voice_schema.py +++ b/tests/test_source_post_voice_schema.py @@ -25,9 +25,10 @@ def test_voice_combination_schema_is_normalized_and_evidence_bearing() -> None: assert "effective_from timestamptz not null" in sql assert "true, 'truth_observed'" in sql assert "where is_primary" in sql - assert "select post.post_id, post.voc_type_code, true, 'truth_observed', post.created_at" in sql + assert "select post.post_id, post.voc_type_code, true, 'truth_observed'" in sql + assert "least(post.created_at, clock_timestamp())" in sql assert "change_at timestamptz := clock_timestamp()" in sql - assert "case when tg_op = 'insert' then new.created_at else change_at end" in sql + assert "case when tg_op = 'insert' then least(new.created_at, change_at) else change_at end" in sql assert "after insert on source_post" in sql assert "after update of voc_type_code on source_post" in sql assert "when (old.voc_type_code is distinct from new.voc_type_code)" in sql diff --git a/tests/test_source_state_serialization.py b/tests/test_source_state_serialization.py index 661c4e2c5..1153ac6e4 100644 --- a/tests/test_source_state_serialization.py +++ b/tests/test_source_state_serialization.py @@ -80,6 +80,7 @@ async def fetch( assert "provenance_assertion_id is not null as evidence_available" in query assert "voice.effective_from <= $2" in query assert "$2 < voice.effective_to" in query + assert "voice.effective_to is null or $2 < voice.effective_to" in query assert post_id == "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" assert effective_cutoff == datetime(2026, 1, 1, tzinfo=UTC) return [