diff --git a/CHANGELOG.d/0.100.0-export-source-ontology-coverage.md b/CHANGELOG.d/0.100.0-export-source-ontology-coverage.md new file mode 100644 index 000000000..decb045ae --- /dev/null +++ b/CHANGELOG.d/0.100.0-export-source-ontology-coverage.md @@ -0,0 +1,15 @@ +# Export-source ontology coverage (ADR 0246) + +- Analyzed an aggregate-only, authorized export source against the published + ontology; all content of its rows is expressible through governed + `postTypeScheme` types, content-semantic classes, PROV attribution, and + raw-code-as-instance lifecycle fields (ADR 0246). +- Closed the one derived-semantic gap: `:Location` now carries the place its + content names (`:locationName`) and an ISO 3166-1 country/region code + (`:countryCode`) as instance data, with a SHACL `:LocationShape` failing + closed on non-code country values. +- No new lookup category is seeded; raw ERP codes remain instance data until + a caller governs their code system (ADR 0145 / 0241 / 0246). +- Aggregate-only reporting and the derived-node (not column-projection) + discipline preserve ADR 0001 / 0207 boundaries; supporting evidence in + `docs/doctoring/export-source-ontology-coverage.md`. \ No newline at end of file diff --git a/CHANGELOG.d/local-scoring-owner-contract.md b/CHANGELOG.d/local-scoring-owner-contract.md new file mode 100644 index 000000000..26acdbef7 --- /dev/null +++ b/CHANGELOG.d/local-scoring-owner-contract.md @@ -0,0 +1,7 @@ +# Local scoring ownership contract + +- Identified active Python lineage-channel, reconstruction-decision, and + corporate-entity similarity paths as migration debt. +- Defined the fail-closed owner envelopes required before those paths can be + removed, without assigning corporate identity or moving existing heuristics + to another repository by assumption. diff --git a/Makefile b/Makefile index d68780720..c98b33011 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # Keep provider credentials outside the repository. Compose interpolation must # read the same home env file as the orchestrator container's env_file. -COMPOSE := docker compose --env-file "$$HOME/.env" +COMPOSE := docker compose --env-file "$$HOME/.env" -p lineageweave up: $(COMPOSE) up -d diff --git a/backend/app/main.py b/backend/app/main.py index 6457bbde1..6c3ad39b3 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1967,6 +1967,9 @@ async def _load_post_semantic_hints(conn: asyncpg.Connection, post_id: str) -> s source_customer.entity_name as source_customer_catalog_name, post.source_project_code, post.source_project_name, + post.voc_type_code, + post.source_stage_code, + post.source_detail_state_code, post.secondary_grouping_key as project_field, customer.entity_name as customer_name, affiliated.entity_name as author_affiliation_name @@ -2038,6 +2041,9 @@ async def _load_post_semantic_hints(conn: asyncpg.Connection, post_id: str) -> s source_customer_catalog_name=first["source_customer_catalog_name"], source_project_code=first["source_project_code"], source_project_name=first["source_project_name"], + source_voc_type_code=first["voc_type_code"], + source_stage_code=first["source_stage_code"], + source_detail_state_code=first["source_detail_state_code"], source_context_present=source_context_present, ) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 892c8231a..bd883a97e 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -15,6 +15,7 @@ import asyncio import math import os +import subprocess import uuid from contextlib import closing from pathlib import Path @@ -383,7 +384,21 @@ def seeded_db(demo_analyst_token): cur.execute(_LEFTOVER_MAP_COVERAGE_MIGRATION.read_text()) cur.execute(_GLOBAL_ASK_JOB_MIGRATION.read_text()) cur.execute(_GLOBAL_ASK_SCOPE_MIGRATION.read_text()) - cur.execute(_GLOBAL_ASK_EVIDENCE_SEARCH_MIGRATION.read_text()) + # Match ADR 0166's production runner exactly: psql keeps + # concurrent indexes outside an implicit transaction and parses + # SQL literals/comments without a fixture-owned splitter. + subprocess.run( + [ + "psql", + "-X", + "-v", + "ON_ERROR_STOP=1", + db_dsn, + "-f", + str(_GLOBAL_ASK_EVIDENCE_SEARCH_MIGRATION), + ], + check=True, + ) cur.execute(_GLOBAL_ASK_KNOWLEDGE_CUTOFF_MIGRATION.read_text()) cur.execute(_GLOBAL_ASK_PUBLIC_VERIFICATION_MIGRATION.read_text()) cur.execute(_EVENT_OCCURRED_AT_MIGRATION.read_text()) diff --git a/docker/contextual-orchestrator/Dockerfile b/docker/contextual-orchestrator/Dockerfile index 0af60f58c..24eb53fb5 100644 --- a/docker/contextual-orchestrator/Dockerfile +++ b/docker/contextual-orchestrator/Dockerfile @@ -15,6 +15,7 @@ RUN mkdir /tmp/contextual-orchestrator \ 'opentelemetry-api>=1.30.0' \ 'opentelemetry-sdk>=1.30.0' \ 'opentelemetry-exporter-otlp-proto-http>=1.30.0' \ + && python -m contextual_orchestrator --help | grep -q -- '--allowed-provider-host' \ && useradd --uid 10001 --no-create-home orchestrator COPY agents.json /app/agents.json diff --git a/docker/contextual-orchestrator/start.py b/docker/contextual-orchestrator/start.py index 01dc5d189..f1a42c743 100644 --- a/docker/contextual-orchestrator/start.py +++ b/docker/contextual-orchestrator/start.py @@ -11,6 +11,7 @@ import sys import json from pathlib import Path +from urllib.parse import urlsplit def _pop_first_env(*names: str) -> str: @@ -23,6 +24,29 @@ def _pop_first_env(*names: str) -> str: return first +def _allowed_provider_hosts(provider_url: str) -> tuple[str, ...]: + """Require the configured gateway host in an explicit outbound allowlist.""" + provider_host = (urlsplit(provider_url).hostname or "").rstrip(".").casefold() + allowed_hosts = tuple( + sorted( + { + value.strip().rstrip(".").casefold() + for value in os.environ.get( + "CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS", "" + ).split(",") + if value.strip() + } + ) + ) + if not provider_host: + raise SystemExit("LLM_GATEWAY_API_URL must contain a hostname") + if not allowed_hosts: + raise SystemExit("CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS is required") + if provider_host not in allowed_hosts: + raise SystemExit("LLM_GATEWAY_API_URL hostname is not in the provider allowlist") + return allowed_hosts + + def main() -> None: """Register the provider credential and delegate to the upstream server.""" gateway_key = _pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY") @@ -48,6 +72,7 @@ def main() -> None: raise SystemExit("LLM_GATEWAY_API_URL or LLM_GATEWAY_URL is required to start the gateway") if not provider_url.rstrip("/").endswith("/v1"): provider_url = provider_url.rstrip("/") + "/v1" + allowed_provider_hosts = _allowed_provider_hosts(provider_url) raw_limit = os.environ.pop("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "4096").strip() try: max_output_tokens = int(raw_limit) @@ -67,6 +92,7 @@ def main() -> None: for agent in agents["agents"]: agent["base_url"] = provider_url agent["credential_key"] = "LLM_GATEWAY_API_KEY" + agent["provider_name"] = "configured_gateway" agent.setdefault("provider_protocol", "auto") os.environ.pop("LLM_GATEWAY_EMBEDDING_MODEL", None) agents_path.write_text(json.dumps(agents), encoding="utf-8") @@ -97,6 +123,8 @@ def main() -> None: "--max-body-bytes", str(max_body_bytes), ] + for allowed_host in allowed_provider_hosts: + sys.argv.extend(("--allowed-provider-host", allowed_host)) del provider_url del auth_token from contextual_orchestrator.__main__ import main as serve diff --git a/docs/adr/0030-external-llm-gateway-environment.md b/docs/adr/0030-external-llm-gateway-environment.md index fccc1636f..505c0602b 100644 --- a/docs/adr/0030-external-llm-gateway-environment.md +++ b/docs/adr/0030-external-llm-gateway-environment.md @@ -67,6 +67,9 @@ must never be returned through a buyer-facing API or persisted failure detail. message that tells them to retry or restore the provider configuration. - `CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS` must explicitly allow the hostname selected by `LLM_GATEWAY_API_URL`; wildcard allowlists are forbidden. + The Compose bootstrap forwards each normalized allowlisted hostname to the + orchestrator's `--allowed-provider-host` boundary so runtime model discovery + and provider calls enforce the same operator policy. - Local Compose development permits only the explicitly enumerated `host.docker.internal:8080` text gateway and `host.docker.internal:18082` Vision gateway when `LINEAGEWEAVE_ALLOW_LOCAL_LLM_HTTP=1`; arbitrary local diff --git a/docs/adr/0208-externalize-local-mathematical-compute.md b/docs/adr/0208-externalize-local-mathematical-compute.md index 42a5a0591..ecb7f34bc 100644 --- a/docs/adr/0208-externalize-local-mathematical-compute.md +++ b/docs/adr/0208-externalize-local-mathematical-compute.md @@ -5,6 +5,9 @@ **Amends:** ADR 0003, ADR 0024, ADR 0064, ADR 0084, ADR 0132, ADR 0145, ADR 0148, ADR 0167, ADR 0168, ADR 0182, ADR 0185, ADR 0200, ADR 0201, and ADR 0205 +**Extended by:** [ADR 0245](0245-lineage-scoring-and-entity-resolution-owner-contract.md), +which names the remaining channel, reconstruction-decision, and corporate- +entity similarity paths and defines their minimum owner envelopes. ## Context @@ -28,9 +31,10 @@ The ecosystem product boundaries are already sufficient: CPU/GPU implementation before LineageWeave treats a new result as governed numerical evidence. -LineageWeave has no standalone canonical PRD file on this exact head. Until -one lands, `ARCHITECTURE.md` and the accepted ADR set are the product baseline; -this absence remains a product-documentation gap, not permission to infer a +At the time of this decision LineageWeave had no standalone canonical PRD. +`docs/product-requirements.md` has since landed as a supporting product +contract and confirms the same consumer-only measurement boundary; accepted +ADRs remain normative. The earlier absence was never permission to infer a different responsibility. ## Decision @@ -113,4 +117,3 @@ https://doi.org/10.1007/s11336-021-09762-5 Roberts, M. E., Stewart, B. M., & Tingley, D. (2019). stm: An R package for structural topic models. *Journal of Statistical Software, 91*(2), 1–40. https://doi.org/10.18637/jss.v091.i02 - diff --git a/docs/adr/0240-explicit-missing-body-import-boundary.md b/docs/adr/0240-explicit-missing-body-import-boundary.md new file mode 100644 index 000000000..50100d3eb --- /dev/null +++ b/docs/adr/0240-explicit-missing-body-import-boundary.md @@ -0,0 +1,56 @@ +# ADR 0240: Explicit missing-body import boundary + +**Status:** Accepted +**Date:** 2026-08-26 +**Extends:** [ADR 0102](0102-semantic-source-unit-boundaries.md) + +## Context + +An authorized source export can contain titles, lifecycle fields, customer and +project codes, lineage keys, actors, timestamps, and source-artifact provenance +while exposing no record body. Requiring a non-empty body makes every such row +unimportable. Copying the title into the body would instead manufacture body +evidence and falsely imply semantic-unit coverage. + +## Decision + +1. The PostgreSQL importer accepts exactly one of a mapped body column or + `--no-body-dimension-evidence` containing a non-blank operator statement. + The importer records that attestation but does not use an arbitrary text- + length threshold as a proxy for evidence quality. +2. A missing body persists as the empty source representation. The title stays + `post_title`; it is never copied into `post_body` or emitted as a paragraph. +3. Content-unit, embedding, summary, VISION, and body-search coverage remain + unavailable until an authoritative body/file source is connected. +4. Structured source fields retain their existing raw provenance columns and + semantic-hint boundaries. Their presence does not prove an entity binding. +5. The import result repeats the evidence statement so an operator can retain + it with private runtime evidence. Repository artifacts contain aggregates + only. +6. `scripts/audit_source_semantic_coverage.py` reproduces availability counts + from caller-mapped columns and emits no source values. +7. RDF `bodyAvailable` and its published SHACL constraint use the same + whitespace predicate as the Python projector, including Unicode separator, + next-line, and legacy information-separator characters. A body containing + only those characters is unavailable; validators must not reinterpret it as + semantic evidence. +8. A no-body-dimension re-import preserves an already-populated target body + atomically in the source-post UPSERT. The preserved body is also the input + to revision and semantic-content persistence; an unavailable source + dimension must not erase evidence acquired from an authoritative body + source. + +## Consequences + +Title-only structured records can participate in explicitly supported +lineage and source-metadata views without fabricated prose. Semantic body +coverage remains honestly incomplete and can be retried after the owning +source supplies bodies. + +## Evidence + +A 2026-08-26 aggregate-only source inspection found 43,814 rows, 43,814 +non-empty titles, zero non-empty bodies, 40,001 customer-code rows, 4,490 +project-code rows, and complete process-unit, sales-pool, actor, and +source-artifact provenance fields. No source value, identifier, organization, +or artifact path was copied into this repository. diff --git a/docs/adr/0241-source-classification-semantic-hints.md b/docs/adr/0241-source-classification-semantic-hints.md new file mode 100644 index 000000000..7589295a2 --- /dev/null +++ b/docs/adr/0241-source-classification-semantic-hints.md @@ -0,0 +1,43 @@ +# ADR 0241: Source classification semantic hints + +**Status:** Accepted +**Date:** 2026-08-26 +**Extends:** [ADR 0004](0004-knowledge-graph-ontology.md), +[ADR 0117](0117-catalog-backed-semantic-hints.md), +[ADR 0159](0159-published-ontology-pages.md), +[ADR 0207](0207-repository-case-ontology-namespace-canonical.md), and +[ADR 0222](0222-project-nodes-in-ontology-neighborhood.md) + +## Context + +The importer preserves a governed VOC type plus caller-mapped source stage and +detail-state codes, but semantic extraction received neither classification. +An authorized source reference catalog currently provides examples, not a +complete code list or authoritative definitions for every observed value. +Dropping the fields loses source evidence; minting ontology concepts from +partial examples invents semantics. + +## Decision + +1. Pass `voc_type_code`, `source_stage_code`, and + `source_detail_state_code` to contextual-orchestrator as labeled raw source + hints with exact `source_post` column provenance. +2. Raw codes are context only. They do not assert a lifecycle transition, + inspection outcome, quality grade, entity relationship, or classified + ontology concept. An RDF source-code literal asserts only the observed raw + value and its predicate, not the value's business meaning. +3. RDF projects the governed five-value VOC type as `:hasPostType` to its + published SKOS concept. Stage and detail-state remain literal properties; + projecting a raw literal preserves evidence without minting a concept. +4. Promoting stage/detail values to ontology concepts requires a complete + source-owned code catalog, stable definitions, mapping provenance, and + SHACL fixtures. Partial screen examples are insufficient. +5. Missing codes remain `none`; no default classification is inferred. Every + RDF post projector therefore requires an explicit governed VOC type. + +## Consequences + +Semantic extraction can consider classifications already preserved by the +import boundary without silently losing them or overclaiming their meaning. +The ontology remains intentionally incomplete for source-specific grades and +inspection states until their authority is available. diff --git a/docs/adr/0242-private-content-semantic-coverage-audit.md b/docs/adr/0242-private-content-semantic-coverage-audit.md new file mode 100644 index 000000000..8088dcdf8 --- /dev/null +++ b/docs/adr/0242-private-content-semantic-coverage-audit.md @@ -0,0 +1,198 @@ +# ADR 0242: Private content semantic-coverage audit + +**Status:** Accepted +**Date:** 2026-08-26 +**Extends:** [ADR 0001](0001-demo-identity-and-data-boundary.md), +[ADR 0004](0004-knowledge-graph-ontology.md), and +[ADR 0089](0089-private-real-data-runtime-boundary.md) + +## Context + +Schema availability does not prove that the ontology expresses the material +meaning of source content. A first runtime request submitted 100 private titles +but returned a claimed sample size of 60. The transport had succeeded, yet the +semantic result was unusable because neither the model nor the caller enforced +cardinality. A later 80-record deterministic-window run proved only that the +pipeline could preserve ordered outputs; it had neither a probability frame nor +known inclusion probabilities and therefore cannot estimate corpus coverage. +Repository artifacts must not retain the private titles. + +## Decision + +1. `scripts/audit_source_content_semantics.py` reads a caller-owned query that + returns exactly `selection_token, content_text` in manifest order and exactly + the declared sample. Each runtime-only owner-issued opaque token must hash to + the corresponding manifest membership digest; neither tokens nor digests are + sent to contextual-orchestrator or printed. +2. Content crosses only the configured contextual-orchestrator boundary in + `conduct` mode. Every accepted batch requires a multi-step trace. The model + receives a deterministic public ontology contract containing each governed + term's IRI, RDF kind, labels, comments, domain, range, and SKOS scheme; local + names alone are not semantic evidence. Coverage evaluates schema + expressibility: source-specific names and values are instance data when a + supplied class/property represents them, not missing private vocabulary. +3. The model classifies only governed semantic dimensions; it does not decide + coverage or select ontology IRIs. The caller accepts a batch only when JSON, + input count, item count, ordered indexes, and non-empty unique governed + dimension codes validate. It then deterministically resolves each dimension + against the current ontology contract: present governed class/property IRIs + become supporting evidence, while a dimension with no current term remains + missing. The contract parses the published PROV-O support + profile with the primary ontology and includes `rdfs:subClassOf` and + `rdfs:subPropertyOf`. It also reuses the runtime's canonical 30 PROV-O class, + 50 property, and qualification-table registries, so standard semantics are + present in the audit rather than reduced to imports or local mappings. + This separation prevents an invented abbreviation or near-match from + becoming a new ontology term while retaining IRI evidence after validation. + Person/actor meaning is a governed dimension distinct from organization role; + collapsing the two would hide whether the ontology identifies an actor or only + an organizational function. Project/initiative meaning is likewise distinct + from product/service meaning because the former denotes organized work while + the latter denotes an offering or deliverable. +4. No retry repairs, lexical rules, inferred categories, source values, + identifiers, or row-level outputs are persisted or printed. Any malformed, + incomplete, single-agent, or unavailable result fails the run. +5. Only complete non-identifying aggregates may enter repository documents. + A sample audit describes the sample, never the full corpus. + The aggregate source audit may additionally compare distinct caller-selected + source and semantic-layer keys. It reports matched and one-sided key counts + only; identifiers and values never enter output or repository artifacts. + When a caller supplies a semantic-edge table, the audit also reports only + aggregate `observed`, `inferred`, `predicted`, ungoverned-status, and + evidence-reference counts. An observed edge without an evidence reference + fails the source-evidence boundary. An inferred or predicted edge may lack a + direct source evidence reference, but that does not make it provenance-free: + its generation or derivation still requires the normalized PROV-O resource, + assertion, and qualification schema from ADR 0011. The audit therefore + reports deployment of that complete schema separately and never relabels an + inference reason, mapping source, confidence, or deterministic identifier as + source evidence or qualified provenance. +6. Missing-dimension counts do not themselves authorize new private ontology + terms. Event/activity candidates must first reconcile with PROV-O; + temporal candidates with OWL-Time; and observed property, asset, system, + or feature-of-interest candidates with the current SOSA/SSN edition. A + source-grounded normalized fact and qualified provenance remain mandatory. +7. A probability-sample audit additionally requires a versioned caller-supplied + sample manifest: a complete population/frame size, simple or stratified + random design, exact inclusion-probability numerator/denominator and frame + digest for every + stratum, ordered selected-unit token digests bound to their strata, a + canonical selection-manifest digest, and + `provider_failures_retained=true`. + Deterministic windows, convenience samples, unknown inclusion probabilities, + and replacement of failed items are pipeline evidence only. +8. NIST/SEMATECH's proportion design begins with + `n0 = z² p(1-p) / delta²`; sampling without replacement applies + `n = n0 / (1 + (n0 - 1) / N)`. Stratified designs determine sample size per + stratum. LineageWeave neither evaluates those equations nor derives sample + weights: a versioned, SHA-256-bound `ContextualWisdomLab/fast-mlsirm` Rust + artifact owns that arithmetic. Manifest contract v3 carries each stratum's + exact `(n_h, N_h)` ratio rather than a rounded decimal. The audit replays the complete Rust-owned + design artifact and requires its population, ordered stratum populations, + total sample size, stratum allocations, and Rust-attested exact inclusion + ratios to match the separately bound selection manifest. The artifact accepts no caller hash or selected + membership. For a complete one-stratum SRSWOR audit, the immutable + `fast-mlsirm.achieved-proportion.v1` artifact binds that design artifact and + attests the sample-proportion estimand, SRSWOR design variance, and exact + Wang/Konijn equal-tailed hypergeometric interval. The script additionally + binds the terminal artifact, selection-manifest digest, ontology SHA-256, + aggregate verdict counts, and trace-count bounds into one audit SHA-256. + The same aggregate-only envelope carries a validated PROV-O graph: the + audit activity `prov:used` the selection manifest, Rust design, completed + attempt, ontology, and Rust terminal entities; the audit entity + `prov:wasGeneratedBy` that activity and `prov:wasDerivedFrom` all five + inputs. The completed attempt must match the selection, design, ontology, + and accepted sample count before terminal evidence can exist. Resource IRIs are content-addressed + URNs, so no private source identifier enters the repository artifact. + Only that complete chain sets `corpus_inference_available=true`. + Stratified terminal inference remains unavailable rather than receiving an + invented variance or interval. Caller-recomputed hashes do not + establish Rust provenance or authorize corpus inference. Confidence, + margin, prior-proportion, or interval fields are not accepted as proof when + no immutable owner artifact attests their computation. +9. Any provider, transport, trace, parse, or item failure invalidates the whole + declared probability sample. The selected item remains in the denominator + and must be retried in place; it is never dropped or replaced by another + record. Only a zero-failure complete run emits a coverage aggregate. The + failed execution itself is not erased: an owner-only aggregate attempt + artifact records the accepted-item count, failed batch index, bounded error + class, and a content-addressed PROV-O graph showing that the attempt activity + used the selection manifest, Rust design artifact, and ontology. This is + execution provenance, not a partial coverage result, and therefore never + sets `corpus_inference_available` or persists row-level verdicts. Each + provider call runs behind a terminable process boundary so the declared + timeout is a wall-clock deadline, not merely a socket inactivity timeout; + a peer that keeps a connection active cannot leave the attempt indefinitely + `in_progress`. Because process spawning does not inherit request context, + every provider call carries the same non-identifying audit session id, + derived from the selection, design, and ontology hashes, through the HTTP + header, request metadata, local telemetry, and attempt artifact, so + orchestration trace and execution PROV stay correlated. +10. The model receives the locally validated semantic-dimension support + profile, not the entire ontology inventory. It classifies source meaning + into governed dimensions but never selects terms or decides coverage. + LineageWeave retains the complete ontology/PROV registry validation and + binds the complete ontology bytes by SHA-256. This keeps each request below + gateway payload limits without weakening the ontology evidence boundary. +11. Requests use contextual-orchestrator's provider-neutral + `orchestrator/auto` route. LineageWeave does not name or rank a provider + model; discovery, agent-pool construction, and routing remain upstream as + required by ADR 0076. +12. Every request supplies a strict JSON Schema whose `input_count`, item-array + minimum/maximum length, index bounds, allowed dimensions, and closed object + fields are bound to that batch. contextual-orchestrator retains + multi-agent synthesis, schema validation, and repair; LineageWeave still + revalidates exact ordered indexes and discards the whole declared sample on + any transport, schema, trace, or cardinality failure. +13. The current corpus acceptance audit declares a two-sided 95% confidence + level, 5-percentage-point margin, and the NIST conservative unknown- + proportion input `p=0.5` before selection. For the current 43,814-record + eligible frame, the Rust finite-population design yields 381 SRSWOR units. + These are explicit audit acceptance inputs, not estimated channel weights; + changing them requires a new design artifact and a new selection manifest. + This one-stratum design supports only overall eligible-frame prevalence; + precision for a rare semantic dimension or category remains unavailable + until governance predeclares the strata, category estimand, and acceptance + precision and fast-mlsirm attests a matching stratified terminal estimator, + covariance, and interval artifact. + +## Consequences + +An HTTP 200 can no longer turn a partial classification into coverage evidence. +The audit remains unavailable when contextual-orchestrator cannot complete all +batches, preserving failures in the declared denominator instead of silently +shrinking the sample. The observed 80-record result remains exploratory pipeline +acceptance evidence. The Rust design artifact proves sample-size, +finite-population-correction, and allocation provenance only. A complete +one-stratum SRSWOR audit becomes corpus inference evidence only when its Rust +terminal artifact and aggregate audit identity also validate. This does not +turn the point estimate into certainty: an all-success sample retains a lower +exact confidence bound below one unless the sample is a census. + +## References + +Cox, S., & Little, C. (Eds.). (2022). *Time ontology in OWL*. World Wide Web +Consortium. https://www.w3.org/TR/owl-time/ + +Cox, S. J. D., Lefrançois, M., Warren, R., Atkinson, R., Moreira de Sousa, L., +Schleidt, K., Grellet, S., & Janowicz, K. (Eds.). (2026). *Semantic Sensor +Network Ontology—2023 edition* (Working Draft). World Wide Web Consortium & +Open Geospatial Consortium. https://www.w3.org/TR/vocab-ssn-2023/ + +Australian Bureau of Statistics. (2022). *Basic survey design: Sample design*. +https://www.abs.gov.au/websitedbs/D3310114.nsf/home/Basic%20Survey%20Design%20-%20Sample%20Design + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV +ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ + +National Institute of Standards and Technology. (n.d.). *Selecting sample +sizes*. In *NIST/SEMATECH e-handbook of statistical methods*. +https://www.itl.nist.gov/div898/handbook/ppc/section3/ppc333.htm + +National Institute of Standards and Technology. (n.d.). *Confidence limits*. +In *NIST/SEMATECH e-handbook of statistical methods*. +https://www.itl.nist.gov/div898/handbook/prc/section2/old.prc271.htm + +Wang, W. (2015). Exact optimal confidence intervals for hypergeometric +parameters. *Journal of the American Statistical Association, 110*(512), +1491–1499. https://doi.org/10.1080/01621459.2014.966191 diff --git a/docs/adr/0245-lineage-scoring-and-entity-resolution-owner-contract.md b/docs/adr/0245-lineage-scoring-and-entity-resolution-owner-contract.md new file mode 100644 index 000000000..190a408b2 --- /dev/null +++ b/docs/adr/0245-lineage-scoring-and-entity-resolution-owner-contract.md @@ -0,0 +1,106 @@ +# ADR 0245 — Lineage scoring and entity resolution require owner artifacts + +**Decision status:** Accepted + +**Date:** 2026-08-26 +**Amends:** ADR 0026, ADR 0064, ADR 0084, and ADR 0208 + +## Context + +ADR 0208 freezes local numerical computation as migration debt, but its audit +does not name every active path. Protected `main` still executes the following +Python decisions during ordinary reconstruction and ingestion: + +- `lineageweave/channels.py` computes an inverse elapsed-day score, a numeric + secondary-key score, and `difflib.SequenceMatcher` text similarity; +- `lineageweave/reconstruct.py` renormalizes weights, limits candidates to the + latest 50 records, applies a fixed `0.3` score floor, and invokes weighted + fusion before selecting a parent; and +- `lineageweave/corporate_hierarchy_resolution.py` deletes a fixed suffix + vocabulary, computes `SequenceMatcher` similarity, and applies a fixed `0.6` + catalog-binding threshold. + +Those paths affect product facts or evidence selection. They are not merely +display formatting or operational resource accounting. The cited record- +linkage literature supports an explicit uncertain outcome, but it does not +validate these particular constants or Python string-similarity rules. + +Current ecosystem contracts do not provide a complete replacement. TEPP's +published LineageWeave project-history exchange returns temporal association, +not candidate-parent scores. RankWeave owns fusion and retrieval, but its +current public contract does not return a Rust-computed Event-Lineage edge +artifact or organization-identity decision. contextual-orchestrator owns +embedding transport and model orchestration, not catalog identity. Keyverse +owns account identity and is not implicitly assigned corporate-master entity +resolution. + +Open PR #704 at audited head `ea6c5c8e9819590dfbc058344435122584947f6e` +publishes a useful external evidence envelope, but its analysis implementation +imports `_best_parent` and `active_weights`, computes candidate-window counts +and per-channel contributions, and applies caller policy score floors. It is +therefore a consumer-contract delivery, not the owner-compute replacement +required by this decision; its local arithmetic must not be cited as closing +this gap. + +## Decision + +1. **No local scoring extension.** The three named modules are frozen migration + debt. No new decay, normalization, token overlap, similarity algorithm, + candidate-order rule, score floor, threshold, or numeric fallback may be + added. Tests may characterize legacy behavior but may not call it calibrated, + paper-grounded, or release-compliant. +2. **Event-Lineage owner envelope.** Replacement activates only from a + versioned owner result containing: + - contract and result-schema versions; + - immutable input snapshot digest and knowledge cutoff; + - every considered record id and the evidence-unit references authorized by + LineageWeave before submission; + - selected parent id or an explicit abstention; + - separate temporal, grouping, semantic, and adjudication evidence with + availability status; + - owner model/method version, convergence or completion status, uncertainty + where the method defines it, and deterministic result digest; and + - an explicit non-causal classification. + TEPP owns calibrated temporal/event criterion evidence. RankWeave may own the + Rust-backed candidate ranking/fusion artifact after its PRD and API accept + that responsibility. LineageWeave validates and persists the envelope; it + never recomputes, renormalizes, thresholds, or repairs it. +3. **Corporate-entity resolution envelope.** No existing repository is assigned + this construct by inference. An owning repository must first accept a PRD/ADR + and publish a versioned result containing the input snapshot digest, bounded + candidate catalog ids, source/alias evidence references, `unique`/`miss`/`tie` + outcome, selected catalog id only for `unique`, method/model version, + uncertainty or review status, and result digest. Until then, no new automatic + similarity binding path may activate. +4. **Resource limits are not evidence.** A bounded request may cap records or + bytes before an owner call, but recency, truncation order, or a fixed window + must not decide scientific relevance. The request records any truncation and + the result remains incomplete rather than silently treating excluded + candidates as negative evidence. +5. **Deletion sequence.** After exact owner contracts land and pass synthetic + recovery/equivalence tests, separate consumer PRs add strict adapters and + persisted provenance. Only then do deletion PRs remove the corresponding + functions and constants. Missing, malformed, stale-digest, incomplete, or + unavailable owner results produce no edge or catalog binding. + +## Consequences + +- Existing runtime behavior remains honestly labeled migration debt rather + than being moved unchanged to another repository. +- LineageWeave cannot claim the affected reconstruction or automatic catalog + binding paths satisfy its product boundary until their local implementation + is deleted. +- Exact equality, sorting, authorization, persistence, and UI projection may + remain local when they do not manufacture a score or identity decision. +- The customer sees an unavailable or review-needed state instead of a guessed + lineage edge or organization identity when the owner artifact is absent. + +## References — APA 7th + +Bhattacharya, I., & Getoor, L. (2007). Collective entity resolution in +relational data. *ACM Transactions on Knowledge Discovery from Data, 1*(1), +Article 5. https://doi.org/10.1145/1217299.1217304 + +Fellegi, I. P., & Sunter, A. B. (1969). A theory for record linkage. *Journal +of the American Statistical Association, 64*(328), 1183–1210. +https://doi.org/10.2307/2286061 diff --git a/docs/adr/0246-export-source-ontology-coverage.md b/docs/adr/0246-export-source-ontology-coverage.md new file mode 100644 index 000000000..b9132c503 --- /dev/null +++ b/docs/adr/0246-export-source-ontology-coverage.md @@ -0,0 +1,66 @@ +# ADR 0246: Export-source ontology coverage + +**Status:** Accepted +**Date:** 2026-08-27 +**Extends:** [ADR 0004](0004-knowledge-graph-ontology.md), [ADR +0207](0207-repository-case-ontology-namespace-canonical.md), and +[ADR 0242](0242-private-content-semantic-coverage-audit.md) + +## Context + +An authorized real PostgreSQL source (`public.zcrht811_export_rows`, a +single-table extract of short timestamped records exported from an ERP +customer-relationship program) was made available at an aggregate-only +analysis boundary. Repository artifacts must not retain any source value, +identifier, organization name, or artifact path (ADR 0001). The question the +audit must answer: is the published Ontology and Semantic Layer sufficient to +express the export's content, and where it is not, does the gap belong to +(a) a missing derived-semantic term, (b) a raw source code that must remain +instance data until its code system is governed, or (c) a column that carries +no semantic meaning of its own? + +The export is a set of business-document rows with a governed source-type +field (VOC / VOCC / VOCO / VOM / VOP), raw ERP lifecycle and classification +codes, an authored document title/body containing the actual site/business +content, a user-attribution and record-timestamp trail, and a caller-owning +geographic country/region field. + +## Decision +1. **Governed document type is the only value now promoted to the SKOS + scheme layer.** The five-value source type field maps 1:1 onto the + governed five-value `voc_type` post-type scheme (ADR 0207, seeded by + migrations/0042): a source `VOC` record is a `:VoiceOfCustomerType` post, + `VOCC` a `:VoiceOfCustomerCustomerType`, and so on. No new document-type + concepts are minted. +2. **All other source lifecycle and classification codes stay instance + literals.** `grade`, `stage`, `detail-state`, reply and deletion flags, + and the product-unit codes are raw ERP codes whose code system a caller + or an upstream project governs; this ontology neither renames nor + interprets them (the documented `sourceStageCode` / `sourceDetailStateCode` + raw projections already exist for the analogous application rows). A code + with no governed system remains instance data, never a fabricated term nor + a local psychometric meaning (ADR 0145 boundaries apply). +3. **Derived semantic content is now locatable.** The audit surfaced that the + semantic layer expressed *that* a post concerns a place yet had no way to + carry the place name or its region. Two derived-semantic datatype + properties, `:locationName` and `:countryCode`, are added to the + `:Location` node with a SHACL node shape. They are properties of the + *derived semantic location node* -- like `:semanticConfidence` on + `:ProjectMention` -- not column projections of the relational tables, so + the ADR 0207 column-only discipline does not apply. +4. **Aggregate coverage reporting only.** All coverage facts in the ADR and + its supporting document are counts and distinct-code-set membership + statements over the export's code sets; no title, person, project number, + or artifact name is narrated. Reproduction uses the caller-supplied + read-only DSN and aggregate-only queries, matching ADR 0242 decision 5. + +## Consequences + +- Posters can name the plant site and country/region a record came from on a + buyer-facing post view without exposing internal source codes. +- Existing round-trip tests (lookup-code coverage) are untouched: the new + terms carry no `:lookupCode`, so there is nothing new for the relational + schema to seed. +- The raw ERP codes remain internal instance data; buying them into a + governed SKOS scheme is a future decision only when a caller governs the + underlying code authority (ADR 0145 measurement boundaries apply). \ No newline at end of file diff --git a/docs/adr/README.md b/docs/adr/README.md index 83e56345c..96c53cbee 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -22,11 +22,13 @@ decision from them. | [`GLOBAL_ASK_KNOWLEDGE_CUTOFF_REFERENCES.md`](../doctoring/GLOBAL_ASK_KNOWLEDGE_CUTOFF_REFERENCES.md) | [0216](0216-global-ask-knowledge-cutoff.md) | | [`GLOBAL_ASK_QUERY_REWRITE_REFERENCES.md`](../doctoring/GLOBAL_ASK_QUERY_REWRITE_REFERENCES.md) | [0217](0217-evidence-constrained-semantic-query-rewrite.md) | | [`MCP_GLOBAL_ASK_REFERENCES.md`](../doctoring/MCP_GLOBAL_ASK_REFERENCES.md) | [0218](0218-current-contract-mcp-global-ask.md) | +| [`SEMANTIC_COVERAGE_SAMPLING_REFERENCES.md`](../doctoring/SEMANTIC_COVERAGE_SAMPLING_REFERENCES.md) | [0242](0242-private-content-semantic-coverage-audit.md) | | [`operability/http-concurrency-evidence.md`](../operability/http-concurrency-evidence.md) | [0204](0204-analysis-run-short-transaction-delivery.md), [0212](0212-single-query-authorized-post-filter-options.md), [0213](0213-global-ask-embedding-pool-release.md) | | [`operability/mcp-concurrency-evidence.md`](../operability/mcp-concurrency-evidence.md) | [0218](0218-current-contract-mcp-global-ask.md) | | Evidence operations Dashboard (`/`) | [0206](0206-evidence-operations-dashboard.md) | | [`temporal-topic-context-influence-research.md`](../temporal-topic-context-influence-research.md) | [0210](0210-temporal-topic-context-influence-dashboard.md) | -| [`python-mathematical-compute-boundary-audit.md`](../doctoring/python-mathematical-compute-boundary-audit.md) | [0208](0208-externalize-local-mathematical-compute.md) | +| [`python-mathematical-compute-boundary-audit.md`](../doctoring/python-mathematical-compute-boundary-audit.md) | [0208](0208-externalize-local-mathematical-compute.md), [0245](0245-lineage-scoring-and-entity-resolution-owner-contract.md) | +| [`export-source-ontology-coverage.md`](../doctoring/export-source-ontology-coverage.md) | [0246](0246-export-source-ontology-coverage.md) | [0011](0011-prov-o-standard-relations.md) and [0065](0065-prov-o-provenance-boundary.md) cite the dated W3C PROV-O and PROV-DM Recommendations (https://www.w3.org/TR/2013/REC-prov-o-20130430/ and https://www.w3.org/TR/2013/REC-prov-dm-20130430/). diff --git a/docs/doctoring/SEMANTIC_COVERAGE_SAMPLING_REFERENCES.md b/docs/doctoring/SEMANTIC_COVERAGE_SAMPLING_REFERENCES.md new file mode 100644 index 000000000..fb69d8a91 --- /dev/null +++ b/docs/doctoring/SEMANTIC_COVERAGE_SAMPLING_REFERENCES.md @@ -0,0 +1,25 @@ +# Semantic coverage sampling references + +This supporting register documents the authorities used by ADR 0242. It does +not make a product decision independently of that ADR. + +Australian Bureau of Statistics. (2022). *Basic survey design: Sample design*. +https://www.abs.gov.au/websitedbs/D3310114.nsf/home/Basic%20Survey%20Design%20-%20Sample%20Design + +National Institute of Standards and Technology. (n.d.). *Selecting sample +sizes*. In *NIST/SEMATECH e-handbook of statistical methods*. +https://www.itl.nist.gov/div898/handbook/ppc/section3/ppc333.htm + +National Institute of Standards and Technology. (n.d.). *Confidence limits*. +In *NIST/SEMATECH e-handbook of statistical methods*. +https://www.itl.nist.gov/div898/handbook/prc/section2/old.prc271.htm + +NIST supplies the proportion sample-size relationship and finite-population +correction; ABS defines probability sampling as requiring determinable +selection probabilities and stratified sampling as independent random +selection within strata. ADR 0242 keeps LineageWeave at structural sample +identity and completeness validation and replays the versioned fast-mlsirm +Rust artifact for sample size, finite-population correction, and allocation. +Its current output is sample-level only; corpus inference remains unavailable +until a terminal Rust owner artifact attests the achieved estimand, estimator, +variance, and interval. diff --git a/docs/doctoring/export-source-ontology-coverage.md b/docs/doctoring/export-source-ontology-coverage.md new file mode 100644 index 000000000..2ad4ef054 --- /dev/null +++ b/docs/doctoring/export-source-ontology-coverage.md @@ -0,0 +1,52 @@ +# Export-source ontology coverage + +Supporting document for [ADR 0246](../adr/0246-export-source-ontology-coverage.md). +Read-only aggregate analysis of an authorized export source against the +published LineageWeave Ontology and Semantic Layer. Per ADR 0001 / ADR 0242 +decision 5, only non-identifying aggregate counts and abstract code-set +statements appear below; no source title, person, project name, organization +name, or artifact path is retained. + +## Coverage + +The export provides short business documents with a governed five-value +document type, a country/region attribute, ERP lifecycle codes, an authored +title/body, a document key, and creator/editor timestamps. The published +ontology covers everything except one derived-semantic location value, which +ADR 0246 closes. + +## Mapping summary + +| Export dimension | Aggregate observation | Ontology / Semantic Layer mapping | Status | +| --- | --- | --- | --- | +| Document type (`VOC`, `VOCC`, `VOCO`, `VOM`, `VOP`) | five governed codes, 43,814 rows | `:postTypeScheme` (ADR 0207, migrations/0042) | covered | +| Authored title / body | non-empty authored title per row; zero non-empty authored bodies | `:postTitle`, `:postBody` + `:bodyAvailable` (ADR 0240), content-semantic classes (ADR 0242) | covered | +| Country / region attribute | curated ISO-3166-1 letters with a region marker (`EU`) | `:Location` + `:countryCode` (new, ADR 0246) | covered after 0246 | +| Place name the content names | authored location mention | `:Location` + `:locationName` (new, ADR 0246) | covered after 0246 | +| Raw ERP codes (grade, stage, detail status, deletion flag, product-unit codes) | raw code sets, sparse deletion marker | raw instance literals only; `sourceStageCode` / `sourceDetailStateCode` discipline (ADR 0241); `StatusStage` stays concept-level | raw (ungoverned, intentional) | +| Record identity / document number | export-internal key | identity via `prov:wasDerivedFrom` provenance (ADR 0011); not a vocabulary fact | covered | +| Creator / editor + timestamps | per-row attribution | `prov:Agent` / `prov:Person` / `prov:Organization` via `prov_agent_type` (ADR 0207) | covered | + +## Relationship to ADR 0242 + +Same private-data discipline: this document carries only aggregate facts. +The earlier semantic-coverage run spoke to title/body completeness and +event/stage/status/KG-edge density; this document focuses on the ontology +vocabulary itself. Both keep the "raw code stays instance data until its +code system is governed" boundary -- no new lookup category is seeded by the +export audit. + +## Standard and literature grounding + +- International Organization for Standardization. (2020). *Codes for the + representation of names of countries and their subdivisions -- Part 1: + Country code* (ISO 3166-1:2020). +- Cyganiak, D., Wood, D., & Lanthaler, M. (Eds.). (2014). *RDF 1.1 Concepts + and Abstract Syntax* (W3C Recommendation, 25 February 2014). + https://www.w3.org/TR/2014/REC-rdf11-concepts-20150225/ +- Lebo, J., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV + Ontology* (W3C Recommendation, 30 April 2013). + https://www.w3.org/TR/2013/REC-prov-o-20130430/ +- Knublauch, H., & Kontokostas, D. (Eds.). (2017). *Shapes Constraint + Language (SHACL)* (W3C Recommendation, 20 July 2017). + https://www.w3.org/TR/2017/REC-shacl-20170720/ \ No newline at end of file diff --git a/docs/doctoring/python-mathematical-compute-boundary-audit.md b/docs/doctoring/python-mathematical-compute-boundary-audit.md index f1dbeee83..4208129b2 100644 --- a/docs/doctoring/python-mathematical-compute-boundary-audit.md +++ b/docs/doctoring/python-mathematical-compute-boundary-audit.md @@ -8,8 +8,9 @@ Python paths satisfy the Rust/GPU requirement. ## Product-boundary sources read -- LineageWeave `ARCHITECTURE.md` and accepted ADRs 0003, 0132, 0145, - 0200, 0201, and 0205. This exact head has no standalone canonical PRD. +- LineageWeave `docs/product-requirements.md`, `ARCHITECTURE.md`, and accepted + ADRs 0003, 0132, 0145, 0200, 0201, 0205, 0208, and 0245. The PRD is a + supporting product contract and ADRs remain normative. - TEPP `docs/product/prd-v0.4-approved.md`, whose approved TRSL-TM scope owns temporal, relational, multilingual, topic, event, and trajectory measurement. @@ -22,12 +23,14 @@ Python paths satisfy the Rust/GPU requirement. | Current LineageWeave path | Local computation | Owner | Consumer replacement | Principal callers / tests | |---|---|---|---|---| +| `lineageweave/channels.py` | inverse elapsed-day scoring, numeric secondary-key scoring, and `SequenceMatcher` label similarity | TEPP for calibrated temporal/event criterion evidence; RankWeave for Rust-backed similarity/ranking only after an accepted owner API | ADR 0245 Event-Lineage owner envelope; no local score fallback | `lineageweave/reconstruct.py`, estimation script; `tests/test_channels.py` | | `lineageweave/channel_weight_estimation.py` | dichotomization, synthetic simulation, MLS2PLM input construction, expected item information and normalization | fast-mlsirm, conditional on TEPP anchor | versioned anchored-weight artifact; strict digest/convergence validation | estimation scripts, seed/server/rebuild paths; `tests/test_channel_weight_estimation.py`, estimator-script tests | | `lineageweave/period_report.py` | response matrix, GRM/GPCM fit/FIPC/EAP, likelihood, category expectation, information ordering | fast-mlsirm | period-measurement artifact with item bank, scores, uncertainty, diagnostics | report ingestion and demo seed; period-report and report API tests | | `lineageweave/leftover_pairs.py` | residual matrix, complete-case selection, SVD/Gabriel coordinates, distances, reconstruction, axis shares | fast-mlsirm | residual-interaction artifact with observed/expected identity and coverage | `period_report.py`, report ingestion/seed; `tests/test_leftover_pairs.py`, report tests | | `lineageweave/embedding_client.py` and `backend/app/post_chat_ingestion.py` | cosine similarity, vector norms, maximum semantic score | RankWeave retrieval-score contract | ranked evidence envelope over ABAC-visible semantic units | reconstruction text channel and Global Ask retrieval; embedding/post-chat tests | | `lineageweave/knowledge_graph.py` | random walk with restart, convergence delta, adaptive relevance cutoff | RankWeave graph-ranking contract | ranked-node artifact with contribution and convergence evidence | related-person/entity API paths; knowledge-graph tests | | `lineageweave/reconstruct.py` | channel-weight renormalization, candidate-score fusion and minimum-score decision | RankWeave fusion; TEPP supplies independent lineage criterion | accepted edge-ranking artifact; LineageWeave persists selected edge and channel provenance | lineage rebuild/start/seed/server; reconstruct, persistence, API tests | +| `lineageweave/corporate_hierarchy_resolution.py` | fixed legal-suffix deletion, `SequenceMatcher` candidate score, and fixed catalog-binding threshold | unassigned; no ecosystem PRD/API currently accepts corporate-master entity resolution | ADR 0245 corporate-entity resolution envelope; remain unbound when unavailable | corporate/team/Keyman/entity-relationship ingestion; corporate resolution and tie tests | | `lineageweave/rankweave_client.py` | channel construction, token overlap, RRF weights and contribution arithmetic | RankWeave | strict ranking artifact exposing owner-computed contributions | `/api/rankings`, frontend Rankings; `tests/test_rankweave_client.py` and frontend tests | `lineageweave/post_evaluation.py` imports fast-mlsirm only for its published @@ -51,6 +54,11 @@ validation, and presentation formatting also remain LineageWeave concerns. - **RankWeave:** Rust-backed similarity, graph ranking, fusion, contribution, evaluation, and policy-selection artifacts. Its present Python calculation core is the correct product owner but not the final execution architecture. +- **Corporate entity resolution:** no owner is designated. A repository must + explicitly accept this product responsibility and ADR 0245's versioned + unique/miss/tie envelope before LineageWeave can replace the current local + candidate scorer. Neither contextual-orchestrator nor Keyverse acquires this + responsibility implicitly. ## Persistence and UI blast radius diff --git a/docs/ontology/lineageweave-kg-shapes.ttl b/docs/ontology/lineageweave-kg-shapes.ttl index 436eb401f..320d1957e 100644 --- a/docs/ontology/lineageweave-kg-shapes.ttl +++ b/docs/ontology/lineageweave-kg-shapes.ttl @@ -33,6 +33,14 @@ a owl:Ontology ; dcterms:title "LineageWeave Knowledge Graph SHACL shapes"@en ; dcterms:description "Closed-world validation shapes over the LineageWeave knowledge-graph ontology: required attributes, confidence bounds, and side disjointness."@en ; + sh:declare [ + sh:prefix "lw" ; + sh:namespace "https://contextualwisdomlab.github.io/LineageWeave/ontology#"^^xsd:anyURI ; + ] ; + sh:declare [ + sh:prefix "rdf" ; + sh:namespace "http://www.w3.org/1999/02/22-rdf-syntax-ns#"^^xsd:anyURI ; + ] ; owl:imports ; owl:versionInfo "1.0.0" . @@ -47,15 +55,66 @@ sh:maxCount 1 ; sh:datatype xsd:string ; sh:minLength 1 ; + sh:pattern "\\S" ; + ] ; + sh:property [ + sh:path :bodyAvailable ; + sh:name "body available" ; + sh:description "Every post distinguishes source body availability from semantic content." ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:datatype xsd:boolean ; + ] ; + sh:property [ + sh:path :hasPostType ; + sh:name "post type" ; + sh:description "The governed VOC type resolves to one published SKOS concept." ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( + :voiceOfCustomerType + :voiceOfCustomersCustomerType + :voiceOfCompetitorType + :voiceOfMarketType + :voiceOfPartnerType + ) ; + ] ; + sh:property [ + sh:path :sourceStageCode ; + sh:name "source stage code" ; + sh:maxCount 1 ; + sh:datatype xsd:string ; + sh:minLength 1 ; + sh:pattern "\\S" ; + ] ; + sh:property [ + sh:path :sourceDetailStateCode ; + sh:name "source detail-state code" ; + sh:maxCount 1 ; + sh:datatype xsd:string ; + sh:minLength 1 ; + sh:pattern "\\S" ; ] ; sh:property [ sh:path :postBody ; sh:name "post body" ; - sh:description "The preserved source representation is never absent." ; + sh:description "The preserved source representation is present as a literal; empty means the body dimension is unavailable (ADR 0240)." ; sh:minCount 1 ; sh:maxCount 1 ; sh:datatype xsd:string ; - sh:minLength 1 ; + ] ; + sh:sparql [ + a sh:SPARQLConstraint ; + sh:prefixes ; + sh:message "bodyAvailable must be true exactly when postBody contains non-whitespace text." ; + sh:select """ + SELECT $this + WHERE { + $this lw:postBody ?body ; lw:bodyAvailable ?available . + BIND(REGEX(STR(?body), "[^\\\\s\\u000B\\u000C\\u001C-\\u001F\\u0085\\u00A0\\u1680\\u2000-\\u200A\\u2028\\u2029\\u202F\\u205F\\u3000]") AS ?actualAvailable) + FILTER(?available != ?actualAvailable) + } + """ ; ] ; sh:property [ sh:path :createdAt ; @@ -183,3 +242,57 @@ a sh:NodeShape ; sh:class :OurSidePerson ; ] . + +:SemanticContentAssertionShape a sh:NodeShape ; + rdfs:label "Semantic content assertion shape" ; + sh:targetClass :SemanticContentAssertion ; + sh:sparql [ + a sh:SPARQLConstraint ; + sh:prefixes ; + sh:message "wasDerivedFromPost must identify the rdf:subject post." ; + sh:select """ + SELECT $this + WHERE { + $this rdf:subject ?subject ; lw:wasDerivedFromPost ?source . + FILTER(?subject != ?source) + } + """ ; + ] ; + sh:property [ + sh:path rdf:subject ; sh:minCount 1 ; sh:maxCount 1 ; sh:class :Post + ] ; + sh:property [ + sh:path rdf:predicate ; sh:minCount 1 ; sh:maxCount 1 ; sh:nodeKind sh:IRI + ] ; + sh:property [ + sh:path rdf:object ; sh:minCount 1 ; sh:maxCount 1 ; sh:nodeKind sh:IRI + ] ; + sh:property [ + sh:path :wasDerivedFromPost ; sh:minCount 1 ; sh:maxCount 1 ; sh:class :Post + ] ; + sh:property [ + sh:path :semanticEvidence ; sh:minCount 1 ; sh:maxCount 1 ; + sh:datatype xsd:string ; sh:minLength 1 ; sh:pattern "\\S" + ] . + +:LocationShape a sh:NodeShape ; + rdfs:label "Location shape" ; + rdfs:comment "A derived semantic location is meaningful only when it carries either the place name the content expresses (:locationName) or a governed region (:countryCode). ADR 0246: the export-source geographic data carries a country/region and an optional descriptor; both stay instance data, never recorded identifiers." ; + sh:targetClass :Location ; + sh:property [ + sh:path :locationName ; + sh:name "location name" ; + sh:description "The place a post's content names; at most one per derived location node." ; + sh:maxCount 1 ; + sh:datatype xsd:string ; + sh:minLength 1 ; + sh:pattern "\\S" ; + ] ; + sh:property [ + sh:path :countryCode ; + sh:name "country code" ; + sh:description "ISO 3166-1 alpha-2 country or region code, optionally allowing the caller-authored 'EU' region marker; keeps raw codes instance data." ; + sh:maxCount 1 ; + sh:datatype xsd:string ; + sh:pattern "^[A-Z]{2}$" ; + ] . diff --git a/docs/ontology/lineageweave-kg.ttl b/docs/ontology/lineageweave-kg.ttl index 0aeb57f9a..e095e0097 100644 --- a/docs/ontology/lineageweave-kg.ttl +++ b/docs/ontology/lineageweave-kg.ttl @@ -6,6 +6,10 @@ @prefix xsd: . @prefix prov: . @prefix org: . +@prefix dcterms: . +@prefix time: . +@prefix sosa: . +@prefix qudt: . ################################################################# # LineageWeave Knowledge Graph Ontology @@ -227,7 +231,31 @@ rdfs:domain :Post ; rdfs:range xsd:string ; rdfs:label "post body" ; - rdfs:comment "source_post.post_body -- the preserved source representation, never flattened into one opaque string by derived views." . + rdfs:comment "source_post.post_body -- the preserved source representation; an empty literal honestly records an evidenced missing-body dimension (ADR 0240)." . + +:bodyAvailable a owl:DatatypeProperty ; + rdfs:domain :Post ; + rdfs:range xsd:boolean ; + rdfs:label "body available" ; + rdfs:comment "True exactly when source_post.post_body contains non-whitespace source content; false is an unavailable state, not negative semantic evidence (ADR 0240)." . + +:hasPostType a owl:ObjectProperty ; + rdfs:domain :Post ; + rdfs:range skos:Concept ; + rdfs:label "has post type" ; + rdfs:comment "The governed source_post.voc_type_code resolved through :postTypeScheme; unknown codes fail before projection." . + +:sourceStageCode a owl:DatatypeProperty ; + rdfs:domain :Post ; + rdfs:range xsd:string ; + rdfs:label "source stage code" ; + rdfs:comment "Raw source_post.source_stage_code with no inferred lifecycle meaning (ADR 0241)." . + +:sourceDetailStateCode a owl:DatatypeProperty ; + rdfs:domain :Post ; + rdfs:range xsd:string ; + rdfs:label "source detail-state code" ; + rdfs:comment "Raw source_post.source_detail_state_code with no inferred inspection or quality meaning (ADR 0241)." . :eventOccurredAt a owl:DatatypeProperty ; rdfs:domain :Post ; @@ -436,3 +464,116 @@ :semanticConfidence a owl:DatatypeProperty ; rdfs:domain :ProjectMention ; rdfs:range xsd:decimal . + +################################################################# +# Content-semantic subjects (ADR 0242) +################################################################# + +:BusinessEventActivity a owl:Class ; + rdfs:subClassOf prov:Activity ; + rdfs:label "Business event or activity"@en ; + rdfs:comment "A source-described occurrence or activity, aligned with PROV-O Activity."@en . + +:ProductOrService a owl:Class ; + rdfs:label "Product or service"@en ; + rdfs:comment "An offering, deliverable, product, or service materially discussed by a post."@en . + +:CommunicationDocument a owl:Class ; + rdfs:subClassOf prov:Entity ; + rdfs:label "Communication or document"@en ; + rdfs:comment "A communication or documentary artifact discussed by a post."@en . + +:OrganizationRole a owl:Class ; + rdfs:subClassOf org:Role ; + rdfs:label "Organization role"@en ; + rdfs:comment "A role borne in an organizational context, aligned with the W3C Organization Ontology."@en . + +:Topic a owl:Class ; + rdfs:subClassOf skos:Concept ; + rdfs:label "Topic or domain"@en ; + rdfs:comment "A governed subject concept materially discussed by a post."@en . + +:Location a owl:Class ; + rdfs:subClassOf dcterms:Location ; + rdfs:label "Location"@en ; + rdfs:comment "A spatial region or named place aligned with DCMI Location."@en . + +:CommercialTransaction a owl:Class ; + rdfs:label "Commercial transaction"@en ; + rdfs:comment "A quotation, order, sale, purchase, contract, or other commercial exchange."@en . + +:FacilityAssetEquipment a owl:Class ; + rdfs:subClassOf sosa:FeatureOfInterest ; + rdfs:label "Facility, asset, or equipment"@en ; + rdfs:comment "A physical facility, asset, component, or equipment item observed or discussed, aligned with SOSA FeatureOfInterest."@en . + +:StatusStage a owl:Class ; + rdfs:subClassOf skos:Concept ; + rdfs:label "Status or stage"@en ; + rdfs:comment "A governed lifecycle state or process stage; raw source codes remain literals until their code system is governed."@en . + +:RequirementIssueRisk a owl:Class ; + rdfs:subClassOf prov:Entity ; + rdfs:label "Requirement, issue, or risk"@en ; + rdfs:comment "An evidence-bearing requirement, issue, constraint, defect, or risk discussed by a post."@en . + +:RelevantTimeInterval a owl:Class ; + rdfs:subClassOf time:TemporalEntity ; + rdfs:label "Relevant time interval or deadline"@en ; + rdfs:comment "A material temporal interval or deadline aligned with OWL-Time."@en . + +:QuantityMeasurement a owl:Class ; + rdfs:subClassOf qudt:QuantityValue ; + rdfs:label "Quantity or measurement"@en ; + rdfs:comment "A quantity value with its unit and provenance, aligned with QUDT QuantityValue."@en . + +:describesActivity a owl:ObjectProperty ; rdfs:domain :Post ; rdfs:range :BusinessEventActivity ; rdfs:label "describes activity"@en . +:concernsProductOrService a owl:ObjectProperty ; rdfs:domain :Post ; rdfs:range :ProductOrService ; rdfs:label "concerns product or service"@en . +:hasCommunicationDocument a owl:ObjectProperty ; rdfs:domain :Post ; rdfs:range :CommunicationDocument ; rdfs:label "has communication or document"@en . +:assignsOrganizationRole a owl:ObjectProperty ; rdfs:domain :Post ; rdfs:range :OrganizationRole ; rdfs:label "assigns organization role"@en . +:concernsTopic a owl:ObjectProperty ; rdfs:domain :Post ; rdfs:range :Topic ; rdfs:label "concerns topic"@en . +:concernsLocation a owl:ObjectProperty ; rdfs:domain :Post ; rdfs:range :Location ; rdfs:label "concerns location"@en . +:concernsTransaction a owl:ObjectProperty ; rdfs:domain :Post ; rdfs:range :CommercialTransaction ; rdfs:label "concerns transaction"@en . +:concernsFacilityAssetEquipment a owl:ObjectProperty ; rdfs:domain :Post ; rdfs:range :FacilityAssetEquipment ; rdfs:label "concerns facility, asset, or equipment"@en . +:hasStatusStage a owl:ObjectProperty ; rdfs:domain :Post ; rdfs:range :StatusStage ; rdfs:label "has status or stage"@en . +:concernsRequirementIssueRisk a owl:ObjectProperty ; rdfs:domain :Post ; rdfs:range :RequirementIssueRisk ; rdfs:label "concerns requirement, issue, or risk"@en . +:hasRelevantTimeInterval a owl:ObjectProperty ; rdfs:domain :Post ; rdfs:range :RelevantTimeInterval ; rdfs:label "has relevant time interval"@en . +:hasQuantityMeasurement a owl:ObjectProperty ; rdfs:domain :Post ; rdfs:range :QuantityMeasurement ; rdfs:label "has quantity or measurement"@en . + +# Semantic-location properties (ADR 0246). A :Location instance names the +# place a post's content actually mentions (a plant site, an office floor, a +# destination city); :countryCode is the ISO 3166-1 alpha-2 country/region +# code of that place as instance data. These are DATATYPE properties of the +# *derived semantic node*, so the "column-projection only" discipline for +# the relational source tables does not apply here -- the same precedent as +# :projectEvidence / :semanticConfidence on the derived :ProjectMention. +# The raw source code set this layer binds (land1_field, admin_txt) stays +# outside the relational ontology vocabulary; source values are instance +# data and are never retained in repository artifacts. +:locationName a owl:DatatypeProperty ; + rdfs:domain :Location ; + rdfs:range xsd:string ; + rdfs:label "location name"@en ; + rdfs:comment "The place a post's content names, as surfaced by derived content semantics. Named locations let a buyer see which site each recorded concern belongs to."@en . + +:countryCode a owl:DatatypeProperty ; + rdfs:domain :Location ; + rdfs:range xsd:string ; + rdfs:label "country code"@en ; + rdfs:comment "ISO 3166-1 alpha-2 code (International Organization for Standardization, 2020) identifying the country or region of the :Location instance. 'EU' is carried as an instance region code where a caller's authored data groups a region; no scheme is minted until a governed code vocabulary is required."@en . + +:SemanticContentAssertion a owl:Class ; + rdfs:subClassOf rdf:Statement, prov:Entity ; + rdfs:label "Semantic content assertion"@en ; + rdfs:comment "A provenance-bearing reification of one content-semantic relation; publication requires a source post and evidence text."@en . + +:semanticEvidence a owl:DatatypeProperty ; + rdfs:domain :SemanticContentAssertion ; + rdfs:range xsd:string ; + rdfs:label "semantic evidence"@en . + +:wasDerivedFromPost a owl:ObjectProperty ; + rdfs:subPropertyOf prov:wasDerivedFrom ; + rdfs:domain :SemanticContentAssertion ; + rdfs:range :Post ; + rdfs:label "was derived from post"@en . diff --git a/docs/product-requirements.md b/docs/product-requirements.md index 0d456f2ae..27a82b8f3 100644 --- a/docs/product-requirements.md +++ b/docs/product-requirements.md @@ -221,7 +221,8 @@ A release claim requires one exact protected-main head that proves: - Asynchronous delivery and database-pool isolation: ADR 0204, ADR 0213. - Knowledge Graph, ontology, and provenance: ADR 0004, ADR 0011, ADR 0065, ADR 0184, ADR 0207, ADR 0222. -- Semantic units and retrieval: ADR 0047, ADR 0062, ADR 0102, ADR 0217. +- Semantic units and retrieval: ADR 0047, ADR 0062, ADR 0102, ADR 0217, + ADR 0223, ADR 0240, ADR 0241, ADR 0242. - LLM/model boundary: ADR 0070, ADR 0072, ADR 0076, ADR 0079. - Measurement: ADR 0003, ADR 0145, ADR 0200, ADR 0205. - UX and publication: ADR 0118, ADR 0159. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 7704fa748..a22b01c50 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,7 +1,231 @@ # Product & Technical Gap Baseline -> Dashboard delivery snapshot: 2026-08-26 07:15 KST. Protected `main` was -> `494b54e2245040bcf02b45376f221c37cd437e76`. This local branch is not +## Authorized source semantic-coverage audit + +An aggregate-only inspection on 2026-08-26 found 43,814 source rows with +43,814 non-empty titles and zero non-empty bodies. Structured coverage was +40,001 customer-code rows (91.3%), 4,490 project-code rows (10.2%), 43,812 +VOC process-unit rows, and complete process-unit, sales-pool, actor, lineage, +and source-artifact provenance fields. No source value, identifier, +organization, table name, or artifact path is retained here. +A 2026-08-27 aggregate follow-up found that the 43,814 rows normalize to +43,707 distinct document keys. The key audit found exactly 43,707 semantic +document nodes, all 43,707 matched, with +zero source-only and zero semantic-only keys. Every semantic document has a +nonblank title, summary, event, stage, status, content manifest, and at least +one knowledge-graph edge carrying that document as evidence. +The aggregate is reproducible with +`scripts/audit_source_semantic_coverage.py`; table and column mappings remain +runtime inputs rather than committed source identifiers. + +Semantic-layer referential checks found no unknown ontology term references, +no missing graph endpoints, and no confidence values outside `[0, 1]`. +All 832,696 observed knowledge-graph edges carry a direct evidence identifier, +and no edge uses an ungoverned evidence status. Of 6,046 inferred edges, 3,024 +lack a direct evidence identifier; all 166 predicted edges lack one. Those +3,190 document-to-document edges span 5,080 distinct documents and have a +nonblank reason, but none joins to the current lineage edge set and only one +joins to an inference candidate with evidence. A direct evidence identifier is +not required to mislabel an inference or prediction as observed. Its generation +or derivation nevertheless requires qualified provenance, and the authorized +source-adjacent analysis database has zero of the 14 normalized PROV-O tables +required by ADR 0011. This is not a reason to mutate the source database: +replaying the application Compose target deployed all 14 tables and populated +the canonical 30 classes, 50 relations, and 14 qualification definitions. The +remaining gap is linkage: the 3,190 analysis edges are not bound to target +PROV resources, activities, assertions, or qualifications. Do not present them +as source-backed facts or qualified derivations until that normalized linkage +is persisted. +The export names 117 distinct opaque source-artifact references. A protected +runtime search found 117 local MHTML files whose complete SHA-256 digests match +the artifact catalog exactly. Replaying all 12.1 GB through the non-rendering +`mhtml-etl-gateway` parser succeeded for 117/117 artifacts, produced exactly +43,814 rows, found exactly one body-candidate header in every artifact, and +found zero nonblank body cells. The database's zero-body result is therefore +faithful to the complete authorized source bytes, not an ETL omission. Body +semantics remain honestly unavailable unless a different authoritative source +is connected; titles must not be copied into bodies. + +The current semantic layer is therefore **not sufficient for the source +content as a whole**. It covers typed Post, Person, CorporateEntity, Team, +Project candidates, raw source context hints, lineage keys, and temporal +provenance, but it cannot derive body semantic units, embeddings, summaries, +VISION evidence, or body-grounded ontology assertions from this export. +ADR 0240 and the PostgreSQL importer now accept an explicitly evidenced +missing-body dimension without copying titles into bodies. This makes the +structured records importable while keeping body-derived capabilities +unavailable instead of fabricated. +ADR 0241 additionally carries the governed VOC type and raw source stage/detail +state into contextual-orchestrator hints with exact column provenance. The +available reference catalog contains examples rather than complete code-system +definitions, so raw stage/detail values are retained only as source-code RDF +literals and hints; they are not minted as classified ontology concepts. + +An ADR 0242 private-content audit then validated eight disjoint deterministic +windows of ten titles (80/80 ordered outputs, four orchestration trace steps +per window). This is pipeline acceptance evidence only: the windows were not a +probability sample, had no known inclusion probabilities, and had no declared +confidence or margin-of-error target. Its observed counts found zero sampled +titles whose material meaning was completely expressible by the published +ontology. Missing dimensions in those 80 pipeline items were observed for +event/activity (55), product/service (37), communication/document type (33), +organization role (26), topic/domain (26), location/geography (24), commercial +transaction (18), facility/asset/equipment (17), status/stage (16), +requirement/issue/risk (15), time interval/deadline (13), and +quantity/measurement (6); a title may contribute to several dimensions. Two +remaining ten-title windows were not accepted after provider failures, so this +is explicitly an 80-record pipeline result, not a probability-sample, 100-record, +or corpus claim. +The reusable audit now rejects the previously observed 100-input/60-output +response, requires every ordered item plus a multi-agent trace, and prints only +complete non-identifying aggregates. It additionally fails closed unless the +caller supplies probability-sample manifest v3 with exact per-stratum +inclusion-probability numerator/denominator pairs, per-stratum frame digests, +ordered owner-token membership +digests, retained provider failures, and a canonical selection-manifest digest. +LineageWeave validates sample identity and completeness. Historical sample +outputs still emit `corpus_inference_available=false` because they predate a +bound terminal estimator/variance/interval artifact and cannot be upgraded by +copying their aggregate counts into a new envelope. +The current candidate consumer separately replays the immutable +`fast-mlsirm.sampling-design.v1` Rust artifact and binds its population, +ordered stratum populations, total sample size, allocation, and exact `(n_h, +N_h)` inclusion ratios to the selected frame manifest. That proves design arithmetic provenance, not achieved +semantic-coverage inference. Merged stacked fast-mlsirm PR #1458 adds the separate +`fast-mlsirm.achieved-proportion.v1` Rust artifact for a complete one-stratum +SRSWOR sample: it binds the design artifact and attests the achieved +sample-proportion estimator, SRSWOR design variance, and exact Wang/Konijn +equal-tailed hypergeometric interval. Its exhaustive small-population tests and +Wang published-table oracle pass; the complete fast-mlsirm Python suite passed +6,677 tests with 15 skips. LineageWeave's candidate consumer additionally +binds the terminal artifact, selection-manifest digest, current ontology +SHA-256, aggregate verdicts, and trace-count bounds into one audit SHA-256. +It also emits a validated aggregate-only PROV-O graph linking the audit entity +and activity to the selection manifest, Rust design, matching completed +attempt, ontology, and terminal Rust entities by content-addressed URNs; no +source record identifier is exposed. A terminal result is rejected when that +attempt does not match the selection, design, ontology, and accepted count. +This capability is not runtime evidence for the historical samples and is not +protected-integrated while prerequisite fast-mlsirm PR #1445 remains open; +stacked PR #1458 is merged into that still-unmerged feature branch, not +protected `main`. + +The next runtime-only acceptance frame contains 43,814 eligible titles. With +the ADR 0242 predeclared 95% confidence, five-percentage-point margin, and NIST +conservative unknown-proportion input `p=0.5`, the pinned Rust design artifact +selected 381 SRSWOR units. The ordered selection tokens and source query remain +outside git. Early executions were correctly rejected rather than partially +persisted: one credential boundary returned 401, an old local orchestrator +exceeded its request-body contract, and the then-current structured path first +omitted the multi-agent trace and later accepted a provider response that +violated the exact batch cardinality schema. contextual-orchestrator PR #891 +now carries the upstream candidate repairs: structured trace disclosure through +the audited trace-read boundary, removal of mock seed agents after successful +provider discovery, and response-format capability selection from explicit +provider catalog evidence. The candidate audit now also retains an owner-only, +aggregate PROV-O attempt artifact from the start of execution: the activity +uses the content-addressed selection manifest, Rust design artifact, and +ontology, while a rejection records only accepted-item count, failed batch, +and bounded error class. A stable non-identifying audit session, derived from +those same input hashes, now crosses the spawned provider boundary and is +bound to the request header, metadata, local telemetry, and attempt so +orchestration trace is not detached from PROV. +It never promotes partial verdicts to corpus inference +or exposes source identifiers. No 381-unit coverage estimate is accepted until +all 39 batches complete against one unchanged manifest. + +A runtime-only simple random sample without replacement then selected 100 new +records from an eligible frame of 43,714. The pre-augmentation audit accepted +all 100 ordered outputs in ten batches, with four contextual-orchestrator trace +steps for every batch and no provider failure; 0/100 were completely covered. +After adding public, standards-aligned content classes/properties plus a +PROV-O-derived semantic-assertion and SHACL evidence contract, the exact same +selection manifest again accepted 100/100 outputs with the same trace bounds: +20/100 were completely covered and 80/100 remained uncovered. The remaining +sample-level missing-dimension counts were event/activity (32), +facility/asset/equipment (35), organization role (35), other unmodeled meaning +(28), product/service (27), requirement/issue/risk (23), location/geography +(16), commercial transaction (14), project/initiative (14), status/stage (14), +topic/domain (10), communication/document type (8), time interval/deadline (5), +person/actor (4), and quantity/measurement (4). These are same-sample audit +counts, not estimated corpus prevalence or a confidence interval. + +A second runtime-only simple random sample without replacement selected 100 +records from the current 43,814-record eligible title frame. Under the old +ambiguous rubric, two complete same-sample runs disagreed materially (20/100 +versus 10/100 covered), so neither result is accepted as stable semantic-gap +evidence. ADR 0242 now distinguishes source-specific instance data from a +missing public schema term and supplies the audit with the existing canonical +PROV-O registry rather than only its import/mapping profile. On the exact same +selection manifest, that revision produced 100/100 covered, zero failures, ten +batches, and four trace steps per batch in two consecutive runs. + +A further independently selected 100-record sample then exposed a remaining +measurement flaw: asking the model to select ontology IRIs yielded 79/100 once, +then failed closed on an invented near-match; replacing long IRIs with numeric +ids avoided that syntax error but changed the verdict to 50/100. The accepted +contract now separates the tasks: contextual-orchestrator classifies only the +governed semantic dimensions, while LineageWeave deterministically resolves +those dimensions against present class/property IRIs. On the unchanged second +manifest, this contract produced 100/100 covered, zero failures, ten batches, +and four trace steps per batch in two consecutive runs. These results prove +repeatable coverage of two sampled title sets only; they are not a corpus +estimate and do not repair the export's zero-body evidence gap. + +A governed-stratum runtime-only audit then used five source-type strata with 20 +records selected without replacement from each stratum (100 total). The exact +manifest produced 100/100 covered, zero failures, ten batches, and four trace +steps per batch in two consecutive runs. Balanced allocation here broadens +type-level diagnostic coverage; it is not a population-weighted estimator and +still emits `corpus_inference_available=false`. + +A separate non-probability diagnostic excluded the first deterministic 100 +records and selected 100 records from each of five event/update-time strata +(500 total). Every stratum again had 100/100 governed VOC type, stage, and +detail-state values but 0/100 non-empty bodies. Across the 500 records, +customer code was present for 447, project code for 50, country code for 486, +and due-date text for 0. Each stratum retained all five governed VOC types; +stage had 2–4 distinct raw values, detail state 3–4, and country 9–14. This +observed the same missing-body boundary and structured hint availability in +each diagnostic stratum; it is not probability-sample or corpus-prevalence +evidence and supplies no confidence interval. +The advertised deployment alias needed by this multi-agent path is repaired in +the canonical contextual-orchestrator PR #868. PR #870 was closed unmerged +after its explicit-conduct regression was composed into #868; until #868's +exact head passes its protected checks and independent review, the runtime path +remains candidate evidence. +LineageWeave's Compose bootstrap now also forwards the normalized configured +provider-host allowlist to the orchestrator CLI; without that handoff, runtime +discovery silently retained a blank model placeholder despite valid gateway +credentials and model inventory. + +Remaining acceptance gaps: + +- protected-integrate fast-mlsirm PR #1445 (including merged stack #1458), then run a new complete + one-stratum probability sample through the exact terminal artifact before + making any corpus coverage estimate; the stratified path remains explicitly + unavailable until it has its own governed estimator, covariance, and + interval; +- continue periodic independent and governed-stratum probability samples as + the source changes; the 381-unit one-stratum design supports only overall + eligible-frame prevalence, while rare semantic-dimension or category + precision remains unavailable until governance predeclares the strata, + category estimand, and acceptance precision and fast-mlsirm attests a matching + stratified terminal estimator, covariance, and interval artifact; reconcile + any newly uncovered meaning with public standards before adding a schema term, + and never mint source-local codes as public concepts; +- connect an authoritative body/file source and prove non-zero, ordered + semantic-unit persistence before claiming PRD-FR-4 corpus coverage; +- obtain governed source definitions before mapping grade, inspection, + lifecycle-detail, country, due-date, or artifact fields to ontology terms; +- publish only terms with domain/range, provenance, SHACL, API, and rendered + acceptance evidence; opaque source codes remain raw hints until then; +- run an authenticated import/backfill and report only non-identifying + aggregate counts for content units, embeddings, proposed/verified facts, + and unavailable channels. + +> Dashboard delivery snapshot: 2026-08-26 19:30 KST. Protected `main` was +> `ff7431bd1851c03e737808d22c6a2d43968582f9`. This local branch is not > protected-main release evidence. ## Operations Dashboard PRD/TRD traceability @@ -60,14 +284,14 @@ only aggregate, non-identifying evidence to this repository. ### Exact open-PR boundary -At this snapshot there were 11 open PRs and 10 open issues. PRs #660 and #659 -merged to protected `main`; PR #666 remains only non-default-branch stack -composition inside #663. Every remaining open head required refreshed hosted -gates and/or independent review after the base changed. These observations are -not merge readiness. Re-fetch exact heads, unresolved threads, checks, -approvals, rulesets, and merge SHA before any lifecycle claim. +At this snapshot there were 17 open PRs and 10 open issues. The exact-head +inventory in section 1 is authoritative for this snapshot. Every open head +remained blocked on hosted gates and/or independent review. These observations +are not merge readiness. Re-fetch exact heads, +unresolved threads, checks, approvals, rulesets, and merge SHA before any +lifecycle claim. -> Audit snapshot: 2026-08-26 07:15 KST (refreshed by the autonomous merge +> Audit snapshot: 2026-08-26 19:30 KST (refreshed by the autonomous merge > loop). This repository records synthetic fixtures and aggregate, > non-identifying runtime evidence only. Open PRs and local checks are not > protected-default-branch release evidence. Identifying post identifiers, @@ -76,25 +300,31 @@ approvals, rulesets, and merge SHA before any lifecycle claim. ## 1. Exact-head and governance evidence -The protected default branch was `494b54e2245040bcf02b45376f221c37cd437e76` -when this baseline was refreshed. The live queue contained 11 open PRs and 10 +The protected default branch was `ff7431bd1851c03e737808d22c6a2d43968582f9` +when this baseline was refreshed. The live queue contained 17 open PRs and 10 open issues. The exact-head inventory below supersedes older per-PR snapshots elsewhere in this document; those older rows remain useful historical delivery context only. | PR | Exact observed head | Merge/check state at this snapshot | | ---: | --- | --- | -| #667 | `3bc662d7` | refreshes protected-main and open-queue documentation evidence; base conflict remains to be repaired | -| #663 | `6fd2f701` | combined Project ontology candidate plus #666's non-default-branch removal of sampled region-coverage arithmetic; base conflict remains to be repaired | -| #658 | `f007a5ed` | evidence-honest Global Ask cutoff; hosted checks and independent review required | -| #657 | `2d9b43b7` | TEPP asynchronous lifecycle persistence while unpublished producer work stays unavailable; hosted checks and independent review required | -| #644 | `ed8d97f3` | native frontend surface code splitting; hosted checks and independent review required | -| #643 | `7fb4d18c` | shared token-backed status notice; hosted checks and independent review required | -| #640 | `2d50fa01` | dashboard case metrics and project journeys; base conflict remains to be repaired | -| #639 | `48065ad1` | restores Running action and Compose contracts; hosted checks and independent review required | -| #632 | `29aee18d` | graph-fact provenance, public verification, MCP admission, and k6 evidence; hosted checks and independent review required | -| #631 | `665046dc` (observed parent) | decomposes closed PR #490; this merge refresh advances its head and restarts hosted review evidence | -| #629 | `0138db5f` | provider-work release and bounded landing reads refreshed onto protected `main`; hosted checks and independent review restarted | +| #703 | `e5a483e2` | stacked on #640; conflicting and not eligible to retarget or merge before the parent reaches protected `main` | +| #702 | `e4c61407` | mergeable but blocked; exact-head checks and independent review required | +| #701 | `cc3351a9` | mergeable but blocked; exact-head checks and independent review required | +| #700 | `495b4504` | mergeable but blocked; exact-head checks and independent review required | +| #680 | `ff4d9eaf` | mergeable but blocked; exact-head checks and independent review required | +| #679 | `e26a7208` | mergeable but blocked; exact-head checks and independent review required | +| #672 | `f78f036c` | mergeable but blocked; exact-head checks and independent review required | +| #668 | `f9c4bd65` | mergeable but blocked; exact-head checks and independent review required | +| #667 | `1754b2c2` | mergeable but blocked; exact-head checks and independent review required | +| #658 | `0ae09b83` | mergeable but blocked; exact-head checks and independent review required | +| #657 | `9f71681c` | mergeable but blocked; exact-head checks and independent review required | +| #644 | `f53dd28e` | mergeable but blocked; exact-head checks and independent review required | +| #643 | `42ba340e` | mergeable but blocked; exact-head checks and independent review required | +| #640 | `26bfea65` | mergeable but blocked; exact-head checks and independent review required; parent of #703 | +| #639 | `f1d7aaaa` | mergeable but blocked; exact-head checks and independent review required | +| #632 | `24262a99` | mergeable but blocked; exact-head checks and independent review required | +| #629 | `b721b0f2` | mergeable but blocked; exact-head checks and independent review required | No row above is merge evidence. Immediately before any lifecycle action, re-fetch the head, unresolved threads, formal reviews, rulesets, and same-head @@ -361,7 +591,8 @@ this file per §3.5 of the prior snapshot). | Gap | Current evidence | Acceptance requirement | | --- | --- | --- | -| Protected release | 12 open PRs at snapshot, all targeting `main` with normal auto-merge enabled. None has the required independent approval, and running checks on #631/#632/#663 are not treated as blockers for safe work on other PRs. #666's merge into the non-default #663 branch is not protected-main delivery | Terminal exact-head checks, no unresolved threads, two independent approvals including last-push approval, protected squash-merge SHA | +| Local lineage and organization scoring ownership | Protected `main` still executes Python elapsed-day decay, numeric secondary-key scoring, `SequenceMatcher` label/entity similarity, weight renormalization, a 50-record recency window, and fixed `0.3`/`0.6` decision floors. ADR 0208 froze this class of work, but no open PR supplies the complete owner artifacts and no repository currently accepts corporate-master entity resolution. TEPP project-history is temporal association only; current RankWeave APIs do not return the ADR 0245 Rust-computed edge or entity-resolution envelopes. Open PR #704 exact head `2948812e` adds a useful external evidence contract but reuses `_best_parent`/`active_weights` and locally computes window counts and contribution arithmetic, so it is not replacement-owner evidence | Land the versioned owner contracts in the construct-owning repositories, including snapshot/cutoff, separate evidence availability, method/model version, uncertainty/completion, digest, abstention/tie, and non-causal status. Then add strict LineageWeave adapters and persisted provenance before deleting `channels.py` scoring, local reconstruction normalization/window/floor decisions, and corporate `SequenceMatcher`/threshold binding. Missing owner evidence must produce no edge or catalog identity; do not transfer the existing heuristics upstream | +| Protected release | 17 open PRs at snapshot. Sixteen target `main` with normal auto-merge enabled; stacked child #703 targets #640 and must wait for its parent, then retarget to `main` and collect fresh evidence. None has the required independent approval, and queued checks are not treated as blockers for safe work on other PRs | Terminal exact-head checks, no unresolved threads, the current ruleset's one independent approval, and protected squash-merge SHA | | CI queue release latency | Two Tests runs for already merged PRs occupied the available runner slots while 54 newer runs remained queued. Manual cancellation released the stale work, but the central close workflow was itself queued behind those runs. #634 merged into #631's non-default branch and reuses the repository's existing per-PR concurrency group so a jobless close event can cancel obsolete Tests work before runner allocation; this is not protected-main delivery | Merge #631 through its refreshed protected gate; close a synthetic PR while its Tests run is active and verify the old run becomes cancelled, the close-event jobs remain skipped, and a newer exact-head run starts without manual intervention | | Evidence-grounded operations workspace | Protected-main #614 delivers governed semantic Ask, live Similar VOC, disjoint pending/failed analysis metrics, full Storybook state inventory, and current desktop/mobile screenshot evidence. Authorized-corpus backfill acceptance remains unavailable | Perform authenticated authorized-corpus acceptance with aggregate evidence and retain fail-closed no-match behavior | | Shared frontend gate | The ADR 0109 login repair is on protected `main`; eight older branches carried the defect and received the same verified repair this loop (#521–#560) | Keep every future branch cut from post-repair bases; re-verify with frontend lint/test/build before push | @@ -506,6 +737,28 @@ review latency are never blockers — keep working while they settle. 7. Fix only evidence-backed failures and repeat the protected merge gate. 8. Refresh this file each loop with the exact queue state. +## 12. Export-source ontology coverage (ADR 0246) + +The authorized export source was audited at the aggregate-only boundary +(ADR 0001 / 0242 decision 5): no source value, title, identifier, +organization, or artifact path is retained here. + +Coverage result: the export rows are expressible through the published +ontology. The governed five-value source document type (VOC family) maps 1:1 +onto the `postTypeScheme` concepts; authored title/body, content-semantic +classes, and PROV attribution cover the business-document content; raw ERP +lifecycle and classification codes stay instance literals under the +`sourceStageCode` / `sourceDetailStateCode` discipline (ADR 0241) without a +minted scheme. + +The single semantic-layer gap the audit surfaced -- a derived place node +could not carry the place name or region -- is closed by ADR 0246: +`:locationName` and an ISO 3166-1 country/region `:countryCode` datatype +properties on the derived `:Location` node, validated closed-world by the +new SHACL `:LocationShape`. Raw codes remain ungoverned instance data until +a caller governs their code system. Supporting evidence is in +`docs/doctoring/export-source-ontology-coverage.md`. + ## 11. Spec pointers (derive, do not fork) - Product/architecture: `ARCHITECTURE.md`, `AGENTS.md`, `CLAUDE.md` diff --git a/lineageweave/ontology.py b/lineageweave/ontology.py index 6c3b15521..5e9786573 100644 --- a/lineageweave/ontology.py +++ b/lineageweave/ontology.py @@ -119,17 +119,59 @@ def ontology_node_iri(node_type_code: str, node_id: str) -> str: ) +def project_source_post_rdf( + *, + post_id: str, + post_title: str, + post_body: str, + post_created_at: datetime, + voc_type_code: str, + source_stage_code: str | None = None, + source_detail_state_code: str | None = None, +) -> Graph: + """Project one authorized ``source_post`` row without interpreting raw codes.""" + canonical_post_id = str(UUID(post_id)) + if not post_title.strip(): + raise ValueError("post_title must be non-empty") + if post_created_at.tzinfo is None or post_created_at.utcoffset() is None: + raise ValueError("post_created_at must be timezone-aware") + post_type = _term_subject(voc_type_code) + if post_type is None or (post_type, SKOS.inScheme, LW.postTypeScheme) not in ONTOLOGY: + raise ValueError("voc_type_code must name a governed post type") + post = URIRef(ontology_node_iri("node_post", canonical_post_id)) + graph = Graph() + graph.bind("lw", LW) + graph.add((post, RDF.type, LW.Post)) + graph.add((post, LW.postTitle, Literal(post_title))) + graph.add((post, LW.postBody, Literal(post_body))) + graph.add((post, LW.bodyAvailable, Literal(bool(post_body.strip()), datatype=XSD.boolean))) + graph.add((post, LW.hasPostType, post_type)) + graph.add((post, LW.createdAt, Literal(post_created_at, datatype=XSD.dateTime))) + for predicate, value in ( + (LW.sourceStageCode, source_stage_code), + (LW.sourceDetailStateCode, source_detail_state_code), + ): + if value is not None: + if not value.strip(): + raise ValueError("source classification codes must be non-empty when provided") + graph.add((post, predicate, Literal(value))) + return graph + + def project_project_mention_rdf( *, post_id: str, post_title: str, post_body: str, post_created_at: datetime, + voc_type_code: str, project_key: str, project_name: str, evidence_text: str, confidence: Decimal | float | str, mention_created_at: datetime, + source_stage_code: str | None = None, + source_detail_state_code: str | None = None, ) -> Graph: """Project one authorized joined Post/Project-mention row to RDF. @@ -141,8 +183,6 @@ def project_project_mention_rdf( if normalize_project_key(project_key) != project_key: raise ValueError("project_key must already be normalized") for field_name, value in ( - ("post_title", post_title), - ("post_body", post_body), ("project_name", project_name), ("evidence_text", evidence_text), ): @@ -167,13 +207,16 @@ def project_project_mention_rdf( mention = URIRef( LW[f"statement/project-mention/{canonical_post_id}/{quote(project_key, safe='')}"] ) - graph = Graph() - graph.bind("lw", LW) + graph = project_source_post_rdf( + post_id=canonical_post_id, + post_title=post_title, + post_body=post_body, + post_created_at=post_created_at, + voc_type_code=voc_type_code, + source_stage_code=source_stage_code, + source_detail_state_code=source_detail_state_code, + ) graph.bind("prov", PROV) - graph.add((post, RDF.type, LW.Post)) - graph.add((post, LW.postTitle, Literal(post_title))) - graph.add((post, LW.postBody, Literal(post_body))) - graph.add((post, LW.createdAt, Literal(post_created_at, datatype=XSD.dateTime))) graph.add((project, RDF.type, LW.Project)) graph.add((project, RDFS.label, Literal(project_name))) graph.add((post, LW.mentionsProject, project)) @@ -202,4 +245,5 @@ def project_project_mention_rdf( "ontology_node_iri", "ontology_annotations", "project_project_mention_rdf", + "project_source_post_rdf", ] diff --git a/lineageweave/semantic_hints.py b/lineageweave/semantic_hints.py index d594eb825..78dc87c8b 100644 --- a/lineageweave/semantic_hints.py +++ b/lineageweave/semantic_hints.py @@ -59,6 +59,9 @@ def format_semantic_hints( source_customer_catalog_name: str | None = None, source_project_code: str | None = None, source_project_name: str | None = None, + source_voc_type_code: str | None = None, + source_stage_code: str | None = None, + source_detail_state_code: str | None = None, source_context_present: bool = False, ) -> str: """Render source-field hints without upgrading them into assertions.""" @@ -183,6 +186,9 @@ def format_semantic_hints( f"source_customer_name_hint_trust={source_customer_name_trust}", f"source_project_code={_value(source_project_code)} [source_field=source_post.source_project_code]", f"source_project_name={_value(source_project_name)} [source_field=source_post.source_project_name]", + f"source_voc_type_code={_value(source_voc_type_code)} [source_field=source_post.voc_type_code]", + f"source_stage_code={_value(source_stage_code)} [source_field=source_post.source_stage_code]", + f"source_detail_state_code={_value(source_detail_state_code)} [source_field=source_post.source_detail_state_code]", *catalog_hints, ) ) diff --git a/pyproject.toml b/pyproject.toml index e98205768..121275647 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ backend = [ # yet; pinned to a specific commit, same pattern as rankweave. Ships a # PyO3/maturin Rust core with no fallback wheel, so building this from # source needs a Rust toolchain -- see backend/Dockerfile. - "fast-mlsirm @ git+https://github.com/ContextualWisdomLab/fast-mlsirm.git@d025b7d237d8db7ca97a5611606c6285d5870895", + "fast-mlsirm @ git+https://github.com/ContextualWisdomLab/fast-mlsirm.git@9cd12d6f74b8145c0d2d405c3ddf1859265fe93e", ] [tool.setuptools.packages.find] diff --git a/scripts/audit_source_content_semantics.py b/scripts/audit_source_content_semantics.py new file mode 100644 index 000000000..ee081b477 --- /dev/null +++ b/scripts/audit_source_content_semantics.py @@ -0,0 +1,1171 @@ +"""Audit private source content against the ontology without emitting source text.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import multiprocessing +import os +import queue +import re +import tempfile +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import asdict +from pathlib import Path +from typing import Any, cast + +import asyncpg +from rdflib import Graph, URIRef +from rdflib.namespace import OWL, RDF, RDFS, SKOS + +from lineageweave.http_client import chat_completion_content, post_json +from lineageweave.llm_context import use_llm_metadata +from lineageweave.prov_o import ( + PROV, + PROV_CLASSES, + PROV_QUALIFICATIONS, + PROV_RELATIONS, + ProvGraph, +) + +SEMANTIC_DIMENSIONS = frozenset( + { + "event_or_activity", + "location_or_geography", + "product_or_service", + "project_or_initiative", + "facility_asset_or_equipment", + "topic_or_domain", + "status_or_stage", + "time_interval_or_deadline", + "organization_role", + "person_or_actor", + "communication_or_document_type", + "commercial_transaction", + "quantity_or_measurement", + "requirement_issue_or_risk", + "other_unmodeled_meaning", + } +) +_ONTOLOGY_NAMESPACE = "https://contextualwisdomlab.github.io/LineageWeave/ontology#" +SEMANTIC_DIMENSION_TERM_IRIS: Mapping[str, tuple[str, ...]] = { + "event_or_activity": ( + _ONTOLOGY_NAMESPACE + "BusinessEventActivity", + _ONTOLOGY_NAMESPACE + "describesActivity", + str(PROV.Activity), + ), + "location_or_geography": ( + _ONTOLOGY_NAMESPACE + "Location", + _ONTOLOGY_NAMESPACE + "concernsLocation", + str(PROV.Location), + ), + "product_or_service": ( + _ONTOLOGY_NAMESPACE + "ProductOrService", + _ONTOLOGY_NAMESPACE + "concernsProductOrService", + ), + "project_or_initiative": ( + _ONTOLOGY_NAMESPACE + "Project", + _ONTOLOGY_NAMESPACE + "mentionsProject", + ), + "facility_asset_or_equipment": ( + _ONTOLOGY_NAMESPACE + "FacilityAssetEquipment", + _ONTOLOGY_NAMESPACE + "concernsFacilityAssetEquipment", + ), + "topic_or_domain": ( + _ONTOLOGY_NAMESPACE + "Topic", + _ONTOLOGY_NAMESPACE + "concernsTopic", + ), + "status_or_stage": ( + _ONTOLOGY_NAMESPACE + "StatusStage", + _ONTOLOGY_NAMESPACE + "hasStatusStage", + ), + "time_interval_or_deadline": ( + _ONTOLOGY_NAMESPACE + "RelevantTimeInterval", + _ONTOLOGY_NAMESPACE + "hasRelevantTimeInterval", + str(PROV.atTime), + ), + "organization_role": ( + _ONTOLOGY_NAMESPACE + "OrganizationRole", + _ONTOLOGY_NAMESPACE + "assignsOrganizationRole", + str(PROV.Role), + ), + "person_or_actor": ( + _ONTOLOGY_NAMESPACE + "Person", + _ONTOLOGY_NAMESPACE + "mentions", + str(PROV.Person), + ), + "communication_or_document_type": ( + _ONTOLOGY_NAMESPACE + "CommunicationDocument", + _ONTOLOGY_NAMESPACE + "hasCommunicationDocument", + str(PROV.Communication), + ), + "commercial_transaction": ( + _ONTOLOGY_NAMESPACE + "CommercialTransaction", + _ONTOLOGY_NAMESPACE + "concernsTransaction", + ), + "quantity_or_measurement": ( + _ONTOLOGY_NAMESPACE + "QuantityMeasurement", + _ONTOLOGY_NAMESPACE + "hasQuantityMeasurement", + ), + "requirement_issue_or_risk": ( + _ONTOLOGY_NAMESPACE + "RequirementIssueRisk", + _ONTOLOGY_NAMESPACE + "concernsRequirementIssueRisk", + ), + "other_unmodeled_meaning": (), +} +_CODE_FENCE = re.compile(r"^```(?:json)?\s*|\s*```$", re.DOTALL) +_SHA256 = re.compile(r"[0-9a-f]{64}$") +_SAMPLE_DESIGNS = { + "simple_random_without_replacement", + "stratified_random_without_replacement", +} + + +def _canonical_sha256(value: object) -> str: + """Hash a JSON-compatible artifact with a stable, whitespace-free encoding.""" + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def validate_probability_sample_manifest( + payload: object, expected_sample_size: int +) -> tuple[dict[str, object], tuple[tuple[str, str], ...]]: + """Validate a caller-supplied probability-sample artifact without doing its math.""" + required = { + "contract_kind", + "contract_version", + "population_size", + "sample_size", + "design_code", + "provider_failures_retained", + "strata", + "selected_units", + "selection_manifest_sha256", + } + if not isinstance(payload, dict) or set(payload) != required: + raise ValueError( + "sample manifest fields do not match the probability-sample contract" + ) + if ( + payload["contract_kind"] != "lineageweave.semantic_coverage_probability_sample" + or payload["contract_version"] != 3 + ): + raise ValueError("unsupported probability-sample manifest contract") + population_size = payload["population_size"] + sample_size = payload["sample_size"] + if ( + type(population_size) is not int + or population_size < 1 + or type(sample_size) is not int + or sample_size != expected_sample_size + or sample_size > population_size + ): + raise ValueError("sample manifest population or sample size is invalid") + if payload["design_code"] not in _SAMPLE_DESIGNS: + raise ValueError("sample manifest must use a supported probability design") + if payload["provider_failures_retained"] is not True: + raise ValueError( + "sample manifest must retain provider failures in the declared sample" + ) + + strata = payload["strata"] + if not isinstance(strata, list) or not strata: + raise ValueError("sample manifest requires at least one probability stratum") + if ( + payload["design_code"] == "simple_random_without_replacement" + and len(strata) != 1 + ) or ( + payload["design_code"] == "stratified_random_without_replacement" + and len(strata) < 2 + ): + raise ValueError("sample manifest strata do not match its probability design") + stratum_fields = { + "stratum_code", + "population_size", + "sample_size", + "inclusion_probability_numerator", + "inclusion_probability_denominator", + "selection_frame_sha256", + } + stratum_codes: set[str] = set() + stratum_populations: dict[str, int] = {} + stratum_samples: dict[str, int] = {} + for stratum in strata: + if not isinstance(stratum, dict) or set(stratum) != stratum_fields: + raise ValueError("sample manifest stratum fields are invalid") + code = stratum["stratum_code"] + stratum_population = stratum["population_size"] + stratum_sample = stratum["sample_size"] + if not isinstance(code, str) or not code.strip() or code in stratum_codes: + raise ValueError( + "sample manifest stratum codes must be unique and nonblank" + ) + stratum_codes.add(code) + if ( + type(stratum_population) is not int + or stratum_population < 1 + or type(stratum_sample) is not int + or stratum_sample < 1 + or stratum_sample > stratum_population + ): + raise ValueError("sample manifest stratum sizes are invalid") + stratum_populations[code] = stratum_population + stratum_samples[code] = stratum_sample + if ( + not isinstance(stratum["selection_frame_sha256"], str) + or _SHA256.fullmatch(stratum["selection_frame_sha256"]) is None + ): + raise ValueError( + "sample manifest requires a selection-frame SHA-256 per stratum" + ) + if sum(stratum_populations.values()) != population_size: + raise ValueError("sample manifest stratum populations must match population_size") + if sum(stratum_samples.values()) != sample_size: + raise ValueError("sample manifest stratum samples must match sample_size") + if any( + type(stratum["inclusion_probability_numerator"]) is not int + or type(stratum["inclusion_probability_denominator"]) is not int + or stratum["inclusion_probability_numerator"] != stratum["sample_size"] + or stratum["inclusion_probability_denominator"] != stratum["population_size"] + for stratum in strata + ): + raise ValueError( + "sample manifest requires the exact sample/population inclusion ratio per stratum" + ) + selected_units = payload["selected_units"] + selected_unit_fields = {"ordinal", "selection_token_sha256", "stratum_code"} + if not isinstance(selected_units, list) or len(selected_units) != sample_size: + raise ValueError("sample manifest selected-unit count must match sample_size") + membership: list[tuple[str, str]] = [] + for ordinal, unit in enumerate(selected_units): + if not isinstance(unit, dict) or set(unit) != selected_unit_fields: + raise ValueError("sample manifest selected-unit fields are invalid") + token_digest = unit["selection_token_sha256"] + stratum_code = unit["stratum_code"] + if unit["ordinal"] != ordinal: + raise ValueError( + "sample manifest selected-unit ordinals must be contiguous and ordered" + ) + if not isinstance(token_digest, str) or _SHA256.fullmatch(token_digest) is None: + raise ValueError("sample manifest selection-token digests must be SHA-256") + if not isinstance(stratum_code, str) or stratum_code not in stratum_codes: + raise ValueError("sample manifest selected unit names an unknown stratum") + membership.append((token_digest, stratum_code)) + if len({token_digest for token_digest, _ in membership}) != sample_size: + raise ValueError("sample manifest selection-token digests must be unique") + if Counter(stratum_code for _, stratum_code in membership) != Counter( + stratum_samples + ): + raise ValueError( + "sample manifest selected-unit strata must match stratum sample sizes" + ) + + selection_digest = payload["selection_manifest_sha256"] + if ( + not isinstance(selection_digest, str) + or _SHA256.fullmatch(selection_digest) is None + or selection_digest != _canonical_sha256(selected_units) + ): + raise ValueError("selected sample does not match its manifest digest") + return ( + { + "design_code": payload["design_code"], + "population_size": population_size, + "sample_size": sample_size, + "stratum_count": len(strata), + "selection_manifest_sha256": selection_digest, + "corpus_inference_available": False, + }, + tuple(membership), + ) + + +def validate_sampling_design_artifact( + payload: object, sample_manifest: Mapping[str, object] +) -> dict[str, object]: + """Replay and bind a package-owned Rust sampling-design artifact.""" + from fast_mlsirm import SamplingStratum, finite_population_proportion_design + + required = { + "schema_version", + "source_identity", + "source_sha256", + "algorithm_version", + "population_size", + "expected_proportion", + "confidence_level", + "margin_of_error", + "critical_value", + "uncorrected_sample_size", + "sample_size", + "finite_population_correction", + "allocation_method", + "strata", + "stratum_sample_sizes", + "stratum_inclusion_probability_ratios", + "input_sha256", + "output_sha256", + "artifact_sha256", + } + if not isinstance(payload, dict) or set(payload) != required: + raise ValueError("sampling design fields do not match the Rust artifact contract") + strata = payload["strata"] + if not isinstance(strata, list) or not strata: + raise ValueError("sampling design artifact requires ordered strata") + if any( + not isinstance(stratum, dict) + or set(stratum) != {"population_size", "expected_proportion"} + for stratum in strata + ): + raise ValueError("sampling design artifact strata are invalid") + try: + replay = finite_population_proportion_design( + payload["population_size"], + payload["confidence_level"], + payload["margin_of_error"], + [ + SamplingStratum( + stratum["population_size"], stratum["expected_proportion"] + ) + for stratum in strata + ], + allocation_method=payload["allocation_method"], + ) + except (TypeError, ValueError) as exc: + raise ValueError("sampling design artifact cannot be replayed by Rust") from exc + replay_payload = json.loads(json.dumps(asdict(replay), sort_keys=True)) + if payload != replay_payload: + raise ValueError("sampling design artifact does not match Rust replay") + + manifest_strata = sample_manifest.get("strata") + if not isinstance(manifest_strata, list): + raise TypeError("sample manifest strata are unavailable for design binding") + if payload["population_size"] != sample_manifest.get("population_size") or payload[ + "sample_size" + ] != sample_manifest.get("sample_size"): + raise ValueError("sampling design artifact totals do not match the sample manifest") + if [stratum["population_size"] for stratum in strata] != [ + stratum.get("population_size") + for stratum in manifest_strata + if isinstance(stratum, dict) + ] or payload["stratum_sample_sizes"] != [ + stratum.get("sample_size") + for stratum in manifest_strata + if isinstance(stratum, dict) + ]: + raise ValueError("sampling design artifact allocation does not match the sample manifest") + if payload["stratum_inclusion_probability_ratios"] != [ + [ + stratum.get("inclusion_probability_numerator"), + stratum.get("inclusion_probability_denominator"), + ] + for stratum in manifest_strata + if isinstance(stratum, dict) + ]: + raise ValueError( + "sampling design artifact inclusion ratios do not match the sample manifest" + ) + return { + "schema_version": payload["schema_version"], + "source_identity": payload["source_identity"], + "source_sha256": payload["source_sha256"], + "algorithm_version": payload["algorithm_version"], + "input_sha256": payload["input_sha256"], + "output_sha256": payload["output_sha256"], + "artifact_sha256": payload["artifact_sha256"], + "confidence_level": payload["confidence_level"], + "margin_of_error": payload["margin_of_error"], + "sample_size": payload["sample_size"], + "stratum_inclusion_probability_ratios": payload[ + "stratum_inclusion_probability_ratios" + ], + "sampling_design_verified": True, + "corpus_inference_available": False, + } + + +def terminal_semantic_coverage_evidence( + sample_design_artifact: Mapping[str, object], + sample_design: Mapping[str, object], + aggregate: Mapping[str, object], + ontology_path: Path, + completed_attempt: Mapping[str, object], +) -> dict[str, object]: + """Build a Rust-owned terminal SRSWOR result and aggregate audit identity.""" + if sample_design.get("design_code") != "simple_random_without_replacement": + return { + "corpus_inference_available": False, + "corpus_inference_unavailable_reason": ( + "stratified_terminal_estimator_not_available" + ), + } + from fast_mlsirm import ( + SamplingStratum, + finite_population_achieved_proportion, + finite_population_proportion_design, + ) + ontology_sha256 = hashlib.sha256(ontology_path.read_bytes()).hexdigest() + if ( + completed_attempt.get("status_code") != "completed" + or completed_attempt.get("accepted_count") != aggregate.get("sample_count") + or not isinstance(completed_attempt.get("attempt_sha256"), str) + or completed_attempt.get("selection_manifest_sha256") + != sample_design.get("selection_manifest_sha256") + or completed_attempt.get("sampling_design_sha256") + != sample_design_artifact.get("artifact_sha256") + or completed_attempt.get("ontology_sha256") != ontology_sha256 + ): + raise ValueError("terminal coverage requires its completed attempt provenance") + + strata = sample_design_artifact.get("strata") + if not isinstance(strata, list) or len(strata) != 1: + raise ValueError("terminal SRSWOR evidence requires one design stratum") + stratum = strata[0] + if not isinstance(stratum, dict): + raise ValueError("terminal SRSWOR evidence requires one design stratum") + design = finite_population_proportion_design( + sample_design_artifact["population_size"], + sample_design_artifact["confidence_level"], + sample_design_artifact["margin_of_error"], + [SamplingStratum(stratum["population_size"], stratum["expected_proportion"])], + allocation_method=sample_design_artifact["allocation_method"], + ) + if design.artifact_sha256 != sample_design_artifact.get("artifact_sha256"): + raise ValueError("terminal coverage design does not match the Rust artifact") + sample_count = aggregate.get("sample_count") + covered_count = aggregate.get("covered_count") + uncovered_count = aggregate.get("uncovered_count") + if ( + aggregate.get("complete") is not True + or type(sample_count) is not int + or sample_count != design.sample_size + or type(covered_count) is not int + or type(uncovered_count) is not int + or covered_count + uncovered_count != sample_count + ): + raise ValueError("terminal coverage requires one complete design-sized audit") + achieved = finite_population_achieved_proportion(design, covered_count) + terminal_artifact = json.loads(json.dumps(asdict(achieved), sort_keys=True)) + audit_identity = { + "selection_manifest_sha256": sample_design["selection_manifest_sha256"], + "ontology_sha256": ontology_sha256, + "terminal_artifact_sha256": terminal_artifact["artifact_sha256"], + "complete": True, + "sample_count": sample_count, + "covered_count": covered_count, + "uncovered_count": uncovered_count, + "missing_semantic_dimension_counts": aggregate.get( + "missing_semantic_dimension_counts" + ), + "batch_count": aggregate.get("batch_count"), + "minimum_trace_step_count": aggregate.get("minimum_trace_step_count"), + "maximum_trace_step_count": aggregate.get("maximum_trace_step_count"), + "attempt_sha256": completed_attempt.get("attempt_sha256"), + } + audit_artifact_sha256 = _canonical_sha256(audit_identity) + resource_iris = { + "selection": "urn:sha256:" + str(sample_design["selection_manifest_sha256"]), + "ontology": "urn:sha256:" + ontology_sha256, + "terminal": "urn:sha256:" + str(terminal_artifact["artifact_sha256"]), + "audit": "urn:sha256:" + audit_artifact_sha256, + "activity": "urn:lineageweave:semantic-coverage-audit:" + audit_artifact_sha256, + "design": "urn:sha256:" + str(sample_design_artifact["artifact_sha256"]), + "attempt": "urn:sha256:" + str(completed_attempt["attempt_sha256"]), + } + provenance = ProvGraph() + for name in ("selection", "design", "ontology", "attempt", "terminal", "audit"): + provenance.add_resource(resource_iris[name], "Entity") + provenance.add_resource(resource_iris["activity"], "Activity") + for name in ("selection", "design", "ontology", "attempt", "terminal"): + provenance.add_assertion(resource_iris["activity"], "used", resource_iris[name]) + provenance.add_assertion(resource_iris["audit"], "wasDerivedFrom", resource_iris[name]) + provenance.add_assertion( + resource_iris["audit"], "wasGeneratedBy", resource_iris["activity"] + ) + prov_o = { + "resource_types": { + iri: sorted(types) for iri, types in sorted(provenance.resource_types.items()) + }, + "assertions": sorted( + ( + { + "subject_iri": assertion.subject_iri, + "relation_iri": str(PROV[assertion.relation]), + "object_iri": assertion.object_resource_iri, + } + for assertion in provenance.explicit_assertions + ), + key=lambda item: ( + item["subject_iri"], + item["relation_iri"], + item["object_iri"], + ), + ), + } + return { + "corpus_inference_available": True, + "rust_terminal_artifact": terminal_artifact, + "ontology_sha256": ontology_sha256, + "audit_artifact_sha256": audit_artifact_sha256, + "prov_o": prov_o, + "prov_o_sha256": _canonical_sha256(prov_o), + } + + +def audit_attempt_provenance( + *, + selection_manifest_sha256: str, + sampling_design_sha256: str, + ontology_sha256: str, + status_code: str, + accepted_count: int, + failed_batch_index: int | None = None, + failure_code: str | None = None, +) -> dict[str, object]: + """Describe an audit attempt without treating partial verdicts as a result.""" + if status_code not in {"in_progress", "completed", "rejected"}: + raise ValueError("unsupported audit-attempt status") + if status_code == "rejected" and not failure_code: + raise ValueError("a rejected audit attempt requires a failure code") + audit_session_id = "semantic-audit:" + _canonical_sha256( + { + "selection_manifest_sha256": selection_manifest_sha256, + "sampling_design_sha256": sampling_design_sha256, + "ontology_sha256": ontology_sha256, + } + ) + identity = { + "selection_manifest_sha256": selection_manifest_sha256, + "sampling_design_sha256": sampling_design_sha256, + "ontology_sha256": ontology_sha256, + "audit_session_id": audit_session_id, + "status_code": status_code, + "accepted_count": accepted_count, + "failed_batch_index": failed_batch_index, + "failure_code": failure_code, + } + attempt_sha256 = _canonical_sha256(identity) + resources = { + "selection": "urn:sha256:" + selection_manifest_sha256, + "design": "urn:sha256:" + sampling_design_sha256, + "ontology": "urn:sha256:" + ontology_sha256, + "attempt": "urn:sha256:" + attempt_sha256, + "activity": "urn:lineageweave:semantic-coverage-attempt:" + attempt_sha256, + } + provenance = ProvGraph() + for name in ("selection", "design", "ontology", "attempt"): + provenance.add_resource(resources[name], "Entity") + provenance.add_resource(resources["activity"], "Activity") + for name in ("selection", "design", "ontology"): + provenance.add_assertion(resources["activity"], "used", resources[name]) + provenance.add_assertion(resources["attempt"], "wasDerivedFrom", resources[name]) + provenance.add_assertion( + resources["attempt"], "wasGeneratedBy", resources["activity"] + ) + prov_o = { + "resource_types": { + iri: sorted(types) for iri, types in sorted(provenance.resource_types.items()) + }, + "assertions": sorted( + ( + { + "subject_iri": assertion.subject_iri, + "relation_iri": str(PROV[assertion.relation]), + "object_iri": assertion.object_resource_iri, + } + for assertion in provenance.explicit_assertions + ), + key=lambda item: ( + item["subject_iri"], + item["relation_iri"], + item["object_iri"], + ), + ), + } + return { + **identity, + "attempt_sha256": attempt_sha256, + "prov_o": prov_o, + "prov_o_sha256": _canonical_sha256(prov_o), + } + + +def _write_private_json(path: Path, payload: object) -> None: + """Atomically replace a runtime-only JSON artifact with owner-only permissions.""" + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(dir=path.parent, prefix=path.name + ".") + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + json.dump(payload, stream, sort_keys=True) + stream.write("\n") + os.chmod(temporary_name, 0o600) + os.replace(temporary_name, path) + finally: + if os.path.exists(temporary_name): + os.unlink(temporary_name) + + +def _post_json_worker( + result_queue: Any, + endpoint: str, + payload: dict[str, object], + headers: dict[str, str], + timeout: float, +) -> None: + """Run one provider request in a terminable child process.""" + try: + session_id = headers.get("x-lineageweave-session-id", "") + if session_id: + with use_llm_metadata({"session_id": session_id}): + response = post_json( + endpoint, payload, headers=headers, timeout=timeout + ) + else: + response = post_json(endpoint, payload, headers=headers, timeout=timeout) + result_queue.put((True, response)) + except Exception as exc: + result_queue.put((False, type(exc).__name__)) + + +def _post_json_with_deadline( + endpoint: str, + payload: dict[str, object], + *, + headers: dict[str, str], + timeout: float, +) -> dict[str, Any]: + """Enforce a wall-clock deadline even when a peer keeps a socket active.""" + if timeout <= 0: + raise ValueError("timeout must be positive") + context = multiprocessing.get_context("spawn") + result_queue = context.Queue(maxsize=1) + process = context.Process( + target=_post_json_worker, + args=(result_queue, endpoint, payload, headers, timeout), + ) + process.start() + try: + try: + succeeded, value = result_queue.get(timeout=timeout) + except queue.Empty as exc: + timed_out = process.is_alive() + if timed_out: + process.terminate() + process.join() + if timed_out: + raise TimeoutError( + "semantic audit provider request exceeded its deadline" + ) from exc + raise RuntimeError( + "semantic audit provider process returned no result" + ) from exc + if process.is_alive(): + process.terminate() + process.join() + if not succeeded: + raise RuntimeError(f"semantic audit provider request failed: {value}") + if not isinstance(value, dict): + raise RuntimeError("semantic audit provider response must be an object") + return value + finally: + result_queue.close() + result_queue.join_thread() + + +def parse_batch_result( + content: str, + expected_count: int, + supporting_terms_by_dimension: Mapping[str, tuple[str, ...]], +) -> tuple[dict[str, Any], ...]: + """Require one ordered, governed verdict for every submitted item.""" + candidate = ( + _CODE_FENCE.sub("", content.strip()) + if content.strip().startswith("```") + else content + ) + try: + payload = json.loads(candidate) + except json.JSONDecodeError as exc: + raise ValueError("semantic audit response must be JSON") from exc + if not isinstance(payload, dict) or payload.get("input_count") != expected_count: + raise ValueError( + "semantic audit input_count does not match the submitted batch" + ) + items = payload.get("items") + if not isinstance(items, list) or len(items) != expected_count: + raise ValueError("semantic audit item count does not match the submitted batch") + expected_indexes = list(range(expected_count)) + if [ + item.get("item_index") for item in items if isinstance(item, dict) + ] != expected_indexes: + raise ValueError( + "semantic audit item indexes are missing, duplicated, or unordered" + ) + for item in items: + if set(item) != {"item_index", "semantic_dimensions"}: + raise ValueError("semantic audit item has an unsupported field") + dimensions = item["semantic_dimensions"] + if not isinstance(dimensions, list) or not dimensions or any( + not isinstance(value, str) or value not in SEMANTIC_DIMENSIONS + for value in dimensions + ): + raise ValueError("semantic audit returned an ungoverned dimension") + if len(dimensions) != len(set(dimensions)): + raise ValueError("semantic audit returned a duplicate semantic dimension") + resolved = [] + for item in items: + dimensions = item["semantic_dimensions"] + missing = [ + dimension + for dimension in dimensions + if not supporting_terms_by_dimension.get(dimension) + ] + resolved.append( + { + "item_index": item["item_index"], + "covered": not missing, + "missing_semantic_dimensions": missing, + "supporting_term_iris": sorted( + { + iri + for dimension in dimensions + for iri in supporting_terms_by_dimension.get(dimension, ()) + } + ), + } + ) + return tuple(resolved) + + +def selected_contents( + records: Sequence[Mapping[str, Any]], selected_membership: Sequence[tuple[str, str]] +) -> list[str]: + """Bind ordered query rows to owner-issued opaque sample-selection tokens.""" + if len(records) != len(selected_membership): + raise ValueError( + f"source query returned {len(records)} rows; expected exactly {len(selected_membership)}" + ) + contents: list[str] = [] + for ordinal, record in enumerate(records): + if tuple(record.keys()) != ("selection_token", "content_text"): + raise ValueError( + "source query must return exactly selection_token, content_text in manifest order" + ) + selection_token = record["selection_token"] + if not isinstance(selection_token, str) or not selection_token.strip(): + raise ValueError( + "source query returned a blank or non-text selection token" + ) + token_digest = hashlib.sha256(selection_token.encode("utf-8")).hexdigest() + if token_digest != selected_membership[ordinal][0]: + raise ValueError( + "source query membership does not match the probability-sample manifest" + ) + content = record["content_text"] + if not isinstance(content, str) or not content.strip(): + raise ValueError("source query returned blank or non-text content") + contents.append(content) + return contents + + +def aggregate_results( + batches: Sequence[Sequence[dict[str, Any]]], trace_counts: Sequence[int] +) -> dict[str, object]: + """Return only non-identifying counts after every batch passed validation.""" + rows = [row for batch in batches for row in batch] + dimensions = Counter( + dimension for row in rows for dimension in row["missing_semantic_dimensions"] + ) + return { + "complete": True, + "sample_count": len(rows), + "covered_count": sum(row["covered"] for row in rows), + "uncovered_count": sum(not row["covered"] for row in rows), + "missing_semantic_dimension_counts": dict(sorted(dimensions.items())), + "batch_count": len(batches), + "minimum_trace_step_count": min(trace_counts), + "maximum_trace_step_count": max(trace_counts), + } + + +def _ontology_terms(path: Path) -> list[dict[str, object]]: + """Return deterministic public semantics for every governed ontology term.""" + graph = Graph().parse(path, format="turtle") + support_profile = path.with_name("prov-o-support-profile.ttl") + if not support_profile.is_file(): + raise FileNotFoundError( + "PROV-O support profile must accompany the ontology audit input: " + f"{support_profile}" + ) + graph.parse(support_profile, format="turtle") + governed_kinds = { + OWL.Class, + OWL.ObjectProperty, + OWL.DatatypeProperty, + SKOS.Concept, + } + terms: list[dict[str, object]] = [] + for subject in sorted( + { + subject + for kind in governed_kinds + for subject in graph.subjects(RDF.type, kind) + if isinstance(subject, URIRef) + }, + key=str, + ): + terms.append( + { + "iri": str(subject), + "kinds": sorted( + str(kind) + for kind in graph.objects(subject, RDF.type) + if kind in governed_kinds + ), + "labels": sorted( + str(value) + for predicate in (RDFS.label, SKOS.prefLabel) + for value in graph.objects(subject, predicate) + ), + "comments": sorted( + str(value) for value in graph.objects(subject, RDFS.comment) + ), + "domains": sorted( + str(value) for value in graph.objects(subject, RDFS.domain) + ), + "ranges": sorted( + str(value) for value in graph.objects(subject, RDFS.range) + ), + "superclasses": sorted( + str(value) + for value in graph.objects(subject, RDFS.subClassOf) + ), + "superproperties": sorted( + str(value) + for value in graph.objects(subject, RDFS.subPropertyOf) + ), + "schemes": sorted( + str(value) for value in graph.objects(subject, SKOS.inScheme) + ), + } + ) + qualifications = { + spec.unqualified_relation: { + "qualification_relation": str(PROV[spec.qualification_relation]), + "influence_class": str(PROV[spec.influence_class]), + "influencer_relation": str(PROV[spec.influencer_relation]), + } + for spec in PROV_QUALIFICATIONS + } + terms.extend( + { + "iri": spec.iri, + "kinds": [str(OWL.Class)], + "labels": [spec.local_name], + "comments": [], + "domains": [], + "ranges": [], + "superclasses": [str(PROV[name]) for name in spec.superclasses], + "superproperties": [], + "schemes": [], + } + for spec in PROV_CLASSES.values() + ) + terms.extend( + { + "iri": spec.iri, + "kinds": [ + str(OWL.ObjectProperty) + if spec.property_kind == "object" + else str(OWL.DatatypeProperty) + ], + "labels": [spec.local_name], + "comments": [], + "domains": [str(PROV[name]) for name in spec.domains], + "ranges": ( + [spec.datatype_iri] + if spec.datatype_iri + else [str(PROV[name]) for name in spec.ranges] + ), + "superclasses": [], + "superproperties": [str(PROV[name]) for name in spec.superproperties], + "schemes": [], + "qualification": qualifications.get(spec.local_name), + } + for spec in PROV_RELATIONS.values() + ) + return sorted(terms, key=lambda term: str(term["iri"])) + + +def _prompt( + supporting_terms_by_dimension: Mapping[str, Sequence[str]], + contents: Sequence[str], +) -> str: + """Build a privacy-constrained exact-cardinality audit request.""" + items = [ + {"item_index": index, "source_content": content} + for index, content in enumerate(contents) + ] + return ( + "Audit whether the supplied OWL/SKOS schema can represent every private item's material meaning. " + "Never quote, paraphrase, reproduce, or expose source content or proper nouns. " + "Treat source-specific people, organizations, places, products, projects, events, and values " + "as instance data, not missing schema terms, when a supplied class/property can represent them. " + "Report a missing dimension only when no supplied class/property can represent it without " + "inventing a new schema term. " + "Return only JSON with input_count and items. Return exactly one ordered item per item_index. " + "Each item has exactly item_index and semantic_dimensions. Classify every material " + "meaning into one or more supplied dimension codes; do not select ontology terms or " + "decide coverage. Never invent a dimension name or synonym; use " + "other_unmodeled_meaning only for material meaning outside every supplied dimension. " + "Do not return a dimension merely because the item is a Post or text. " + "Semantic dimensions may use only: " + + ", ".join(sorted(SEMANTIC_DIMENSIONS)) + + ". If uncertain, use other_unmodeled_meaning.\nPUBLIC SUPPORT PROFILE:\n" + + json.dumps(supporting_terms_by_dimension, ensure_ascii=False, sort_keys=True) + + "\nPRIVATE INPUT (never repeat):\n" + + json.dumps(items, ensure_ascii=False) + ) + + +def _response_format(expected_count: int) -> dict[str, object]: + """Return the strict structured-output contract for one complete batch.""" + return { + "type": "json_schema", + "json_schema": { + "name": "semantic_coverage_batch", + "strict": True, + "schema": { + "type": "object", + "properties": { + "input_count": {"const": expected_count}, + "items": { + "type": "array", + "minItems": expected_count, + "maxItems": expected_count, + "items": { + "type": "object", + "properties": { + "item_index": { + "type": "integer", + "minimum": 0, + "maximum": expected_count - 1, + }, + "semantic_dimensions": { + "type": "array", + "minItems": 1, + "uniqueItems": True, + "items": { + "type": "string", + "enum": sorted(SEMANTIC_DIMENSIONS), + }, + }, + }, + "required": ["item_index", "semantic_dimensions"], + "additionalProperties": False, + }, + }, + }, + "required": ["input_count", "items"], + "additionalProperties": False, + }, + }, + } + + +async def audit_source_content( + *, + source_dsn: str, + query: str, + sample_size: int, + sample_manifest: object, + sample_design_artifact: object, + batch_size: int, + ontology_path: Path, + gateway_url: str, + gateway_api_key: str, + timeout: float, + attempt_evidence_path: Path | None = None, +) -> dict[str, object]: + """Run a fail-closed multi-agent audit and return aggregate evidence only.""" + if sample_size < 1 or not 1 <= batch_size <= 10: + raise ValueError( + "sample_size must be positive and batch_size must be between 1 and 10" + ) + sample_design, selected_membership = validate_probability_sample_manifest( + sample_manifest, sample_size + ) + sample_design["rust_artifact"] = validate_sampling_design_artifact( + sample_design_artifact, cast(Mapping[str, object], sample_manifest) + ) + rust_artifact = cast(Mapping[str, object], sample_design["rust_artifact"]) + ontology_sha256 = hashlib.sha256(ontology_path.read_bytes()).hexdigest() + attempt_inputs = { + "selection_manifest_sha256": str(sample_design["selection_manifest_sha256"]), + "sampling_design_sha256": str(rust_artifact["artifact_sha256"]), + "ontology_sha256": ontology_sha256, + } + accepted_count = 0 + failed_batch_index: int | None = None + + def retain_attempt(status_code: str, failure_code: str | None = None) -> dict[str, object]: + evidence = audit_attempt_provenance( + **attempt_inputs, + status_code=status_code, + accepted_count=accepted_count, + failed_batch_index=( + failed_batch_index if status_code == "rejected" else None + ), + failure_code=failure_code, + ) + if attempt_evidence_path is not None: + _write_private_json(attempt_evidence_path, evidence) + return evidence + + initial_attempt = retain_attempt("in_progress") + audit_session_id = str(initial_attempt["audit_session_id"]) + try: + connection = await asyncpg.connect(source_dsn) + try: + records = await connection.fetch(query) + finally: + await connection.close() + contents = selected_contents(records, selected_membership) + + terms = _ontology_terms(ontology_path) + allowed_term_iris = {str(term["iri"]) for term in terms} + supporting_terms_by_dimension = { + dimension: tuple(iri for iri in expected_iris if iri in allowed_term_iris) + for dimension, expected_iris in SEMANTIC_DIMENSION_TERM_IRIS.items() + } + batches: list[tuple[dict[str, Any], ...]] = [] + trace_counts: list[int] = [] + endpoint = gateway_url.rstrip("/") + "/v1/chat/completions" + for start in range(0, len(contents), batch_size): + failed_batch_index = start // batch_size + window = contents[start : start + batch_size] + response = await asyncio.to_thread( + _post_json_with_deadline, + endpoint, + { + "model": "orchestrator/auto", + "messages": [ + { + "role": "developer", + "content": "Preserve privacy and exact cardinality. Output JSON only.", + }, + { + "role": "user", + "content": _prompt(supporting_terms_by_dimension, window), + }, + ], + "orchestration_mode": "conduct", + "include_orchestration_trace": True, + "response_format": _response_format(len(window)), + }, + headers={ + "authorization": f"Bearer {gateway_api_key}", + "x-lineageweave-session-id": audit_session_id, + }, + timeout=timeout, + ) + orchestration = response.get("orchestration") + trace = orchestration.get("trace") if isinstance(orchestration, dict) else None + if not isinstance(trace, list) or len(trace) < 2: + raise ValueError("semantic audit did not return multi-agent trace evidence") + try: + parsed_batch = parse_batch_result( + chat_completion_content(response), + len(window), + supporting_terms_by_dimension, + ) + except ValueError as exc: + raise ValueError( + f"semantic audit batch {start // batch_size} failed validation" + ) from exc + batches.append(parsed_batch) + trace_counts.append(len(trace)) + accepted_count += len(window) + retain_attempt("in_progress") + failed_batch_index = None + result = aggregate_results(batches, trace_counts) + if result["sample_count"] != sample_size: + raise AssertionError( + "validated semantic audit total does not match source sample" + ) + completed_attempt = retain_attempt("completed") + sample_design.update( + terminal_semantic_coverage_evidence( + cast(Mapping[str, object], sample_design_artifact), + sample_design, + result, + ontology_path, + completed_attempt, + ) + ) + result["sample_design"] = sample_design + result["attempted_count"] = sample_size + result["failed_count"] = 0 + result["attempt_provenance"] = completed_attempt + return result + except Exception as exc: + retain_attempt("rejected", type(exc).__name__) + raise + + +def _parser() -> argparse.ArgumentParser: + """Build the private-content, aggregate-output CLI contract.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-dsn", required=True) + parser.add_argument("--query-file", type=Path, required=True) + parser.add_argument("--sample-size", type=int, required=True) + parser.add_argument("--sample-manifest-file", type=Path, required=True) + parser.add_argument("--sample-design-artifact-file", type=Path, required=True) + parser.add_argument("--batch-size", type=int, default=10) + parser.add_argument( + "--ontology-path", + type=Path, + default=Path("docs/ontology/lineageweave-kg.ttl"), + ) + parser.add_argument("--gateway-url", required=True) + parser.add_argument( + "--gateway-api-key-env", default="CONTEXTUAL_ORCHESTRATOR_TOKEN" + ) + parser.add_argument("--timeout", type=float, default=300.0) + parser.add_argument("--attempt-evidence-file", type=Path, required=True) + return parser + + +def main() -> None: + """Run the audit and print no source-derived text, even on failure.""" + args = _parser().parse_args() + api_key = os.environ.get(args.gateway_api_key_env, "").strip() + if not api_key: + raise SystemExit(f"{args.gateway_api_key_env} is required") + result = asyncio.run( + audit_source_content( + source_dsn=args.source_dsn, + query=args.query_file.read_text(encoding="utf-8"), + sample_size=args.sample_size, + sample_manifest=json.loads( + args.sample_manifest_file.read_text(encoding="utf-8") + ), + sample_design_artifact=json.loads( + args.sample_design_artifact_file.read_text(encoding="utf-8") + ), + batch_size=args.batch_size, + ontology_path=args.ontology_path, + gateway_url=args.gateway_url, + gateway_api_key=api_key, + timeout=args.timeout, + attempt_evidence_path=args.attempt_evidence_file, + ) + ) + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/audit_source_semantic_coverage.py b/scripts/audit_source_semantic_coverage.py new file mode 100644 index 000000000..73b24b25b --- /dev/null +++ b/scripts/audit_source_semantic_coverage.py @@ -0,0 +1,244 @@ +"""Report aggregate-only source-field availability for semantic coverage.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import re +from collections.abc import Mapping + +import asyncpg + +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_PROVENANCE_TABLES = frozenset( + { + "provenance_assertion", + "provenance_assertion_derivation", + "provenance_class_definition", + "provenance_class_hierarchy", + "provenance_inverse_definition", + "provenance_literal_value", + "provenance_qualification_definition", + "provenance_relation_definition", + "provenance_relation_domain", + "provenance_relation_hierarchy", + "provenance_relation_resource_range", + "provenance_resource", + "provenance_resource_binding", + "provenance_resource_type", + } +) + + +def _identifier(value: str) -> str: + """Quote one validated PostgreSQL identifier without accepting SQL syntax.""" + if not _IDENTIFIER.fullmatch(value): + raise ValueError(f"invalid PostgreSQL identifier: {value!r}") + return f'"{value}"' + + +def _table(value: str) -> str: + """Quote a required schema-qualified PostgreSQL table name.""" + parts = value.split(".") + if len(parts) != 2: + raise ValueError("source table must be schema-qualified") + return ".".join(_identifier(part) for part in parts) + + +async def audit_source_semantic_coverage( + dsn: str, + table: str, + columns: Mapping[str, str], + *, + source_key: str | None = None, + coverage_table: str | None = None, + coverage_key: str | None = None, + assertion_table: str | None = None, + assertion_status: str | None = None, + assertion_evidence: str | None = None, + provenance_schema: str | None = None, +) -> dict[str, object]: + """Return row and nonblank counts without reading source values.""" + coverage_options = (source_key, coverage_table, coverage_key) + if any(value is not None for value in coverage_options) and not all( + value is not None for value in coverage_options + ): + raise ValueError("source_key, coverage_table, and coverage_key are all required") + source_key_column = _identifier(source_key) if source_key else None + coverage_table_sql = _table(coverage_table) if coverage_table else None + coverage_key_column = _identifier(coverage_key) if coverage_key else None + assertion_options = (assertion_table, assertion_status, assertion_evidence) + if any(value is not None for value in assertion_options) and not all( + value is not None for value in assertion_options + ): + raise ValueError( + "assertion_table, assertion_status, and assertion_evidence are all required" + ) + assertion_table_sql = _table(assertion_table) if assertion_table else None + assertion_status_column = ( + _identifier(assertion_status) if assertion_status else None + ) + assertion_evidence_column = ( + _identifier(assertion_evidence) if assertion_evidence else None + ) + provenance_schema_name = ( + _identifier(provenance_schema) if provenance_schema else None + ) + projections = ["count(*)::bigint as row_count"] + for role, column in columns.items(): + if not _IDENTIFIER.fullmatch(role): + raise ValueError(f"invalid semantic role: {role!r}") + quoted = _identifier(column) + projections.append( + f"count(*) filter (where nullif(btrim({quoted}::text), '') is not null)::bigint " + f'as "{role}_nonblank_count"' + ) + query = f"select {', '.join(projections)} from {_table(table)}" + connection = await asyncpg.connect(dsn) + try: + # SQL identifiers cannot be bind parameters; every interpolated token + # passed _identifier/_table's strict ASCII identifier grammar above. + # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + row = await connection.fetchrow(query) + result: dict[str, object] = { + "row_count": row["row_count"], + "semantic_role_nonblank_counts": { + role: row[f"{role}_nonblank_count"] for role in columns + }, + } + if source_key and coverage_table and coverage_key: + assert source_key_column and coverage_table_sql and coverage_key_column + # Same identifier-only boundary as the aggregate query; no source + # value is interpolated into this statement. + # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + key_row = await connection.fetchrow( + f""" + with source_keys as ( + select distinct {source_key_column}::text as key_value + from {_table(table)} + where nullif(btrim({source_key_column}::text), '') is not null + ), coverage_keys as ( + select distinct {coverage_key_column}::text as key_value + from {coverage_table_sql} + where nullif(btrim({coverage_key_column}::text), '') is not null + ) + select count(source_keys.key_value)::bigint as source_key_count, + count(coverage_keys.key_value)::bigint as coverage_key_count, + count(*) filter ( + where source_keys.key_value is not null + and coverage_keys.key_value is not null + )::bigint as matched_key_count, + count(*) filter ( + where source_keys.key_value is not null + and coverage_keys.key_value is null + )::bigint as source_without_coverage_count, + count(*) filter ( + where source_keys.key_value is null + and coverage_keys.key_value is not null + )::bigint as coverage_without_source_count + from source_keys + full join coverage_keys using (key_value) + """ + ) + result["semantic_key_coverage"] = dict(key_row) + if assertion_table_sql: + assert assertion_status_column and assertion_evidence_column + # All interpolated values are validated identifiers. Status values + # are fixed governed literals, not caller or source data. + # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + assertion_row = await connection.fetchrow( + f""" + select count(*)::bigint as assertion_count, + count(*) filter (where {assertion_status_column} = 'observed')::bigint as observed_count, + count(*) filter (where {assertion_status_column} = 'observed' and {assertion_evidence_column} is null)::bigint as observed_without_evidence_count, + count(*) filter (where {assertion_status_column} = 'inferred')::bigint as inferred_count, + count(*) filter (where {assertion_status_column} = 'inferred' and {assertion_evidence_column} is null)::bigint as inferred_without_direct_evidence_count, + count(*) filter (where {assertion_status_column} = 'predicted')::bigint as predicted_count, + count(*) filter (where {assertion_status_column} = 'predicted' and {assertion_evidence_column} is null)::bigint as predicted_without_direct_evidence_count, + count(*) filter (where {assertion_status_column} not in ('observed', 'inferred', 'predicted') or {assertion_status_column} is null)::bigint as ungoverned_status_count + from {assertion_table_sql} + """ + ) + assertion_counts = dict(assertion_row) + assertion_counts["source_evidence_boundary_complete"] = ( + assertion_counts["observed_without_evidence_count"] == 0 + and assertion_counts["ungoverned_status_count"] == 0 + ) + result["semantic_assertion_evidence"] = assertion_counts + if provenance_schema_name: + table_rows = await connection.fetch( + """ + select table_name + from information_schema.tables + where table_schema = $1 + and table_name = any($2::text[]) + """, + provenance_schema, + sorted(_PROVENANCE_TABLES), + ) + present = {row["table_name"] for row in table_rows} + result["normalized_provenance_schema"] = { + "required_table_count": len(_PROVENANCE_TABLES), + "present_table_count": len(present), + "schema_complete": present == _PROVENANCE_TABLES, + } + return result + finally: + await connection.close() + + +def _parser() -> argparse.ArgumentParser: + """Build the aggregate-only audit command contract.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dsn", required=True) + parser.add_argument("--table", required=True) + parser.add_argument("--source-key") + parser.add_argument("--coverage-table") + parser.add_argument("--coverage-key") + parser.add_argument("--assertion-table") + parser.add_argument("--assertion-status") + parser.add_argument("--assertion-evidence") + parser.add_argument("--provenance-schema") + parser.add_argument( + "--column", + action="append", + default=[], + metavar="ROLE=COLUMN", + help="repeatable semantic role to source-column mapping", + ) + return parser + + +def main() -> None: + """Run the audit and print only aggregate JSON.""" + args = _parser().parse_args() + columns: dict[str, str] = {} + for mapping in args.column: + role, separator, column = mapping.partition("=") + if not separator or not role or not column: + raise SystemExit("--column must use ROLE=COLUMN") + columns[role] = column + print( + json.dumps( + asyncio.run( + audit_source_semantic_coverage( + args.dsn, + args.table, + columns, + source_key=args.source_key, + coverage_table=args.coverage_table, + coverage_key=args.coverage_key, + assertion_table=args.assertion_table, + assertion_status=args.assertion_status, + assertion_evidence=args.assertion_evidence, + provenance_schema=args.provenance_schema, + ) + ), + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/backfill_post_summaries.py b/scripts/backfill_post_summaries.py index 2c008fea0..3441d3227 100644 --- a/scripts/backfill_post_summaries.py +++ b/scripts/backfill_post_summaries.py @@ -92,6 +92,9 @@ def _semantic_hints(row: asyncpg.Record) -> str: source_customer_catalog_name=row["source_customer_catalog_name"], source_project_code=row["source_project_code"], source_project_name=row["source_project_name"], + source_voc_type_code=row["voc_type_code"], + source_stage_code=row["source_stage_code"], + source_detail_state_code=row["source_detail_state_code"], ) @@ -124,6 +127,9 @@ async def _load_posts( source_customer.entity_name as source_customer_catalog_name, post.source_project_code, post.source_project_name, + post.voc_type_code, + post.source_stage_code, + post.source_detail_state_code, post.secondary_grouping_key as project_field, customer.entity_name as customer_name, coalesce( diff --git a/scripts/import_postgresql_posts.py b/scripts/import_postgresql_posts.py index 26de5f856..08e202305 100644 --- a/scripts/import_postgresql_posts.py +++ b/scripts/import_postgresql_posts.py @@ -14,7 +14,7 @@ import sys import uuid from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -52,12 +52,10 @@ } -def _normalize_voc_type(value: Any, *, mapped: bool) -> str: +def _normalize_voc_type(value: Any) -> str: """Preserve the governed source VOC vocabulary as canonical target codes.""" if value is None or not str(value).strip(): - if mapped: - raise ValueError("mapped source VOC type is empty") - return "voc" + raise ValueError("mapped source VOC type is empty") normalized = _VOC_TYPE_ALIASES.get(str(value).strip().casefold()) if normalized is None: raise ValueError(f"unsupported source VOC type {value!r}") @@ -71,7 +69,7 @@ class ColumnMapping: record_key: str post_id: str | None title: str - body: str + body: str | None created_at: str updated_at: str | None event_occurred_at: str | None @@ -112,14 +110,23 @@ def _parser() -> argparse.ArgumentParser: help="optional source UUID column for post_id; otherwise derive it from record key", ) parser.add_argument("--title-column", required=True) - parser.add_argument("--body-column", required=True) + body_group = parser.add_mutually_exclusive_group(required=True) + body_group.add_argument("--body-column") + body_group.add_argument( + "--no-body-dimension-evidence", + default="", + help=( + "written evidence that the source export has no body dimension; " + "the title remains a title and no semantic body is invented" + ), + ) parser.add_argument("--created-at-column", required=True) parser.add_argument("--updated-at-column") parser.add_argument( "--event-occurred-at-column", help="optional source-system event instant; Global Ask falls back to created_at when omitted", ) - parser.add_argument("--voc-type-column") + parser.add_argument("--voc-type-column", required=True) parser.add_argument("--visibility-column") parser.add_argument("--stage-column") parser.add_argument("--detail-state-column") @@ -197,7 +204,7 @@ def _value(row: Any, column: str | None, default: Any = None) -> Any: """Read an optional mapped field without guessing absent source data.""" if column is None: return default - if column not in row.keys(): + if column not in row: raise KeyError(f"source query did not return mapped column {column!r}") return row[column] @@ -219,7 +226,7 @@ def _timestamp(value: Any) -> datetime: """Normalize a source timestamp for asyncpg timestamptz parameters.""" if not isinstance(value, datetime): raise TypeError("created/updated source values must be datetime instances") - return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value + return value.replace(tzinfo=UTC) if value.tzinfo is None else value def _source_code_matches( @@ -299,11 +306,6 @@ def _validate_publication_state( "mapped draft column or --exclude-draft-value; pick one " "publication-state door" ) - if len(evidence) < 40: - raise ValueError( - "--no-draft-dimension-evidence must actually state the " - "evidence (at least 40 characters), not a placeholder" - ) return if mapping.draft is None: raise ValueError("source draft status column is required for publication-state preflight") @@ -319,11 +321,22 @@ def _validate_source_rows( excluded_draft_values: list[str], excluded_deleted_values: list[str], no_draft_dimension_evidence: str = "", + no_body_dimension_evidence: str = "", ) -> None: """Reject incomplete source evidence before the target is mutated.""" _validate_publication_state( rows, mapping, excluded_draft_values, no_draft_dimension_evidence ) + body_evidence = no_body_dimension_evidence.strip() + if mapping.body is None: + if not body_evidence: + raise ValueError( + "--no-body-dimension-evidence requires an operator evidence statement" + ) + elif body_evidence: + raise ValueError( + "--no-body-dimension-evidence cannot be combined with a mapped body column" + ) seen_record_keys: dict[str, int] = {} seen_post_ids: dict[uuid.UUID, int] = {} post_id_column = getattr(mapping, "post_id", None) @@ -351,14 +364,13 @@ def _validate_source_rows( ) seen_post_ids[post_id] = row_number body = str(_value(row, mapping.body) or "") - if not body.strip(): + if mapping.body is not None and not body.strip(): raise ValueError(f"source post body cannot be empty at source row {row_number}") voc_type_column = getattr(mapping, "voc_type", None) + if voc_type_column is None: + raise ValueError("source VOC type column is required") try: - _normalize_voc_type( - _value(row, voc_type_column, "voc"), - mapped=voc_type_column is not None, - ) + _normalize_voc_type(_value(row, voc_type_column)) except ValueError as exc: raise ValueError(f"invalid source VOC type at source row {row_number}: {exc}") from exc @@ -457,6 +469,7 @@ async def import_rows(args: argparse.Namespace) -> dict[str, object]: args.exclude_draft_value, args.exclude_deleted_value, args.no_draft_dimension_evidence, + args.no_body_dimension_evidence, ) account_id, corporate_id, process_unit_id = await _ensure_scope(target, args) vision_client = orchestrator_vision_client( @@ -504,10 +517,7 @@ async def import_rows(args: argparse.Namespace) -> dict[str, object]: post_id = _source_post_id(row, mapping, args.source_system_code, record_key) title = str(_value(row, mapping.title, "") or "") body = str(_value(row, mapping.body, "") or "") - voc_type_code = _normalize_voc_type( - _value(row, mapping.voc_type, "voc"), - mapped=mapping.voc_type is not None, - ) + voc_type_code = _normalize_voc_type(_value(row, mapping.voc_type)) ( source_thread_group_key, source_secondary_grouping_key, @@ -519,7 +529,8 @@ async def import_rows(args: argparse.Namespace) -> dict[str, object]: record_key=record_key, default_group=args.process_unit_code, ) - await target.execute( + preserve_existing_body = mapping.body is None + effective_body = await target.fetchval( """ insert into source_post (post_id, author_account_id, corporate_entity_id, process_unit_id, @@ -541,7 +552,10 @@ async def import_rows(args: argparse.Namespace) -> dict[str, object]: corporate_entity_id = excluded.corporate_entity_id, process_unit_id = excluded.process_unit_id, post_title = excluded.post_title, - post_body = excluded.post_body, + post_body = case + when $34 then source_post.post_body + else excluded.post_body + end, voc_type_code = excluded.voc_type_code, visibility_code = excluded.visibility_code, source_stage_code = excluded.source_stage_code, @@ -570,6 +584,7 @@ async def import_rows(args: argparse.Namespace) -> dict[str, object]: created_at = excluded.created_at, updated_at = excluded.updated_at, event_occurred_at = excluded.event_occurred_at + returning post_body """, post_id, account_id, @@ -604,20 +619,7 @@ async def import_rows(args: argparse.Namespace) -> dict[str, object]: created_at, updated_at, event_occurred_at, - ) - await target.execute( - """ - insert into source_post_revision (post_id, post_title, post_body, written_at, superseded_at) - select $1, $2, $3, $4, null - where not exists ( - select 1 from source_post_revision - where post_id = $1 and written_at = $4 and superseded_at is null - ) - """, - post_id, - title, - body, - updated_at, + preserve_existing_body, ) metadata = build_post_llm_metadata( str(post_id), @@ -635,7 +637,7 @@ async def import_rows(args: argparse.Namespace) -> dict[str, object]: await persist_post_content( target, str(post_id), - body, + effective_body, vision_client=vision_client, embedding_client=embedding_client, structure_client=structure_client, @@ -671,6 +673,10 @@ async def import_rows(args: argparse.Namespace) -> dict[str, object]: summary["no_draft_dimension_evidence"] = ( args.no_draft_dimension_evidence.strip() ) + if args.no_body_dimension_evidence.strip(): + summary["no_body_dimension_evidence"] = ( + args.no_body_dimension_evidence.strip() + ) return summary finally: await source.close() diff --git a/tests/test_audit_source_content_semantics.py b/tests/test_audit_source_content_semantics.py new file mode 100644 index 000000000..5286a8cda --- /dev/null +++ b/tests/test_audit_source_content_semantics.py @@ -0,0 +1,751 @@ +import hashlib +import json +import threading +from copy import deepcopy +from dataclasses import asdict +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +from fast_mlsirm import SamplingStratum, finite_population_proportion_design +from lineageweave.prov_o import PROV + +from scripts.audit_source_content_semantics import ( + SEMANTIC_DIMENSION_TERM_IRIS, + SEMANTIC_DIMENSIONS, + _ontology_terms, + _parser, + _post_json_with_deadline, + _prompt, + _response_format, + aggregate_results, + audit_attempt_provenance, + parse_batch_result, + selected_contents, + terminal_semantic_coverage_evidence, + validate_probability_sample_manifest, + validate_sampling_design_artifact, +) + +_TERM_IRI = "https://example.test/ontology#Event" +_SUPPORTING_TERMS = { + "event_or_activity": (_TERM_IRI,), + "other_unmodeled_meaning": (), +} + + +def test_cli_defaults_to_the_internal_orchestrator_credential() -> None: + """The audit must not send a provider credential to the internal service.""" + action = next( + action + for action in _parser()._actions + if action.dest == "gateway_api_key_env" + ) + + assert action.default == "CONTEXTUAL_ORCHESTRATOR_TOKEN" + design_action = next( + action + for action in _parser()._actions + if action.dest == "sample_design_artifact_file" + ) + assert design_action.required is True + attempt_action = next( + action for action in _parser()._actions if action.dest == "attempt_evidence_file" + ) + assert attempt_action.required is True + + +def test_audit_uses_the_orchestrator_owned_auto_route() -> None: + """The audit names no provider model and leaves discovery upstream.""" + source = Path("scripts/audit_source_content_semantics.py").read_text() + + assert '"model": "orchestrator/auto"' in source + assert '"model": "contextual-orchestrator"' not in source + + +def test_provider_deadline_must_be_positive() -> None: + """The hard wall-clock boundary cannot be disabled accidentally.""" + with pytest.raises(ValueError, match="timeout must be positive"): + _post_json_with_deadline( + "https://example.test", + {}, + headers={}, + timeout=0, + ) + + +def test_provider_deadline_accepts_response_larger_than_queue_pipe() -> None: + """The parent drains a valid large result before waiting for child shutdown.""" + body = json.dumps({"trace": "x" * 262_144}).encode() + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format: str, *_args: object) -> None: + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + result = _post_json_with_deadline( + f"http://127.0.0.1:{server.server_port}/conduct", + {}, + headers={}, + timeout=60, + ) + finally: + server.shutdown() + thread.join() + server.server_close() + + assert len(result["trace"]) == 262_144 + + +def test_provider_deadline_preserves_explicit_audit_session() -> None: + """A spawned request retains the non-identifying audit correlation id.""" + received_session_id = "" + received_metadata: dict[str, object] = {} + body = b'{"ok":true}' + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + nonlocal received_metadata, received_session_id + received_session_id = self.headers.get( + "x-lineageweave-session-id", "" + ) + received = json.loads( + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + ) + received_metadata = received.get("metadata", {}) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format: str, *_args: object) -> None: + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + result = _post_json_with_deadline( + f"http://127.0.0.1:{server.server_port}/conduct", + {}, + headers={"x-lineageweave-session-id": "semantic-audit:synthetic"}, + timeout=60, + ) + finally: + server.shutdown() + thread.join() + server.server_close() + + assert result == {"ok": True} + assert received_session_id == "semantic-audit:synthetic" + assert received_metadata == {"session_id": "semantic-audit:synthetic"} + + +def test_audit_contract_distinguishes_instance_data_from_schema_gaps() -> None: + """Private names and values do not require private ontology vocabulary.""" + prompt = _prompt( + {"event_or_activity": (_TERM_IRI,)}, + ["Synthetic event at a synthetic facility"], + ) + + assert "as instance data, not missing schema terms" in prompt + assert "no supplied class/property can represent it" in prompt + assert "do not select ontology terms or decide coverage" in prompt + assert "PUBLIC SUPPORT PROFILE" in prompt + assert "ONTOLOGY TERMS" not in prompt + + +def test_audit_structured_output_binds_the_exact_batch_cardinality() -> None: + """Schema validation and the parser both retain every submitted item.""" + response_format = _response_format(10) + schema = response_format["json_schema"]["schema"] + + assert schema["properties"]["input_count"] == {"const": 10} + items = schema["properties"]["items"] + assert items["minItems"] == items["maxItems"] == 10 + assert items["items"]["properties"]["item_index"] == { + "type": "integer", + "minimum": 0, + "maximum": 9, + } + + +def _probability_manifest() -> dict[str, object]: + """Return a synthetic stratified sample-audit contract.""" + digest = "a" * 64 + manifest: dict[str, object] = { + "contract_kind": "lineageweave.semantic_coverage_probability_sample", + "contract_version": 3, + "population_size": 1000, + "sample_size": 80, + "design_code": "stratified_random_without_replacement", + "provider_failures_retained": True, + "strata": [ + { + "stratum_code": "synthetic-a", + "population_size": 600, + "sample_size": 48, + "inclusion_probability_numerator": 48, + "inclusion_probability_denominator": 600, + "selection_frame_sha256": digest, + }, + { + "stratum_code": "synthetic-b", + "population_size": 400, + "sample_size": 32, + "inclusion_probability_numerator": 32, + "inclusion_probability_denominator": 400, + "selection_frame_sha256": "b" * 64, + }, + ], + "selected_units": [ + { + "ordinal": ordinal, + "selection_token_sha256": hashlib.sha256( + f"synthetic-token-{ordinal}".encode() + ).hexdigest(), + "stratum_code": "synthetic-a" if ordinal < 48 else "synthetic-b", + } + for ordinal in range(80) + ], + "selection_manifest_sha256": "", + } + manifest["selection_manifest_sha256"] = hashlib.sha256( + json.dumps( + manifest["selected_units"], sort_keys=True, separators=(",", ":") + ).encode() + ).hexdigest() + return manifest + + +def _rust_design_artifact() -> dict[str, object]: + """Return the Rust-owned design matching the synthetic sample manifest.""" + design = finite_population_proportion_design( + 1000, + 0.95, + 0.1055, + [SamplingStratum(600, 0.5), SamplingStratum(400, 0.5)], + allocation_method="proportional", + ) + return json.loads(json.dumps(asdict(design), sort_keys=True)) + + +def _simple_probability_evidence() -> tuple[dict[str, object], dict[str, object]]: + """Return a synthetic one-stratum manifest and its exact Rust design.""" + design = finite_population_proportion_design( + 1000, + 0.95, + 0.1, + [SamplingStratum(1000, 0.5)], + allocation_method="proportional", + ) + selected_units = [ + { + "ordinal": ordinal, + "selection_token_sha256": hashlib.sha256( + f"synthetic-simple-token-{ordinal}".encode() + ).hexdigest(), + "stratum_code": "synthetic-simple", + } + for ordinal in range(design.sample_size) + ] + manifest: dict[str, object] = { + "contract_kind": "lineageweave.semantic_coverage_probability_sample", + "contract_version": 3, + "population_size": design.population_size, + "sample_size": design.sample_size, + "design_code": "simple_random_without_replacement", + "provider_failures_retained": True, + "strata": [ + { + "stratum_code": "synthetic-simple", + "population_size": design.population_size, + "sample_size": design.sample_size, + "inclusion_probability_numerator": design.sample_size, + "inclusion_probability_denominator": design.population_size, + "selection_frame_sha256": "c" * 64, + } + ], + "selected_units": selected_units, + "selection_manifest_sha256": hashlib.sha256( + json.dumps(selected_units, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest(), + } + return manifest, json.loads(json.dumps(asdict(design), sort_keys=True)) + + +def test_parser_rejects_the_observed_100_to_60_cardinality_mismatch() -> None: + payload = { + "input_count": 60, + "items": [ + { + "item_index": index, + "semantic_dimensions": ["event_or_activity"], + } + for index in range(60) + ], + } + + with pytest.raises(ValueError, match="input_count"): + parse_batch_result(json.dumps(payload), 100, _SUPPORTING_TERMS) + + +def test_valid_batches_aggregate_without_source_values() -> None: + rows = parse_batch_result( + '{"input_count":2,"items":[' + '{"item_index":0,"semantic_dimensions":["other_unmodeled_meaning"]},' + '{"item_index":1,"semantic_dimensions":["event_or_activity"]}]}', + expected_count=2, + supporting_terms_by_dimension=_SUPPORTING_TERMS, + ) + + result = aggregate_results([rows], [4]) + + assert rows[1]["supporting_term_iris"] == [_TERM_IRI] + assert result == { + "complete": True, + "sample_count": 2, + "covered_count": 1, + "uncovered_count": 1, + "missing_semantic_dimension_counts": {"other_unmodeled_meaning": 1}, + "batch_count": 1, + "minimum_trace_step_count": 4, + "maximum_trace_step_count": 4, + } + + +def test_parser_rejects_ungoverned_dimensions() -> None: + with pytest.raises(ValueError, match="ungoverned"): + parse_batch_result( + '{"input_count":1,"items":[' + '{"item_index":0,"semantic_dimensions":["invented"]}]}', + expected_count=1, + supporting_terms_by_dimension=_SUPPORTING_TERMS, + ) + + +def test_parser_reports_a_dimension_when_its_ontology_terms_are_absent() -> None: + """Classification and deterministic schema support remain separate evidence.""" + rows = parse_batch_result( + '{"input_count":1,"items":[' + '{"item_index":0,"semantic_dimensions":["event_or_activity"]}]}', + expected_count=1, + supporting_terms_by_dimension={"event_or_activity": ()}, + ) + + assert rows[0]["covered"] is False + assert rows[0]["missing_semantic_dimensions"] == ["event_or_activity"] + + +@pytest.mark.parametrize( + ("dimensions", "message"), + [ + ([], "ungoverned dimension"), + (["event_or_activity", "event_or_activity"], "duplicate semantic"), + ], +) +def test_parser_requires_auditable_noncontradictory_verdicts( + dimensions: list[str], + message: str, +) -> None: + """Empty or duplicated content classifications fail closed.""" + payload = { + "input_count": 1, + "items": [ + { + "item_index": 0, + "semantic_dimensions": dimensions, + } + ], + } + + with pytest.raises(ValueError, match=message): + parse_batch_result(json.dumps(payload), 1, _SUPPORTING_TERMS) + + +def test_ontology_contract_contains_public_semantics_not_only_local_names() -> None: + """Coverage decisions receive term kinds and meaning-bearing RDF relations.""" + terms = _ontology_terms(Path("docs/ontology/lineageweave-kg.ttl")) + + assert terms + assert all(term["iri"] and term["kinds"] for term in terms) + assert any(term["labels"] for term in terms) + assert any(term["domains"] or term["ranges"] for term in terms) + by_iri = {term["iri"]: term for term in terms} + namespace = "https://contextualwisdomlab.github.io/LineageWeave/ontology#" + assert "http://www.w3.org/ns/prov#Entity" in by_iri[namespace + "Post"]["superclasses"] + assert "http://www.w3.org/ns/prov#Person" in by_iri[namespace + "Person"]["superclasses"] + assert "http://www.w3.org/ns/prov#wasDerivedFrom" in by_iri[ + namespace + "wasDerivedFromPost" + ]["superproperties"] + assert by_iri["http://www.w3.org/ns/prov#Activity"]["kinds"] == [ + "http://www.w3.org/2002/07/owl#Class" + ] + assert by_iri["http://www.w3.org/ns/prov#wasInformedBy"]["qualification"] == { + "qualification_relation": "http://www.w3.org/ns/prov#qualifiedCommunication", + "influence_class": "http://www.w3.org/ns/prov#Communication", + "influencer_relation": "http://www.w3.org/ns/prov#activity", + } + assert set(SEMANTIC_DIMENSION_TERM_IRIS) == SEMANTIC_DIMENSIONS + assert SEMANTIC_DIMENSION_TERM_IRIS["other_unmodeled_meaning"] == () + for dimension in SEMANTIC_DIMENSIONS - {"other_unmodeled_meaning"}: + assert set(SEMANTIC_DIMENSION_TERM_IRIS[dimension]) <= set(by_iri) + + +def test_ontology_contract_requires_the_governed_prov_support_profile( + tmp_path: Path, +) -> None: + """A partial ontology bundle fails instead of changing coverage semantics.""" + ontology_path = tmp_path / "lineageweave-kg.ttl" + ontology_path.write_bytes(Path("docs/ontology/lineageweave-kg.ttl").read_bytes()) + + with pytest.raises(FileNotFoundError, match="PROV-O support profile"): + _ontology_terms(ontology_path) + + +def test_probability_sample_manifest_preserves_design_evidence() -> None: + """The audit preserves selection evidence without claiming corpus inference.""" + manifest = _probability_manifest() + result, membership = validate_probability_sample_manifest(manifest, 80) + + assert result == { + "design_code": "stratified_random_without_replacement", + "population_size": 1000, + "sample_size": 80, + "stratum_count": 2, + "selection_manifest_sha256": manifest["selection_manifest_sha256"], + "corpus_inference_available": False, + } + assert len(membership) == 80 + + +def test_rust_sampling_design_replays_and_binds_the_manifest() -> None: + """Caller hashes cannot replace exact package-owned Rust replay evidence.""" + manifest = _probability_manifest() + artifact = _rust_design_artifact() + + result = validate_sampling_design_artifact(artifact, manifest) + + assert result["sampling_design_verified"] is True + assert result["corpus_inference_available"] is False + assert result["artifact_sha256"] == artifact["artifact_sha256"] + artifact["sample_size"] = 79 + with pytest.raises(ValueError, match="Rust replay"): + validate_sampling_design_artifact(artifact, manifest) + + +def test_rust_sampling_design_rejects_every_unbound_boundary() -> None: + """Malformed, unreplayable, or manifest-divergent artifacts fail closed.""" + manifest = _probability_manifest() + artifact = _rust_design_artifact() + + wrong_fields = deepcopy(artifact) + wrong_fields.pop("source_sha256") + with pytest.raises(ValueError, match="fields"): + validate_sampling_design_artifact(wrong_fields, manifest) + + no_strata = deepcopy(artifact) + no_strata["strata"] = [] + with pytest.raises(ValueError, match="ordered strata"): + validate_sampling_design_artifact(no_strata, manifest) + + malformed_stratum = deepcopy(artifact) + malformed_stratum["strata"][0]["unsupported"] = True + with pytest.raises(ValueError, match="strata are invalid"): + validate_sampling_design_artifact(malformed_stratum, manifest) + + unreplayable = deepcopy(artifact) + unreplayable["allocation_method"] = "caller_guess" + with pytest.raises(ValueError, match="cannot be replayed"): + validate_sampling_design_artifact(unreplayable, manifest) + + missing_manifest_strata = deepcopy(manifest) + missing_manifest_strata["strata"] = None + with pytest.raises(TypeError, match="strata are unavailable"): + validate_sampling_design_artifact(artifact, missing_manifest_strata) + + wrong_total = deepcopy(manifest) + wrong_total["sample_size"] = 79 + with pytest.raises(ValueError, match="totals"): + validate_sampling_design_artifact(artifact, wrong_total) + + wrong_allocation = deepcopy(manifest) + wrong_allocation["strata"][0]["sample_size"] = 47 + wrong_allocation["strata"][1]["sample_size"] = 33 + with pytest.raises(ValueError, match="allocation"): + validate_sampling_design_artifact(artifact, wrong_allocation) + + wrong_ratio = deepcopy(manifest) + wrong_ratio["strata"][0]["inclusion_probability_numerator"] = 47 + with pytest.raises(ValueError, match="inclusion ratios"): + validate_sampling_design_artifact(artifact, wrong_ratio) + + +def test_terminal_semantic_coverage_binds_exact_interval_and_audit() -> None: + """A complete SRSWOR result receives Rust inference and an aggregate identity.""" + manifest, artifact = _simple_probability_evidence() + sample_design, _ = validate_probability_sample_manifest( + manifest, manifest["sample_size"] + ) + validate_sampling_design_artifact(artifact, manifest) + sample_count = manifest["sample_size"] + assert isinstance(sample_count, int) + aggregate = { + "complete": True, + "sample_count": sample_count, + "covered_count": sample_count, + "uncovered_count": 0, + "missing_semantic_dimension_counts": {}, + "batch_count": 10, + "minimum_trace_step_count": 2, + "maximum_trace_step_count": 4, + } + ontology_path = Path("docs/ontology/lineageweave-kg.ttl") + ontology_sha256 = hashlib.sha256(ontology_path.read_bytes()).hexdigest() + completed_attempt = audit_attempt_provenance( + selection_manifest_sha256=sample_design["selection_manifest_sha256"], + sampling_design_sha256=artifact["artifact_sha256"], + ontology_sha256=ontology_sha256, + status_code="completed", + accepted_count=sample_count, + ) + + result = terminal_semantic_coverage_evidence( + artifact, + sample_design, + aggregate, + ontology_path, + completed_attempt, + ) + + assert result["corpus_inference_available"] is True + terminal = result["rust_terminal_artifact"] + assert isinstance(terminal, dict) + assert terminal["design_artifact_sha256"] == artifact["artifact_sha256"] + assert terminal["estimated_proportion"] == 1.0 + assert terminal["lower_proportion"] < 1.0 + assert terminal["upper_proportion"] == 1.0 + assert len(result["ontology_sha256"]) == 64 + assert len(result["audit_artifact_sha256"]) == 64 + assert len(result["prov_o_sha256"]) == 64 + prov_o = result["prov_o"] + assert isinstance(prov_o, dict) + relations = {assertion["relation_iri"] for assertion in prov_o["assertions"]} + assert relations == { + str(PROV.used), + str(PROV.wasDerivedFrom), + str(PROV.wasGeneratedBy), + } + assert len(prov_o["resource_types"]) == 7 + assert len(prov_o["assertions"]) == 11 + + +def test_rejected_attempt_retains_prov_without_becoming_a_coverage_result() -> None: + """A failed batch remains auditable but cannot claim terminal inference.""" + evidence = audit_attempt_provenance( + selection_manifest_sha256="a" * 64, + sampling_design_sha256="b" * 64, + ontology_sha256="c" * 64, + status_code="rejected", + accepted_count=30, + failed_batch_index=3, + failure_code="ValueError", + ) + + assert evidence["status_code"] == "rejected" + assert evidence["accepted_count"] == 30 + assert evidence["failed_batch_index"] == 3 + assert evidence["audit_session_id"].startswith("semantic-audit:") + completed = audit_attempt_provenance( + selection_manifest_sha256="a" * 64, + sampling_design_sha256="b" * 64, + ontology_sha256="c" * 64, + status_code="completed", + accepted_count=381, + ) + assert completed["audit_session_id"] == evidence["audit_session_id"] + assert "corpus_inference_available" not in evidence + assert len(evidence["prov_o_sha256"]) == 64 + prov_o = evidence["prov_o"] + assert isinstance(prov_o, dict) + relations = {assertion["relation_iri"] for assertion in prov_o["assertions"]} + assert relations == { + str(PROV.used), + str(PROV.wasDerivedFrom), + str(PROV.wasGeneratedBy), + } + assert len(prov_o["resource_types"]) == 5 + assert len(prov_o["assertions"]) == 7 + + +def test_terminal_semantic_coverage_fails_closed_or_stays_unavailable() -> None: + """Partial/tampered SRSWOR and unsupported stratified inference never open.""" + stratified_manifest = _probability_manifest() + stratified_design, _ = validate_probability_sample_manifest( + stratified_manifest, 80 + ) + unavailable = terminal_semantic_coverage_evidence( + _rust_design_artifact(), + stratified_design, + {"complete": True, "sample_count": 80}, + Path("docs/ontology/lineageweave-kg.ttl"), + {}, + ) + assert unavailable == { + "corpus_inference_available": False, + "corpus_inference_unavailable_reason": ( + "stratified_terminal_estimator_not_available" + ), + } + + manifest, artifact = _simple_probability_evidence() + sample_design, _ = validate_probability_sample_manifest( + manifest, manifest["sample_size"] + ) + sample_count = manifest["sample_size"] + assert isinstance(sample_count, int) + ontology_path = Path("docs/ontology/lineageweave-kg.ttl") + ontology_sha256 = hashlib.sha256(ontology_path.read_bytes()).hexdigest() + incomplete = { + "complete": True, + "sample_count": sample_count - 1, + "covered_count": sample_count - 1, + "uncovered_count": 0, + } + incomplete_attempt = audit_attempt_provenance( + selection_manifest_sha256=sample_design["selection_manifest_sha256"], + sampling_design_sha256=artifact["artifact_sha256"], + ontology_sha256=ontology_sha256, + status_code="completed", + accepted_count=sample_count - 1, + ) + with pytest.raises(ValueError, match="complete design-sized"): + terminal_semantic_coverage_evidence( + artifact, + sample_design, + incomplete, + ontology_path, + incomplete_attempt, + ) + tampered = deepcopy(artifact) + tampered["artifact_sha256"] = "f" * 64 + complete_attempt = audit_attempt_provenance( + selection_manifest_sha256=sample_design["selection_manifest_sha256"], + sampling_design_sha256=tampered["artifact_sha256"], + ontology_sha256=ontology_sha256, + status_code="completed", + accepted_count=sample_count, + ) + with pytest.raises(ValueError, match="does not match"): + terminal_semantic_coverage_evidence( + tampered, + sample_design, + { + "complete": True, + "sample_count": sample_count, + "covered_count": sample_count, + "uncovered_count": 0, + }, + ontology_path, + complete_attempt, + ) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("design_code", "deterministic_windows", "probability design"), + ("provider_failures_retained", False, "retain provider failures"), + ("contract_version", 1, "unsupported"), + ], +) +def test_probability_sample_manifest_rejects_noninferential_contracts( + field: str, value: object, message: str +) -> None: + """Deterministic windows and undocumented targets cannot imply corpus coverage.""" + manifest = _probability_manifest() + manifest[field] = value + + with pytest.raises(ValueError, match=message): + validate_probability_sample_manifest(manifest, 80) + + +def test_probability_sample_manifest_requires_known_stratum_inclusion_probability() -> ( + None +): + """Every stratum retains an exact inclusion ratio and frame digest.""" + manifest = _probability_manifest() + strata = manifest["strata"] + assert isinstance(strata, list) and isinstance(strata[0], dict) + strata[0]["inclusion_probability_numerator"] = "unknown" + + with pytest.raises(ValueError, match="exact sample/population inclusion ratio"): + validate_probability_sample_manifest(manifest, 80) + + +def test_probability_manifest_inclusion_probability_matches_sampling_fraction() -> None: + """A declared probability cannot contradict the selected stratum fraction.""" + manifest = _probability_manifest() + strata = manifest["strata"] + assert isinstance(strata, list) and isinstance(strata[0], dict) + strata[0]["inclusion_probability_numerator"] = 300 + + with pytest.raises(ValueError, match="exact sample/population inclusion ratio"): + validate_probability_sample_manifest(manifest, 80) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("population_size", 601, "stratum populations"), + ("sample_size", 47, "stratum samples"), + ], +) +def test_probability_manifest_stratum_totals_match_declared_totals( + field: str, value: int, message: str +) -> None: + """Stratum totals cannot contradict the declared sample population.""" + manifest = _probability_manifest() + strata = manifest["strata"] + assert isinstance(strata, list) and isinstance(strata[0], dict) + strata[0][field] = value + + with pytest.raises(ValueError, match=message): + validate_probability_sample_manifest(manifest, 80) + + +def test_probability_manifest_selected_units_match_each_stratum_sample() -> None: + """Selected-unit membership must realize every declared stratum count.""" + manifest = _probability_manifest() + selected_units = manifest["selected_units"] + assert isinstance(selected_units, list) and isinstance(selected_units[0], dict) + selected_units[0]["stratum_code"] = "synthetic-b" + + with pytest.raises(ValueError, match="selected-unit strata"): + validate_probability_sample_manifest(manifest, 80) + + +def test_selected_contents_bind_query_order_to_owner_tokens() -> None: + """A different query row cannot masquerade as the Rust-selected member.""" + token = "synthetic-owner-token" + membership = ((hashlib.sha256(token.encode()).hexdigest(), "synthetic-a"),) + + assert selected_contents( + [{"selection_token": token, "content_text": "Synthetic content"}], membership + ) == ["Synthetic content"] + with pytest.raises(ValueError, match="membership"): + selected_contents( + [{"selection_token": "replacement", "content_text": "Synthetic content"}], + membership, + ) diff --git a/tests/test_audit_source_semantic_coverage.py b/tests/test_audit_source_semantic_coverage.py new file mode 100644 index 000000000..80b608396 --- /dev/null +++ b/tests/test_audit_source_semantic_coverage.py @@ -0,0 +1,204 @@ +import asyncio + +import pytest + +from scripts.audit_source_semantic_coverage import ( + _identifier, + audit_source_semantic_coverage, +) + + +def test_audit_returns_aggregate_roles_without_source_values(monkeypatch) -> None: + class Connection: + closed = False + + async def fetchrow(self, query: str): + assert 'from "source_schema"."source_rows"' in query + assert '"body_text"' in query + return {"row_count": 3, "body_nonblank_count": 2} + + async def close(self) -> None: + self.closed = True + + connection = Connection() + + async def connect(_dsn: str): + return connection + + monkeypatch.setattr("scripts.audit_source_semantic_coverage.asyncpg.connect", connect) + + result = asyncio.run( + audit_source_semantic_coverage( + "postgresql://synthetic", + "source_schema.source_rows", + {"body": "body_text"}, + ) + ) + + assert result == { + "row_count": 3, + "semantic_role_nonblank_counts": {"body": 2}, + } + assert connection.closed + + +def test_audit_rejects_sql_syntax_in_identifiers() -> None: + with pytest.raises(ValueError, match="invalid PostgreSQL identifier"): + _identifier('source_rows; select secret') + + +@pytest.mark.parametrize( + "value", + ["document_key; select secret", 'document_key" from private_table --'], +) +def test_key_coverage_rejects_sql_syntax_in_identifiers(value: str) -> None: + with pytest.raises(ValueError, match="invalid PostgreSQL identifier"): + asyncio.run( + audit_source_semantic_coverage( + "postgresql://synthetic", + "source_schema.source_rows", + {}, + source_key=value, + coverage_table="semantic_schema.document_nodes", + coverage_key="document_key", + ) + ) + + +def test_audit_reports_distinct_semantic_key_coverage(monkeypatch) -> None: + """Coverage compares normalized keys and emits counts, never source values.""" + class Connection: + calls = 0 + + async def fetchrow(self, query: str): + self.calls += 1 + if self.calls == 1: + return {"row_count": 3} + assert 'from "source_schema"."source_rows"' in query + assert 'from "semantic_schema"."document_nodes"' in query + assert 'full join coverage_keys using (key_value)' in query + return { + "source_key_count": 2, + "coverage_key_count": 2, + "matched_key_count": 2, + "source_without_coverage_count": 0, + "coverage_without_source_count": 0, + } + + async def close(self) -> None: + pass + + async def connect(_dsn: str): + return Connection() + + monkeypatch.setattr("scripts.audit_source_semantic_coverage.asyncpg.connect", connect) + + result = asyncio.run( + audit_source_semantic_coverage( + "postgresql://synthetic", + "source_schema.source_rows", + {}, + source_key="document_key", + coverage_table="semantic_schema.document_nodes", + coverage_key="document_key", + ) + ) + + assert result["semantic_key_coverage"] == { + "source_key_count": 2, + "coverage_key_count": 2, + "matched_key_count": 2, + "source_without_coverage_count": 0, + "coverage_without_source_count": 0, + } + + +def test_audit_requires_complete_key_coverage_configuration() -> None: + with pytest.raises(ValueError, match="all required"): + asyncio.run( + audit_source_semantic_coverage( + "postgresql://synthetic", + "source_schema.source_rows", + {}, + source_key="document_key", + ) + ) + + +def test_audit_reports_assertion_evidence_and_prov_schema_without_values( + monkeypatch, +) -> None: + """Assertion evidence stays aggregate and PROV deployment is exact.""" + + class Connection: + calls = 0 + + async def fetchrow(self, query: str): + self.calls += 1 + if self.calls == 1: + return {"row_count": 2} + assert 'from "semantic_schema"."edge_assertions"' in query + return { + "assertion_count": 9, + "observed_count": 6, + "observed_without_evidence_count": 0, + "inferred_count": 2, + "inferred_without_direct_evidence_count": 1, + "predicted_count": 1, + "predicted_without_direct_evidence_count": 1, + "ungoverned_status_count": 0, + } + + async def fetch(self, query: str, schema: str, required: list[str]): + assert "information_schema.tables" in query + assert schema == "semantic_schema" + return [{"table_name": name} for name in required] + + async def close(self) -> None: + pass + + async def connect(_dsn: str): + return Connection() + + monkeypatch.setattr("scripts.audit_source_semantic_coverage.asyncpg.connect", connect) + + result = asyncio.run( + audit_source_semantic_coverage( + "postgresql://synthetic", + "source_schema.source_rows", + {}, + assertion_table="semantic_schema.edge_assertions", + assertion_status="evidence_status", + assertion_evidence="evidence_id", + provenance_schema="semantic_schema", + ) + ) + + assert result["semantic_assertion_evidence"] == { + "assertion_count": 9, + "observed_count": 6, + "observed_without_evidence_count": 0, + "inferred_count": 2, + "inferred_without_direct_evidence_count": 1, + "predicted_count": 1, + "predicted_without_direct_evidence_count": 1, + "ungoverned_status_count": 0, + "source_evidence_boundary_complete": True, + } + assert result["normalized_provenance_schema"] == { + "required_table_count": 14, + "present_table_count": 14, + "schema_complete": True, + } + + +def test_audit_requires_complete_assertion_configuration() -> None: + with pytest.raises(ValueError, match="assertion_table.*all required"): + asyncio.run( + audit_source_semantic_coverage( + "postgresql://synthetic", + "source_schema.source_rows", + {}, + assertion_table="semantic_schema.edge_assertions", + ) + ) diff --git a/tests/test_contextual_orchestrator_start.py b/tests/test_contextual_orchestrator_start.py index ad65a4c95..d83b552dc 100644 --- a/tests/test_contextual_orchestrator_start.py +++ b/tests/test_contextual_orchestrator_start.py @@ -37,6 +37,19 @@ def test_bootstrap_does_not_patch_upstream_model_classes() -> None: assert "_apply_provider_models" not in module.__dict__ +def test_container_pin_verifies_provider_host_cli_contract() -> None: + """The image build fails if its immutable upstream drops the forwarded flag.""" + dockerfile = ( + Path(__file__).parents[1] + / "docker" + / "contextual-orchestrator" + / "Dockerfile" + ).read_text(encoding="utf-8") + + assert "python -m contextual_orchestrator --help" in dockerfile + assert "grep -q -- '--allowed-provider-host'" in dockerfile + + def test_provider_api_url_is_canonical_over_compatibility_aliases(monkeypatch) -> None: module = _load_start_module() monkeypatch.setenv("LLM_GATEWAY_API_URL", "https://canonical.example/v1") @@ -55,6 +68,29 @@ def test_gateway_api_key_accepts_local_compatibility_alias(monkeypatch) -> None: assert module._pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY") == "compatibility-key" +def test_gateway_host_requires_an_explicit_matching_allowlist(monkeypatch) -> None: + """The bootstrap cannot inherit upstream's open public-host default.""" + module = _load_start_module() + monkeypatch.delenv("CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS", raising=False) + with pytest.raises(SystemExit, match="ALLOWED_PROVIDER_HOSTS is required"): + module._allowed_provider_hosts("https://gateway.example/v1") + + monkeypatch.setenv( + "CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS", "other.example" + ) + with pytest.raises(SystemExit, match="not in the provider allowlist"): + module._allowed_provider_hosts("https://gateway.example/v1") + + monkeypatch.setenv( + "CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS", + " gateway.example., OTHER.EXAMPLE ", + ) + assert module._allowed_provider_hosts("https://GATEWAY.EXAMPLE/v1") == ( + "gateway.example", + "other.example", + ) + + def test_provider_key_is_not_aliased_as_gateway_transport(monkeypatch) -> None: module = _load_start_module() for name in ("LLM_GATEWAY_API_KEY", "LLM_API_KEY"): @@ -112,6 +148,10 @@ def serve() -> None: monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "orchestrator-token") monkeypatch.setenv("LLM_GATEWAY_API_URL", "https://gateway.example") monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "embedding-model") + monkeypatch.setenv( + "CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS", + " secondary.example, gateway.example,secondary.example ", + ) module.main() @@ -119,6 +159,11 @@ def serve() -> None: assert isinstance(argv, list) assert "--embedding-provider-url" not in argv assert "--embedding-model" not in argv + assert argv.count("--allowed-provider-host") == 2 + assert argv[argv.index("--allowed-provider-host") + 1] == "gateway.example" + assert argv[argv.index("--allowed-provider-host", argv.index("--allowed-provider-host") + 1) + 1] == ( + "secondary.example" + ) assert captured["credentials"] == [ ("LLM_GATEWAY_API_KEY", "provider-key"), ("OPENAI_API_KEY", "openai-key"), @@ -138,5 +183,6 @@ def serve() -> None: } & os.environ.keys() agents = captured["agents"] assert isinstance(agents, dict) + assert agents["agents"][0]["provider_name"] == "configured_gateway" assert not [agent for agent in agents["agents"] if "embedding" in agent.get("tags", [])] assert "LLM_GATEWAY_EMBEDDING_MODEL" not in os.environ diff --git a/tests/test_import_postgresql_posts.py b/tests/test_import_postgresql_posts.py index cd35d4f57..d39e60c2e 100644 --- a/tests/test_import_postgresql_posts.py +++ b/tests/test_import_postgresql_posts.py @@ -51,9 +51,20 @@ def test_real_source_grouping_remains_the_derived_grouping() -> None: ) == ("thread-a", "secondary-a", "thread-a", "secondary-a") +@pytest.mark.parametrize( + ("body_column", "source_body", "effective_body", "preserve_existing_body"), + [ + ("body", "Synthetic customer-safe evidence body.", "Synthetic customer-safe evidence body.", False), + (None, None, "Previously imported authoritative body.", True), + ], +) def test_import_rows_persists_raw_and_derived_grouping_values( monkeypatch, tmp_path: Path, + body_column: str | None, + source_body: str | None, + effective_body: str, + preserve_existing_body: bool, ) -> None: """One synthetic import carries provenance and reconstruction fields together.""" query_file = tmp_path / "synthetic-query.sql" @@ -61,13 +72,15 @@ def test_import_rows_persists_raw_and_derived_grouping_values( row = { "record_key": "record-1", "title": "Synthetic lineage follow-up", - "body": "Synthetic customer-safe evidence body.", "created_at": datetime(2026, 1, 2, tzinfo=UTC), "draft_state": "published", "thread": "record-1", "secondary": "document-1", "project": "project-1", + "voc_type": "VOC", } + if source_body is not None: + row["body"] = source_body class FakeConnection: def __init__(self, *, source: bool) -> None: @@ -82,6 +95,10 @@ async def fetch(self, _query: str): async def execute(self, query: str, *args: object): self.executions.append((query, args)) + async def fetchval(self, query: str, *args: object): + self.executions.append((query, args)) + return effective_body + async def close(self) -> None: self.closed = True @@ -95,8 +112,10 @@ async def fake_connect(_dsn: str): async def fake_scope(_conn, _args): return "account-1", "corporate-1", "process-unit-1" - async def no_content(*_args, **_kwargs) -> None: - return None + persisted_bodies: list[str] = [] + + async def no_content(_conn, _post_id, body: str, **_kwargs) -> None: + persisted_bodies.append(body) async def no_cleanup(*_args, **_kwargs) -> dict[str, int]: return {"synthetic_rows_removed": 0} @@ -118,8 +137,7 @@ async def no_edges(_conn, *, llm=None) -> list[object]: lambda *_args: object(), ) - args = _parser().parse_args( - [ + command = [ "--source-dsn", "postgresql://synthetic-source", "--target-dsn", @@ -132,10 +150,10 @@ async def no_edges(_conn, *, llm=None) -> list[object]: "record_key", "--title-column", "title", - "--body-column", - "body", "--created-at-column", "created_at", + "--voc-type-column", + "voc_type", "--draft-column", "draft_state", "--exclude-draft-value", @@ -153,7 +171,13 @@ async def no_edges(_conn, *, llm=None) -> list[object]: "--process-unit-code", "synthetic-pu", ] - ) + if body_column is None: + command.extend( + ["--no-body-dimension-evidence", "synthetic export has no body dimension"] + ) + else: + command.extend(["--body-column", body_column]) + args = _parser().parse_args(command) result = asyncio.run(import_rows(args)) @@ -168,14 +192,25 @@ async def no_edges(_conn, *, llm=None) -> list[object]: "", "project-1", ) - assert source_post_args[-1] is None - assert result == { + assert source_post_args[32] is None + assert source_post_args[33] is preserve_existing_body + assert not any( + "insert into source_post_revision" in query + for query, _call_args in target.executions + ) + assert persisted_bodies == [effective_body] + expected_result: dict[str, object] = { "source_rows": 1, "imported_rows": 1, "skipped_rows": 0, "lineage_edges": 0, "synthetic_rows_removed": 0, } + if preserve_existing_body: + expected_result["no_body_dimension_evidence"] = ( + "synthetic export has no body dimension" + ) + assert result == expected_result assert source.closed and target.closed @@ -184,14 +219,14 @@ async def no_edges(_conn, *, llm=None) -> list[object]: [("VOC", "voc"), ("VOCC", "vocc"), ("VOCO", "voco"), ("VOM", "vom"), ("VOP", "vop")], ) def test_importer_preserves_source_voc_type_vocabulary(source_value: str, expected: str) -> None: - assert _normalize_voc_type(source_value, mapped=True) == expected + assert _normalize_voc_type(source_value) == expected def test_importer_rejects_unknown_or_empty_mapped_voc_type() -> None: with pytest.raises(ValueError, match="unsupported source VOC type"): - _normalize_voc_type("not-a-voc-type", mapped=True) + _normalize_voc_type("not-a-voc-type") with pytest.raises(ValueError, match="mapped source VOC type is empty"): - _normalize_voc_type("", mapped=True) + _normalize_voc_type("") def test_source_state_exclusion_uses_only_explicit_caller_values() -> None: @@ -215,13 +250,19 @@ def test_importer_rejects_mapping_the_pu_column_as_sales_pool() -> None: def test_importer_preflights_identity_and_body_before_target_mutation() -> None: - mapping = SimpleNamespace(record_key="record_key", body="body", draft="draft_state", deleted=None) + mapping = SimpleNamespace( + record_key="record_key", + body="body", + voc_type="voc_type", + draft="draft_state", + deleted=None, + ) with pytest.raises(ValueError, match="source record key cannot be empty at source row 2"): _validate_source_rows( [ - {"record_key": "one", "body": "body", "draft_state": "N"}, - {"record_key": "", "body": "body", "draft_state": "N"}, + {"record_key": "one", "body": "body", "voc_type": "VOC", "draft_state": "N"}, + {"record_key": "", "body": "body", "voc_type": "VOC", "draft_state": "N"}, ], mapping, ["Y"], @@ -230,16 +271,64 @@ def test_importer_preflights_identity_and_body_before_target_mutation() -> None: with pytest.raises(ValueError, match="source post body cannot be empty at source row 1"): _validate_source_rows( - [{"record_key": "one", "body": "", "draft_state": "N"}], mapping, ["Y"], [] + [{"record_key": "one", "body": "", "voc_type": "VOC", "draft_state": "N"}], mapping, ["Y"], [] + ) + + +def test_importer_accepts_explicitly_evidenced_missing_body_dimension() -> None: + mapping = SimpleNamespace( + record_key="record_key", body=None, voc_type="voc_type", draft="draft_state", deleted=None + ) + evidence = ( + "aggregate source inspection found no non-empty body values while " + "titles and structured dimensions remained populated" + ) + + _validate_source_rows( + [{"record_key": "one", "voc_type": "VOC", "draft_state": "published"}], + mapping, + ["draft"], + [], + "", + evidence, + ) + + with pytest.raises(ValueError, match="requires an operator evidence statement"): + _validate_source_rows( + [{"record_key": "one", "voc_type": "VOC", "draft_state": "published"}], + mapping, + ["draft"], + [], + "", + " ", ) +def test_importer_requires_exactly_one_body_evidence_door() -> None: + with pytest.raises(SystemExit): + _parser().parse_args( + [ + "--source-dsn", "postgresql://source", + "--target-dsn", "postgresql://target", + "--query-file", "query.sql", + "--source-system-code", "source", + "--record-key-column", "record_key", + "--title-column", "title", + "--body-column", "body", + "--no-body-dimension-evidence", "body is absent from this source export", + "--created-at-column", "created_at", + "--author-subject-id", "subject", + "--corporate-entity-code", "corp", + "--process-unit-code", "pu", + ] + ) + def test_importer_keeps_source_record_key_separate_from_source_uuid() -> None: - mapping = SimpleNamespace(post_id="guid_field") + mapping = SimpleNamespace(post_id="synthetic_post_id") source_uuid = "01234567-89ab-cdef-0123-456789abcdef" assert _source_post_id( - {"guid_field": source_uuid}, mapping, "source", "human-entered-source-key" + {"synthetic_post_id": source_uuid}, mapping, "source", "human-entered-source-key" ) == uuid.UUID(source_uuid) @@ -252,13 +341,15 @@ def test_importer_derives_legacy_post_uuid_without_a_post_id_mapping() -> None: def test_importer_rejects_duplicate_active_source_identity() -> None: - mapping = SimpleNamespace(record_key="record_key", body="body", draft="draft_state", deleted=None) + mapping = SimpleNamespace( + record_key="record_key", body="body", voc_type="voc_type", draft="draft_state", deleted=None + ) with pytest.raises(ValueError, match="duplicate source record key at source rows 1 and 2"): _validate_source_rows( [ - {"record_key": "same", "body": "first", "draft_state": "N"}, - {"record_key": "same", "body": "second", "draft_state": "N"}, + {"record_key": "same", "body": "first", "voc_type": "VOC", "draft_state": "N"}, + {"record_key": "same", "body": "second", "voc_type": "VOC", "draft_state": "N"}, ], mapping, ["Y"], @@ -271,6 +362,7 @@ def test_importer_allows_repeated_lookup_keys_when_source_uuids_are_distinct() - record_key="record_key", post_id="post_id", body="body", + voc_type="voc_type", draft="draft_state", deleted=None, ) @@ -281,12 +373,14 @@ def test_importer_allows_repeated_lookup_keys_when_source_uuids_are_distinct() - "record_key": "same", "post_id": "01234567-89ab-cdef-0123-456789abcdef", "body": "first", + "voc_type": "VOC", "draft_state": "N", }, { "record_key": "same", "post_id": "11234567-89ab-cdef-0123-456789abcdef", "body": "second", + "voc_type": "VOC", "draft_state": "N", }, ], @@ -331,11 +425,12 @@ def test_importer_has_no_unknown_publication_state_bypass() -> None: def test_no_draft_dimension_evidence_is_an_explicit_audited_door() -> None: """An export with no authorship-draft dimension passes only with the - operator's written evidence; the note cannot be a placeholder and - cannot be combined with a mapped draft column. + operator's written evidence and cannot be combined with a mapped draft + column. Evidence quality remains an operator/governance responsibility; + text length is not a validity proxy. """ no_draft_mapping = SimpleNamespace( - record_key="record_key", body="body", draft=None, deleted=None + record_key="record_key", body="body", voc_type="voc_type", draft=None, deleted=None ) evidence = ( "every candidate draft column is NULL across the export and the " @@ -343,21 +438,20 @@ def test_no_draft_dimension_evidence_is_an_explicit_audited_door() -> None: "real document" ) _validate_source_rows( - [{"record_key": "one", "body": "body"}], + [{"record_key": "one", "body": "body", "voc_type": "VOC"}], no_draft_mapping, [], [], evidence, ) - with pytest.raises(ValueError, match="at least 40 characters"): - _validate_source_rows( - [{"record_key": "one", "body": "body"}], - no_draft_mapping, - [], - [], - "no drafts", - ) + _validate_source_rows( + [{"record_key": "one", "body": "body", "voc_type": "VOC"}], + no_draft_mapping, + [], + [], + "operator attestation", + ) draft_mapping = SimpleNamespace( record_key="record_key", body="body", draft="draft_state", deleted=None @@ -392,6 +486,7 @@ def test_importer_does_not_select_a_provider_embedding_model(monkeypatch) -> Non "--title-column", "title", "--body-column", "body", "--created-at-column", "created_at", + "--voc-type-column", "voc_type", "--author-subject-id", "subject", "--corporate-entity-code", "corp", "--process-unit-code", "pu", @@ -412,6 +507,7 @@ def test_importer_accepts_explicit_source_name_mappings() -> None: "--title-column", "title", "--body-column", "body", "--created-at-column", "created_at", + "--voc-type-column", "voc_type", "--author-subject-id", "subject", "--corporate-entity-code", "corp", "--process-unit-code", "pu", diff --git a/tests/test_ontology.py b/tests/test_ontology.py index e883916f7..9dc8e86c4 100644 --- a/tests/test_ontology.py +++ b/tests/test_ontology.py @@ -14,7 +14,7 @@ import pytest from rdflib import Graph, Literal, URIRef -from rdflib.namespace import OWL, RDF, RDFS, SKOS, XSD +from rdflib.namespace import OWL, PROV, RDF, RDFS, SKOS, XSD from lineageweave.knowledge_graph import ( EDGE_AFFILIATION, @@ -95,6 +95,13 @@ def test_ontology_parses_as_valid_turtle() -> None: assert len(graph) > 0 +def test_semantic_content_provenance_specializes_prov_o() -> None: + """Content assertions remain PROV entities derived from their source posts.""" + graph = load_ontology() + assert (LW.SemanticContentAssertion, RDFS.subClassOf, PROV.Entity) in graph + assert (LW.wasDerivedFromPost, RDFS.subPropertyOf, PROV.wasDerivedFrom) in graph + + def test_every_seeded_lookup_code_is_declared_in_the_ontology() -> None: seeded = _seeded_lookup_codes_for_covered_categories() declared = all_declared_lookup_codes() @@ -340,6 +347,9 @@ def test_node_attribute_datatype_properties_project_real_columns() -> None: expected = { (LW.postTitle, LW.Post, XSD.string), (LW.postBody, LW.Post, XSD.string), + (LW.bodyAvailable, LW.Post, XSD.boolean), + (LW.sourceStageCode, LW.Post, XSD.string), + (LW.sourceDetailStateCode, LW.Post, XSD.string), (LW.eventOccurredAt, LW.Post, XSD.dateTime), (LW.personName, LW.Person, XSD.string), (LW.lastKnownJobTitle, LW.Person, XSD.string), @@ -350,6 +360,25 @@ def test_node_attribute_datatype_properties_project_real_columns() -> None: assert (prop, RDF.type, OWL.DatatypeProperty) in graph, str(prop) assert (prop, RDFS.domain, domain) in graph, str(prop) assert (prop, RDFS.range, datatype_range) in graph, str(prop) + assert (LW.hasPostType, RDF.type, OWL.ObjectProperty) in graph + assert (LW.hasPostType, RDFS.domain, LW.Post) in graph + assert (LW.hasPostType, RDFS.range, SKOS.Concept) in graph + + +def test_semantic_location_properties_carry_derived_place_and_region() -> None: + """ADR 0246: derived semantic locations bind the place name a post + expresses and the ISO 3166-1 alpha-2 region as instance data -- they + are DATATYPE properties of the derived :Location node, not relational + column projections, so the ADR 0207 column discipline does not apply. + """ + graph = load_ontology() + assert (LW.locationName, RDF.type, OWL.DatatypeProperty) in graph, "locationName" + assert (LW.locationName, RDFS.domain, LW.Location) in graph + assert (LW.locationName, RDFS.range, XSD.string) in graph + assert (LW.countryCode, RDF.type, OWL.DatatypeProperty) in graph + assert (LW.countryCode, RDFS.domain, LW.Location) in graph + assert (LW.countryCode, RDFS.range, XSD.string) in graph + assert "ISO 3166-1" in str(graph.value(LW.countryCode, RDFS.comment)) def test_shared_timestamps_declare_no_domain_to_avoid_multi_domain_entailment() -> None: diff --git a/tests/test_ontology_shapes.py b/tests/test_ontology_shapes.py index 6c74ebc63..f0691a98f 100644 --- a/tests/test_ontology_shapes.py +++ b/tests/test_ontology_shapes.py @@ -19,11 +19,11 @@ from pathlib import Path import pytest -from rdflib import Graph, Literal, Namespace, URIRef -from rdflib.namespace import RDF, XSD from pyshacl import validate as shacl_validate +from rdflib import Graph, Literal, Namespace, URIRef +from rdflib.namespace import RDF, SH, XSD -from lineageweave.ontology import project_project_mention_rdf +from lineageweave.ontology import project_project_mention_rdf, project_source_post_rdf ROOT = Path(__file__).resolve().parents[1] KG_PATH = ROOT / "docs" / "ontology" / "lineageweave-kg.ttl" @@ -66,6 +66,8 @@ def _representative_projection() -> Graph: data.add((post, RDF.type, LWn.Post)) data.add((post, LWn.postTitle, Literal("Line 3 downtime window"))) data.add((post, LWn.postBody, Literal("Customer reported a stoppage after changeover."))) + data.add((post, LWn.bodyAvailable, Literal(True, datatype=XSD.boolean))) + data.add((post, LWn.hasPostType, LWn.voiceOfCustomerType)) data.add( ( post, @@ -112,12 +114,77 @@ def test_shipped_shapes_conform_to_shacl_specification() -> None: assert conforms, report_text +def test_sparql_constraints_declare_their_prefixes() -> None: + """Portable SHACL-SPARQL constraints never rely on parser prefix fallback.""" + shapes = _load_shapes() + constraints = set(shapes.subjects(RDF.type, SH.SPARQLConstraint)) + assert constraints + assert all((constraint, SH.prefixes, None) in shapes for constraint in constraints) + declared_prefixes = { + str(prefix) + for declaration in shapes.objects(None, SH.declare) + for prefix in shapes.objects(declaration, SH.prefix) + } + assert {"lw", "rdf"} <= declared_prefixes + + def test_representative_db_projection_passes_validation() -> None: """A realistic projection of real schema rows validates cleanly.""" conforms, report_text = _conforms(_representative_projection()) assert conforms, report_text +def test_semantic_content_assertion_requires_source_post_provenance() -> None: + """A semantic assertion without its source-post derivation fails closed.""" + data = _representative_projection() + LWn = Namespace(LW) + assertion = URIRef(LW + "semantic-assertion-alpha") + post = URIRef(LW + "post-alpha") + activity = URIRef(LW + "activity-alpha") + data.add((assertion, RDF.type, LWn.SemanticContentAssertion)) + data.add((assertion, RDF.subject, post)) + data.add((assertion, RDF.predicate, LWn.describesActivity)) + data.add((assertion, RDF.object, activity)) + data.add((assertion, LWn.semanticEvidence, Literal("Synthetic activity evidence."))) + conforms, report_text = _conforms(data) + assert not conforms + assert "wasDerivedFromPost" in report_text + + +def test_semantic_content_assertion_derives_from_its_subject_post() -> None: + """A different valid post cannot be substituted as assertion provenance.""" + data = _representative_projection() + LWn = Namespace(LW) + subject_post = URIRef(LW + "post-alpha") + other_post = URIRef(LW + "post-beta") + assertion = URIRef(LW + "semantic-assertion-alpha") + activity = URIRef(LW + "activity-alpha") + data += project_source_post_rdf( + post_id="bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbb2", + post_title="Synthetic alternate post", + post_body="Synthetic unrelated source evidence.", + post_created_at=datetime(2026, 8, 25, 2, 0, tzinfo=timezone.utc), + voc_type_code="vop", + ) + projected_other = URIRef( + LW + "node/node_post/bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbb2" + ) + data.add((other_post, RDF.type, LWn.Post)) + for predicate, value in data.predicate_objects(projected_other): + data.add((other_post, predicate, value)) + data.add((assertion, RDF.type, LWn.SemanticContentAssertion)) + data.add((assertion, RDF.subject, subject_post)) + data.add((assertion, RDF.predicate, LWn.describesActivity)) + data.add((assertion, RDF.object, activity)) + data.add((assertion, LWn.wasDerivedFromPost, other_post)) + data.add((assertion, LWn.semanticEvidence, Literal("Synthetic evidence."))) + + conforms, report_text = _conforms(data) + + assert not conforms + assert "wasDerivedFromPost must identify the rdf:subject post" in report_text + + def test_schema_shaped_project_row_projection_passes_validation() -> None: """The production projector emits the complete SHACL-governed chain.""" data = project_project_mention_rdf( @@ -125,11 +192,14 @@ def test_schema_shaped_project_row_projection_passes_validation() -> None: post_title="Synthetic commissioning review", post_body="The synthetic source names the grid-upgrade project.", post_created_at=datetime(2026, 8, 25, 1, 23, 45, tzinfo=timezone.utc), + voc_type_code="vop", project_key="grid-upgrade", project_name="Grid Upgrade", evidence_text="grid-upgrade project", confidence=Decimal("0.870"), mention_created_at=datetime(2026, 8, 25, 1, 24, tzinfo=timezone.utc), + source_stage_code="synthetic-stage", + source_detail_state_code="synthetic-detail", ) conforms, report_text = _conforms(data) assert conforms, report_text @@ -143,6 +213,54 @@ def test_schema_shaped_project_row_projection_passes_validation() -> None: assert (mention, RDF.subject, None) in data assert (mention, RDF.predicate, LWn.mentionsProject) in data assert (mention, RDF.object, project) in data + post = URIRef(LW + "node/node_post/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1") + assert (post, Namespace(LW).hasPostType, Namespace(LW).voiceOfPartnerType) in data + assert (post, Namespace(LW).sourceStageCode, Literal("synthetic-stage")) in data + assert (post, Namespace(LW).sourceDetailStateCode, Literal("synthetic-detail")) in data + + +def test_missing_body_source_post_projection_passes_without_fabricated_text() -> None: + """An evidenced missing body remains an empty literal and explicit false state.""" + data = project_source_post_rdf( + post_id="aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa2", + post_title="Synthetic title-only record", + post_body="", + post_created_at=datetime(2026, 8, 25, 1, 23, 45, tzinfo=timezone.utc), + voc_type_code="voc", + source_stage_code="synthetic-stage", + source_detail_state_code="synthetic-detail", + ) + conforms, report_text = _conforms(data) + assert conforms, report_text + LWn = Namespace(LW) + post = URIRef(LW + "node/node_post/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa2") + assert (post, LWn.postBody, Literal("")) in data + assert (post, LWn.bodyAvailable, Literal(False, datatype=XSD.boolean)) in data + assert (post, LWn.hasPostType, LWn.voiceOfCustomerType) in data + + +@pytest.mark.parametrize("post_body", ["\u00a0", "\u202f", "\u0085"]) +def test_unicode_whitespace_body_matches_explicit_unavailable_state( + post_body: str, +) -> None: + """Unicode separators and NEL remain unavailable in RDF and SHACL.""" + data = project_source_post_rdf( + post_id="aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa3", + post_title="Synthetic whitespace record", + post_body=post_body, + post_created_at=datetime(2026, 8, 25, 1, 23, 45, tzinfo=timezone.utc), + voc_type_code="voc", + ) + + conforms, report_text = _conforms(data) + + assert conforms, report_text + post = URIRef(LW + "node/node_post/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa3") + assert ( + post, + Namespace(LW).bodyAvailable, + Literal(False, datatype=XSD.boolean), + ) in data @pytest.mark.parametrize( @@ -168,6 +286,7 @@ def test_project_row_projection_rejects_invalid_source_values( "post_title": "Synthetic commissioning review", "post_body": "The synthetic source names the grid-upgrade project.", "post_created_at": datetime(2026, 8, 25, 1, 23, 45, tzinfo=timezone.utc), + "voc_type_code": "voc", "project_key": "grid-upgrade", "project_name": "Grid Upgrade", "evidence_text": "grid-upgrade project", @@ -193,6 +312,50 @@ def test_project_row_projection_rejects_invalid_source_values( "postTitle", id="missing-required-post-title", ), + pytest.param( + lambda g: g.set( + ( + URIRef(LW + "post-alpha"), + URIRef(LW + "postTitle"), + Literal(" "), + ) + ), + "postTitle", + id="whitespace-only-post-title", + ), + pytest.param( + lambda g: g.set( + ( + URIRef(LW + "post-alpha"), + URIRef(LW + "bodyAvailable"), + Literal(False, datatype=XSD.boolean), + ) + ), + "bodyAvailable must be true", + id="nonempty-body-marked-unavailable", + ), + pytest.param( + lambda g: g.set( + ( + URIRef(LW + "post-alpha"), + URIRef(LW + "postBody"), + Literal(" \n\t"), + ) + ), + "bodyAvailable must be true", + id="whitespace-body-marked-available", + ), + pytest.param( + lambda g: g.add( + ( + URIRef(LW + "post-alpha"), + URIRef(LW + "sourceStageCode"), + Literal(" "), + ) + ), + "sourceStageCode", + id="whitespace-only-source-stage-code", + ), pytest.param( lambda g: g.remove( ( @@ -290,3 +453,29 @@ def test_confidence_boundary_values_are_inclusive() -> None: ) conforms, report_text = _conforms(data) assert conforms, f"{value} rejected:\n{report_text}" + + +def test_derived_location_with_place_and_region_passes_validation() -> None: + """ADR 0246: a semantic :Location instance may carry the derived place + name plus an ISO 3166-1 alpha-2 country/region code as instance data.""" + data = _representative_projection() + LWn = Namespace(LW) + site = URIRef(LW + "location-plant-a") + data.add((site, RDF.type, LWn.Location)) + data.add((site, LWn.locationName, Literal("Anaerobic digester plant site"))) + data.add((site, LWn.countryCode, Literal("KR"))) + conforms, report_text = _conforms(data) + assert conforms, report_text + + +def test_derived_location_rejects_a_non_uppercase_country_code() -> None: + """A :Location carrying a phrase instead of an ISO 3166-1 alpha-2 code + fails closed with the country-code property named.""" + data = _representative_projection() + LWn = Namespace(LW) + site = URIRef(LW + "location-plant") + data.add((site, RDF.type, LWn.Location)) + data.add((site, LWn.countryCode, Literal("korea"))) + conforms, report_text = _conforms(data) + assert not conforms + assert "countryCode" in report_text diff --git a/tests/test_semantic_hints.py b/tests/test_semantic_hints.py index 97e180377..9ded4ba9d 100644 --- a/tests/test_semantic_hints.py +++ b/tests/test_semantic_hints.py @@ -27,6 +27,9 @@ def test_semantic_hints_keep_explicit_project_pool_and_author_sources() -> None: source_project_code="SOURCE-PROJECT", source_company_name="Named company", source_process_unit_name="Named PU", + source_voc_type_code="vocc", + source_stage_code="medium", + source_detail_state_code="inspection-report", ) assert "project_field=PROJECT-42" in hints @@ -46,6 +49,9 @@ def test_semantic_hints_keep_explicit_project_pool_and_author_sources() -> None: assert "source_project_code=SOURCE-PROJECT" in hints assert "source_company_name=Named company [source_field=source_post.source_company_name]" in hints assert "source_process_unit_name=Named PU [source_field=source_post.source_process_unit_name]" in hints + assert "source_voc_type_code=vocc [source_field=source_post.voc_type_code]" in hints + assert "source_stage_code=medium [source_field=source_post.source_stage_code]" in hints + assert "source_detail_state_code=inspection-report [source_field=source_post.source_detail_state_code]" in hints assert "source_company_catalog_name=Catalog Company [source_lookup=corporate_entity.corporate_entity_code]" in hints assert "source_process_unit_catalog_name=Catalog PU [source_lookup=process_unit.process_unit_code]" in hints assert "source_customer_catalog_name=Catalog Customer [source_lookup=corporate_entity.corporate_entity_code]" in hints @@ -168,3 +174,25 @@ def test_without_source_context_account_affiliation_is_kept_as_keyman_hint() -> assert "author_affiliations=Synthetic Corp" in hints assert "author_side_hint=our_side_candidate" in hints + + +def test_classification_only_hints_do_not_suppress_identity_context() -> None: + """Classification evidence is not a substitute source identity boundary.""" + hints = format_semantic_hints( + author_name="Synthetic Analyst", + author_account_id="demo-account", + author_account_name="Synthetic Analyst", + author_affiliations=["Synthetic Corp"], + order_pool_code=None, + order_pool_name=None, + project_field=None, + customer_name="Synthetic Customer", + source_voc_type_code="vop", + source_stage_code="synthetic-stage", + source_detail_state_code="synthetic-detail", + ) + + assert "author_affiliations=Synthetic Corp" in hints + assert "customer=Synthetic Customer" in hints + assert "author_side_hint=our_side_candidate" in hints + assert "source_voc_type_code=vop" in hints diff --git a/uv.lock b/uv.lock index a29ef2708..bfc653d61 100644 --- a/uv.lock +++ b/uv.lock @@ -483,8 +483,8 @@ wheels = [ [[package]] name = "fast-mlsirm" -version = "0.8.0" -source = { git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=d025b7d237d8db7ca97a5611606c6285d5870895#d025b7d237d8db7ca97a5611606c6285d5870895" } +version = "0.9.1" +source = { git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=9cd12d6f74b8145c0d2d405c3ddf1859265fe93e#9cd12d6f74b8145c0d2d405c3ddf1859265fe93e" } dependencies = [ { name = "numpy" }, ] @@ -725,7 +725,7 @@ requires-dist = [ { name = "certifi", specifier = ">=2024.0.0" }, { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.6" }, { name = "cryptography", specifier = ">=42.0" }, - { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=d025b7d237d8db7ca97a5611606c6285d5870895" }, + { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=9cd12d6f74b8145c0d2d405c3ddf1859265fe93e" }, { name = "fastapi", marker = "extra == 'backend'", specifier = ">=0.141.1" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27.0" }, { name = "httpx2", marker = "extra == 'dev'", specifier = ">=2.12.0" },