diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f9194e28..b364318e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -43,6 +43,11 @@ flowchart LR Every boundary must be independently usable and expose versioned contracts for integration with organization repositories, `naruon`, and `contextual-orchestrator`. +The `analysis_engine` vertical slice is intentionally separate from `tepp_api`: +the API owns wire contracts while the engine owns deterministic execution. It +does not replace the future topic or psychometric estimators and does not read +another service's application tables. + ## Implemented foundation topology Task 1 materializes the first storage-independent workspace boundaries. The @@ -61,6 +66,9 @@ boundaries above remain the target modular MSA architecture. | `persistence_postgres` | PostgreSQL repositories and migrations | | `corpus_split` | cutoff-safe, relation-aware partitioning | | `tepp_simulation` | known-truth temporal/event data generation | +| `validation_core` | RMSE, bias, coverage, graph, and Monte Carlo metrics | +| `tepp_api` | versioned DTO, schema, terminal-result, and export contracts | +| `location_membership` | location is not entity identity and not a language channel | | `validation_core` | RMSE, bias, coverage, graph, Monte Carlo, and exact-head claim-promotion metrics | | `tepp_api` | versioned DTO, schema, and export contracts | | `prompt_source` | prompt boilerplate is not unique latent content and not stopword deletion | @@ -102,6 +110,8 @@ boundaries above remain the target modular MSA architecture. | `compute_backend` | VRAM-budgeted streamed planning, executable OOM retry plans, and a compensated CPU `f64` reference | | `episode_membership` | episode membership cannot escape the episode event-time interval | | `membership_target` | language, episode, template, department, and opportunity-pool targets cannot collapse into entity or project | +| `topic_measurement` | logistic-normal ALR and sequential Egozcue ILR topic coordinates | +| `analysis_engine` | bounded cutoff-safe temporal evidence readiness execution and digest-bound terminal artifacts | | `psychometric_core` | posterior-aware structural input gates, CWC within/between OLS plus the contextual effect, event-time log-rate, unequal-interval discrete-lag remapping, constant-predictor discrete effect, time-varying-predictor discrete effect (Eq. 14), exact scalar discrete process noise (Driver et al., 2017, Eq. 3), lagged latent covariance and unconditional latent variance (Driver et al., 2017, Eq. 3–4), stationary within-subject variance (Driver et al., 2017, Eq. 4 as `Δt → ∞`; `asymDIFFUSION`), trait-plus-state variance (Driver et al., 2017, §4.3 `TRAITVAR`; not process noise), observed-indicator variance and lagged observed covariance (Driver et al., 2017, Eq. 5; Table 2 `MANIFESTVAR` is `Θ`, not `Var(y)`; `MANIFESTTRAITVAR` is not `MANIFESTVAR`; `Θ` does not enter lagged observed covariance; observed-indicator mean is `τ + λ μ`; `MANIFESTMEANS` is not `E(y)`; `CINT` is not `MANIFESTMEANS`; discrete latent mean is `exp(a Δt) μ_0 + (exp(a Δt) − 1)/a κ`; `T0MEANS` is not `μ_t`; `CINT` is not the discrete increment; evolved observed mean is `τ + λ μ_t`; `τ + λ μ_0` is not `E(y_t)`; contemporaneous `TDPREDEFFECT` impulse is `m x`, not `CINT`, not `TIPREDEFFECT`, and not Voelkle Eq. 14; Eq. 5 of that contemporaneous impulse is `τ + λ(μ_t + m x)`, and `τ + λ μ_t` is not that observed mean; time-independent `TIPREDEFFECT` increment is `A^{-1}[e^{A Δt} − I] B z`, not `CINT`, not `M x`, not Voelkle Eq. 14, and not the coefficient `B`; Eq. 5 of that increment is `τ + λ(μ_t + A^{-1}[e^{A Δt} − I] B z)`, and `τ + λ μ_t` is not that observed mean; `τ + λ(μ_t + m x)` is not that observed mean; `τ + λ(μ_t + e^{a(t−u)} m x)` is not that observed mean when `u ≠ t`; within-interval `TDPREDEFFECT` carry is `e^{A(t−u)} M x` for `t0 < u < t`, not the contemporaneous Dirac, not `CINT`, not `TIPREDEFFECT`, and not Voelkle Eq. 14; Eq. 5 of that carry is `τ + λ(μ_t + e^{a(t−u)} m x)`, and `τ + λ μ_t` is not that observed mean; `τ + λ(μ_t + m x)` is not that carried observed mean when `u ≠ t`; first-occasion `T0TIPREDEFFECT` shift is `t0_b z` and Eq. 3 first-summand carry is `e^{A Δt} t0_b z` (`T0TIPREDEFFECT` is not `TIPREDEFFECT` `B`; `t0_b z` is not `A^{-1}[e^{A Δt} − I] B z`; `e^{A Δt} t0_b z` is not `t0_b z`; Eq. 5 of that carry is `τ + λ(μ_t + e^{a Δt} t0_b z)`, and `τ + λ μ_t` is not that observed mean; `τ + λ(μ_t + A^{-1}[e^{A Δt} − I] B z)` is not that observed mean), first-occasion `T0TDPREDEFFECT` shift is `t0_m x0` and Eq. 3 first-summand carry is `e^{A Δt} t0_m x0` (`T0TDPREDEFFECT` is not `TDPREDEFFECT` `M`; `t0_m x0` is not `M x`; `e^{A Δt} t0_m x0` is not `t0_m x0`; `e^{A Δt} t0_m x0` is not `e^{A(t−u)} M x` for `t0 < u < t`; `t0_m x0` is not `t0_b z`; an impulse at `u ≤ t0` that used `M` is already in `η(t0)` as `TDPREDEFFECT`, not as `T0TDPREDEFFECT`; Eq. 5 of that carry is `τ + λ(μ_t + e^{a Δt} t0_m x0)`, and `τ + λ μ_t` is not that observed mean; `τ + λ(μ_t + A^{-1}[e^{A Δt} − I] B z)` is not that observed mean; `τ + λ(μ_t + e^{a Δt} t0_b z)` is not that observed mean; §7.2 level-change `CINT` is `κ = −a m x` with `a < 0` so `−κ / a = m x` (`−a m x` is not the dissipating Dirac, not a free `CINT`, not `TIPREDEFFECT`, and not the extra near-zero-drift latent process also named in §7.2; Eq. 3 of that setting is `(1 − e^{a Δt}) m x`, which is not `m x`, not `κ`, and not `TIPREDEFFECT`; §7.2 extra-process contribution is `a_{ηξ} x (e^{ε Δt} − e^{a Δt}) / (ε − a)` (`ε = a` is `a_{ηξ} x Δt e^{a Δt}`; identification `TDPREDEFFECT` on the extra process is 1; printed extra `DRIFT` is `−0.000001`; not `κ = −a m x`, not `(1 − e^{a Δt}) m x`, and not the dissipating Dirac `m x`; `ε ≥ 0` fails closed; Eq. 5 of that contribution is `τ + λ(μ_t + a_{ηξ} x (e^{ε Δt} − e^{a Δt}) / (ε − a)`; the extra process has `LAMBDA` 0 and is not an observed indicator; `τ + λ μ_t` is not that observed mean; `τ + λ(μ_t + m x)` is not that observed mean; the contribution is not `E(y_t)`; the evolved-plus-contribution latent mean is not `E(y_t)`; after-t0 extra-process `TDPREDEFFECT` is `a_{ηξ} x (e^{ε(t−u)} − e^{a(t−u)}) / (ε − a)` for `t0 < u < t` while `μ_t` uses `Δt`; Eq. 5 of that after-t0 contribution is `τ + λ(μ_t + a_{ηξ} x (e^{ε(t−u)} − e^{a(t−u)}) / (ε − a)`; the first-occasion extra-process observed mean is not that observed mean when `u ≠ t0`; `e^{a(t−u)} m x` is a Dirac on the original process, not this `DRIFT` drive; §7.2 `asymTIPREDEFFECT` is `-B z / a` for `a < 0` (`-B z / a` is not the coefficient `B`, not `A^{-1}[e^{A Δt} − I] B z`, not `CINT`, and not `M x`; §7.2 `addedTIPREDVAR` is `(B / a)² v`, not `TRAITVAR`, not `asymDIFFUSION`, and not `-B z / a`; Table 2 `asymCINT` is `-κ / a` for `a < 0` and is not `κ`, not `A^{-1}[e^{A Δt} − I] κ`, not `T0MEANS`, and not `-B z / a`; p. 16 stationary `T0MEANS` is `-κ / a + −B z / a` and is not free `T0MEANS`, not `asymCINT` alone, not `asymTIPREDEFFECT` alone, and not the finite-interval discrete latent mean; Eq. 5 of that constrained mean is `τ + λ(−κ / a + −B z / a)`; `τ + λ μ_0` is not that observed mean; `τ + λ(−κ / a)` is not that observed mean when `B z ≠ 0`; `τ + λ μ_t` is not that observed mean; `MANIFESTMEANS` is not `E(y_0)`; the constrained latent mean is not `E(y_0)`; stationary `T0VAR` is `trait + −q / (2 a) + (B / a)² v` (not free `T0VAR`, not `asymDIFFUSION` alone, not `TRAITVAR` alone, not `addedTIPREDVAR` alone, and not the finite-interval discrete latent variance. Eq. 5 of that constrained variance is `λ²(trait + −q / (2 a) + (B / a)² v) + θ + ψ` (JSS PDF re-opened 2026-08-22T03:20Z; form the stationary latent variance first, then `λ² p + θ + ψ`; `λ² p_0` is not that observed variance; `λ²(−q / (2 a)) + θ` is not that observed variance when `TRAITVAR` or `addedTIPREDVAR` is nonzero; `MANIFESTVAR` is not `Var(y_0)`; the constrained latent variance is not `Var(y_0)`); lagged stationary `T0VAR` is `trait + e^{a Δt}(−q / (2 a)) + (B / a)² v` (trait and `addedTIPREDVAR` do not decay; contemporaneous `T0VAR` is not that lagged map; decaying the constrained total as if it were all state is not that lagged map; Eq. 5 of that lagged covariance is `λ²(trait + e^{a Δt}(−q / (2 a)) + (B / a)² v) + ψ`; `Θ` does not enter; contemporaneous `Var(y_0)` is not that lagged observed covariance; the lagged latent covariance is not that observed covariance); later-occasion stationary `T0VAR` is `trait + e^{2 a Δt}(−q / (2 a)) + Q_Δt + (B / a)² v` (trait and `addedTIPREDVAR` do not enter `Q_Δt`; under stationarity that composition equals contemporaneous `T0VAR`; evolving the constrained total as if it were all state is not that later map; the lagged covariance omits `Q_Δt`; `Q_Δt` is not that later map; Eq. 5 of that later-occasion variance is `λ²(trait + e^{2 a Δt}(−q / (2 a)) + Q_Δt + (B / a)² v) + θ + ψ`; lagged observed covariance omits `Q_Δt` and `θ`; `MANIFESTVAR` is not `Var(y_t)`; the later-occasion latent variance is not `Var(y_t)`))), irregular already-centered residual lag, Rubin `T` on OLS loadings, and strong-gated latent means (two-observation residual variance is identically `0` and caps at strong/scalar; Putnick & Bornstein, 2016) | diff --git a/CHANGELOG.md b/CHANGELOG.md index f5fcc159..c45ef29d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,21 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang - `psychometric_core` posterior-aware structural input gates: construct classification, refusal of raw-proportion Pearson/OLS, explicit ALR-versus-ILR geometry boundaries, CPU `f64` OLS recovery, posterior-draw loading point-estimate averaging without Rubin uncertainty claims, invariance-gated latent-mean comparison, and causal-heuristic refusal (ADR 0005 first production slice; no new migration). ### Added +- `topic_measurement` bounded deterministic CPU `f64` TRSL-TM reference estimator: canonical CSR/CSC inputs, cutoff-safe documents, standardized event time, weighted multiple memberships, prevalence covariates, explicit predecessor/successor regularization, multi-seed generalized EM, diagonal Laplace uncertainty, and fitted topic-lineage counts with known-truth RMSE plus exact line/branch coverage (ADR 0012; no persistence or accelerated-backend claim). +- `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, pairwise CLR Aitchison distance recovered by ILR Euclidean isometry for valid composition pairs, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). +- Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. +- Coverage source classification now scans Rust normal/raw/byte strings, comments, and character literals with escape-aware state, preserving executable string method calls and ignoring quoted comments. +- Quality-gate coverage tests now exercise blank-predecessor structural commas and escaped character literals in the authored-line scanner, including the past-EOF fail-closed path. +- Restored Graham Neubig's correct APA 7 initial in the Liu et al. (2023) prompting-survey register entry after the protected-main rebase. +- After protected-main consolidation #215, the analysis-run execution decision is recorded as ADR 0022 so it does not collide with ADR 0021 LineageWeave project-history. +- Registered the analysis-engine gap-closure doctoring in the canonical documentation map so its product and scientific traceability record is discoverable. +- Authored Rust coverage classification now ignores standalone structural closing parentheses, preventing formatting-only LCOV rows from appearing as uncovered production behavior. +- `analysis_engine` vertical slice (ADR 0022): bounded Rust execution from an accepted analysis run to either a cutoff-safe readiness result or a validated `tepp.trsl_topic_lineage.v1` artifact from the ADR-0012 estimator. Topic artifacts preserve fitted predecessor/successor edges, connectable-post and lineage counts, request/snapshot/cutoff bindings, SHA-256 identity, and fail-closed non-convergence/tamper behavior with exact line/branch coverage. This remains active-PR evidence and does not claim causal or psychometric authority. +- Coverage classification preserves the final expression line of multiline Rust `match` guards while respecting preceding-arm boundaries, keeping the 100% authored-line gate conservative. +- `tepp_api` fail-closed analysis-result boundaries: status constructors reject + terminal envelopes that cannot fit the default 64 KiB status limit, and + standalone terminal results reject knowledge cutoffs in the future. +- `tepp_api` request-bound terminal analysis results and typed analysis-run status/read responses: accepted/running states cannot carry measurement evidence, terminal results bind exact request and receipt identities, and succeeded/failed payloads remain digest-bound or content-redacted. - `evidence_core` embedded-image units: `data:image/;base64,...` URIs keep their original source spans and media types, and cannot be used as lexical inference text. - `persistence_postgres` entity/project target SQL now rejects empty, oversized, or hostile type/status labels before insert; interpolated codes are restricted to lowercase ASCII `snake_case` characters so membership foreign keys remain referentially safe (ADR 0003 / ADR 0013). - `persistence_postgres` live SQLx transport retains one pool-backed PostgreSQL connection per session so tenant binding and the following statement share a session, and closes the connection and owned runtime safely from another Tokio runtime (ADR 0013). @@ -181,7 +196,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). - `persistence_postgres` concurrent document-write stress: atomic revise `DO` block that requires exactly one open `system_to` close, SQLSTATE mapping onto `ConcurrentWriteConflict` / `DuplicateDocumentRecord`, and live multi-session insert/revise/append-only proofs. No new migration number. -- `tepp_api` naruon HTTP interchange: versioned `https` POST contracts for analysis-run create and modular export authorization that refuse table-access URLs, review/Copilot credential headers, reserved standard-header redefinition, principal-only export idempotency keys, and lexical inference claims (ADR 0011). +- `tepp_api` naruon HTTP interchange: versioned `https` POST contracts for analysis-run create and modular export authorization that refuse table-access URLs, provider-specific API-key/secret and review/Copilot credential headers, malformed extra HTTP fields, reserved standard-header redefinition, principal-only export idempotency keys, and lexical inference claims (ADR 0011). - `persistence_postgres` audit-event SQL contracts: append-only insert that refuses empty, oversized, or hostile `action_code` values before SQL is rendered. - `network_analysis` compositional cluster gates: raw topic proportions cannot be treated as Euclidean coordinates; recovered clusters are scored with label-invariant pair precision and recall against known truth. - `persistence_postgres` event-instance SQL contracts: bitemporal insert and as-known-at lookup that refuse inverted valid/system windows and hostile type/lifecycle labels before SQL is rendered. @@ -247,6 +262,9 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Changed +- ALR and ILR inverse normalization now fails closed when division would turn + a representable subnormal weight into a zero simplex part; runtime images + are pinned to the reviewed multi-platform Rust and Debian OCI digests. - `psychometric_core` scalar forward map `φ(Δt) = exp(a Δt)` now refuses binary64 underflow to `+0`. Voelkle et al. (2012, Eq. 7; ZORA accepted manuscript p. 16) write discrete auto-effects as `e^{a Δt}`, which are strictly positive; `a = ln(φ) / Δt` requires `φ > 0`. Direct overflow already failed closed. The Newton residual path refuses a mapped `+0` the same way. - The docstring discovery test compares crate-root names to `EXPECTED_CRATES` instead of a hardcoded count of 10, so `semantic_core` is required and an unapproved extra crate fails closed. - The LineageWeave temporal-context read exchange no longer emits a fabricated @@ -273,6 +291,11 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang head SHA to that exact-head register, including #201 `6afd650667e1` (RFC 5646 cited once; first GAP-005 slice, not implemented-main), stacked drafts #202–#204, and #164 `ff2e645b1785` as the predecessor register head. Duplicate + PR #179 remains closed. Stacked-merged heads and queued Checks are not + implemented-main. +- Rust LCOV quality gating now ignores visibility-qualified function signatures + and structural match-arm labels that LLVM reports as zero-hit non-executable + lines. PR #179 remains closed. Stacked-merged heads and queued Checks are not implemented-main. - Removed the completed one-shot PR #51 repair job from `docs-quality.yml`; the diff --git a/Cargo.lock b/Cargo.lock index e10e5441..7bbfc51b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,6 +58,22 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "analysis_engine" +version = "0.1.0" +dependencies = [ + "corpus_split", + "membership_core", + "relation_graph", + "serde", + "serde_json", + "sha2", + "temporal_core", + "tepp_api", + "topic_measurement", + "uuid", +] + [[package]] name = "assertion_clock" version = "0.1.0" @@ -1706,6 +1722,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "topic_measurement" +version = "0.1.0" +dependencies = [ + "corpus_split", + "membership_core", + "relation_graph", + "temporal_core", + "uuid", + "validation_core", +] + [[package]] name = "tracing" version = "0.1.44" diff --git a/Cargo.toml b/Cargo.toml index cfe6ee51..b36593bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,8 @@ members = [ "crates/compute_backend", "crates/episode_membership", "crates/membership_target", + "crates/topic_measurement", + "crates/analysis_engine", "crates/psychometric_core", ] default-members = [ @@ -108,6 +110,8 @@ default-members = [ "crates/compute_backend", "crates/episode_membership", "crates/membership_target", + "crates/topic_measurement", + "crates/analysis_engine", "crates/psychometric_core", ] diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 9a30447d..6a9cbca8 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -56,7 +56,10 @@ TEPP's approved PRD v0.4 and implementation plan are the primary product baselin | Stopword-deletion doctoring | [`docs/research/stopword-deletion.md`](docs/research/stopword-deletion.md) | | Provider-payload minimization doctoring | [`docs/research/provider-payload-minimization.md`](docs/research/provider-payload-minimization.md) | | Adaptive orchestration router doctoring | [`docs/research/adaptive-orchestration-router.md`](docs/research/adaptive-orchestration-router.md) | +| Topic log-ratio coordinate doctoring | [`docs/research/topic-logratio-coordinates.md`](docs/research/topic-logratio-coordinates.md) | | Hourly NIM OpenCode doctoring | [`docs/doctoring/hourly-nim-opencode-development.md`](docs/doctoring/hourly-nim-opencode-development.md) | +| Analysis engine v1 doctoring | [`docs/doctoring/analysis-engine-v1.md`](docs/doctoring/analysis-engine-v1.md) | +| Analysis engine gap-closure doctoring | [`docs/doctoring/analysis-engine-gap-closure.md`](docs/doctoring/analysis-engine-gap-closure.md) | | Corpus-split leakage-audit wire doctoring | [`docs/research/corpus-split-manifest-wire.md`](docs/research/corpus-split-manifest-wire.md) | | Unicode canonical-identity doctoring | [`docs/research/unicode-canonical-identity.md`](docs/research/unicode-canonical-identity.md) | | Change history | [`CHANGELOG.md`](CHANGELOG.md) | diff --git a/README.md b/README.md index f58a3487..c0110375 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,24 @@ implemented in Rust. ## Current implementation state -The current workspace contains 50 independently documented Rust crates. Each -crate exposes a bounded, tested contract for evidence, temporal semantics, -event and relation reasoning, membership, persistence, simulation, validation, -API exchange, compute planning, or evidence-grounded interpretation. Numerical -and psychometric authority remains on the CPU `f64` reference path; streamed -accelerator plans must preserve the full observation set and fail closed to the -reference path when resources or validation are insufficient. +The repository currently implements 53 independently documented crates rather +than a full commercial release. The implemented crates include topic measurement +and the analysis engine; they do not claim a complete commercial estimator, +operator workspace, or supported release. +- `topic_measurement`: the first production topic-measurement crate. It + estimates topic proportions from observed counts, maps those proportions into + additive log-ratio coordinates, and keeps posterior uncertainty attached so + later psychometric models do not treat raw topic proportions as ordinary + Euclidean indicators. +- `analysis_engine`: the first production analysis-run crate. It assembles one + cutoff-safe run from a validated design, documented evidence graph, and + estimator contract; persists the run with the six TEPP clocks; and emits a + typed terminal result. The crate does not claim buyer-visible product + completeness. + +```text +crates/analysis_engine These are production contracts, not a claim that the complete commercial estimator, operator workspace, or supported release already exists. Read the [product and technical gap baseline](docs/product-technical-gap-baseline.md) @@ -38,19 +48,39 @@ crates/assertion_clock crates/available_clock crates/checkpoint_authority crates/citation_edge +crates/compute_backend +crates/copied_text +crates/copy_identity +crates/corpus_background +crates/corpus_split +crates/cutoff_clock +crates/derived_sensitivity +crates/document_clocks +crates/encrypted_mapping +crates/episode_membership +crates/event_clock crates/evidence_core crates/semantic_core crates/temporal_core crates/event_core -crates/relation_graph +crates/evidence_core +crates/inferred_status +crates/intake_authorization +crates/interpretation_gateway +crates/location_membership +crates/longitudinal_core crates/membership_core +crates/membership_target +crates/modality_source +crates/model_selection +crates/network_analysis +crates/operational_log +crates/outcome_order +crates/payload_bound crates/persistence_postgres -crates/corpus_split -crates/tepp_simulation -crates/validation_core -crates/tepp_api -crates/location_membership +crates/prediction_contradiction crates/prompt_source +crates/provider_receipt crates/corpus_background crates/modality_source crates/copied_text @@ -99,6 +129,7 @@ crates/temporal_core crates/tepp_api crates/tepp_simulation crates/topic_lineage +crates/topic_measurement crates/validation_core crates/network_analysis crates/interpretation_gateway @@ -141,6 +172,12 @@ uncovered production behavior. - `docs/superpowers/plans/2026-08-05-temporal-event-foundation.md` - `docs/research/standards-and-literature.md` +No release, production-readiness, GPU, database, or statistical-recovery claim is +made by this foundation slice. + +The active stacked analysis-engine slice adds a bounded executable readiness path +from an accepted run to a digest-bound terminal artifact. It is not yet +implemented-main and does not replace scientific estimator contracts. Validated statistical-recovery APIs exist only inside `psychometric_core`: OLS loading recovery on already-mapped coordinates, posterior-draw point estimates, the Rubin total-variance identity `T = U_bar + (1 + 1/m) B`, CWC/event-time/ diff --git a/crates/analysis_engine/Cargo.toml b/crates/analysis_engine/Cargo.toml new file mode 100644 index 00000000..995ad914 --- /dev/null +++ b/crates/analysis_engine/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "analysis_engine" +description = "Deterministic cutoff-safe temporal evidence readiness execution." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +tepp_api = { path = "../tepp_api", version = "0.1.0" } +temporal_core = { path = "../temporal_core", version = "0.1.0" } +topic_measurement = { path = "../topic_measurement", version = "0.1.0" } +uuid.workspace = true + +[dev-dependencies] +corpus_split = { path = "../corpus_split", version = "0.1.0" } +membership_core = { path = "../membership_core", version = "0.1.0" } +relation_graph = { path = "../relation_graph", version = "0.1.0" } + +[lints] +workspace = true diff --git a/crates/analysis_engine/src/lib.rs b/crates/analysis_engine/src/lib.rs new file mode 100644 index 00000000..b807e607 --- /dev/null +++ b/crates/analysis_engine/src/lib.rs @@ -0,0 +1,726 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +//! Deterministic, cutoff-safe execution for the first TEPP analysis vertical slice. +//! +//! The engine consumes identity-free evidence metadata, excludes evidence that +//! was unavailable at the requested knowledge cutoff, counts multiple-membership +//! assignments without collapsing them, and emits a digest-bound terminal result +//! through [`tepp_api`]. It deliberately does not claim latent-variable or topic +//! estimation authority; it invokes estimators through their scientific crate +//! contracts and preserves their artifact meaning. + +mod topic_lineage_artifact; + +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use std::fmt; +use std::fmt::Write as _; +use temporal_core::{AvailableTime, EventTime, KnowledgeCutoff}; +use tepp_api::{ + AnalysisResultSummary, AnalysisRunAccepted, AnalysisRunRequest, AnalysisRunTerminalResult, + ApiError, +}; +use topic_measurement::TopicMeasurementError; + +/// Topic-lineage artifact and execution contracts from this engine. +pub use topic_lineage_artifact::{ + TOPIC_LINEAGE_ARTIFACT_BYTE_LIMIT, TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION, + TOPIC_LINEAGE_MODEL_CONTRACT_VERSION, TOPIC_LINEAGE_OUTPUT_PROFILE, TopicLineageArtifact, + TopicLineageArtifactEdge, TopicLineageExecution, execute_topic_lineage_run, +}; + +/// Versioned artifact schema emitted by this engine. +pub const ANALYSIS_ARTIFACT_SCHEMA_VERSION: &str = "tepp.temporal_evidence_readiness.v1"; +/// Number of deterministic statistics represented in the artifact summary. +pub const ANALYSIS_STATISTIC_COUNT: u64 = 4; +/// Maximum number of evidence units accepted by one in-memory execution. +pub const MAX_EVIDENCE_UNITS: usize = 100_000; +/// Maximum UTF-8 byte length of one snapshot or opaque evidence identifier. +pub const MAX_ANALYSIS_IDENTIFIER_BYTES: usize = 256; + +/// A bounded identity-free evidence unit offered to one analysis run. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AnalysisEvidenceUnit { + evidence_id: String, + event_time: EventTime, + available_time: AvailableTime, + membership_count: u32, +} + +impl AnalysisEvidenceUnit { + /// Construct an evidence unit with explicit event, availability, and + /// multiple-membership metadata. + /// + /// # Errors + /// + /// Returns [`AnalysisEngineError::InvalidEvidence`] when the identity is + /// empty or no membership assignment is supplied. + pub fn new( + evidence_id: impl Into, + event_time: EventTime, + available_time: AvailableTime, + membership_count: u32, + ) -> Result { + let evidence_id = evidence_id.into(); + if !valid_identifier(&evidence_id) || membership_count == 0 { + return Err(AnalysisEngineError::InvalidEvidence); + } + Ok(Self { + evidence_id, + event_time, + available_time, + membership_count, + }) + } + + /// Return the opaque evidence identity. + #[must_use] + pub fn evidence_id(&self) -> &str { + &self.evidence_id + } + + /// Return the event-valid time. + #[must_use] + pub const fn event_time(&self) -> EventTime { + self.event_time + } + + /// Return the evidence availability time. + #[must_use] + pub const fn available_time(&self) -> AvailableTime { + self.available_time + } + + /// Return the number of simultaneous membership assignments. + #[must_use] + pub const fn membership_count(&self) -> u32 { + self.membership_count + } +} + +/// A bounded snapshot of evidence metadata for one analysis run. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AnalysisCorpus { + snapshot_id: String, + evidence_units: Vec, +} + +impl AnalysisCorpus { + /// Construct a snapshot-owned corpus. + /// + /// # Errors + /// + /// Returns [`AnalysisEngineError::InvalidEvidence`] for an empty snapshot + /// identity or [`AnalysisEngineError::LimitExceeded`] for an oversized + /// in-memory corpus. + pub fn new( + snapshot_id: impl Into, + evidence_units: Vec, + ) -> Result { + let snapshot_id = snapshot_id.into(); + if !valid_identifier(&snapshot_id) { + return Err(AnalysisEngineError::InvalidEvidence); + } + if evidence_units.len() > MAX_EVIDENCE_UNITS { + return Err(AnalysisEngineError::LimitExceeded); + } + Ok(Self { + snapshot_id, + evidence_units, + }) + } + + /// Return the immutable snapshot identity. + #[must_use] + pub fn snapshot_id(&self) -> &str { + &self.snapshot_id + } + + /// Return the evidence units in source order. + #[must_use] + pub fn evidence_units(&self) -> &[AnalysisEvidenceUnit] { + &self.evidence_units + } +} + +/// Digest-bound, identity-free output artifact for one successful execution. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct AnalysisArtifact { + /// Versioned artifact schema. + pub schema_version: String, + /// Opaque accepted-run identity. + pub run_id: String, + /// Immutable source snapshot identity. + pub snapshot_id: String, + /// Historical cutoff applied to availability. + pub knowledge_cutoff: String, + /// Number of evidence units available by the cutoff. + pub eligible_evidence_count: u64, + /// Sum of preserved multiple-membership assignments. + pub eligible_membership_count: u64, + /// Earliest event-valid time among eligible evidence. + pub earliest_event_time: String, + /// Latest event-valid time among eligible evidence. + pub latest_event_time: String, +} + +impl AnalysisArtifact { + /// Serialize the canonical artifact bytes used for digesting. + /// + /// # Errors + /// + /// Returns [`AnalysisEngineError::SerializationFailure`] if serialization + /// unexpectedly fails. + pub fn to_json(&self) -> Result { + serde_json::to_string(self).map_err(|_| AnalysisEngineError::SerializationFailure) + } + + /// Return the lowercase SHA-256 digest of the canonical artifact JSON. + /// + /// # Errors + /// + /// Returns [`AnalysisEngineError::SerializationFailure`] if serialization + /// unexpectedly fails. + pub fn sha256(&self) -> Result { + self.to_json() + .map(|json| format_digest(Sha256::digest(json.into_bytes()))) + } +} + +/// One complete execution response, including the internal artifact and the +/// request-bound terminal wire result. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AnalysisExecution { + /// Digest-bound artifact, present only when the terminal result succeeded. + pub artifact: Option, + /// Request-bound terminal result returned to the service boundary. + pub terminal_result: AnalysisRunTerminalResult, +} + +/// Fail-closed errors from the deterministic analysis vertical slice. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AnalysisEngineError { + /// Request or accepted receipt failed its API contract. + Api(ApiError), + /// Evidence metadata was empty or structurally invalid. + InvalidEvidence, + /// Two evidence units reused one opaque identity. + DuplicateEvidence, + /// Corpus snapshot identity differed from the request snapshot. + SnapshotMismatch, + /// A bounded integer aggregation overflowed. + ArithmeticOverflow, + /// A serialized artifact could not be produced. + SerializationFailure, + /// The in-memory corpus exceeded the execution bound. + LimitExceeded, + /// A topic-measurement estimator rejected or could not complete the fit. + TopicMeasurement(TopicMeasurementError), + /// A topic-lineage artifact violated its bounded schema or count invariants. + InvalidTopicLineageArtifact, +} + +impl fmt::Display for AnalysisEngineError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::Api(error) => return error.fmt(formatter), + Self::InvalidEvidence => "invalid analysis evidence", + Self::DuplicateEvidence => "duplicate analysis evidence identity", + Self::SnapshotMismatch => "analysis snapshot identity mismatch", + Self::ArithmeticOverflow => "analysis evidence count overflow", + Self::SerializationFailure => "analysis artifact serialization failed", + Self::LimitExceeded => "analysis corpus exceeded its execution bound", + Self::TopicMeasurement(error) => return error.fmt(formatter), + Self::InvalidTopicLineageArtifact => "invalid topic lineage artifact", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for AnalysisEngineError {} + +impl From for AnalysisEngineError { + fn from(error: ApiError) -> Self { + Self::Api(error) + } +} + +impl From for AnalysisEngineError { + fn from(error: TopicMeasurementError) -> Self { + Self::TopicMeasurement(error) + } +} + +/// Execute the cutoff-safe temporal evidence readiness analysis. +/// +/// Evidence whose `available_time` is later than the request cutoff is excluded +/// before aggregation. Event time remains a separate clock, and all membership +/// assignments are summed rather than collapsed to one group. Successful output +/// contains only bounded counts and temporal extrema; source text, credentials, +/// and direct identities never enter the artifact. +/// +/// # Errors +/// +/// Returns a fail-closed error for invalid contracts, snapshot mismatch, +/// duplicate evidence identities, or invalid arithmetic/serialization state. +pub fn execute_analysis_run( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + corpus: &AnalysisCorpus, + completed_at: impl Into, +) -> Result { + request.to_json()?; + accepted.to_json()?; + require_receipt_identity(request, accepted)?; + if request.snapshot_id != corpus.snapshot_id { + return Err(AnalysisEngineError::SnapshotMismatch); + } + let cutoff = KnowledgeCutoff::parse_rfc3339(&request.knowledge_cutoff) + .map_err(|_| AnalysisEngineError::Api(ApiError::InvalidWirePayload))?; + let mut identities = BTreeSet::new(); + let mut eligible = Vec::new(); + for unit in &corpus.evidence_units { + if !identities.insert(unit.evidence_id.clone()) { + return Err(AnalysisEngineError::DuplicateEvidence); + } + if unit.available_time.instant() <= cutoff.instant() { + eligible.push(unit); + } + } + + let completed_at = completed_at.into(); + if eligible.is_empty() { + let terminal_result = AnalysisRunTerminalResult::failed( + request, + accepted, + completed_at, + "no_eligible_evidence", + )?; + return Ok(AnalysisExecution { + artifact: None, + terminal_result, + }); + } + + // The corpus bound makes this conversion and sum strictly smaller than + // `u64::MAX`: 100,000 * u32::MAX is below the 64-bit range. + let eligible_evidence_count = eligible.len() as u64; + let eligible_membership_count = eligible + .iter() + .fold(0_u64, |sum, unit| sum + u64::from(unit.membership_count)); + let (earliest, latest) = eligible.iter().fold( + (eligible[0].event_time, eligible[0].event_time), + |(earliest, latest), unit| (earliest.min(unit.event_time), latest.max(unit.event_time)), + ); + let artifact = AnalysisArtifact { + schema_version: ANALYSIS_ARTIFACT_SCHEMA_VERSION.to_owned(), + run_id: accepted.run_id.clone(), + snapshot_id: request.snapshot_id.clone(), + knowledge_cutoff: cutoff.to_rfc3339(), + eligible_evidence_count, + eligible_membership_count, + earliest_event_time: earliest.to_rfc3339(), + latest_event_time: latest.to_rfc3339(), + }; + artifact.sha256().and_then(move |digest| { + let artifact_id = format!("analysis_artifact_{}", &digest[..16]); + let summary = AnalysisResultSummary { + analysis_family: "temporal_evidence_readiness".to_owned(), + evidence_count: eligible_evidence_count, + statistic_count: ANALYSIS_STATISTIC_COUNT, + validation_status: "validated".to_owned(), + }; + let terminal_result = AnalysisRunTerminalResult::succeeded( + request, + accepted, + artifact_id, + digest, + ANALYSIS_ARTIFACT_SCHEMA_VERSION, + completed_at, + summary, + ) + .map_err(AnalysisEngineError::from)?; + Ok(AnalysisExecution { + artifact: Some(artifact), + terminal_result, + }) + }) +} + +/// Require the accepted receipt to carry the request's idempotency identity. +fn require_receipt_identity( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, +) -> Result<(), AnalysisEngineError> { + if request.idempotency_key != accepted.idempotency_key { + return Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)); + } + Ok(()) +} + +fn format_digest(digest: impl AsRef<[u8]>) -> String { + let mut output = String::with_capacity(digest.as_ref().len() * 2); + for byte in digest.as_ref() { + let _ = write!(output, "{byte:02x}"); + } + output +} + +fn valid_identifier(value: &str) -> bool { + !value.trim().is_empty() + && value.len() <= MAX_ANALYSIS_IDENTIFIER_BYTES + && !value.chars().any(char::is_control) +} + +#[cfg(test)] +mod tests { + use super::{ + ANALYSIS_ARTIFACT_SCHEMA_VERSION, ANALYSIS_STATISTIC_COUNT, AnalysisCorpus, + AnalysisEngineError, AnalysisEvidenceUnit, MAX_ANALYSIS_IDENTIFIER_BYTES, + MAX_EVIDENCE_UNITS, TopicMeasurementError, execute_analysis_run, + }; + use temporal_core::{AvailableTime, EventTime}; + use tepp_api::{AnalysisRunAccepted, AnalysisRunRequest, AnalysisRunTerminalState, ApiError}; + + fn request() -> AnalysisRunRequest { + AnalysisRunRequest { + contract_version: 1, + idempotency_key: "idem-analysis-1".into(), + tenant_workspace_id: "tenant-workspace-1".into(), + snapshot_id: "snapshot-1".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "temporal-evidence-v1".into(), + output_profile: "validation-report".into(), + } + } + + fn accepted() -> AnalysisRunAccepted { + AnalysisRunAccepted::new("run-1", "accepted", "idem-analysis-1").expect("accepted") + } + + fn unit(id: &str, event: &str, available: &str, memberships: u32) -> AnalysisEvidenceUnit { + AnalysisEvidenceUnit::new( + id, + EventTime::parse_rfc3339(event).expect("event"), + AvailableTime::parse_rfc3339(available).expect("available"), + memberships, + ) + .expect("unit") + } + + #[test] + fn successful_run_is_cutoff_safe_and_preserves_multiple_memberships() { + let corpus = AnalysisCorpus::new( + "snapshot-1", + vec![ + unit( + "evidence-1", + "2026-07-01T00:00:00Z", + "2026-07-15T00:00:00Z", + 2, + ), + unit( + "evidence-2", + "2026-07-20T00:00:00Z", + "2026-08-01T00:00:00Z", + 3, + ), + unit( + "late-evidence", + "2026-07-25T00:00:00Z", + "2026-08-02T00:00:00Z", + 9, + ), + ], + ) + .expect("corpus"); + let execution = + execute_analysis_run(&request(), &accepted(), &corpus, "2026-08-03T00:00:00Z") + .expect("execution"); + let artifact = execution.artifact.expect("artifact"); + assert_eq!(artifact.schema_version, ANALYSIS_ARTIFACT_SCHEMA_VERSION); + assert_eq!(artifact.eligible_evidence_count, 2); + assert_eq!(artifact.eligible_membership_count, 5); + assert_eq!(artifact.earliest_event_time, "2026-07-01T00:00:00Z"); + assert_eq!(artifact.latest_event_time, "2026-07-20T00:00:00Z"); + assert_eq!( + execution.terminal_result.run_state, + AnalysisRunTerminalState::Succeeded + ); + let summary = execution.terminal_result.summary.as_ref().expect("summary"); + assert_eq!(summary.evidence_count, 2); + assert_eq!(summary.statistic_count, ANALYSIS_STATISTIC_COUNT); + assert_eq!(summary.validation_status, "validated"); + assert!(execution.terminal_result.result_sha256.is_some()); + assert!(execution.terminal_result.to_json().is_ok()); + } + + #[test] + fn no_eligible_evidence_returns_a_redacted_failure_result() { + let corpus = AnalysisCorpus::new( + "snapshot-1", + vec![unit( + "late", + "2026-07-25T00:00:00Z", + "2026-08-02T00:00:00Z", + 1, + )], + ) + .expect("corpus"); + let execution = + execute_analysis_run(&request(), &accepted(), &corpus, "2026-08-03T00:00:00Z") + .expect("failure result"); + assert!(execution.artifact.is_none()); + assert_eq!( + execution.terminal_result.run_state, + AnalysisRunTerminalState::Failed + ); + assert_eq!( + execution.terminal_result.failure_code.as_deref(), + Some("no_eligible_evidence") + ); + assert!(execution.terminal_result.summary.is_none()); + } + + #[test] + fn trust_boundary_and_shape_errors_fail_closed() { + assert_eq!( + AnalysisEvidenceUnit::new( + "", + EventTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("event"), + AvailableTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("available"), + 1, + ), + Err(AnalysisEngineError::InvalidEvidence) + ); + assert_eq!( + AnalysisCorpus::new("", Vec::new()), + Err(AnalysisEngineError::InvalidEvidence) + ); + assert_eq!( + AnalysisCorpus::new("\n", Vec::new()), + Err(AnalysisEngineError::InvalidEvidence) + ); + assert_eq!( + AnalysisCorpus::new("s".repeat(MAX_ANALYSIS_IDENTIFIER_BYTES + 1), Vec::new()), + Err(AnalysisEngineError::InvalidEvidence) + ); + assert_eq!( + AnalysisEvidenceUnit::new( + "e", + EventTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("event"), + AvailableTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("available"), + 0, + ), + Err(AnalysisEngineError::InvalidEvidence) + ); + assert_eq!( + AnalysisEvidenceUnit::new( + "e".repeat(MAX_ANALYSIS_IDENTIFIER_BYTES + 1), + EventTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("event"), + AvailableTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("available"), + 1, + ), + Err(AnalysisEngineError::InvalidEvidence) + ); + let corpus = AnalysisCorpus::new( + "snapshot-2", + vec![unit( + "evidence-1", + "2026-07-01T00:00:00Z", + "2026-07-01T00:00:00Z", + 1, + )], + ) + .expect("corpus"); + assert_eq!( + execute_analysis_run(&request(), &accepted(), &corpus, "2026-08-03T00:00:00Z"), + Err(AnalysisEngineError::SnapshotMismatch) + ); + let mismatched_receipt = + AnalysisRunAccepted::new("run-1", "accepted", "other-idempotency").expect("receipt"); + let matching_corpus = AnalysisCorpus::new( + "snapshot-1", + vec![unit( + "evidence-1", + "2026-07-01T00:00:00Z", + "2026-07-01T00:00:00Z", + 1, + )], + ) + .expect("corpus"); + assert_eq!( + execute_analysis_run( + &request(), + &mismatched_receipt, + &matching_corpus, + "2026-08-03T00:00:00Z" + ), + Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)) + ); + let duplicate = AnalysisCorpus::new( + "snapshot-1", + vec![ + unit("same", "2026-07-01T00:00:00Z", "2026-07-01T00:00:00Z", 1), + unit("same", "2026-07-02T00:00:00Z", "2026-07-02T00:00:00Z", 1), + ], + ) + .expect("corpus"); + assert_eq!( + execute_analysis_run(&request(), &accepted(), &duplicate, "2026-08-03T00:00:00Z"), + Err(AnalysisEngineError::DuplicateEvidence) + ); + assert_eq!( + AnalysisEngineError::Api(ApiError::LimitExceeded).to_string(), + "API request exceeded configured limits" + ); + assert_eq!( + AnalysisEngineError::SerializationFailure.to_string(), + "analysis artifact serialization failed" + ); + } + + #[test] + fn public_accessors_limits_and_error_messages_are_executable() { + let evidence = unit( + "evidence-accessor", + "2026-07-01T00:00:00Z", + "2026-07-01T00:00:00Z", + 4, + ); + assert_eq!(evidence.evidence_id(), "evidence-accessor"); + assert_eq!( + evidence.event_time(), + EventTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("event") + ); + assert_eq!( + evidence.available_time(), + AvailableTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("available") + ); + assert_eq!(evidence.membership_count(), 4); + let corpus = + AnalysisCorpus::new("snapshot-accessor", vec![evidence.clone()]).expect("corpus"); + assert_eq!(corpus.snapshot_id(), "snapshot-accessor"); + assert_eq!(corpus.evidence_units(), &[evidence]); + + let oversized = AnalysisCorpus::new( + "snapshot-limit", + vec![ + unit("bounded", "2026-07-01T00:00:00Z", "2026-07-01T00:00:00Z", 1,); + MAX_EVIDENCE_UNITS + 1 + ], + ); + assert_eq!(oversized, Err(AnalysisEngineError::LimitExceeded)); + + let messages = [ + ( + AnalysisEngineError::InvalidEvidence, + "invalid analysis evidence", + ), + ( + AnalysisEngineError::DuplicateEvidence, + "duplicate analysis evidence identity", + ), + ( + AnalysisEngineError::SnapshotMismatch, + "analysis snapshot identity mismatch", + ), + ( + AnalysisEngineError::ArithmeticOverflow, + "analysis evidence count overflow", + ), + ( + AnalysisEngineError::SerializationFailure, + "analysis artifact serialization failed", + ), + ( + AnalysisEngineError::LimitExceeded, + "analysis corpus exceeded its execution bound", + ), + ( + AnalysisEngineError::TopicMeasurement(TopicMeasurementError::DidNotConverge), + "topic estimator did not converge", + ), + ( + AnalysisEngineError::InvalidTopicLineageArtifact, + "invalid topic lineage artifact", + ), + ]; + for (error, message) in messages { + assert_eq!(error.to_string(), message); + } + let converted: AnalysisEngineError = ApiError::InvalidWirePayload.into(); + assert_eq!(converted.to_string(), "invalid API wire payload"); + } + + #[test] + fn malformed_request_receipt_cutoff_and_completion_fail_closed() { + let corpus = AnalysisCorpus::new( + "snapshot-1", + vec![unit( + "evidence-1", + "2026-07-01T00:00:00Z", + "2026-07-01T00:00:00Z", + 1, + )], + ) + .expect("corpus"); + + let mut invalid_request = request(); + invalid_request.idempotency_key.clear(); + assert_eq!( + execute_analysis_run( + &invalid_request, + &accepted(), + &corpus, + "2026-08-03T00:00:00Z" + ), + Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)) + ); + + let mut invalid_accepted = accepted(); + invalid_accepted.run_id.clear(); + assert_eq!( + execute_analysis_run( + &request(), + &invalid_accepted, + &corpus, + "2026-08-03T00:00:00Z" + ), + Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)) + ); + + let mut invalid_cutoff = request(); + invalid_cutoff.knowledge_cutoff = "not-a-time".into(); + assert_eq!( + execute_analysis_run( + &invalid_cutoff, + &accepted(), + &corpus, + "2026-08-03T00:00:00Z" + ), + Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)) + ); + + assert_eq!( + execute_analysis_run(&request(), &accepted(), &corpus, "not-a-time"), + Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)) + ); + + let no_evidence = AnalysisCorpus::new( + "snapshot-1", + vec![unit( + "late", + "2026-07-01T00:00:00Z", + "2026-08-02T00:00:00Z", + 1, + )], + ) + .expect("corpus"); + assert_eq!( + execute_analysis_run(&request(), &accepted(), &no_evidence, "not-a-time"), + Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)) + ); + } +} diff --git a/crates/analysis_engine/src/topic_lineage_artifact.rs b/crates/analysis_engine/src/topic_lineage_artifact.rs new file mode 100644 index 00000000..9b33ce17 --- /dev/null +++ b/crates/analysis_engine/src/topic_lineage_artifact.rs @@ -0,0 +1,471 @@ +//! Digest-bound completed artifacts from the ADR-0012 topic estimator. + +use std::collections::BTreeSet; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use temporal_core::KnowledgeCutoff; +use tepp_api::{ + AnalysisResultSummary, AnalysisRunAccepted, AnalysisRunRequest, AnalysisRunTerminalResult, +}; +use topic_measurement::{ + ReferenceTopicInput, ReferenceTopicModelConfig, fit_reference_topic_model, +}; +use uuid::Uuid; + +use crate::{AnalysisEngineError, format_digest, require_receipt_identity, valid_identifier}; + +/// Versioned schema for a completed TRSL topic-lineage artifact. +pub const TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION: &str = "tepp.trsl_topic_lineage.v1"; +/// Model contract required by the CPU `f64` reference execution path. +pub const TOPIC_LINEAGE_MODEL_CONTRACT_VERSION: &str = "trsl_tm_cpu_f64_v1"; +/// Analysis-run output profile required for a topic-lineage artifact. +pub const TOPIC_LINEAGE_OUTPUT_PROFILE: &str = "trsl_topic_lineage_v1"; +/// Maximum canonical artifact JSON size. +pub const TOPIC_LINEAGE_ARTIFACT_BYTE_LIMIT: usize = 256 * 1024; +const TOPIC_LINEAGE_EDGE_LIMIT: usize = 100_000; + +/// One fitted same-topic predecessor/successor association. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TopicLineageArtifactEdge { + /// Opaque predecessor document identity. + pub predecessor_document_id: String, + /// Opaque successor document identity. + pub successor_document_id: String, + /// Artifact-local global topic index. + pub topic_index: u64, + /// Minimum dominant-topic posterior mean across the linked documents. + pub association_strength: f64, +} + +/// Completed, bounded topic-lineage result consumed by product-history clients. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TopicLineageArtifact { + /// Exact versioned schema identity. + pub schema_version: String, + /// Opaque accepted-run identity. + pub run_id: String, + /// Immutable source snapshot identity. + pub snapshot_id: String, + /// Historical evidence cutoff used by the estimator. + pub knowledge_cutoff: String, + /// Selected deterministic initialization seed. + pub selected_seed: u64, + /// Iterations used by the selected converged fit. + pub iterations: u64, + /// Final finite penalized objective. + pub objective: f64, + /// Number of global topics in the fitted model. + pub topic_count: u64, + /// Number of modeled evidence documents. + pub evidence_count: u64, + /// Documents incident to at least one fitted same-topic sequence edge. + pub connected_post_count: u64, + /// Topics represented by at least one fitted sequence edge. + pub lineage_count: u64, + /// Fitted edges restricted to explicit forward predecessor/successor input. + pub sequence_edges: Vec, + /// Fixed claim boundary for consumer copy. + pub inference_status: String, +} + +impl TopicLineageArtifact { + /// Parse and fully validate a bounded artifact JSON payload. + /// + /// # Errors + /// + /// Returns [`AnalysisEngineError::InvalidTopicLineageArtifact`] when the + /// schema, dimensions, identifiers, counts, edges, or claim boundary fail. + pub fn from_json(payload: &str) -> Result { + if payload.len() > TOPIC_LINEAGE_ARTIFACT_BYTE_LIMIT { + return Err(AnalysisEngineError::LimitExceeded); + } + let artifact: Self = serde_json::from_str(payload) + .map_err(|_| AnalysisEngineError::InvalidTopicLineageArtifact)?; + artifact.validate()?; + Ok(artifact) + } + + /// Serialize canonical validated artifact JSON. + /// + /// # Errors + /// + /// Returns a typed validation, serialization, or size failure. + pub fn to_json(&self) -> Result { + self.validate()?; + let payload = + serde_json::to_string(self).map_err(|_| AnalysisEngineError::SerializationFailure)?; + if payload.len() > TOPIC_LINEAGE_ARTIFACT_BYTE_LIMIT { + return Err(AnalysisEngineError::LimitExceeded); + } + Ok(payload) + } + + /// Return the lowercase SHA-256 digest of canonical artifact JSON. + /// + /// # Errors + /// + /// Returns a typed validation or serialization failure. + pub fn sha256(&self) -> Result { + self.to_json() + .map(|json| format_digest(Sha256::digest(json.into_bytes()))) + } + + fn validate(&self) -> Result<(), AnalysisEngineError> { + if self.schema_version != TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION + || !valid_identifier(&self.run_id) + || !valid_identifier(&self.snapshot_id) + || KnowledgeCutoff::parse_rfc3339(&self.knowledge_cutoff).is_err() + || self.iterations == 0 + || !self.objective.is_finite() + || self.topic_count < 2 + || self.evidence_count < 2 + || self.connected_post_count > self.evidence_count + || self.lineage_count > self.topic_count + || self.sequence_edges.len() > TOPIC_LINEAGE_EDGE_LIMIT + || self.inference_status != "fitted_topic_association_not_causation" + { + return Err(AnalysisEngineError::InvalidTopicLineageArtifact); + } + let mut pairs = BTreeSet::new(); + let mut connected = BTreeSet::new(); + let mut lineages = BTreeSet::new(); + for edge in &self.sequence_edges { + let predecessor = Uuid::parse_str(&edge.predecessor_document_id) + .map_err(|_| AnalysisEngineError::InvalidTopicLineageArtifact)?; + let successor = Uuid::parse_str(&edge.successor_document_id) + .map_err(|_| AnalysisEngineError::InvalidTopicLineageArtifact)?; + if predecessor == successor + || edge.topic_index >= self.topic_count + || !edge.association_strength.is_finite() + || edge.association_strength <= 0.0 + || edge.association_strength > 1.0 + || !pairs.insert((predecessor, successor)) + { + return Err(AnalysisEngineError::InvalidTopicLineageArtifact); + } + connected.insert(predecessor); + connected.insert(successor); + lineages.insert(edge.topic_index); + } + if self.connected_post_count != connected.len() as u64 + || self.lineage_count != lineages.len() as u64 + { + return Err(AnalysisEngineError::InvalidTopicLineageArtifact); + } + Ok(()) + } +} + +/// One completed topic-lineage artifact and its request-bound terminal result. +#[derive(Clone, Debug, PartialEq)] +pub struct TopicLineageExecution { + /// Digest-bound completed model artifact. + pub artifact: TopicLineageArtifact, + /// Terminal result carrying the artifact identity, digest, and schema. + pub terminal_result: AnalysisRunTerminalResult, +} + +/// Execute the validated ADR-0012 CPU `f64` reference estimator. +/// +/// The caller supplies the exact snapshot identity and cutoff used to construct +/// `input`; both must exactly match the already-validated analysis request. +/// This executor preserves the estimator result and does not select `K`, infer +/// causal edges, or emit a partial artifact. +/// +/// # Errors +/// +/// Returns a request/receipt/snapshot/cutoff/profile error, estimator failure, +/// arithmetic error, or invalid/oversized artifact error. +pub fn execute_topic_lineage_run( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + snapshot_id: &str, + knowledge_cutoff: KnowledgeCutoff, + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + completed_at: impl Into, +) -> Result { + request.to_json()?; + accepted.to_json()?; + require_receipt_identity(request, accepted)?; + if request.snapshot_id != snapshot_id { + return Err(AnalysisEngineError::SnapshotMismatch); + } + if request.knowledge_cutoff != knowledge_cutoff.to_rfc3339() + || request.model_contract_version != TOPIC_LINEAGE_MODEL_CONTRACT_VERSION + || request.output_profile != TOPIC_LINEAGE_OUTPUT_PROFILE + { + return Err(AnalysisEngineError::InvalidEvidence); + } + + let model = fit_reference_topic_model(input, config)?; + let topic_count = u64::try_from(model.topic_term_probabilities.len()) + .map_err(|_| AnalysisEngineError::ArithmeticOverflow)?; + let evidence_count = u64::try_from(input.document_count()) + .map_err(|_| AnalysisEngineError::ArithmeticOverflow)?; + let connected_post_count = u64::try_from(model.connected_post_count) + .map_err(|_| AnalysisEngineError::ArithmeticOverflow)?; + let lineage_count = + u64::try_from(model.lineage_count).map_err(|_| AnalysisEngineError::ArithmeticOverflow)?; + let sequence_edges: Vec<_> = model + .sequence_edges + .iter() + .map(|edge| { + Ok(TopicLineageArtifactEdge { + predecessor_document_id: edge.predecessor_document_id.to_string(), + successor_document_id: edge.successor_document_id.to_string(), + topic_index: u64::try_from(edge.topic_index) + .map_err(|_| AnalysisEngineError::ArithmeticOverflow)?, + association_strength: edge.association_strength, + }) + }) + .collect::>()?; + let artifact = TopicLineageArtifact { + schema_version: TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION.into(), + run_id: accepted.run_id.clone(), + snapshot_id: snapshot_id.to_owned(), + knowledge_cutoff: knowledge_cutoff.to_rfc3339(), + selected_seed: model.seed, + iterations: u64::try_from(model.iterations) + .map_err(|_| AnalysisEngineError::ArithmeticOverflow)?, + objective: model.objective, + topic_count, + evidence_count, + connected_post_count, + lineage_count, + sequence_edges, + inference_status: "fitted_topic_association_not_causation".into(), + }; + let digest = artifact.sha256()?; + let statistic_count = u64::try_from(artifact.sequence_edges.len()) + .map_err(|_| AnalysisEngineError::ArithmeticOverflow)? + .checked_add(2) + .ok_or(AnalysisEngineError::ArithmeticOverflow)?; + let summary = AnalysisResultSummary::new( + "trsl_topic_lineage", + evidence_count, + statistic_count, + "reference_estimator_converged", + ); + let summary = summary?; + let terminal_result = AnalysisRunTerminalResult::succeeded( + request, + accepted, + format!("topic_lineage_artifact_{}", &digest[..16]), + digest, + TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION, + completed_at, + summary, + ); + let terminal_result = terminal_result?; + Ok(TopicLineageExecution { + artifact, + terminal_result, + }) +} + +#[cfg(test)] +mod tests { + use super::{ + TOPIC_LINEAGE_ARTIFACT_BYTE_LIMIT, TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION, + TOPIC_LINEAGE_EDGE_LIMIT, TopicLineageArtifact, TopicLineageArtifactEdge, + }; + use crate::AnalysisEngineError; + + fn artifact() -> TopicLineageArtifact { + TopicLineageArtifact { + schema_version: TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION.into(), + run_id: "run-1".into(), + snapshot_id: "snapshot-1".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + selected_seed: 7, + iterations: 4, + objective: -1.0, + topic_count: 2, + evidence_count: 2, + connected_post_count: 2, + lineage_count: 1, + sequence_edges: vec![TopicLineageArtifactEdge { + predecessor_document_id: "00000000-0000-0000-0000-000000000001".into(), + successor_document_id: "00000000-0000-0000-0000-000000000002".into(), + topic_index: 0, + association_strength: 0.8, + }], + inference_status: "fitted_topic_association_not_causation".into(), + } + } + + fn assert_invalid(artifact: &TopicLineageArtifact) { + assert_eq!( + artifact.to_json(), + Err(AnalysisEngineError::InvalidTopicLineageArtifact) + ); + } + + #[test] + fn artifact_round_trip_and_size_bounds_fail_closed() { + let artifact = artifact(); + let payload = artifact.to_json().expect("json"); + assert_eq!( + TopicLineageArtifact::from_json(&payload), + Ok(artifact.clone()) + ); + assert_eq!(artifact.sha256().expect("digest").len(), 64); + assert_eq!( + TopicLineageArtifact::from_json("{}"), + Err(AnalysisEngineError::InvalidTopicLineageArtifact) + ); + assert_eq!( + TopicLineageArtifact::from_json(&"x".repeat(TOPIC_LINEAGE_ARTIFACT_BYTE_LIMIT + 1)), + Err(AnalysisEngineError::LimitExceeded) + ); + + let mut oversized = artifact; + oversized.sequence_edges = (1_u128..=2_000) + .map(|index| TopicLineageArtifactEdge { + predecessor_document_id: uuid::Uuid::from_u128(index).to_string(), + successor_document_id: uuid::Uuid::from_u128(index + 1).to_string(), + topic_index: 0, + association_strength: 0.8, + }) + .collect(); + oversized.evidence_count = 2_001; + oversized.connected_post_count = 2_001; + assert_eq!(oversized.to_json(), Err(AnalysisEngineError::LimitExceeded)); + } + + #[test] + fn artifact_metadata_tampering_fails_closed() { + let artifact = artifact(); + let invalid_artifacts = [ + { + let mut value = artifact.clone(); + value.schema_version.clear(); + value + }, + { + let mut value = artifact.clone(); + value.run_id.clear(); + value + }, + { + let mut value = artifact.clone(); + value.snapshot_id.clear(); + value + }, + { + let mut value = artifact.clone(); + value.knowledge_cutoff = "invalid".into(); + value + }, + { + let mut value = artifact.clone(); + value.iterations = 0; + value + }, + { + let mut value = artifact.clone(); + value.objective = f64::NAN; + value + }, + { + let mut value = artifact.clone(); + value.topic_count = 1; + value + }, + { + let mut value = artifact.clone(); + value.evidence_count = 1; + value + }, + { + let mut value = artifact.clone(); + value.connected_post_count = 3; + value + }, + { + let mut value = artifact.clone(); + value.lineage_count = 3; + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges = + vec![value.sequence_edges[0].clone(); TOPIC_LINEAGE_EDGE_LIMIT + 1]; + value + }, + { + let mut value = artifact.clone(); + value.inference_status.clear(); + value + }, + ]; + for invalid in invalid_artifacts { + assert_invalid(&invalid); + } + } + + #[test] + fn artifact_edge_tampering_fails_closed() { + let artifact = artifact(); + let invalid_artifacts = [ + { + let mut value = artifact.clone(); + value.sequence_edges[0].successor_document_id = + value.sequence_edges[0].predecessor_document_id.clone(); + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges[0].topic_index = 2; + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges[0].association_strength = f64::NAN; + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges[0].association_strength = 0.0; + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges[0].association_strength = 1.1; + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges.push(value.sequence_edges[0].clone()); + value + }, + { + let mut value = artifact.clone(); + value.connected_post_count = 1; + value + }, + { + let mut value = artifact.clone(); + value.lineage_count = 0; + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges[0].predecessor_document_id = "invalid".into(); + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges[0].successor_document_id = "invalid".into(); + value + }, + ]; + for invalid in invalid_artifacts { + assert_invalid(&invalid); + } + } +} diff --git a/crates/analysis_engine/tests/crate_contract.rs b/crates/analysis_engine/tests/crate_contract.rs new file mode 100644 index 00000000..c401c287 --- /dev/null +++ b/crates/analysis_engine/tests/crate_contract.rs @@ -0,0 +1,6 @@ +//! Package identity contract for the analysis engine. + +#[test] +fn package_identity_is_stable() { + assert_eq!(env!("CARGO_PKG_NAME"), "analysis_engine"); +} diff --git a/crates/analysis_engine/tests/end_to_end_contract.rs b/crates/analysis_engine/tests/end_to_end_contract.rs new file mode 100644 index 00000000..829e56f5 --- /dev/null +++ b/crates/analysis_engine/tests/end_to_end_contract.rs @@ -0,0 +1,107 @@ +//! Realistic cutoff-safe end-to-end analysis execution. + +use analysis_engine::{ + AnalysisCorpus, AnalysisEngineError, AnalysisEvidenceUnit, execute_analysis_run, +}; +use temporal_core::{AvailableTime, EventTime}; +use tepp_api::{AnalysisRunAccepted, AnalysisRunRequest, AnalysisRunTerminalState}; + +fn evidence(id: &str, available: &str, memberships: u32) -> AnalysisEvidenceUnit { + AnalysisEvidenceUnit::new( + id, + EventTime::parse_rfc3339("2026-07-10T12:00:00Z").expect("event time"), + AvailableTime::parse_rfc3339(available).expect("available time"), + memberships, + ) + .expect("evidence") +} + +#[test] +fn production_shape_run_excludes_future_available_evidence() { + let request = AnalysisRunRequest { + contract_version: 1, + idempotency_key: "customer-run-2026-08-01".into(), + tenant_workspace_id: "workspace-opaque-1".into(), + snapshot_id: "snapshot-customer-2026-08-01".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "temporal-evidence-v1".into(), + output_profile: "validation-report".into(), + }; + let accepted = + AnalysisRunAccepted::new("run-customer-1", "accepted", "customer-run-2026-08-01") + .expect("accepted"); + let corpus = AnalysisCorpus::new( + "snapshot-customer-2026-08-01", + vec![ + evidence("invoice-renewal", "2026-07-31T23:59:59Z", 2), + evidence("later-correction", "2026-08-01T00:00:01Z", 4), + ], + ) + .expect("snapshot"); + let execution = execute_analysis_run(&request, &accepted, &corpus, "2026-08-01T00:01:00Z") + .expect("execute"); + assert_eq!( + execution.terminal_result.run_state, + AnalysisRunTerminalState::Succeeded + ); + assert_eq!( + execution + .artifact + .expect("artifact") + .eligible_evidence_count, + 1 + ); +} + +#[test] +fn snapshot_identity_is_not_inferred_from_customer_payload() { + let request = AnalysisRunRequest { + contract_version: 1, + idempotency_key: "run".into(), + tenant_workspace_id: "workspace".into(), + snapshot_id: "request-snapshot".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "model-v1".into(), + output_profile: "report".into(), + }; + let accepted = AnalysisRunAccepted::new("run", "accepted", "run").expect("accepted"); + let corpus = AnalysisCorpus::new( + "other-snapshot", + vec![evidence("evidence", "2026-07-01T00:00:00Z", 1)], + ) + .expect("snapshot"); + assert_eq!( + execute_analysis_run(&request, &accepted, &corpus, "2026-08-01T00:01:00Z"), + Err(AnalysisEngineError::SnapshotMismatch) + ); +} + +#[test] +fn mismatched_receipt_identity_is_rejected_before_corpus_scan() { + let request = AnalysisRunRequest { + contract_version: 1, + idempotency_key: "request-idempotency".into(), + tenant_workspace_id: "workspace".into(), + snapshot_id: "snapshot".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "model-v1".into(), + output_profile: "report".into(), + }; + let accepted = + AnalysisRunAccepted::new("run", "accepted", "receipt-idempotency").expect("accepted"); + let corpus = AnalysisCorpus::new( + "snapshot", + vec![ + evidence("duplicate-evidence", "2026-07-01T00:00:00Z", 1), + evidence("duplicate-evidence", "2026-07-02T00:00:00Z", 1), + ], + ) + .expect("snapshot"); + + assert_eq!( + execute_analysis_run(&request, &accepted, &corpus, "2026-08-01T00:01:00Z"), + Err(AnalysisEngineError::Api( + tepp_api::ApiError::InvalidWirePayload + )) + ); +} diff --git a/crates/analysis_engine/tests/topic_lineage_execution_contract.rs b/crates/analysis_engine/tests/topic_lineage_execution_contract.rs new file mode 100644 index 00000000..045a8946 --- /dev/null +++ b/crates/analysis_engine/tests/topic_lineage_execution_contract.rs @@ -0,0 +1,238 @@ +//! End-to-end contract for the completed TRSL topic-lineage artifact. + +use analysis_engine::{ + AnalysisEngineError, TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION, + TOPIC_LINEAGE_MODEL_CONTRACT_VERSION, TOPIC_LINEAGE_OUTPUT_PROFILE, execute_topic_lineage_run, +}; +use corpus_split::{CorpusDocument, CorpusSnapshot}; +use membership_core::{ + GroupId, MemberId, MembershipAssignment, MembershipNetwork, MembershipRole, MembershipWeight, +}; +use relation_graph::{ + RelationEdge, RelationEndpointId, RelationEvidenceStatus, RelationGraph, RelationKind, +}; +use temporal_core::{ + AvailableTime, EventTime, KnowledgeCutoff, TemporalBoundary, TemporalInterval, + TemporalPrecision, +}; +use tepp_api::{AnalysisRunAccepted, AnalysisRunRequest, AnalysisRunTerminalState}; +use topic_measurement::{ReferenceTopicInput, ReferenceTopicModelConfig, SparseMatrix}; +use uuid::Uuid; + +fn event_time(day: u8) -> EventTime { + EventTime::parse_rfc3339(&format!("2026-07-{day:02}T00:00:00Z")).expect("event time") +} + +fn fixture() -> ( + CorpusSnapshot, + Vec, + Vec, + MembershipNetwork, + RelationGraph, +) { + let ids: Vec<_> = (1_u128..=4).map(Uuid::from_u128).collect(); + let times: Vec<_> = (1_u8..=4).map(event_time).collect(); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-08-01T00:00:00Z").expect("cutoff"); + let available = AvailableTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("available"); + let mut snapshot = CorpusSnapshot::new(); + let mut memberships = MembershipNetwork::new(); + for id in &ids { + snapshot + .insert_if_eligible(CorpusDocument::new(*id, available), &cutoff) + .expect("eligible"); + memberships + .insert( + MembershipAssignment::new( + MemberId::from_uuid(*id), + GroupId::from_uuid(Uuid::from_u128(100)), + MembershipRole::Project, + MembershipWeight::full().expect("weight"), + event_time(1), + event_time(9), + ) + .expect("membership"), + ) + .expect("insert"); + } + let mut relations = RelationGraph::new(); + for (source, target, source_day, target_day) in [(0, 1, 1, 2), (1, 2, 2, 3), (2, 3, 3, 4)] { + let interval = |day| { + TemporalInterval::bounded( + TemporalBoundary::Included(event_time(day)), + TemporalBoundary::Included( + EventTime::parse_rfc3339(&format!("2026-07-{day:02}T12:00:00Z")).expect("end"), + ), + TemporalPrecision::Second, + ) + .expect("interval") + }; + relations + .insert( + RelationEdge::new( + RelationKind::TransitionsTo, + RelationEndpointId::from_uuid(ids[source]), + RelationEndpointId::from_uuid(ids[target]), + RelationEvidenceStatus::Observed, + interval(source_day), + interval(target_day), + ) + .expect("relation"), + ) + .expect("insert relation"); + } + (snapshot, ids, times, memberships, relations) +} + +fn request() -> AnalysisRunRequest { + AnalysisRunRequest { + contract_version: 1, + idempotency_key: "topic-lineage-idem".into(), + tenant_workspace_id: "tenant-workspace".into(), + snapshot_id: "snapshot-topic-lineage".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: TOPIC_LINEAGE_MODEL_CONTRACT_VERSION.into(), + output_profile: TOPIC_LINEAGE_OUTPUT_PROFILE.into(), + } +} + +#[test] +fn fitted_topics_emit_digest_bound_predecessor_successor_counts() { + let (snapshot, ids, times, memberships, relations) = fixture(); + let counts = SparseMatrix::from_csr( + 4, + 4, + vec![0, 2, 4, 6, 8], + vec![0, 1, 0, 1, 2, 3, 2, 3], + vec![90.0, 10.0, 85.0, 15.0, 10.0, 90.0, 15.0, 85.0], + ) + .expect("counts"); + let input = ReferenceTopicInput::new( + &snapshot, + ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("input"); + let config = ReferenceTopicModelConfig::new(2, vec![7, 11], 2_000, 1e-5) + .expect("config") + .with_hyperparameters(1.0, 0.5, 0.01, 0.05, 0.2) + .expect("hyperparameters"); + let request = request(); + let accepted = + AnalysisRunAccepted::new("run-topic-lineage", "accepted", &request.idempotency_key) + .expect("accepted"); + let execution = execute_topic_lineage_run( + &request, + &accepted, + "snapshot-topic-lineage", + KnowledgeCutoff::parse_rfc3339("2026-08-01T00:00:00Z").expect("cutoff"), + &input, + &config, + "2026-08-02T00:00:00Z", + ) + .expect("execution"); + + assert_eq!( + execution.artifact.schema_version, + TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION + ); + assert_eq!(execution.artifact.connected_post_count, 4); + assert_eq!(execution.artifact.lineage_count, 2); + assert_eq!(execution.artifact.sequence_edges.len(), 2); + assert_eq!( + execution.terminal_result.run_state, + AnalysisRunTerminalState::Succeeded + ); + assert_eq!( + execution.terminal_result.result_sha256.as_deref(), + Some(execution.artifact.sha256().expect("digest").as_str()) + ); + assert_eq!( + execution.terminal_result.result_schema_version.as_deref(), + Some(TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION) + ); + assert!(execution.artifact.to_json().is_ok()); +} + +#[test] +fn execution_refuses_binding_and_nonconvergence_without_an_artifact() { + let (snapshot, ids, times, memberships, relations) = fixture(); + let counts = SparseMatrix::from_csr(4, 2, vec![0, 1, 2, 3, 4], vec![0, 0, 1, 1], vec![1.0; 4]) + .expect("counts"); + let input = ReferenceTopicInput::new( + &snapshot, + ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("input"); + let request = request(); + let accepted = + AnalysisRunAccepted::new("run-topic-lineage", "accepted", &request.idempotency_key) + .expect("accepted"); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-08-01T00:00:00Z").expect("cutoff"); + let config = ReferenceTopicModelConfig::new(2, vec![1], 2, 1e-12).expect("config"); + + assert_eq!( + execute_topic_lineage_run( + &request, + &accepted, + "other-snapshot", + cutoff, + &input, + &config, + "2026-08-02T00:00:00Z", + ), + Err(AnalysisEngineError::SnapshotMismatch) + ); + for invalid_request in [ + { + let mut value = request.clone(); + value.knowledge_cutoff = "2026-08-02T00:00:00Z".into(); + value + }, + { + let mut value = request.clone(); + value.model_contract_version = "other-model".into(); + value + }, + { + let mut value = request.clone(); + value.output_profile = "other-profile".into(); + value + }, + ] { + assert_eq!( + execute_topic_lineage_run( + &invalid_request, + &accepted, + "snapshot-topic-lineage", + cutoff, + &input, + &config, + "2026-08-02T00:00:00Z", + ), + Err(AnalysisEngineError::InvalidEvidence) + ); + } + assert_eq!( + execute_topic_lineage_run( + &request, + &accepted, + "snapshot-topic-lineage", + cutoff, + &input, + &config, + "2026-08-02T00:00:00Z", + ), + Err(AnalysisEngineError::TopicMeasurement( + topic_measurement::TopicMeasurementError::DidNotConverge + )) + ); +} diff --git a/crates/tepp_api/src/analysis_result.rs b/crates/tepp_api/src/analysis_result.rs new file mode 100644 index 00000000..61bd13db --- /dev/null +++ b/crates/tepp_api/src/analysis_result.rs @@ -0,0 +1,362 @@ +//! Versioned terminal analysis-run result contracts. +//! +//! Submission acceptance and scientific completion are separate facts. +//! [`AnalysisRunAccepted`] is only a durable receipt. This module defines a +//! distinct, request-bound terminal result with a digest-bound artifact or a +//! redacted failure code. + +use crate::analysis_run::require_rfc3339_knowledge_cutoff; +use crate::wire::{ + from_json, require_byte_limit, require_contract_version, require_nonempty, to_json, +}; +use crate::{AnalysisRunAccepted, AnalysisRunRequest, ApiError}; +use serde::{Deserialize, Serialize}; +use temporal_core::SystemTime; + +/// Supported terminal analysis-result contract version. +pub const ANALYSIS_RESULT_CONTRACT_VERSION: u16 = 1; + +/// Default maximum terminal analysis-result JSON payload size in bytes. +pub const DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT: usize = 64 * 1024; + +const MAXIMUM_SUMMARY_COUNT: u64 = 1_000_000_000; +const MAXIMUM_FAILURE_CODE_BYTES: usize = 64; + +/// Canonical terminal lifecycle state for an analysis run. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AnalysisRunTerminalState { + /// Computation completed with a digest-bound result artifact. + Succeeded, + /// Computation ended without a result artifact. + Failed, +} + +/// Bounded, identity-free summary of one completed measurement artifact. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AnalysisResultSummary { + /// Versioned analysis family. + pub analysis_family: String, + /// Number of evidence units represented by the result. + pub evidence_count: u64, + /// Number of reported statistics or parameters. + pub statistic_count: u64, + /// Provider-authored validation status. + pub validation_status: String, +} + +impl AnalysisResultSummary { + /// Construct and validate a bounded, identity-free summary. + /// + /// # Errors + /// + /// Returns a fail-closed contract error for empty labels or unbounded + /// counts. + pub fn new( + analysis_family: impl Into, + evidence_count: u64, + statistic_count: u64, + validation_status: impl Into, + ) -> Result { + let value = Self { + analysis_family: analysis_family.into(), + evidence_count, + statistic_count, + validation_status: validation_status.into(), + }; + value.validate()?; + Ok(value) + } + + fn validate(&self) -> Result<(), ApiError> { + require_nonempty(&self.analysis_family)?; + require_nonempty(&self.validation_status)?; + if self.evidence_count > MAXIMUM_SUMMARY_COUNT + || self.statistic_count > MAXIMUM_SUMMARY_COUNT + { + return Err(ApiError::LimitExceeded); + } + Ok(()) + } +} + +/// Request-bound terminal outcome for one accepted analysis run. +/// +/// The succeeded shape excludes source text, credentials, direct identity, +/// respondent/item records, and unrestricted model output. The failed shape +/// contains no measurement artifact. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AnalysisRunTerminalResult { + /// Semantic contract version. + pub contract_version: u16, + /// Opaque remote run identity from [`AnalysisRunAccepted`]. + pub run_id: String, + /// Terminal lifecycle state. + pub run_state: AnalysisRunTerminalState, + /// Exact request idempotency key. + pub idempotency_key: String, + /// Authorized tenant/workspace opaque identity. + pub tenant_workspace_id: String, + /// Immutable corpus/evidence snapshot identity. + pub snapshot_id: String, + /// Exact request knowledge cutoff. + pub knowledge_cutoff: String, + /// Exact model/backend contract identity. + pub model_contract_version: String, + /// Exact requested output profile. + pub output_profile: String, + /// Opaque result artifact identity for a succeeded run. + pub result_artifact_id: Option, + /// Canonical lowercase SHA-256 result digest. + pub result_sha256: Option, + /// Versioned result-schema identity. + pub result_schema_version: Option, + /// Strict RFC 3339 system time at terminal completion. + pub completed_at: String, + /// Bounded summary for a succeeded run. + pub summary: Option, + /// Stable snake-case code for a failed run. + pub failure_code: Option, +} + +impl AnalysisRunTerminalResult { + /// Construct a succeeded terminal result bound to request and receipt. + /// + /// # Errors + /// + /// Returns a fail-closed error for invalid shape, digest, time, summary, or + /// request/receipt binding. + pub fn succeeded( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + result_artifact_id: impl Into, + result_sha256: impl Into, + result_schema_version: impl Into, + completed_at: impl Into, + summary: AnalysisResultSummary, + ) -> Result { + let value = Self { + contract_version: ANALYSIS_RESULT_CONTRACT_VERSION, + run_id: accepted.run_id.clone(), + run_state: AnalysisRunTerminalState::Succeeded, + idempotency_key: request.idempotency_key.clone(), + tenant_workspace_id: request.tenant_workspace_id.clone(), + snapshot_id: request.snapshot_id.clone(), + knowledge_cutoff: request.knowledge_cutoff.clone(), + model_contract_version: request.model_contract_version.clone(), + output_profile: request.output_profile.clone(), + result_artifact_id: Some(result_artifact_id.into()), + result_sha256: Some(result_sha256.into()), + result_schema_version: Some(result_schema_version.into()), + completed_at: completed_at.into(), + summary: Some(summary), + failure_code: None, + }; + value.validate()?; + require_terminal_binding(request, accepted, &value)?; + Ok(value) + } + + /// Construct a failed terminal result bound to request and receipt. + /// + /// # Errors + /// + /// Returns a fail-closed error for invalid time, failure code, or + /// request/receipt binding. + pub fn failed( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + completed_at: impl Into, + failure_code: impl Into, + ) -> Result { + let value = Self { + contract_version: ANALYSIS_RESULT_CONTRACT_VERSION, + run_id: accepted.run_id.clone(), + run_state: AnalysisRunTerminalState::Failed, + idempotency_key: request.idempotency_key.clone(), + tenant_workspace_id: request.tenant_workspace_id.clone(), + snapshot_id: request.snapshot_id.clone(), + knowledge_cutoff: request.knowledge_cutoff.clone(), + model_contract_version: request.model_contract_version.clone(), + output_profile: request.output_profile.clone(), + result_artifact_id: None, + result_sha256: None, + result_schema_version: None, + completed_at: completed_at.into(), + summary: None, + failure_code: Some(failure_code.into()), + }; + value.validate()?; + require_terminal_binding(request, accepted, &value)?; + Ok(value) + } + + /// Parse and validate a terminal result with the default byte limit. + /// + /// # Errors + /// + /// Returns wire, version, limit, time, digest, shape, or field errors. + pub fn from_json(payload: &str) -> Result { + Self::from_json_with_limit(payload, DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT) + } + + /// Parse and validate a terminal result with a caller-supplied byte limit. + /// + /// # Errors + /// + /// Returns wire, version, limit, time, digest, shape, or field errors. + pub fn from_json_with_limit(payload: &str, maximum_bytes: usize) -> Result { + require_byte_limit(payload, maximum_bytes)?; + let value: Self = from_json(payload)?; + value.validate()?; + Ok(value) + } + + /// Serialize this terminal result after complete validation. + /// + /// # Errors + /// + /// Returns validation or serialization errors. + pub fn to_json(&self) -> Result { + self.validate()?; + let payload = to_json(self)?; + require_byte_limit(&payload, DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT)?; + Ok(payload) + } + + pub(crate) fn validate(&self) -> Result<(), ApiError> { + require_contract_version(self.contract_version, ANALYSIS_RESULT_CONTRACT_VERSION)?; + for value in [ + &self.run_id, + &self.idempotency_key, + &self.tenant_workspace_id, + &self.snapshot_id, + &self.knowledge_cutoff, + &self.model_contract_version, + &self.output_profile, + &self.completed_at, + ] { + require_nonempty(value)?; + } + require_rfc3339_knowledge_cutoff(&self.knowledge_cutoff)?; + SystemTime::parse_rfc3339(&self.completed_at).map_err(|_| ApiError::InvalidWirePayload)?; + + match self.run_state { + AnalysisRunTerminalState::Succeeded => self.validate_succeeded(), + AnalysisRunTerminalState::Failed => self.validate_failed(), + } + } + + fn validate_succeeded(&self) -> Result<(), ApiError> { + let artifact_id = self + .result_artifact_id + .as_deref() + .ok_or(ApiError::InvalidWirePayload)?; + let digest = self + .result_sha256 + .as_deref() + .ok_or(ApiError::InvalidWirePayload)?; + let schema = self + .result_schema_version + .as_deref() + .ok_or(ApiError::InvalidWirePayload)?; + let summary = self.summary.as_ref().ok_or(ApiError::InvalidWirePayload)?; + require_nonempty(artifact_id)?; + require_nonempty(schema)?; + require_canonical_sha256(digest)?; + summary.validate()?; + if self.failure_code.is_some() { + return Err(ApiError::InvalidWirePayload); + } + Ok(()) + } + + fn validate_failed(&self) -> Result<(), ApiError> { + if self.result_artifact_id.is_some() + || self.result_sha256.is_some() + || self.result_schema_version.is_some() + || self.summary.is_some() + { + return Err(ApiError::InvalidWirePayload); + } + require_failure_code( + self.failure_code + .as_deref() + .ok_or(ApiError::InvalidWirePayload)?, + ) + } +} + +/// Return whether a terminal result exactly binds to its submitted request. +#[must_use] +pub fn terminal_result_matches_request( + request: &AnalysisRunRequest, + result: &AnalysisRunTerminalResult, +) -> bool { + result.idempotency_key == request.idempotency_key + && result.tenant_workspace_id == request.tenant_workspace_id + && result.snapshot_id == request.snapshot_id + && result.knowledge_cutoff == request.knowledge_cutoff + && result.model_contract_version == request.model_contract_version + && result.output_profile == request.output_profile +} + +/// Return whether a terminal result exactly binds to an accepted receipt. +#[must_use] +pub fn terminal_result_matches_accepted( + accepted: &AnalysisRunAccepted, + result: &AnalysisRunTerminalResult, +) -> bool { + result.run_id == accepted.run_id && result.idempotency_key == accepted.idempotency_key +} + +/// Require exact request and accepted-receipt binding. +/// +/// # Errors +/// +/// Returns [`ApiError::InvalidWirePayload`] when either binding differs. +pub fn require_terminal_binding( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + result: &AnalysisRunTerminalResult, +) -> Result<(), ApiError> { + request.validate()?; + accepted.validate()?; + result.validate()?; + if terminal_result_matches_request(request, result) + && terminal_result_matches_accepted(accepted, result) + { + Ok(()) + } else { + Err(ApiError::InvalidWirePayload) + } +} + +fn require_canonical_sha256(value: &str) -> Result<(), ApiError> { + let valid = value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)); + if valid { + Ok(()) + } else { + Err(ApiError::InvalidWirePayload) + } +} + +fn require_failure_code(value: &str) -> Result<(), ApiError> { + let bytes = value.as_bytes(); + let valid = !bytes.is_empty() + && bytes.len() <= MAXIMUM_FAILURE_CODE_BYTES + && bytes[0].is_ascii_lowercase() + && bytes + .iter() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'_'); + if valid { + Ok(()) + } else { + Err(ApiError::InvalidWirePayload) + } +} diff --git a/crates/tepp_api/src/analysis_run.rs b/crates/tepp_api/src/analysis_run.rs index b263a1a3..2dd80374 100644 --- a/crates/tepp_api/src/analysis_run.rs +++ b/crates/tepp_api/src/analysis_run.rs @@ -4,6 +4,7 @@ use crate::ApiError; use crate::wire::{ from_json, require_byte_limit, require_contract_version, require_nonempty, to_json, }; +use crate::{AnalysisRunTerminalResult, AnalysisRunTerminalState, require_terminal_binding}; use jiff::Timestamp; use serde::{Deserialize, Serialize}; use temporal_core::KnowledgeCutoff; @@ -14,6 +15,9 @@ pub const ANALYSIS_RUN_CONTRACT_VERSION: u16 = 1; /// Default maximum analysis-run JSON payload size in bytes. pub const DEFAULT_ANALYSIS_RUN_BYTE_LIMIT: usize = 64 * 1024; +/// Supported analysis-run status/read contract version. +pub const ANALYSIS_RUN_STATUS_CONTRACT_VERSION: u16 = 1; + /// Request to create a durable analysis run. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(deny_unknown_fields)] @@ -48,6 +52,36 @@ pub struct AnalysisRunAccepted { pub idempotency_key: String, } +/// Lifecycle state returned by the typed analysis-run status contract. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AnalysisRunStatusState { + /// The server durably accepted the run. + Accepted, + /// The server is processing the accepted run. + Running, + /// The run completed with a measurement artifact. + Succeeded, + /// The run completed without a measurement artifact. + Failed, +} + +/// Typed status/read response for an accepted analysis run. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AnalysisRunStatus { + /// Semantic contract version for this status payload family. + pub contract_version: u16, + /// Opaque server-assigned run identity. + pub run_id: String, + /// Current lifecycle state. + pub run_state: AnalysisRunStatusState, + /// Exact request idempotency key. + pub idempotency_key: String, + /// Validated terminal result, present only for terminal states. + pub terminal_result: Option, +} + impl AnalysisRunRequest { /// Parse and validate a JSON analysis-run request with default size limit. /// @@ -77,10 +111,12 @@ impl AnalysisRunRequest { /// Returns field-validation or serialization errors. pub fn to_json(&self) -> Result { self.validate()?; - to_json(self) + let payload = to_json(self)?; + require_byte_limit(&payload, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT)?; + Ok(payload) } - fn validate(&self) -> Result<(), ApiError> { + pub(crate) fn validate(&self) -> Result<(), ApiError> { require_contract_version(self.contract_version, ANALYSIS_RUN_CONTRACT_VERSION)?; require_nonempty(&self.idempotency_key)?; require_nonempty(&self.tenant_workspace_id)?; @@ -96,7 +132,7 @@ impl AnalysisRunRequest { /// /// A buyer cannot claim analysis of evidence that is not yet available. The /// request receipt instant is treated as availability of the command itself. -fn require_rfc3339_knowledge_cutoff(knowledge_cutoff: &str) -> Result<(), ApiError> { +pub(crate) fn require_rfc3339_knowledge_cutoff(knowledge_cutoff: &str) -> Result<(), ApiError> { require_nonempty(knowledge_cutoff)?; let cutoff = KnowledgeCutoff::parse_rfc3339(knowledge_cutoff) .map_err(|_| ApiError::InvalidWirePayload)?; @@ -135,6 +171,16 @@ impl AnalysisRunAccepted { /// /// Returns wire, version, or field-validation errors. pub fn from_json(payload: &str) -> Result { + Self::from_json_with_limit(payload, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT) + } + + /// Parse an accepted-run payload with a caller-supplied byte limit. + /// + /// # Errors + /// + /// Returns wire, version, limit, or field-validation errors. + pub fn from_json_with_limit(payload: &str, maximum_bytes: usize) -> Result { + require_byte_limit(payload, maximum_bytes)?; let accepted: Self = from_json(payload)?; accepted.validate()?; Ok(accepted) @@ -147,14 +193,143 @@ impl AnalysisRunAccepted { /// Returns validation or serialization errors. pub fn to_json(&self) -> Result { self.validate()?; - to_json(self) + let payload = to_json(self)?; + require_byte_limit(&payload, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT)?; + Ok(payload) } - fn validate(&self) -> Result<(), ApiError> { + pub(crate) fn validate(&self) -> Result<(), ApiError> { require_contract_version(self.contract_version, ANALYSIS_RUN_CONTRACT_VERSION)?; require_nonempty(&self.run_id)?; - require_nonempty(&self.run_state)?; + if self.run_state != "accepted" { + return Err(ApiError::InvalidWirePayload); + } + require_nonempty(&self.idempotency_key)?; + Ok(()) + } +} + +impl AnalysisRunStatus { + /// Construct an accepted status from a durable receipt. + /// + /// # Errors + /// + /// Returns a fail-closed error when the receipt is invalid. + pub fn accepted(accepted: &AnalysisRunAccepted) -> Result { + Self::new(accepted, AnalysisRunStatusState::Accepted, None) + } + + /// Construct a running status from a durable receipt. + /// + /// # Errors + /// + /// Returns a fail-closed error when the receipt is invalid. + pub fn running(accepted: &AnalysisRunAccepted) -> Result { + Self::new(accepted, AnalysisRunStatusState::Running, None) + } + + /// Construct a terminal status bound to the submitted request and receipt. + /// + /// # Errors + /// + /// Returns a fail-closed error when the result or its binding is invalid. + pub fn terminal( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + result: AnalysisRunTerminalResult, + ) -> Result { + require_terminal_binding(request, accepted, &result)?; + let state = match result.run_state { + AnalysisRunTerminalState::Succeeded => AnalysisRunStatusState::Succeeded, + AnalysisRunTerminalState::Failed => AnalysisRunStatusState::Failed, + }; + Self::new(accepted, state, Some(result)) + } + + /// Parse and validate a status/read payload with the default byte limit. + /// + /// # Errors + /// + /// Returns wire, version, limit, shape, or field-validation errors. + pub fn from_json(payload: &str) -> Result { + Self::from_json_with_limit(payload, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT) + } + + /// Parse and validate a status/read payload with a caller-supplied limit. + /// + /// # Errors + /// + /// Returns wire, version, limit, shape, or field-validation errors. + pub fn from_json_with_limit(payload: &str, maximum_bytes: usize) -> Result { + require_byte_limit(payload, maximum_bytes)?; + let status: Self = from_json(payload)?; + status.validate()?; + Ok(status) + } + + /// Serialize a status/read payload after complete validation. + /// + /// # Errors + /// + /// Returns validation or serialization errors. + pub fn to_json(&self) -> Result { + self.validate()?; + let payload = to_json(self)?; + require_byte_limit(&payload, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT)?; + Ok(payload) + } + + fn new( + accepted: &AnalysisRunAccepted, + run_state: AnalysisRunStatusState, + terminal_result: Option, + ) -> Result { + accepted.validate()?; + let status = Self { + contract_version: ANALYSIS_RUN_STATUS_CONTRACT_VERSION, + run_id: accepted.run_id.clone(), + run_state, + idempotency_key: accepted.idempotency_key.clone(), + terminal_result, + }; + status.validate()?; + status.require_serialized_size()?; + Ok(status) + } + + fn require_serialized_size(&self) -> Result<(), ApiError> { + let payload = to_json(self)?; + require_byte_limit(&payload, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT) + } + + fn validate(&self) -> Result<(), ApiError> { + require_contract_version(self.contract_version, ANALYSIS_RUN_STATUS_CONTRACT_VERSION)?; + require_nonempty(&self.run_id)?; require_nonempty(&self.idempotency_key)?; + match self.run_state { + AnalysisRunStatusState::Accepted | AnalysisRunStatusState::Running => { + if self.terminal_result.is_some() { + return Err(ApiError::InvalidWirePayload); + } + } + AnalysisRunStatusState::Succeeded | AnalysisRunStatusState::Failed => { + let result = self + .terminal_result + .as_ref() + .ok_or(ApiError::InvalidWirePayload)?; + result.validate()?; + let expected_state = match result.run_state { + AnalysisRunTerminalState::Succeeded => AnalysisRunStatusState::Succeeded, + AnalysisRunTerminalState::Failed => AnalysisRunStatusState::Failed, + }; + if expected_state != self.run_state + || result.run_id != self.run_id + || result.idempotency_key != self.idempotency_key + { + return Err(ApiError::InvalidWirePayload); + } + } + } Ok(()) } } @@ -168,6 +343,32 @@ pub fn requests_are_idempotent_matches( left == right } +/// Require exact status identity and, for terminal states, request binding. +/// +/// # Errors +/// +/// Returns [`ApiError::InvalidWirePayload`] when the status does not match the +/// receipt or its terminal result does not match the request. +pub fn require_status_binding( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + status: &AnalysisRunStatus, +) -> Result<(), ApiError> { + request.validate()?; + accepted.validate()?; + status.validate()?; + if request.idempotency_key != accepted.idempotency_key + || status.run_id != accepted.run_id + || status.idempotency_key != accepted.idempotency_key + { + return Err(ApiError::InvalidWirePayload); + } + if let Some(result) = status.terminal_result.as_ref() { + require_terminal_binding(request, accepted, result)?; + } + Ok(()) +} + #[cfg(test)] mod tests { use super::{ diff --git a/crates/tepp_api/src/lib.rs b/crates/tepp_api/src/lib.rs index 0bd0c954..8b9e4dd3 100644 --- a/crates/tepp_api/src/lib.rs +++ b/crates/tepp_api/src/lib.rs @@ -11,6 +11,7 @@ //! Loopback listeners prove the HTTP boundary without claiming production TLS, //! causality, or completed psychometric model results. +mod analysis_result; mod analysis_run; mod analysis_run_live; mod authorization; @@ -28,16 +29,40 @@ mod provider_payload; mod temporal_context; mod wire; +/// Terminal analysis-result contract version constant. +pub use analysis_result::ANALYSIS_RESULT_CONTRACT_VERSION; +/// Bounded identity-free terminal result summary. +pub use analysis_result::AnalysisResultSummary; +/// Request-bound terminal analysis outcome. +pub use analysis_result::AnalysisRunTerminalResult; +/// Canonical terminal analysis-run state. +pub use analysis_result::AnalysisRunTerminalState; +/// Default terminal analysis-result payload byte limit. +pub use analysis_result::DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT; +/// Require exact terminal result binding to request and accepted receipt. +pub use analysis_result::require_terminal_binding; +/// Compare a terminal result with an accepted receipt. +pub use analysis_result::terminal_result_matches_accepted; +/// Compare a terminal result with its submitted request. +pub use analysis_result::terminal_result_matches_request; /// Analysis-run contract version constant. pub use analysis_run::ANALYSIS_RUN_CONTRACT_VERSION; +/// Analysis-run status/read contract version constant. +pub use analysis_run::ANALYSIS_RUN_STATUS_CONTRACT_VERSION; /// Accepted analysis-run response. pub use analysis_run::AnalysisRunAccepted; /// Analysis-run create request. pub use analysis_run::AnalysisRunRequest; +/// Typed analysis-run status/read response. +pub use analysis_run::AnalysisRunStatus; +/// Analysis-run status/read lifecycle state. +pub use analysis_run::AnalysisRunStatusState; /// Default analysis-run payload byte limit. pub use analysis_run::DEFAULT_ANALYSIS_RUN_BYTE_LIMIT; /// Idempotent request equality helper. pub use analysis_run::requests_are_idempotent_matches; +/// Require exact status binding to a request and accepted receipt. +pub use analysis_run::require_status_binding; /// Consumer-neutral loopback analysis-run service. pub use analysis_run_live::AnalysisRunLiveService; /// Corpus-split leakage-audit contract version. diff --git a/crates/tepp_api/src/naruon_http.rs b/crates/tepp_api/src/naruon_http.rs index eb442844..c5e277ec 100644 --- a/crates/tepp_api/src/naruon_http.rs +++ b/crates/tepp_api/src/naruon_http.rs @@ -154,9 +154,16 @@ pub(crate) fn header_is_credential(name: &str) -> bool { lowered == "authorization" || lowered == "proxy-authorization" || lowered == "cookie" + || lowered == "x-api-key" || lowered.contains("api-key") || lowered.contains("api_key") || lowered.contains("apikey") + || lowered.contains("secret") + || lowered.contains("credential") + || lowered.contains("openai") + || lowered.contains("anthropic") + || lowered.contains("bytez") + || lowered.contains("openrouter") || lowered.contains("token") || lowered.contains("copilot") || lowered.contains("github") @@ -165,7 +172,10 @@ pub(crate) fn header_is_credential(name: &str) -> bool { } fn refuse_credential_headers(extra_headers: &[(&str, &str)]) -> Result<(), ApiError> { - for (name, _) in extra_headers { + for (name, value) in extra_headers { + if !is_http_field_name(name) || value.chars().any(char::is_control) { + return Err(ApiError::InvalidWirePayload); + } if header_is_reserved_standard(name) { return Err(ApiError::InvalidWirePayload); } @@ -176,6 +186,30 @@ fn refuse_credential_headers(extra_headers: &[(&str, &str)]) -> Result<(), ApiEr Ok(()) } +fn is_http_field_name(name: &str) -> bool { + !name.is_empty() + && name.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'!' | b'#' + | b'$' + | b'%' + | b'&' + | b'\'' + | b'*' + | b'+' + | b'-' + | b'.' + | b'^' + | b'_' + | b'`' + | b'|' + | b'~' + ) + }) +} + pub(crate) fn standard_headers(idempotency_key: &str) -> Vec<(String, String)> { vec![ ("content-type".into(), "application/json".into()), @@ -330,6 +364,41 @@ mod tests { refuse_credential_headers(&[("x-nvidia-nim-key", "nvapi-x")]), Err(ApiError::AuthorizationDenied) ); + for name in [ + "x-openai-api-key", + "x-anthropic-key", + "x-bytez-api-key", + "x-openrouter-api-key", + "x-api_key", + "x-secret", + "x-credential", + "x-openai", + "x-bytez", + "x-openrouter", + "x-provider-api_key", + "x-provider-secret", + "x-provider-credential", + "x-provider-openai", + "x-provider-bytez", + "x-provider-openrouter", + ] { + assert_eq!( + refuse_credential_headers(&[(name, "provider-secret")]), + Err(ApiError::AuthorizationDenied), + "header={name}" + ); + } + for (name, value) in [ + ("", "value"), + ("bad name", "value"), + ("x-trace", "ok\r\nx-injected: 1"), + ] { + assert_eq!( + refuse_credential_headers(&[(name, value)]), + Err(ApiError::InvalidWirePayload), + "header={name:?}" + ); + } } #[test] diff --git a/crates/tepp_api/src/wire.rs b/crates/tepp_api/src/wire.rs index 6a5ee6d5..3586f10f 100644 --- a/crates/tepp_api/src/wire.rs +++ b/crates/tepp_api/src/wire.rs @@ -148,7 +148,7 @@ mod tests { assert_eq!(require_nonempty(" "), Err(ApiError::InvalidWirePayload)); assert_eq!(require_nonempty(""), Err(ApiError::InvalidWirePayload)); assert_eq!( - require_nonempty("topic\u{1f}unit"), + require_nonempty("tenant\u{1f}workspace"), Err(ApiError::InvalidWirePayload) ); require_byte_limit("abc", 3).expect("ok"); diff --git a/crates/tepp_api/tests/analysis_result_contract.rs b/crates/tepp_api/tests/analysis_result_contract.rs new file mode 100644 index 00000000..c18e536e --- /dev/null +++ b/crates/tepp_api/tests/analysis_result_contract.rs @@ -0,0 +1,574 @@ +//! Contract tests for request-bound terminal analysis results. + +use tepp_api::{ + ANALYSIS_RESULT_CONTRACT_VERSION, ANALYSIS_RUN_CONTRACT_VERSION, + ANALYSIS_RUN_STATUS_CONTRACT_VERSION, AnalysisResultSummary, AnalysisRunAccepted, + AnalysisRunRequest, AnalysisRunStatus, AnalysisRunStatusState, AnalysisRunTerminalResult, + AnalysisRunTerminalState, ApiError, DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT, + DEFAULT_ANALYSIS_RUN_BYTE_LIMIT, require_status_binding, require_terminal_binding, + terminal_result_matches_accepted, terminal_result_matches_request, +}; + +const DIGEST: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +fn request() -> AnalysisRunRequest { + AnalysisRunRequest { + contract_version: ANALYSIS_RUN_CONTRACT_VERSION, + idempotency_key: "idem-1".into(), + tenant_workspace_id: "tenant-ws-1".into(), + snapshot_id: "snapshot-1".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "temporal-model-v1".into(), + output_profile: "validation-report".into(), + } +} + +fn accepted() -> AnalysisRunAccepted { + AnalysisRunAccepted::new("run-1", "accepted", "idem-1").expect("accepted") +} + +#[test] +fn accepted_receipt_rejects_non_accepted_lifecycle_states() { + assert_eq!( + AnalysisRunAccepted::new("run-1", "running", "idem-1"), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + AnalysisRunAccepted::new("run-1", "failed", "idem-1"), + Err(ApiError::InvalidWirePayload) + ); +} + +fn summary() -> AnalysisResultSummary { + AnalysisResultSummary::new("temporal_topic_measurement", 120, 42, "validated").expect("summary") +} + +fn succeeded() -> AnalysisRunTerminalResult { + AnalysisRunTerminalResult::succeeded( + &request(), + &accepted(), + "artifact-1", + DIGEST, + "tepp-result-v1", + "2026-08-02T03:04:05Z", + summary(), + ) + .expect("succeeded") +} + +fn failed() -> AnalysisRunTerminalResult { + AnalysisRunTerminalResult::failed( + &request(), + &accepted(), + "2026-08-02T03:04:05Z", + "estimation_failed", + ) + .expect("failed") +} + +#[test] +fn terminal_success_and_failure_round_trip_without_receipt_confusion() { + let success = succeeded(); + assert_eq!(success.run_state, AnalysisRunTerminalState::Succeeded); + assert!(terminal_result_matches_request(&request(), &success)); + assert!(terminal_result_matches_accepted(&accepted(), &success)); + assert_eq!( + require_terminal_binding(&request(), &accepted(), &success), + Ok(()) + ); + let json = success.to_json().expect("json"); + assert!(json.len() <= DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT); + assert_eq!( + AnalysisRunTerminalResult::from_json(&json).expect("decoded"), + success + ); + + let failure = failed(); + assert_eq!(failure.run_state, AnalysisRunTerminalState::Failed); + assert_eq!(failure.result_artifact_id, None); + assert_eq!(failure.summary, None); + let json = failure.to_json().expect("json"); + assert_eq!( + AnalysisRunTerminalResult::from_json(&json).expect("decoded"), + failure + ); + + let accepted_json = accepted().to_json().expect("accepted json"); + assert_eq!( + AnalysisRunTerminalResult::from_json(&accepted_json), + Err(ApiError::InvalidWirePayload) + ); +} + +#[test] +fn wire_version_limit_extension_and_time_validation_fail_closed() { + let mut value: serde_json::Value = + serde_json::from_str(&succeeded().to_json().expect("json")).expect("value"); + value["extra"] = serde_json::json!(true); + assert_eq!( + AnalysisRunTerminalResult::from_json(&value.to_string()), + Err(ApiError::InvalidWirePayload) + ); + + let json = succeeded().to_json().expect("json"); + assert_eq!( + AnalysisRunTerminalResult::from_json_with_limit(&json, 8), + Err(ApiError::LimitExceeded) + ); + + let mut value = succeeded(); + value.contract_version = ANALYSIS_RESULT_CONTRACT_VERSION + 1; + assert_eq!(value.to_json(), Err(ApiError::UnsupportedContractVersion)); + + let mut value = succeeded(); + value.knowledge_cutoff = "yesterday".into(); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.knowledge_cutoff = "2099-01-01T00:00:00Z".into(); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.completed_at = "2026-99-99T25:00:00Z".into(); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + // System time is distinct from the knowledge cutoff; a pre-cutoff run may + // legitimately publish a result for a historical snapshot. + let mut value = succeeded(); + value.completed_at = "2026-07-31T23:59:59Z".into(); + assert!(value.to_json().is_ok()); +} + +#[test] +fn serialization_enforces_default_result_and_status_limits() { + let mut result = succeeded(); + result.summary.as_mut().expect("summary").analysis_family = + "x".repeat(DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT); + assert_eq!(result.to_json(), Err(ApiError::LimitExceeded)); + + let oversized_accepted = AnalysisRunAccepted::new( + "x".repeat(DEFAULT_ANALYSIS_RUN_BYTE_LIMIT), + "accepted", + "idem-1", + ) + .expect("accepted"); + assert_eq!(oversized_accepted.to_json(), Err(ApiError::LimitExceeded)); + assert_eq!( + AnalysisRunAccepted::from_json(&format!( + "{{\"contract_version\":1,\"run_id\":\"{}\",\"run_state\":\"accepted\",\"idempotency_key\":\"idem-1\"}}", + "x".repeat(DEFAULT_ANALYSIS_RUN_BYTE_LIMIT) + )), + Err(ApiError::LimitExceeded) + ); + assert_eq!( + AnalysisRunStatus::accepted(&oversized_accepted), + Err(ApiError::LimitExceeded) + ); + + let mut near_limit_result = succeeded(); + let initial_size = near_limit_result.to_json().expect("initial result").len(); + near_limit_result + .summary + .as_mut() + .expect("summary") + .analysis_family + .push_str(&"x".repeat(DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT - 1 - initial_size)); + let near_limit_json = near_limit_result.to_json().expect("near-limit result"); + assert!(near_limit_json.len() < DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT); + assert_eq!( + AnalysisRunStatus::terminal(&request(), &accepted(), near_limit_result), + Err(ApiError::LimitExceeded) + ); +} + +#[test] +fn every_required_binding_field_is_nonempty() { + for index in 0..8 { + let mut value = succeeded(); + match index { + 0 => value.run_id.clear(), + 1 => value.idempotency_key.clear(), + 2 => value.tenant_workspace_id.clear(), + 3 => value.snapshot_id.clear(), + 4 => value.knowledge_cutoff.clear(), + 5 => value.model_contract_version.clear(), + 6 => value.output_profile.clear(), + 7 => value.completed_at.clear(), + _ => unreachable!(), + } + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + } +} + +#[test] +fn succeeded_shape_requires_complete_digest_bound_result() { + let mut value = succeeded(); + value.result_artifact_id = None; + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.result_artifact_id = Some(String::new()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.result_sha256 = None; + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + for digest in [ + String::new(), + "abcd".into(), + DIGEST.to_uppercase(), + format!("{DIGEST}0"), + format!("g{}", &DIGEST[1..]), + ] { + let mut value = succeeded(); + value.result_sha256 = Some(digest); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + } + + let mut value = succeeded(); + value.result_schema_version = None; + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.result_schema_version = Some(String::new()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.summary = None; + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.failure_code = Some("unexpected_failure".into()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); +} + +#[test] +fn summary_is_nonempty_and_bounded_in_constructor_and_wire_shape() { + assert_eq!( + AnalysisResultSummary::new("", 0, 0, "validated"), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + AnalysisResultSummary::new("family", 0, 0, ""), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + AnalysisResultSummary::new("family", 1_000_000_001, 0, "validated"), + Err(ApiError::LimitExceeded) + ); + assert_eq!( + AnalysisResultSummary::new("family", 0, 1_000_000_001, "validated"), + Err(ApiError::LimitExceeded) + ); + + let invalid_summaries = [ + AnalysisResultSummary { + analysis_family: String::new(), + evidence_count: 0, + statistic_count: 0, + validation_status: "validated".into(), + }, + AnalysisResultSummary { + analysis_family: "family".into(), + evidence_count: 0, + statistic_count: 0, + validation_status: String::new(), + }, + AnalysisResultSummary { + analysis_family: "family".into(), + evidence_count: 1_000_000_001, + statistic_count: 0, + validation_status: "validated".into(), + }, + AnalysisResultSummary { + analysis_family: "family".into(), + evidence_count: 0, + statistic_count: 1_000_000_001, + validation_status: "validated".into(), + }, + ]; + for invalid_summary in invalid_summaries { + let mut value = succeeded(); + value.summary = Some(invalid_summary); + assert!(matches!( + value.to_json(), + Err(ApiError::InvalidWirePayload | ApiError::LimitExceeded) + )); + } +} + +#[test] +fn failed_shape_refuses_measurement_fields_and_invalid_failure_codes() { + let base = failed(); + let digit_code = AnalysisRunTerminalResult::failed( + &request(), + &accepted(), + "2026-08-02T03:04:05Z", + "estimation_failed_2", + ) + .expect("digits are valid failure-code characters"); + assert_eq!( + digit_code.failure_code.as_deref(), + Some("estimation_failed_2") + ); + + let mut value = base.clone(); + value.result_artifact_id = Some("artifact".into()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = base.clone(); + value.result_sha256 = Some(DIGEST.into()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = base.clone(); + value.result_schema_version = Some("schema".into()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = base.clone(); + value.summary = Some(summary()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + for code in [ + None, + Some(String::new()), + Some("UPPER_CASE".into()), + Some("_leading".into()), + Some("contains-hyphen".into()), + Some("x".repeat(65)), + ] { + let mut value = base.clone(); + value.failure_code = code; + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + } +} + +#[test] +fn every_request_binding_dimension_and_receipt_identity_is_checked() { + let result = succeeded(); + + let mut tampered = result.clone(); + tampered.failure_code = Some("late_failure".into()); + assert_eq!( + require_terminal_binding(&request(), &accepted(), &tampered), + Err(ApiError::InvalidWirePayload) + ); + + for index in 0..6 { + let mut mismatched = request(); + match index { + 0 => mismatched.idempotency_key = "other".into(), + 1 => mismatched.tenant_workspace_id = "other".into(), + 2 => mismatched.snapshot_id = "other".into(), + 3 => mismatched.knowledge_cutoff = "2026-07-31T00:00:00Z".into(), + 4 => mismatched.model_contract_version = "other".into(), + 5 => mismatched.output_profile = "other".into(), + _ => unreachable!(), + } + assert!(!terminal_result_matches_request(&mismatched, &result)); + assert_eq!( + require_terminal_binding(&mismatched, &accepted(), &result), + Err(ApiError::InvalidWirePayload) + ); + } + + let other_run = AnalysisRunAccepted::new("other-run", "accepted", "idem-1").expect("accepted"); + assert!(!terminal_result_matches_accepted(&other_run, &result)); + assert_eq!( + require_terminal_binding(&request(), &other_run, &result), + Err(ApiError::InvalidWirePayload) + ); + + let other_key = AnalysisRunAccepted::new("run-1", "accepted", "other-key").expect("accepted"); + assert!(!terminal_result_matches_accepted(&other_key, &result)); + assert_eq!( + require_terminal_binding(&request(), &other_key, &result), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + AnalysisRunTerminalResult::succeeded( + &request(), + &other_key, + "artifact", + DIGEST, + "schema", + "2026-08-02T03:04:05Z", + summary(), + ), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + AnalysisRunTerminalResult::failed( + &request(), + &other_key, + "2026-08-02T03:04:05Z", + "provider_timeout", + ), + Err(ApiError::InvalidWirePayload) + ); + + let mut invalid_request = request(); + invalid_request.contract_version += 1; + assert_eq!( + require_terminal_binding(&invalid_request, &accepted(), &result), + Err(ApiError::UnsupportedContractVersion) + ); + let mut invalid_accepted = accepted(); + invalid_accepted.contract_version += 1; + assert_eq!( + require_terminal_binding(&request(), &invalid_accepted, &result), + Err(ApiError::UnsupportedContractVersion) + ); +} + +#[test] +fn status_read_contract_round_trips_lifecycle_and_terminal_results() { + let accepted_status = AnalysisRunStatus::accepted(&accepted()).expect("accepted status"); + assert_eq!(accepted_status.run_state, AnalysisRunStatusState::Accepted); + assert_eq!(accepted_status.terminal_result, None); + assert_eq!( + require_status_binding(&request(), &accepted(), &accepted_status), + Ok(()) + ); + let accepted_json = accepted_status.to_json().expect("accepted json"); + assert_eq!( + AnalysisRunStatus::from_json(&accepted_json).expect("accepted decode"), + accepted_status + ); + + let running_status = AnalysisRunStatus::running(&accepted()).expect("running status"); + assert_eq!(running_status.run_state, AnalysisRunStatusState::Running); + assert_eq!( + require_status_binding(&request(), &accepted(), &running_status), + Ok(()) + ); + + for (result, expected_state) in [ + (succeeded(), AnalysisRunStatusState::Succeeded), + (failed(), AnalysisRunStatusState::Failed), + ] { + let status = + AnalysisRunStatus::terminal(&request(), &accepted(), result).expect("terminal status"); + assert_eq!(status.run_state, expected_state); + assert!(status.terminal_result.is_some()); + assert_eq!( + require_status_binding(&request(), &accepted(), &status), + Ok(()) + ); + let json = status.to_json().expect("terminal json"); + assert_eq!( + AnalysisRunStatus::from_json(&json).expect("terminal decode"), + status + ); + } + + assert_eq!( + ANALYSIS_RUN_STATUS_CONTRACT_VERSION, + ANALYSIS_RUN_CONTRACT_VERSION + ); +} + +#[test] +fn status_read_contract_rejects_unknown_oversized_and_invalid_shapes() { + let accepted_status = AnalysisRunStatus::accepted(&accepted()).expect("status"); + let mut value: serde_json::Value = + serde_json::from_str(&accepted_status.to_json().expect("json")).expect("value"); + value["extra"] = serde_json::json!(true); + assert_eq!( + AnalysisRunStatus::from_json(&value.to_string()), + Err(ApiError::InvalidWirePayload) + ); + let json = accepted_status.to_json().expect("json"); + assert_eq!( + AnalysisRunStatus::from_json_with_limit(&json, 1), + Err(ApiError::LimitExceeded) + ); + + let mut invalid = accepted_status.clone(); + invalid.contract_version = ANALYSIS_RUN_STATUS_CONTRACT_VERSION + 1; + assert_eq!(invalid.to_json(), Err(ApiError::UnsupportedContractVersion)); + invalid = accepted_status.clone(); + invalid.run_id.clear(); + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + invalid = accepted_status.clone(); + invalid.idempotency_key.clear(); + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut invalid = accepted_status.clone(); + invalid.terminal_result = Some(succeeded()); + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + + let terminal = + AnalysisRunStatus::terminal(&request(), &accepted(), succeeded()).expect("terminal status"); + let mut invalid = terminal.clone(); + invalid.terminal_result = None; + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + invalid = terminal.clone(); + invalid.run_state = AnalysisRunStatusState::Failed; + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut invalid = terminal.clone(); + invalid.terminal_result.as_mut().expect("result").run_id = "other".into(); + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + let mut invalid = terminal; + invalid + .terminal_result + .as_mut() + .expect("result") + .idempotency_key = "other".into(); + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); +} + +#[test] +fn status_read_binding_rejects_receipt_and_request_mismatches() { + let accepted_status = AnalysisRunStatus::accepted(&accepted()).expect("status"); + let other_run = AnalysisRunAccepted::new("other-run", "accepted", "idem-1").expect("run"); + assert_eq!( + require_status_binding(&request(), &other_run, &accepted_status), + Err(ApiError::InvalidWirePayload) + ); + let other_key = AnalysisRunAccepted::new("run-1", "accepted", "other-key").expect("key"); + assert_eq!( + require_status_binding(&request(), &other_key, &accepted_status), + Err(ApiError::InvalidWirePayload) + ); + + let terminal = + AnalysisRunStatus::terminal(&request(), &accepted(), succeeded()).expect("terminal status"); + let mut other_request = request(); + other_request.snapshot_id = "other-snapshot".into(); + assert_eq!( + require_status_binding(&other_request, &accepted(), &terminal), + Err(ApiError::InvalidWirePayload) + ); + + let accepted_status = AnalysisRunStatus::accepted(&accepted()).expect("status"); + let mut other_idempotency = request(); + other_idempotency.idempotency_key = "other-key".into(); + assert_eq!( + require_status_binding(&other_idempotency, &accepted(), &accepted_status), + Err(ApiError::InvalidWirePayload) + ); + + let mut invalid_status = accepted_status.clone(); + invalid_status.idempotency_key = "other-key".into(); + assert_eq!( + require_status_binding(&request(), &accepted(), &invalid_status), + Err(ApiError::InvalidWirePayload) + ); + + let mut invalid_request = request(); + invalid_request.contract_version += 1; + assert_eq!( + require_status_binding(&invalid_request, &accepted(), &accepted_status), + Err(ApiError::UnsupportedContractVersion) + ); + assert_eq!( + AnalysisRunStatus::terminal( + &other_request, + &accepted(), + terminal.terminal_result.unwrap() + ), + Err(ApiError::InvalidWirePayload) + ); +} diff --git a/crates/tepp_api/tests/naruon_http_contract.rs b/crates/tepp_api/tests/naruon_http_contract.rs index cb096cb5..1623e5bd 100644 --- a/crates/tepp_api/tests/naruon_http_contract.rs +++ b/crates/tepp_api/tests/naruon_http_contract.rs @@ -57,6 +57,7 @@ fn table_access_and_non_https_origins_fail_closed() { let run = sample_run(); for origin in [ "", + "https://", "postgres://tepp.example.test/tepp", "postgresql://tepp.example.test/tepp", "jdbc:postgresql://tepp.example.test/tepp", @@ -119,6 +120,49 @@ fn review_and_copilot_headers_are_authorization_denied() { ), Err(ApiError::AuthorizationDenied) ); + for name in [ + "x-openai-api-key", + "x-anthropic-key", + "x-bytez-api-key", + "x-openrouter-api-key", + "x-apikey", + "x_api_key", + "x-secret", + "x-credential", + "x_openai", + "x_bytez", + "x_openrouter", + ] { + assert_eq!( + naruon_analysis_run_exchange_with_headers( + "https://tepp.example.test", + &run, + &[(name, "provider-secret")] + ), + Err(ApiError::AuthorizationDenied), + "header={name}" + ); + } +} + +#[test] +fn malformed_extra_headers_fail_closed_before_forwarding() { + let run = sample_run(); + for (name, value) in [ + ("", "value"), + ("bad name", "value"), + ("x-trace", "ok\r\nx-injected: 1"), + ] { + assert_eq!( + naruon_analysis_run_exchange_with_headers( + "https://tepp.example.test", + &run, + &[(name, value)] + ), + Err(ApiError::InvalidWirePayload), + "header={name:?}" + ); + } } #[test] diff --git a/crates/topic_measurement/Cargo.toml b/crates/topic_measurement/Cargo.toml new file mode 100644 index 00000000..8d990d48 --- /dev/null +++ b/crates/topic_measurement/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "topic_measurement" +description = "Logistic-normal and log-ratio coordinates for compositional topics." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[dependencies] +corpus_split = { path = "../corpus_split", version = "0.1.0" } +membership_core = { path = "../membership_core", version = "0.1.0" } +relation_graph = { path = "../relation_graph", version = "0.1.0" } +temporal_core = { path = "../temporal_core", version = "0.1.0" } +uuid.workspace = true + +[dev-dependencies] +validation_core = { path = "../validation_core", version = "0.1.0" } + +[lints] +workspace = true diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs new file mode 100644 index 00000000..aa4922a7 --- /dev/null +++ b/crates/topic_measurement/src/coordinates.rs @@ -0,0 +1,274 @@ +//! Additive and isometric log-ratio maps for compositional topic coordinates. + +use crate::error::TopicMeasurementError; + +const UNIT_SUM_TOLERANCE: f64 = 1e-12; + +/// Map a strictly positive unit simplex vector to additive log-ratio coordinates. +/// +/// For a `K`-part composition `θ` the image is the `K-1` vector +/// `y_k = ln(θ_k / θ_K)`. This reference-dependent, full-rank coordinate map +/// supports logistic-normal regression and ESEM/DSEM interfaces. It is not an +/// orthonormal isometry for Aitchison distance; use ILR coordinates when that +/// Euclidean geometry is the estimand. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidComposition`] when the vector is +/// empty, has fewer than two parts, contains a non-finite or non-positive +/// entry, or does not sum to one within a tight absolute tolerance. +pub fn additive_log_ratio(proportions: &[f64]) -> Result, TopicMeasurementError> { + let last = require_composition(proportions)?; + let reference_log = last.ln(); + Ok(proportions[..proportions.len() - 1] + .iter() + .map(|part| part.ln() - reference_log) + .collect()) +} + +/// Invert additive log-ratio coordinates back to the unit simplex. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the +/// coordinate vector is empty, non-finite, or would underflow a part to zero +/// in the strictly positive `f64` simplex representation. +pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { + if coordinates.is_empty() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut maximum = 0.0_f64; + for &value in coordinates { + if !value.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + maximum = maximum.max(value); + } + + let reference_weight = (-maximum).exp(); + if reference_weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut shifted_weights = Vec::with_capacity(coordinates.len() + 1); + for &value in coordinates { + let weight = (value - maximum).exp(); + if weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + shifted_weights.push(weight); + } + shifted_weights.push(reference_weight); + normalize_positive_weights(shifted_weights) +} + +/// Map a strictly positive unit simplex vector to isometric log-ratio coordinates. +/// +/// The sequential Egozcue orthonormal basis sends a `K`-part composition to +/// the `K-1` vector whose Euclidean distance from another composition's ILR +/// vector equals their Aitchison distance. A vector norm is only the distance +/// from the equal-share origin. This is the coordinate system for distance-based +/// topic geometry. It is not the reference-dependent logistic-normal map; use +/// [`additive_log_ratio`] when that regression interface is the estimand. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidComposition`] when the vector is +/// empty, has fewer than two parts, contains a non-finite or non-positive +/// entry, or does not sum to one within a tight absolute tolerance. +pub fn isometric_log_ratio(proportions: &[f64]) -> Result, TopicMeasurementError> { + require_composition(proportions)?; + let dimension = proportions.len(); + let logs: Vec = proportions.iter().map(|part| part.ln()).collect(); + let mut coordinates = Vec::with_capacity(dimension - 1); + for index in 0..(dimension - 1) { + let remaining = dimension - index - 1; + #[allow(clippy::cast_precision_loss)] + let remaining_f = remaining as f64; + let scale = (remaining_f / (remaining_f + 1.0)).sqrt(); + let mut rest_sum = 0.0_f64; + for log_part in &logs[index + 1..] { + rest_sum += *log_part; + } + coordinates.push(scale * (logs[index] - rest_sum / remaining_f)); + } + Ok(coordinates) +} + +/// Invert isometric log-ratio coordinates back to the unit simplex. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the +/// coordinate vector is empty, non-finite, or would underflow a part to zero +/// in the strictly positive `f64` simplex representation. +pub fn from_isometric_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { + if coordinates.is_empty() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + for &value in coordinates { + if !value.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + } + + let dimension = coordinates.len() + 1; + let mut centered_logs = vec![0.0_f64; dimension]; + for (index, &coordinate) in coordinates.iter().enumerate() { + let remaining = dimension - index - 1; + #[allow(clippy::cast_precision_loss)] + let remaining_f = remaining as f64; + let scale = (remaining_f / (remaining_f + 1.0)).sqrt(); + let negative = -1.0 / (remaining_f * (remaining_f + 1.0)).sqrt(); + centered_logs[index] += scale * coordinate; + for centered in &mut centered_logs[index + 1..] { + *centered += negative * coordinate; + } + } + + let mut maximum = centered_logs[0]; + for &value in ¢ered_logs[1..] { + maximum = maximum.max(value); + } + if !maximum.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + + let mut weights = Vec::with_capacity(dimension); + for &value in ¢ered_logs { + let weight = (value - maximum).exp(); + if weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + weights.push(weight); + } + normalize_positive_weights(weights) +} + +fn normalize_positive_weights(weights: Vec) -> Result, TopicMeasurementError> { + let denominator: f64 = weights.iter().sum(); + let simplex: Vec = weights + .into_iter() + .map(|weight| weight / denominator) + .collect(); + if simplex.iter().any(|part| *part <= 0.0) { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + Ok(simplex) +} + +/// Aitchison distance between two strictly positive unit simplex vectors. +/// +/// The distance is the Euclidean norm of the clr residual +/// `clr(x) − clr(y)`. Sequential ILR is an isometry for this distance, so +/// `‖ilr(x) − ilr(y)‖` must recover the same value. A single ILR vector +/// norm is only the distance from the equal-share origin. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidComposition`] when the vectors +/// have unequal length or fail simplex validation. +pub fn aitchison_distance(left: &[f64], right: &[f64]) -> Result { + if left.len() != right.len() { + return Err(TopicMeasurementError::InvalidComposition); + } + require_composition(left)?; + require_composition(right)?; + #[allow(clippy::cast_precision_loss)] + let parts = left.len() as f64; + let mut left_log_sum = 0.0_f64; + let mut right_log_sum = 0.0_f64; + for index in 0..left.len() { + left_log_sum += left[index].ln(); + right_log_sum += right[index].ln(); + } + let left_mean = left_log_sum / parts; + let right_mean = right_log_sum / parts; + let mut square_sum = 0.0_f64; + for index in 0..left.len() { + let residual = (left[index].ln() - left_mean) - (right[index].ln() - right_mean); + square_sum += residual * residual; + } + Ok(square_sum.sqrt()) +} + +fn require_composition(proportions: &[f64]) -> Result { + if proportions.len() < 2 { + return Err(TopicMeasurementError::InvalidComposition); + } + let mut sum = 0.0_f64; + let mut compensation = 0.0_f64; + for &part in proportions { + if !part.is_finite() || part <= 0.0 { + return Err(TopicMeasurementError::InvalidComposition); + } + let next = sum + part; + compensation += if sum.abs() >= part.abs() { + (sum - next) + part + } else { + (part - next) + sum + }; + sum = next; + } + let compensated_sum = sum + compensation; + if !compensated_sum.is_finite() || (compensated_sum - 1.0).abs() > UNIT_SUM_TOLERANCE { + return Err(TopicMeasurementError::InvalidComposition); + } + Ok(proportions[proportions.len() - 1]) +} + +#[cfg(test)] +mod tests { + use super::{ + additive_log_ratio, aitchison_distance, from_additive_log_ratio, from_isometric_log_ratio, + isometric_log_ratio, + }; + use crate::error::TopicMeasurementError; + + #[test] + fn two_part_equal_shares_are_zero_and_unrepresentable_extremes_fail_closed() { + let pair = additive_log_ratio(&[0.5, 0.5]).expect("pair"); + assert_eq!(pair.len(), 1); + assert!(pair[0].abs() < 1e-15); + let recovered = from_additive_log_ratio(&pair).expect("inverse"); + assert!((recovered[0] - 0.5).abs() < 1e-15); + assert!((recovered[1] - 0.5).abs() < 1e-15); + assert_eq!( + from_additive_log_ratio(&[1.0e9]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + additive_log_ratio(&[f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidComposition), + "overflowing finite parts must fail closed because compensated mass is non-finite" + ); + let origin = isometric_log_ratio(&[0.5, 0.5]).expect("ilr origin"); + assert!(origin[0].abs() < 1e-15); + let recovered_ilr = from_isometric_log_ratio(&origin).expect("ilr inverse"); + assert!((recovered_ilr[0] - 0.5).abs() < 1e-15); + assert_eq!( + from_isometric_log_ratio(&[1000.0]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_isometric_log_ratio(&[-f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + let three = isometric_log_ratio(&[2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]).expect("ilr three"); + assert!((three[1] - (0.5_f64).sqrt() * 3.0_f64.ln()).abs() < 1e-15); + let recovered_three = from_isometric_log_ratio(&three).expect("ilr three inverse"); + assert!((recovered_three.iter().sum::() - 1.0).abs() < 1e-15); + assert!(aitchison_distance(&[0.5, 0.5], &[0.5, 0.5]).expect("self") < 1e-15); + assert_eq!( + aitchison_distance(&[0.5, 0.5], &[1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + aitchison_distance(&[0.0, 1.0], &[0.5, 0.5]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + aitchison_distance(&[0.5, 0.5], &[0.0, 1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + } +} diff --git a/crates/topic_measurement/src/error.rs b/crates/topic_measurement/src/error.rs new file mode 100644 index 00000000..abf0b225 --- /dev/null +++ b/crates/topic_measurement/src/error.rs @@ -0,0 +1,78 @@ +//! Fail-closed topic-coordinate errors. + +use std::fmt; + +/// A fail-closed topic-measurement error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum TopicMeasurementError { + /// Composition is empty, has fewer than two parts, is non-positive, + /// non-finite, or does not sum to one. + InvalidComposition, + /// Log-ratio vector is empty, non-finite, or not representable as a strictly positive `f64` simplex. + InvalidLogRatioDimension, + /// TF-IDF, BM25, or keyword scores were offered as inferential coordinates. + LexicalWeightForbidden, + /// A sparse matrix violated its compressed-storage contract. + InvalidSparseMatrix, + /// A reference-estimator input or configuration violated its scientific contract. + InvalidModelInput, + /// The estimator produced a non-finite intermediate and failed closed. + NonFiniteEstimate, + /// No seeded initialization converged within the bounded iteration budget. + DidNotConverge, +} + +impl fmt::Display for TopicMeasurementError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::InvalidComposition => "invalid compositional topic vector", + Self::InvalidLogRatioDimension => "invalid log-ratio dimension", + Self::LexicalWeightForbidden => "lexical inferential weights are forbidden", + Self::InvalidSparseMatrix => "invalid sparse matrix", + Self::InvalidModelInput => "invalid topic model input", + Self::NonFiniteEstimate => "non-finite topic estimate", + Self::DidNotConverge => "topic estimator did not converge", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for TopicMeasurementError {} + +#[cfg(test)] +mod tests { + use super::TopicMeasurementError; + + #[test] + fn messages_are_stable() { + assert_eq!( + TopicMeasurementError::InvalidComposition.to_string(), + "invalid compositional topic vector" + ); + assert_eq!( + TopicMeasurementError::InvalidLogRatioDimension.to_string(), + "invalid log-ratio dimension" + ); + assert_eq!( + TopicMeasurementError::LexicalWeightForbidden.to_string(), + "lexical inferential weights are forbidden" + ); + assert_eq!( + TopicMeasurementError::InvalidSparseMatrix.to_string(), + "invalid sparse matrix" + ); + assert_eq!( + TopicMeasurementError::InvalidModelInput.to_string(), + "invalid topic model input" + ); + assert_eq!( + TopicMeasurementError::NonFiniteEstimate.to_string(), + "non-finite topic estimate" + ); + assert_eq!( + TopicMeasurementError::DidNotConverge.to_string(), + "topic estimator did not converge" + ); + } +} diff --git a/crates/topic_measurement/src/lexical.rs b/crates/topic_measurement/src/lexical.rs new file mode 100644 index 00000000..588f60f9 --- /dev/null +++ b/crates/topic_measurement/src/lexical.rs @@ -0,0 +1,36 @@ +//! Refusal of lexical heuristics as inferential topic coordinates. + +use crate::error::TopicMeasurementError; + +/// Refuse TF-IDF, BM25, and keyword scores as topic-estimator coordinates. +/// +/// ADR 0012 forbids treating lexical retrieval weights as inferential topic +/// coordinates. A recognized statistical method name is accepted so callers +/// can share one vocabulary gate. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::LexicalWeightForbidden`] for empty labels +/// and for `tfidf`, `bm25`, and `keyword` after alphanumeric folding. +pub fn refuse_lexical_inferential_weight(method: &str) -> Result<(), TopicMeasurementError> { + let folded: String = method + .chars() + .filter(char::is_ascii_alphanumeric) + .flat_map(char::to_lowercase) + .collect(); + if folded.is_empty() || matches!(folded.as_str(), "tfidf" | "bm25" | "keyword") { + return Err(TopicMeasurementError::LexicalWeightForbidden); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::refuse_lexical_inferential_weight; + + #[test] + fn statistical_method_names_are_allowed() { + refuse_lexical_inferential_weight("tepp_topic_measurement").expect("allowed"); + refuse_lexical_inferential_weight("logistic_normal").expect("allowed"); + } +} diff --git a/crates/topic_measurement/src/lib.rs b/crates/topic_measurement/src/lib.rs new file mode 100644 index 00000000..38cd38ae --- /dev/null +++ b/crates/topic_measurement/src/lib.rs @@ -0,0 +1,48 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +//! Logistic-normal and log-ratio coordinates for compositional topic proportions. +//! +//! Raw topic proportions are compositional rather than unconstrained Euclidean +//! indicators. ALR supplies a reference-dependent full-rank logistic-normal map +//! for regression and psychometric interfaces; it is not an orthonormal +//! Aitchison-distance isometry. Distance-based Aitchison geometry uses the +//! sequential Egozcue ILR basis, whose pairwise Euclidean distance recovers +//! CLR Aitchison distance. TF-IDF, BM25, and keyword scores remain +//! forbidden inferential coordinates. + +mod coordinates; +mod error; +mod lexical; +mod reference; +mod sparse; + +/// Additive log-ratio map from a simplex vector. +pub use coordinates::additive_log_ratio; +/// Aitchison distance between two simplex vectors. +pub use coordinates::aitchison_distance; +/// Inverse additive log-ratio map back to the simplex. +pub use coordinates::from_additive_log_ratio; +/// Inverse isometric log-ratio map back to the simplex. +pub use coordinates::from_isometric_log_ratio; +/// Isometric log-ratio map from a simplex vector. +pub use coordinates::isometric_log_ratio; +/// Fail-closed topic-coordinate errors. +pub use error::TopicMeasurementError; +/// Refuse lexical retrieval weights as inferential coordinates. +pub use lexical::refuse_lexical_inferential_weight; +/// One admitted structural prevalence feature. +pub use reference::PrevalenceFeature; +/// Validated input for the CPU `f64` reference estimator. +pub use reference::ReferenceTopicInput; +/// A converged topic-model result with uncertainty and lineage counts. +pub use reference::ReferenceTopicModel; +/// Bounded deterministic reference-estimator configuration. +pub use reference::ReferenceTopicModelConfig; +/// One inferred predecessor/successor association within a fitted topic. +pub use reference::TopicSequenceEdge; +/// Fit the bounded deterministic CPU `f64` TRSL-TM reference estimator. +pub use reference::fit_reference_topic_model; +/// Validated compressed sparse numeric matrix. +pub use sparse::SparseMatrix; +/// Whether compressed values are grouped by row or by column. +pub use sparse::SparseOrientation; diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs new file mode 100644 index 00000000..ec5f223c --- /dev/null +++ b/crates/topic_measurement/src/reference.rs @@ -0,0 +1,877 @@ +//! Bounded CPU `f64` reference estimator for TRSL-TM prevalence and lineage. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use corpus_split::CorpusSnapshot; +use membership_core::{GroupId, MemberId, MembershipNetwork, MembershipRole}; +use relation_graph::RelationGraph; +use temporal_core::EventTime; +use uuid::Uuid; + +use crate::{SparseMatrix, TopicMeasurementError, from_additive_log_ratio}; + +const DEFAULT_PRIOR_VARIANCE: f64 = 1.0; +const DEFAULT_RELATION_STRENGTH: f64 = 0.25; +const DEFAULT_RIDGE: f64 = 0.01; +const DEFAULT_TOPIC_SMOOTHING: f64 = 0.05; +const DEFAULT_STEP_SIZE: f64 = 0.2; + +/// One column in the structural prevalence mean. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PrevalenceFeature { + /// Constant intercept. + Intercept, + /// Standardized event-time offset. + EventTime, + /// Caller-supplied admitted prevalence covariate. + Covariate(usize), + /// One active weighted cross-classified membership context. + Membership { + /// Contextual membership role. + role: MembershipRole, + /// Opaque analytical group identity. + group_id: GroupId, + }, +} + +/// Validated input for the CPU `f64` reference estimator. +#[derive(Clone, Debug)] +pub struct ReferenceTopicInput { + document_ids: Vec, + term_rows: Vec>, + vocabulary_size: usize, + design: Vec>, + features: Vec, + transition_pairs: Vec<(usize, usize)>, +} + +impl ReferenceTopicInput { + /// Build a cutoff-, membership-, time-, and relation-validated model input. + /// + /// `document_term` may be CSR or CSC. `covariates`, when present, may also + /// use either orientation. Every document must occur in `snapshot`, have a + /// nonempty nonnegative term row, span at least two event times, and have at + /// least one active membership across the modeled corpus. Only validated + /// forward transition edges with both endpoints in the corpus affect the + /// relational objective; all other relation kinds remain provenance only. + /// + /// # Errors + /// + /// Returns [`TopicMeasurementError::InvalidModelInput`] when any dimension, + /// cutoff, count, time, membership, covariate, or transition invariant fails. + pub fn new( + snapshot: &CorpusSnapshot, + document_ids: Vec, + document_term: &SparseMatrix, + event_times: &[EventTime], + covariates: Option<&SparseMatrix>, + memberships: &MembershipNetwork, + relations: &RelationGraph, + ) -> Result { + let document_count = document_ids.len(); + if document_count < 2 + || document_term.rows() != document_count + || document_term.columns() < 2 + || event_times.len() != document_count + || document_ids.iter().any(|id| !snapshot.contains(*id)) + { + return Err(TopicMeasurementError::InvalidModelInput); + } + let index_by_id: HashMap = document_ids + .iter() + .copied() + .enumerate() + .map(|(index, id)| (id, index)) + .collect(); + if index_by_id.len() != document_count { + return Err(TopicMeasurementError::InvalidModelInput); + } + + let term_rows = document_term.row_entries(); + for row in &term_rows { + if row.is_empty() + || row.iter().any(|(_, value)| *value < 0.0) + || row.iter().map(|(_, value)| value).sum::() <= 0.0 + { + return Err(TopicMeasurementError::InvalidModelInput); + } + } + + let (design, features) = build_design(&document_ids, event_times, covariates, memberships)?; + let transition_pairs = collect_transition_pairs(&index_by_id, relations)?; + + Ok(Self { + document_ids, + term_rows, + vocabulary_size: document_term.columns(), + design, + features, + transition_pairs, + }) + } + + /// Return the number of modeled documents. + #[must_use] + pub fn document_count(&self) -> usize { + self.document_ids.len() + } + + /// Return the vocabulary size. + #[must_use] + pub const fn vocabulary_size(&self) -> usize { + self.vocabulary_size + } + + /// Return the ordered structural prevalence features. + #[must_use] + pub fn features(&self) -> &[PrevalenceFeature] { + &self.features + } +} + +fn build_design( + document_ids: &[Uuid], + event_times: &[EventTime], + covariates: Option<&SparseMatrix>, + memberships: &MembershipNetwork, +) -> Result<(Vec>, Vec), TopicMeasurementError> { + let document_count = document_ids.len(); + let standardized_time = standardize_event_time(event_times)?; + let covariate_rows = match covariates { + Some(matrix) if matrix.rows() == document_count => Some(matrix.row_entries()), + Some(_) => return Err(TopicMeasurementError::InvalidModelInput), + None => None, + }; + let covariate_count = covariate_rows.as_ref().map_or(0, |rows| { + rows.iter() + .flatten() + .map(|(column, _)| *column) + .max() + .map_or(0, |value| value + 1) + }); + + let active: Vec<_> = document_ids + .iter() + .zip(event_times) + .map(|(id, time)| memberships.active_memberships_for(MemberId::from_uuid(*id), *time)) + .collect(); + let membership_keys: BTreeSet<_> = active + .iter() + .flatten() + .map(|assignment| (assignment.role(), assignment.group_id())) + .collect(); + if membership_keys.is_empty() { + return Err(TopicMeasurementError::InvalidModelInput); + } + let membership_columns: BTreeMap<_, _> = membership_keys + .iter() + .copied() + .enumerate() + .map(|(index, key)| (key, index)) + .collect(); + + let mut features = vec![PrevalenceFeature::Intercept, PrevalenceFeature::EventTime]; + features.extend((0..covariate_count).map(PrevalenceFeature::Covariate)); + features.extend( + membership_keys + .iter() + .map(|(role, group_id)| PrevalenceFeature::Membership { + role: *role, + group_id: *group_id, + }), + ); + let mut design = vec![vec![0.0; features.len()]; document_count]; + for row in 0..document_count { + design[row][0] = 1.0; + design[row][1] = standardized_time[row]; + if let Some(covariates) = &covariate_rows { + for &(column, value) in &covariates[row] { + design[row][2 + column] = value; + } + } + for assignment in &active[row] { + let column = membership_columns[&(assignment.role(), assignment.group_id())]; + design[row][2 + covariate_count + column] = assignment.weight().value(); + } + } + Ok((design, features)) +} + +fn collect_transition_pairs( + index_by_id: &HashMap, + relations: &RelationGraph, +) -> Result, TopicMeasurementError> { + let mut transition_pairs = BTreeSet::new(); + for edge in relations.edges().filter(|edge| edge.is_transition_edge()) { + let Some(&source) = index_by_id.get(&edge.source().as_uuid()) else { + continue; + }; + let Some(&target) = index_by_id.get(&edge.target().as_uuid()) else { + continue; + }; + transition_pairs.insert((source, target)); + } + if transition_pairs.is_empty() { + Err(TopicMeasurementError::InvalidModelInput) + } else { + Ok(transition_pairs.into_iter().collect()) + } +} + +/// Bounded deterministic reference-estimator configuration. +#[derive(Clone, Debug, PartialEq)] +pub struct ReferenceTopicModelConfig { + topic_count: usize, + seeds: Vec, + maximum_iterations: usize, + tolerance: f64, + prior_variance: f64, + relation_strength: f64, + ridge: f64, + topic_smoothing: f64, + step_size: f64, +} + +impl ReferenceTopicModelConfig { + /// Construct a reference configuration with ADR-owned v1 hyperparameters. + /// + /// # Errors + /// + /// Returns [`TopicMeasurementError::InvalidModelInput`] unless `topic_count` + /// is at least two, seeds are nonempty, the iteration budget is at least + /// two, and tolerance is finite and positive. + pub fn new( + topic_count: usize, + seeds: Vec, + maximum_iterations: usize, + tolerance: f64, + ) -> Result { + let value = Self { + topic_count, + seeds, + maximum_iterations, + tolerance, + prior_variance: DEFAULT_PRIOR_VARIANCE, + relation_strength: DEFAULT_RELATION_STRENGTH, + ridge: DEFAULT_RIDGE, + topic_smoothing: DEFAULT_TOPIC_SMOOTHING, + step_size: DEFAULT_STEP_SIZE, + }; + value.validate()?; + Ok(value) + } + + /// Replace numerical hyperparameters while retaining dimensional controls. + /// + /// # Errors + /// + /// Returns [`TopicMeasurementError::InvalidModelInput`] for any non-finite + /// or non-positive value, except `relation_strength` and `ridge`, which may + /// be exactly zero for a declared ablation. + pub fn with_hyperparameters( + mut self, + prior_variance: f64, + relation_strength: f64, + ridge: f64, + topic_smoothing: f64, + step_size: f64, + ) -> Result { + self.prior_variance = prior_variance; + self.relation_strength = relation_strength; + self.ridge = ridge; + self.topic_smoothing = topic_smoothing; + self.step_size = step_size; + self.validate()?; + Ok(self) + } + + fn validate(&self) -> Result<(), TopicMeasurementError> { + if self.topic_count < 2 + || self.seeds.is_empty() + || self.maximum_iterations < 2 + || !self.tolerance.is_finite() + || self.tolerance <= 0.0 + || !self.prior_variance.is_finite() + || self.prior_variance <= 0.0 + || !self.relation_strength.is_finite() + || self.relation_strength < 0.0 + || !self.ridge.is_finite() + || self.ridge < 0.0 + || !self.topic_smoothing.is_finite() + || self.topic_smoothing <= 0.0 + || !self.step_size.is_finite() + || self.step_size <= 0.0 + { + return Err(TopicMeasurementError::InvalidModelInput); + } + Ok(()) + } +} + +/// One inferred predecessor/successor association within a dominant topic. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct TopicSequenceEdge { + /// Opaque predecessor document identity. + pub predecessor_document_id: Uuid, + /// Opaque successor document identity. + pub successor_document_id: Uuid, + /// Artifact-local global topic index. + pub topic_index: usize, + /// Minimum dominant-topic posterior mean across the two documents. + pub association_strength: f64, +} + +/// A converged topic-model result with uncertainty and lineage counts. +#[derive(Clone, Debug, PartialEq)] +pub struct ReferenceTopicModel { + /// Selected deterministic initialization seed. + pub seed: u64, + /// Iterations used by the selected converged fit. + pub iterations: usize, + /// Final finite penalized objective. + pub objective: f64, + /// Global topic-by-term probability matrix. + pub topic_term_probabilities: Vec>, + /// Document-by-topic posterior mean proportions. + pub document_topic_proportions: Vec>, + /// Diagonal Laplace variance for each document ALR coordinate. + pub document_coordinate_variances: Vec>, + /// Structural prevalence coefficient matrix, feature by ALR coordinate. + pub prevalence_coefficients: Vec>, + /// Ordered structural feature meanings for coefficient rows. + pub prevalence_features: Vec, + /// Inferred dominant-topic links restricted to explicit forward transitions. + pub sequence_edges: Vec, + /// Distinct documents incident to at least one inferred sequence edge. + pub connected_post_count: usize, + /// Distinct global topics represented by at least one sequence edge. + pub lineage_count: usize, +} + +#[derive(Clone)] +struct FitState { + seed: u64, + iterations: usize, + objective: f64, + beta: Vec>, + eta: Vec>, + coefficients: Vec>, +} + +/// Fit the bounded deterministic CPU `f64` TRSL-TM reference estimator. +/// +/// # Errors +/// +/// Returns a typed invalid-input, non-finite, or convergence failure. The +/// function never returns a partial fit. +pub fn fit_reference_topic_model( + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, +) -> Result { + config.validate()?; + if config.topic_count > input.vocabulary_size { + return Err(TopicMeasurementError::InvalidModelInput); + } + let mut best = None; + for &seed in &config.seeds { + match fit_seed(input, config, seed) { + Ok(candidate) + if best.as_ref().is_none_or(|incumbent: &FitState| { + candidate.objective > incumbent.objective + }) => + { + best = Some(candidate); + } + Ok(_) | Err(TopicMeasurementError::DidNotConverge) => {} + Err(error) => return Err(error), + } + } + let state = best.ok_or(TopicMeasurementError::DidNotConverge)?; + build_result(input, config, state) +} + +fn fit_seed( + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + seed: u64, +) -> Result { + let document_count = input.document_ids.len(); + let coordinate_count = config.topic_count - 1; + let mut rng = seed.max(1); + let mut beta = vec![vec![0.0; input.vocabulary_size]; config.topic_count]; + for topic in &mut beta { + for value in topic.iter_mut() { + *value = config.topic_smoothing + next_unit(&mut rng); + } + normalize(topic)?; + } + let mut eta = vec![vec![0.0; coordinate_count]; document_count]; + for row in &mut eta { + for value in row { + *value = (next_unit(&mut rng) - 0.5) * 0.1; + } + } + let mut coefficients = vec![vec![0.0; coordinate_count]; input.features.len()]; + let mut previous = None; + + for iteration in 1..=config.maximum_iterations { + let theta = topic_proportions(&eta)?; + let (document_topic_counts, beta_counts, ll) = + expectation(input, &theta, &beta, config.topic_count)?; + let means = prevalence_means(&input.design, &coefficients); + let objective = objective(input, config, &theta, &eta, &means, &coefficients, ll)?; + if previous.is_some_and(|value: f64| { + (objective - value).abs() / (1.0 + value.abs()) <= config.tolerance + }) && iteration > 3 + { + return Ok(FitState { + seed, + iterations: iteration, + objective, + beta, + eta, + coefficients, + }); + } + previous = Some(objective); + beta = update_beta(beta_counts, config.topic_smoothing)?; + update_coefficients(input, config, &eta, &means, &mut coefficients)?; + let counts = &document_topic_counts; + update_eta(input, config, counts, &theta, &means, &mut eta)?; + } + Err(TopicMeasurementError::DidNotConverge) +} + +fn topic_proportions(eta: &[Vec]) -> Result>, TopicMeasurementError> { + eta.iter().map(|row| from_additive_log_ratio(row)).collect() +} + +type ExpectationOutput = (Vec>, Vec>, f64); + +fn expectation( + input: &ReferenceTopicInput, + theta: &[Vec], + beta: &[Vec], + topic_count: usize, +) -> Result { + let mut document_topic_counts = vec![vec![0.0; topic_count]; input.document_ids.len()]; + let mut beta_counts = vec![vec![0.0; input.vocabulary_size]; topic_count]; + let mut log_likelihood = 0.0; + for (document, terms) in input.term_rows.iter().enumerate() { + for &(term, count) in terms { + let probability = (0..topic_count) + .map(|topic| theta[document][topic] * beta[topic][term]) + .sum::(); + let log_probability = probability.ln(); + require_finite(log_probability)?; + log_likelihood += count * log_probability; + for topic in 0..topic_count { + let expected = count * theta[document][topic] * beta[topic][term] / probability; + document_topic_counts[document][topic] += expected; + beta_counts[topic][term] += expected; + } + } + } + if !log_likelihood.is_finite() { + return Err(TopicMeasurementError::NonFiniteEstimate); + } + Ok((document_topic_counts, beta_counts, log_likelihood)) +} + +fn update_beta( + mut counts: Vec>, + smoothing: f64, +) -> Result>, TopicMeasurementError> { + for topic in &mut counts { + for value in topic.iter_mut() { + *value += smoothing; + } + normalize(topic)?; + } + Ok(counts) +} + +fn prevalence_means(design: &[Vec], coefficients: &[Vec]) -> Vec> { + let coordinate_count = coefficients[0].len(); + design + .iter() + .map(|row| { + let mut mean = vec![0.0; coordinate_count]; + for (feature, value) in row.iter().enumerate() { + for (coordinate, target) in mean.iter_mut().enumerate() { + *target += value * coefficients[feature][coordinate]; + } + } + mean + }) + .collect() +} + +fn objective( + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + theta: &[Vec], + eta: &[Vec], + means: &[Vec], + coefficients: &[Vec], + log_likelihood: f64, +) -> Result { + let prior = eta + .iter() + .zip(means) + .flat_map(|(row, mean)| row.iter().zip(mean)) + .map(|(value, mean)| (value - mean).powi(2)) + .sum::() + / (2.0 * config.prior_variance); + let relation = input + .transition_pairs + .iter() + .map(|&(source, target)| { + theta[source] + .iter() + .zip(&theta[target]) + .map(|(left, right)| (left - right).powi(2)) + .sum::() + }) + .sum::() + * config.relation_strength + / 2.0; + let ridge = coefficients + .iter() + .flatten() + .map(|value| value * value) + .sum::() + * config.ridge + / 2.0; + let value = log_likelihood - prior - relation - ridge; + if value.is_finite() { + Ok(value) + } else { + Err(TopicMeasurementError::NonFiniteEstimate) + } +} + +fn update_coefficients( + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + eta: &[Vec], + means: &[Vec], + coefficients: &mut [Vec], +) -> Result<(), TopicMeasurementError> { + let scale = config.step_size / bounded_count(input.document_ids.len())?; + for feature in 0..coefficients.len() { + for coordinate in 0..coefficients[feature].len() { + let gradient = input + .design + .iter() + .enumerate() + .map(|(document, row)| { + row[feature] * (eta[document][coordinate] - means[document][coordinate]) + / config.prior_variance + }) + .sum::() + - config.ridge * coefficients[feature][coordinate]; + coefficients[feature][coordinate] += scale * gradient; + require_finite(coefficients[feature][coordinate])?; + } + } + Ok(()) +} + +fn update_eta( + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + counts: &[Vec], + theta: &[Vec], + means: &[Vec], + eta: &mut [Vec], +) -> Result<(), TopicMeasurementError> { + let coordinate_count = config.topic_count - 1; + let mut relation_gradient = vec![vec![0.0; coordinate_count]; input.document_ids.len()]; + for &(source, target) in &input.transition_pairs { + let delta: Vec = theta[source] + .iter() + .zip(&theta[target]) + .map(|(left, right)| left - right) + .collect(); + let source_dot = dot(&delta, &theta[source]); + let target_dot = dot(&delta, &theta[target]); + for coordinate in 0..coordinate_count { + relation_gradient[source][coordinate] -= config.relation_strength + * theta[source][coordinate] + * (delta[coordinate] - source_dot); + relation_gradient[target][coordinate] += config.relation_strength + * theta[target][coordinate] + * (delta[coordinate] - target_dot); + } + } + for document in 0..eta.len() { + let token_count = counts[document].iter().sum::(); + let scale = config.step_size / (1.0 + token_count); + for coordinate in 0..coordinate_count { + let gradient = counts[document][coordinate] + - token_count * theta[document][coordinate] + - (eta[document][coordinate] - means[document][coordinate]) / config.prior_variance + + relation_gradient[document][coordinate]; + eta[document][coordinate] += scale * gradient; + require_finite(eta[document][coordinate])?; + } + } + Ok(()) +} + +fn build_result( + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + state: FitState, +) -> Result { + let theta = topic_proportions(&state.eta)?; + let mut degrees = vec![0_usize; input.document_ids.len()]; + for &(source, target) in &input.transition_pairs { + degrees[source] += 1; + degrees[target] += 1; + } + let mut variances = Vec::with_capacity(theta.len()); + for (document, proportions) in theta.iter().enumerate() { + let token_count = input.term_rows[document] + .iter() + .map(|(_, count)| count) + .sum::(); + let degree = bounded_count(degrees[document])?; + variances.push( + proportions[..config.topic_count - 1] + .iter() + .map(|value| { + 1.0 / (token_count * value * (1.0 - value) + + 1.0 / config.prior_variance + + degree * config.relation_strength) + }) + .collect(), + ); + } + let dominant: Vec = theta.iter().map(|row| argmax(row)).collect(); + let mut sequence_edges = Vec::new(); + let mut connected = BTreeSet::new(); + let mut lineages = BTreeSet::new(); + for &(source, target) in &input.transition_pairs { + if dominant[source] != dominant[target] { + continue; + } + let topic = dominant[source]; + connected.insert(input.document_ids[source]); + connected.insert(input.document_ids[target]); + lineages.insert(topic); + sequence_edges.push(TopicSequenceEdge { + predecessor_document_id: input.document_ids[source], + successor_document_id: input.document_ids[target], + topic_index: topic, + association_strength: theta[source][topic].min(theta[target][topic]), + }); + } + Ok(ReferenceTopicModel { + seed: state.seed, + iterations: state.iterations, + objective: state.objective, + topic_term_probabilities: state.beta, + document_topic_proportions: theta, + document_coordinate_variances: variances, + prevalence_coefficients: state.coefficients, + prevalence_features: input.features.clone(), + sequence_edges, + connected_post_count: connected.len(), + lineage_count: lineages.len(), + }) +} + +#[allow(clippy::cast_precision_loss)] +fn standardize_event_time(times: &[EventTime]) -> Result, TopicMeasurementError> { + let origin = times[0].instant().as_nanosecond(); + let offsets: Vec = times + .iter() + .map(|time| (time.instant().as_nanosecond() - origin) as f64 / 1_000_000_000.0) + .collect(); + let mean = offsets.iter().sum::() / offsets.len() as f64; + let variance = offsets + .iter() + .map(|value| (value - mean).powi(2)) + .sum::() + / offsets.len() as f64; + if variance <= 0.0 { + return Err(TopicMeasurementError::InvalidModelInput); + } + let deviation = variance.sqrt(); + Ok(offsets + .iter() + .map(|value| (value - mean) / deviation) + .collect()) +} + +fn normalize(values: &mut [f64]) -> Result<(), TopicMeasurementError> { + let sum = values.iter().sum::(); + require_finite(sum.ln())?; + for value in values { + *value /= sum; + } + Ok(()) +} + +fn require_finite(value: f64) -> Result<(), TopicMeasurementError> { + value + .is_finite() + .then_some(()) + .ok_or(TopicMeasurementError::NonFiniteEstimate) +} + +fn bounded_count(value: usize) -> Result { + u32::try_from(value) + .map(f64::from) + .map_err(|_| TopicMeasurementError::InvalidModelInput) +} + +fn dot(left: &[f64], right: &[f64]) -> f64 { + left.iter().zip(right).map(|(a, b)| a * b).sum() +} + +fn argmax(values: &[f64]) -> usize { + values + .iter() + .enumerate() + .max_by(|left, right| left.1.total_cmp(right.1)) + .map_or(0, |(index, _)| index) +} + +fn next_unit(state: &mut u64) -> f64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + #[allow(clippy::cast_precision_loss)] + let value = (*state >> 11) as f64 / ((1_u64 << 53) as f64); + value.max(f64::EPSILON) +} + +#[cfg(test)] +mod tests { + use super::{ + FitState, PrevalenceFeature, ReferenceTopicInput, ReferenceTopicModelConfig, argmax, + bounded_count, build_result, dot, expectation, next_unit, normalize, objective, + require_finite, standardize_event_time, + }; + use crate::TopicMeasurementError; + use temporal_core::EventTime; + use uuid::Uuid; + + #[test] + fn numeric_helpers_are_deterministic_and_fail_closed() { + let mut seed = 1; + let first = next_unit(&mut seed); + assert!(first > 0.0); + assert!(first < 1.0); + assert!((dot(&[1.0, 2.0], &[3.0, 4.0]) - 11.0).abs() < f64::EPSILON); + assert_eq!(argmax(&[0.1, 0.8, 0.1]), 1); + assert_eq!(argmax(&[]), 0); + let mut values = [1.0, 3.0]; + normalize(&mut values).expect("normalize"); + assert!((values[0] - 0.25).abs() < f64::EPSILON); + assert!((values[1] - 0.75).abs() < f64::EPSILON); + assert_eq!( + normalize(&mut [0.0, 0.0]), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + assert_eq!( + normalize(&mut [f64::INFINITY]), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + assert_eq!(require_finite(1.0), Ok(())); + assert_eq!( + require_finite(f64::NAN), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + } + + #[test] + fn impossible_numeric_states_and_mixed_topic_edges_fail_closed() { + let input = ReferenceTopicInput { + document_ids: vec![Uuid::from_u128(1), Uuid::from_u128(2)], + term_rows: vec![vec![(0, f64::MAX)], vec![(0, f64::MAX)]], + vocabulary_size: 2, + design: vec![vec![1.0], vec![1.0]], + features: vec![PrevalenceFeature::Intercept], + transition_pairs: vec![(0, 1)], + }; + let theta = vec![vec![0.5, 0.5], vec![0.5, 0.5]]; + let zero_beta = vec![vec![0.0, 0.0], vec![0.0, 0.0]]; + assert_eq!( + expectation(&input, &theta, &zero_beta, 2), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + let infinite_beta = vec![vec![f64::INFINITY, 0.0], vec![0.0, 0.0]]; + assert_eq!( + expectation(&input, &theta, &infinite_beta, 2), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + let finite_beta = vec![vec![0.5, 0.5], vec![0.5, 0.5]]; + assert_eq!( + expectation(&input, &theta, &finite_beta, 2), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + let finite_input = ReferenceTopicInput { + term_rows: vec![vec![(0, 1.0)], vec![(0, 1.0)]], + ..input.clone() + }; + assert!(expectation(&finite_input, &theta, &finite_beta, 2).is_ok()); + + let config = ReferenceTopicModelConfig::new(2, vec![1], 10, 1e-6).expect("config"); + assert_eq!( + objective( + &input, + &config, + &theta, + &[vec![0.0], vec![0.0]], + &[vec![0.0], vec![0.0]], + &[vec![0.0]], + f64::INFINITY, + ), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + assert!( + objective( + &input, + &config, + &theta, + &[vec![0.0], vec![0.0]], + &[vec![0.0], vec![0.0]], + &[vec![0.0]], + 0.0, + ) + .is_ok() + ); + + let result = build_result( + &input, + &config, + FitState { + seed: 1, + iterations: 4, + objective: -1.0, + beta: finite_beta, + eta: vec![vec![10.0], vec![-10.0]], + coefficients: vec![vec![0.0]], + }, + ) + .expect("mixed-topic result"); + assert!(result.sequence_edges.is_empty()); + assert_eq!(result.connected_post_count, 0); + assert_eq!(result.lineage_count, 0); + + let same_time = EventTime::parse_rfc3339("2026-01-01T00:00:00Z").expect("time"); + assert_eq!( + standardize_event_time(&[same_time, same_time]), + Err(TopicMeasurementError::InvalidModelInput) + ); + #[cfg(target_pointer_width = "64")] + assert_eq!( + bounded_count(usize::try_from(u64::from(u32::MAX) + 1).expect("wide usize")), + Err(TopicMeasurementError::InvalidModelInput) + ); + } +} diff --git a/crates/topic_measurement/src/sparse.rs b/crates/topic_measurement/src/sparse.rs new file mode 100644 index 00000000..9749bd0a --- /dev/null +++ b/crates/topic_measurement/src/sparse.rs @@ -0,0 +1,222 @@ +//! Validated compressed sparse matrices used by the reference estimator. + +use crate::TopicMeasurementError; + +/// Whether compressed values are grouped by row or by column. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SparseOrientation { + /// Compressed sparse row storage. + Row, + /// Compressed sparse column storage. + Column, +} + +/// A finite numeric matrix in canonical CSR or CSC form. +#[derive(Clone, Debug, PartialEq)] +pub struct SparseMatrix { + rows: usize, + columns: usize, + offsets: Vec, + indices: Vec, + values: Vec, + orientation: SparseOrientation, +} + +impl SparseMatrix { + /// Construct and validate a compressed sparse row matrix. + /// + /// # Errors + /// + /// Returns [`TopicMeasurementError::InvalidSparseMatrix`] for zero + /// dimensions, malformed offsets, unsorted or repeated inner indices, + /// out-of-range indices, or non-finite values. + pub fn from_csr( + rows: usize, + columns: usize, + offsets: Vec, + indices: Vec, + values: Vec, + ) -> Result { + Self::new( + rows, + columns, + offsets, + indices, + values, + SparseOrientation::Row, + ) + } + + /// Construct and validate a compressed sparse column matrix. + /// + /// # Errors + /// + /// Returns [`TopicMeasurementError::InvalidSparseMatrix`] under the same + /// canonical-storage rules as [`Self::from_csr`]. + pub fn from_csc( + rows: usize, + columns: usize, + offsets: Vec, + indices: Vec, + values: Vec, + ) -> Result { + Self::new( + rows, + columns, + offsets, + indices, + values, + SparseOrientation::Column, + ) + } + + fn new( + rows: usize, + columns: usize, + offsets: Vec, + indices: Vec, + values: Vec, + orientation: SparseOrientation, + ) -> Result { + let outer = match orientation { + SparseOrientation::Row => rows, + SparseOrientation::Column => columns, + }; + let inner = match orientation { + SparseOrientation::Row => columns, + SparseOrientation::Column => rows, + }; + if rows == 0 + || columns == 0 + || offsets.len() != outer + 1 + || offsets.first() != Some(&0) + || offsets.last().copied() != Some(indices.len()) + || indices.len() != values.len() + || values.iter().any(|value| !value.is_finite()) + { + return Err(TopicMeasurementError::InvalidSparseMatrix); + } + for bounds in offsets.windows(2) { + if bounds[0] > bounds[1] || bounds[1] > indices.len() { + return Err(TopicMeasurementError::InvalidSparseMatrix); + } + let mut previous = None; + for &index in &indices[bounds[0]..bounds[1]] { + if index >= inner || previous.is_some_and(|value| index <= value) { + return Err(TopicMeasurementError::InvalidSparseMatrix); + } + previous = Some(index); + } + } + Ok(Self { + rows, + columns, + offsets, + indices, + values, + orientation, + }) + } + + /// Return the matrix row count. + #[must_use] + pub const fn rows(&self) -> usize { + self.rows + } + + /// Return the matrix column count. + #[must_use] + pub const fn columns(&self) -> usize { + self.columns + } + + /// Return the compressed orientation. + #[must_use] + pub const fn orientation(&self) -> SparseOrientation { + self.orientation + } + + pub(crate) fn row_entries(&self) -> Vec> { + let mut rows = vec![Vec::new(); self.rows]; + match self.orientation { + SparseOrientation::Row => { + for (row, bounds) in self.offsets.windows(2).enumerate() { + for index in bounds[0]..bounds[1] { + rows[row].push((self.indices[index], self.values[index])); + } + } + } + SparseOrientation::Column => { + for (column, bounds) in self.offsets.windows(2).enumerate() { + for index in bounds[0]..bounds[1] { + rows[self.indices[index]].push((column, self.values[index])); + } + } + } + } + rows + } +} + +#[cfg(test)] +mod tests { + use super::{SparseMatrix, SparseOrientation}; + use crate::TopicMeasurementError; + + #[test] + fn csr_and_csc_produce_the_same_rows() { + let csr = SparseMatrix::from_csr(2, 3, vec![0, 2, 3], vec![0, 2, 1], vec![1.0, 2.0, 3.0]) + .expect("csr"); + let csc = + SparseMatrix::from_csc(2, 3, vec![0, 1, 2, 3], vec![0, 1, 0], vec![1.0, 3.0, 2.0]) + .expect("csc"); + assert_eq!(csr.rows(), 2); + assert_eq!(csr.columns(), 3); + assert_eq!(csr.orientation(), SparseOrientation::Row); + assert_eq!(csc.orientation(), SparseOrientation::Column); + assert_eq!(csr.row_entries(), csc.row_entries()); + } + + #[test] + fn malformed_sparse_storage_fails_closed() { + let error = Err(TopicMeasurementError::InvalidSparseMatrix); + assert_eq!(SparseMatrix::from_csr(0, 1, vec![0], vec![], vec![]), error); + assert_eq!( + SparseMatrix::from_csr(1, 0, vec![0, 0], vec![], vec![]), + error + ); + assert_eq!(SparseMatrix::from_csr(1, 1, vec![0], vec![], vec![]), error); + assert_eq!( + SparseMatrix::from_csr(1, 1, vec![1, 1], vec![], vec![]), + error + ); + assert_eq!( + SparseMatrix::from_csr(1, 1, vec![0, 2], vec![0], vec![1.0]), + error + ); + assert_eq!( + SparseMatrix::from_csr(1, 1, vec![0, 1], vec![0], vec![]), + error + ); + assert_eq!( + SparseMatrix::from_csr(1, 1, vec![0, 1], vec![1], vec![1.0]), + error + ); + assert_eq!( + SparseMatrix::from_csr(1, 2, vec![0, 2], vec![1, 1], vec![1.0, 2.0]), + error + ); + assert_eq!( + SparseMatrix::from_csr(1, 1, vec![0, 1], vec![0], vec![f64::NAN]), + error + ); + assert_eq!( + SparseMatrix::from_csr(2, 1, vec![0, 2, 1], vec![0], vec![1.0]), + error + ); + assert_eq!( + SparseMatrix::from_csr(3, 2, vec![0, 2, 1, 3], vec![0, 1, 1], vec![1.0; 3]), + error + ); + } +} diff --git a/crates/topic_measurement/tests/composition_sum_precision_contract.rs b/crates/topic_measurement/tests/composition_sum_precision_contract.rs new file mode 100644 index 00000000..f76cb763 --- /dev/null +++ b/crates/topic_measurement/tests/composition_sum_precision_contract.rs @@ -0,0 +1,39 @@ +//! Composition validation must not lose tiny positive mass after a dominant part. + +use topic_measurement::{TopicMeasurementError, additive_log_ratio}; + +#[test] +fn compensated_sum_rejects_mass_hidden_by_naive_floating_point_addition() { + let mut composition = Vec::with_capacity(20_001); + composition.push(1.0); + composition.extend(std::iter::repeat_n(1.0e-16, 20_000)); + + assert_eq!( + additive_log_ratio(&composition), + Err(TopicMeasurementError::InvalidComposition), + "the true mass exceeds one by 2e-12 even though naive ordered addition rounds to one" + ); +} + +#[test] +fn overflowing_finite_parts_fail_closed_as_non_finite_mass() { + assert_eq!( + additive_log_ratio(&[f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidComposition), + "Kahan-compensated MAX+MAX is non-finite; NaN cannot pass a unit-sum tolerance comparison" + ); +} + +#[test] +fn compensated_sum_accepts_a_valid_many_part_composition() { + let tiny_mass = 1.0e-16; + let tiny_parts = 10_000_usize; + let dominant = 1.0 - tiny_mass * 10_000.0; + let mut composition = Vec::with_capacity(tiny_parts + 1); + composition.push(dominant); + composition.extend(std::iter::repeat_n(tiny_mass, tiny_parts)); + + let coordinates = additive_log_ratio(&composition).expect("valid unit simplex"); + assert_eq!(coordinates.len(), tiny_parts); + assert!(coordinates.iter().all(|value| value.is_finite())); +} diff --git a/crates/topic_measurement/tests/crate_contract.rs b/crates/topic_measurement/tests/crate_contract.rs new file mode 100644 index 00000000..8f9cfd6a --- /dev/null +++ b/crates/topic_measurement/tests/crate_contract.rs @@ -0,0 +1,7 @@ +//! Integration contract for the `topic_measurement` package identity. + +#[test] +fn package_identity_is_stable() { + let observed = std::hint::black_box(env!("CARGO_PKG_NAME")); + assert_eq!(observed, "topic_measurement"); +} diff --git a/crates/topic_measurement/tests/ilr_recovery_contract.rs b/crates/topic_measurement/tests/ilr_recovery_contract.rs new file mode 100644 index 00000000..74b94d46 --- /dev/null +++ b/crates/topic_measurement/tests/ilr_recovery_contract.rs @@ -0,0 +1,180 @@ +//! True-parameter recovery of isometric log-ratio topic coordinates. +#![allow(clippy::cast_precision_loss)] + +use topic_measurement::{ + TopicMeasurementError, additive_log_ratio, aitchison_distance, from_isometric_log_ratio, + isometric_log_ratio, +}; + +fn euclidean(left: &[f64], right: &[f64]) -> f64 { + left.iter() + .zip(right) + .map(|(left_value, right_value)| { + let residual = left_value - right_value; + residual * residual + }) + .sum::() + .sqrt() +} + +fn rmse(truth: &[f64], recovered: &[f64]) -> f64 { + let n = truth.len() as f64; + let sum_sq: f64 = truth + .iter() + .zip(recovered) + .map(|(left, right)| { + let residual = left - right; + residual * residual + }) + .sum(); + (sum_sq / n).sqrt() +} + +#[test] +fn known_simplex_recovers_through_ilr_with_computed_rmse() { + // Closed-form simplex: (2, 3, 1) / 6. + // Sequential Egozcue ILR: y1 = √(2/3) ln(2√3 / 3), y2 = √(1/2) ln 3. + let truth = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; + let true_parameters = [ + (2.0_f64 / 3.0).sqrt() * (2.0 * 3.0_f64.sqrt() / 3.0).ln(), + (1.0_f64 / 2.0).sqrt() * 3.0_f64.ln(), + ]; + let coordinates = isometric_log_ratio(&truth).expect("ilr"); + assert_eq!(coordinates.len(), 2); + let parameter_rmse = rmse(&true_parameters, &coordinates); + assert!( + parameter_rmse < 1e-15, + "true-parameter ILR RMSE {parameter_rmse} exceeded machine-scale bound" + ); + + let recovered = from_isometric_log_ratio(&coordinates).expect("inverse"); + let simplex_rmse = rmse(&truth, &recovered); + assert!( + simplex_rmse < 1e-15, + "ILR round-trip RMSE {simplex_rmse} exceeded machine-scale bound" + ); + let sum: f64 = recovered.iter().sum(); + assert!((sum - 1.0).abs() < 1e-15); + + let alr = additive_log_ratio(&truth).expect("alr"); + assert!( + (alr[0] - coordinates[0]).abs() > 1e-6, + "ILR must not collapse to the reference-dependent ALR map" + ); +} + +#[test] +fn equal_shares_are_the_ilr_origin_and_preserve_aitchison_distance() { + let halves = [0.5, 0.5]; + let origin = isometric_log_ratio(&halves).expect("origin"); + assert_eq!(origin.len(), 1); + assert!(origin[0].abs() < 1e-15); + + let unbalanced = [0.8, 0.2]; + let coordinates = isometric_log_ratio(&unbalanced).expect("pair"); + let direct_aitchison_distance = (0.5_f64).sqrt() + * ((unbalanced[0] / unbalanced[1]).ln() - (halves[0] / halves[1]).ln()).abs(); + let ilr_euclidean_distance = coordinates + .iter() + .zip(&origin) + .map(|(left, right)| (left - right).powi(2)) + .sum::() + .sqrt(); + assert!((ilr_euclidean_distance - direct_aitchison_distance).abs() < 1e-15); + + let recovered = from_isometric_log_ratio(&coordinates).expect("inverse"); + assert!(rmse(&unbalanced, &recovered) < 1e-15); + assert!( + (aitchison_distance(&unbalanced, &halves).expect("clr") - direct_aitchison_distance).abs() + < 1e-15 + ); +} + +#[test] +fn pairwise_ilr_euclidean_recovers_aitchison_distance_away_from_the_origin() { + let left = [0.70, 0.30]; + let right = [0.20, 0.80]; + let left_ilr = isometric_log_ratio(&left).expect("left"); + let right_ilr = isometric_log_ratio(&right).expect("right"); + let ilr_euclidean = euclidean(&left_ilr, &right_ilr); + let distance = aitchison_distance(&left, &right).expect("aitchison"); + assert!( + (ilr_euclidean - distance).abs() < 1e-15, + "two-part ILR Euclidean {ilr_euclidean} must equal Aitchison {distance}" + ); + assert!((distance - aitchison_distance(&right, &left).expect("symmetric")).abs() < 1e-15); + assert!(aitchison_distance(&left, &left).expect("self") < 1e-15); + + let three_left = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; + let three_right = [1.0 / 6.0, 1.0 / 6.0, 4.0 / 6.0]; + let isometric_left = isometric_log_ratio(&three_left).expect("three left"); + let isometric_right = isometric_log_ratio(&three_right).expect("three right"); + let three_ilr = euclidean(&isometric_left, &isometric_right); + let three_distance = aitchison_distance(&three_left, &three_right).expect("three"); + assert!( + (three_ilr - three_distance).abs() < 1e-15, + "three-part ILR Euclidean {three_ilr} must equal Aitchison {three_distance}" + ); + + let additive_left = additive_log_ratio(&three_left).expect("alr left"); + let additive_right = additive_log_ratio(&three_right).expect("alr right"); + let alr_euclidean = euclidean(&additive_left, &additive_right); + assert!( + (alr_euclidean - three_distance).abs() > 1e-6, + "ALR Euclidean must not be treated as Aitchison distance" + ); +} + +#[test] +fn large_finite_ilr_coordinates_round_trip_or_fail_closed() { + let representable = from_isometric_log_ratio(&[40.0]).expect("representable"); + assert!( + representable + .iter() + .all(|part| part.is_finite() && *part > 0.0) + ); + let recovered = isometric_log_ratio(&representable).expect("forward"); + assert!(rmse(&[40.0], &recovered) < 1e-12); + + assert_eq!( + from_isometric_log_ratio(&[1000.0]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "ILR inverse must not return a zero simplex part after underflow" + ); + assert_eq!( + from_isometric_log_ratio(&[-f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "overflowing CLR reconstruction must fail closed" + ); + let gap = 745.0_f64; + let division_underflow = [ + (3.0_f64 / 4.0).sqrt() * gap / 3.0, + (2.0_f64 / 3.0).sqrt() * gap / 2.0, + (1.0_f64 / 2.0).sqrt() * gap, + ]; + assert_eq!( + from_isometric_log_ratio(&division_underflow), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "normalization must not underflow a nonzero weight to a zero simplex part" + ); +} + +#[test] +fn invalid_ilr_inputs_fail_closed() { + assert_eq!( + isometric_log_ratio(&[]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + from_isometric_log_ratio(&[]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_isometric_log_ratio(&[f64::NAN]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_isometric_log_ratio(&[f64::INFINITY]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); +} diff --git a/crates/topic_measurement/tests/logratio_recovery_contract.rs b/crates/topic_measurement/tests/logratio_recovery_contract.rs new file mode 100644 index 00000000..e08a497e --- /dev/null +++ b/crates/topic_measurement/tests/logratio_recovery_contract.rs @@ -0,0 +1,132 @@ +//! True-parameter recovery of logistic-normal topic coordinates. +#![allow(clippy::cast_precision_loss)] + +use topic_measurement::{ + TopicMeasurementError, additive_log_ratio, from_additive_log_ratio, + refuse_lexical_inferential_weight, +}; + +fn rmse(truth: &[f64], recovered: &[f64]) -> f64 { + let n = truth.len() as f64; + let sum_sq: f64 = truth + .iter() + .zip(recovered) + .map(|(left, right)| { + let residual = left - right; + residual * residual + }) + .sum(); + (sum_sq / n).sqrt() +} + +#[test] +fn known_simplex_recovers_through_alr_with_computed_rmse() { + // Closed-form simplex: (2, 3, 1) / 6. ALR is (ln 2, ln 3). + let truth = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; + let coordinates = additive_log_ratio(&truth).expect("alr"); + let true_parameters = [2.0_f64.ln(), 3.0_f64.ln()]; + assert_eq!(coordinates.len(), 2); + let parameter_rmse = rmse(&true_parameters, &coordinates); + assert!( + parameter_rmse < 1e-15, + "true-parameter ALR RMSE {parameter_rmse} exceeded machine-scale bound" + ); + + let recovered = from_additive_log_ratio(&coordinates).expect("inverse"); + let simplex_rmse = rmse(&truth, &recovered); + assert!( + simplex_rmse < 1e-15, + "ALR round-trip RMSE {simplex_rmse} exceeded machine-scale bound" + ); + let sum: f64 = recovered.iter().sum(); + assert!((sum - 1.0).abs() < 1e-15); +} + +#[test] +fn large_finite_coordinates_round_trip_without_exponential_overflow() { + let truth = [710.0, 709.0]; + let simplex = from_additive_log_ratio(&truth) + .expect("finite representable ALR coordinates must use a stable inverse"); + assert!(simplex.iter().all(|part| part.is_finite() && *part > 0.0)); + assert!((simplex.iter().sum::() - 1.0).abs() < 1e-15); + + let recovered = additive_log_ratio(&simplex) + .expect("forward ALR must subtract logs instead of overflowing the ratio"); + assert!( + rmse(&truth, &recovered) < 1e-10, + "large-coordinate round trip must retain the true parameters" + ); +} + +#[test] +fn equal_shares_map_to_zero_alr_and_refuse_raw_euclidean_use() { + let thirds = [1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0]; + let coordinates = additive_log_ratio(&thirds).expect("equal"); + assert!(coordinates.iter().all(|value| value.abs() < 1e-15)); + let recovered = from_additive_log_ratio(&[0.0, 0.0]).expect("zeros"); + assert!(rmse(&thirds, &recovered) < 1e-15); +} + +#[test] +fn invalid_compositions_and_lexical_weights_fail_closed() { + // K=2 is valid; zero/negative/non-unit-sum/non-finite/K<2 are not. + assert_eq!( + additive_log_ratio(&[0.0, 1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[-0.1, 1.1]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[0.2, 0.2, 0.2]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[f64::NAN, 1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + from_additive_log_ratio(&[]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_additive_log_ratio(&[f64::INFINITY]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_additive_log_ratio(&[1.0e9]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "max-shifted reference weight must fail closed when it underflows to zero" + ); + assert_eq!( + from_additive_log_ratio(&[-1.0e9]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "inverse must not return a zero simplex part after underflow" + ); + assert_eq!( + from_additive_log_ratio(&[0.0, 0.0, 0.0, -744.0]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "normalization must not underflow a nonzero weight to a zero simplex part" + ); + assert_eq!( + additive_log_ratio(&[f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidComposition), + "overflowing finite parts must fail closed because compensated mass is non-finite" + ); + + for method in ["tfidf", "bm25", "keyword", "TF-IDF", ""] { + assert_eq!( + refuse_lexical_inferential_weight(method), + Err(TopicMeasurementError::LexicalWeightForbidden) + ); + } +} diff --git a/crates/topic_measurement/tests/reference_estimator_contract.rs b/crates/topic_measurement/tests/reference_estimator_contract.rs new file mode 100644 index 00000000..2aa6de87 --- /dev/null +++ b/crates/topic_measurement/tests/reference_estimator_contract.rs @@ -0,0 +1,516 @@ +//! Known-truth recovery contract for the CPU `f64` TRSL-TM reference. + +use corpus_split::{CorpusDocument, CorpusSnapshot}; +use membership_core::{ + GroupId, MemberId, MembershipAssignment, MembershipNetwork, MembershipRole, MembershipWeight, +}; +use relation_graph::{ + RelationEdge, RelationEndpointId, RelationEvidenceStatus, RelationGraph, RelationKind, +}; +use temporal_core::{ + AvailableTime, EventTime, KnowledgeCutoff, TemporalBoundary, TemporalInterval, + TemporalPrecision, +}; +use topic_measurement::{ + PrevalenceFeature, ReferenceTopicInput, ReferenceTopicModelConfig, SparseMatrix, + fit_reference_topic_model, +}; +use uuid::Uuid; +use validation_core::root_mean_square_error; + +fn event_time(day: u8) -> EventTime { + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T00:00:00Z")).expect("event time") +} + +fn relation(source: Uuid, target: Uuid, source_day: u8, target_day: u8) -> RelationEdge { + let interval = |day| { + TemporalInterval::bounded( + TemporalBoundary::Included(event_time(day)), + TemporalBoundary::Included( + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T12:00:00Z")) + .expect("interval end"), + ), + TemporalPrecision::Second, + ) + .expect("bounded interval") + }; + RelationEdge::new( + RelationKind::TransitionsTo, + RelationEndpointId::from_uuid(source), + RelationEndpointId::from_uuid(target), + RelationEvidenceStatus::Observed, + interval(source_day), + interval(target_day), + ) + .expect("forward relation") +} + +fn fixture() -> ( + CorpusSnapshot, + Vec, + Vec, + MembershipNetwork, + RelationGraph, +) { + let document_ids: Vec<_> = (1_u128..=6).map(Uuid::from_u128).collect(); + let times: Vec<_> = (1_u8..=6).map(event_time).collect(); + let available = AvailableTime::parse_rfc3339("2026-01-10T00:00:00Z").expect("available"); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"); + let mut snapshot = CorpusSnapshot::new(); + for id in &document_ids { + snapshot + .insert_if_eligible(CorpusDocument::new(*id, available), &cutoff) + .expect("eligible"); + } + + let organization = GroupId::from_uuid(Uuid::from_u128(100)); + let projects = [ + GroupId::from_uuid(Uuid::from_u128(101)), + GroupId::from_uuid(Uuid::from_u128(102)), + ]; + let validity_start = event_time(1); + let validity_end = event_time(9); + let mut memberships = MembershipNetwork::new(); + for (index, id) in document_ids.iter().enumerate() { + let member = MemberId::from_uuid(*id); + memberships + .insert( + MembershipAssignment::new( + member, + organization, + MembershipRole::Organization, + MembershipWeight::full().expect("full"), + validity_start, + validity_end, + ) + .expect("organization membership"), + ) + .expect("insert organization"); + memberships + .insert( + MembershipAssignment::new( + member, + projects[usize::from(index >= 3)], + MembershipRole::Project, + MembershipWeight::new(0.75).expect("partial"), + validity_start, + validity_end, + ) + .expect("project membership"), + ) + .expect("insert project"); + } + + let mut relations = RelationGraph::new(); + for (source, target, source_day, target_day) in [ + (0, 1, 1, 2), + (1, 2, 2, 3), + (2, 3, 3, 4), + (3, 4, 4, 5), + (4, 5, 5, 6), + ] { + relations + .insert(relation( + document_ids[source], + document_ids[target], + source_day, + target_day, + )) + .expect("insert relation"); + } + (snapshot, document_ids, times, memberships, relations) +} + +fn separated_counts() -> SparseMatrix { + SparseMatrix::from_csr( + 6, + 4, + vec![0, 2, 4, 6, 8, 10, 12], + vec![0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3], + vec![ + 90.0, 10.0, 85.0, 15.0, 80.0, 20.0, 10.0, 90.0, 15.0, 85.0, 20.0, 80.0, + ], + ) + .expect("counts") +} + +#[test] +fn separated_topics_recover_and_emit_predecessor_successor_counts() { + let (snapshot, document_ids, times, memberships, relations) = fixture(); + let counts = separated_counts(); + let input = ReferenceTopicInput::new( + &snapshot, + document_ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("input"); + assert_eq!(input.document_count(), 6); + assert_eq!(input.vocabulary_size(), 4); + assert!(matches!(input.features()[0], PrevalenceFeature::Intercept)); + assert!(matches!(input.features()[1], PrevalenceFeature::EventTime)); + + let config = ReferenceTopicModelConfig::new(2, vec![7, 11, 19], 2_000, 1e-5) + .expect("configuration") + .with_hyperparameters(1.0, 0.5, 0.01, 0.05, 0.2) + .expect("hyperparameters"); + let result = fit_reference_topic_model(&input, &config).expect("converged fit"); + assert!(result.objective.is_finite()); + assert!(result.iterations <= 2_000); + assert_eq!(result.connected_post_count, 6); + assert_eq!(result.lineage_count, 2); + assert_eq!(result.sequence_edges.len(), 4); + assert!( + result + .sequence_edges + .iter() + .all(|edge| edge.association_strength > 0.5) + ); + assert!( + result + .document_coordinate_variances + .iter() + .flatten() + .all(|value| *value > 0.0) + ); + + let recovered: Vec = result + .document_topic_proportions + .iter() + .map(|row| row[0]) + .collect(); + let truth_a = [0.9, 0.85, 0.8, 0.1, 0.15, 0.2]; + let truth_b = [0.1, 0.15, 0.2, 0.9, 0.85, 0.8]; + let rmse = root_mean_square_error(&truth_a, &recovered) + .expect("rmse") + .min(root_mean_square_error(&truth_b, &recovered).expect("label-swapped rmse")); + assert!(rmse < 0.25, "known-truth topic RMSE {rmse} exceeded 0.25"); +} + +#[test] +fn invalid_configuration_and_topic_dimension_fail_closed() { + let (snapshot, document_ids, times, memberships, relations) = fixture(); + let counts = SparseMatrix::from_csr( + 6, + 2, + vec![0, 1, 2, 3, 4, 5, 6], + vec![0, 0, 0, 1, 1, 1], + vec![1.0; 6], + ) + .expect("counts"); + let input = ReferenceTopicInput::new( + &snapshot, + document_ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("input"); + assert!(ReferenceTopicModelConfig::new(1, vec![1], 10, 1e-6).is_err()); + let too_many = ReferenceTopicModelConfig::new(3, vec![1], 10, 1e-6).expect("config"); + assert!(fit_reference_topic_model(&input, &too_many).is_err()); + + for (topics, seeds, iterations, tolerance) in [ + (2, vec![], 10, 1e-6), + (2, vec![1], 1, 1e-6), + (2, vec![1], 10, f64::NAN), + (2, vec![1], 10, 0.0), + ] { + assert!(ReferenceTopicModelConfig::new(topics, seeds, iterations, tolerance).is_err()); + } + let base = ReferenceTopicModelConfig::new(2, vec![1], 10, 1e-6).expect("base"); + for values in [ + (f64::NAN, 0.5, 0.01, 0.05, 0.2), + (0.0, 0.5, 0.01, 0.05, 0.2), + (1.0, f64::NAN, 0.01, 0.05, 0.2), + (1.0, -1.0, 0.01, 0.05, 0.2), + (1.0, 0.5, f64::NAN, 0.05, 0.2), + (1.0, 0.5, -1.0, 0.05, 0.2), + (1.0, 0.5, 0.01, f64::NAN, 0.2), + (1.0, 0.5, 0.01, 0.0, 0.2), + (1.0, 0.5, 0.01, 0.05, f64::NAN), + (1.0, 0.5, 0.01, 0.05, 0.0), + ] { + assert!( + base.clone() + .with_hyperparameters(values.0, values.1, values.2, values.3, values.4) + .is_err() + ); + } +} + +#[test] +#[allow(clippy::too_many_lines)] +fn invalid_structural_inputs_and_nonconvergence_fail_closed() { + let (snapshot, document_ids, times, memberships, relations) = fixture(); + let counts = separated_counts(); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids[..1].to_vec(), + &counts, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + let wrong_rows = + SparseMatrix::from_csr(2, 2, vec![0, 1, 2], vec![0, 1], vec![1.0; 2]).expect("wrong rows"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &wrong_rows, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + let one_column = + SparseMatrix::from_csr(6, 1, vec![0, 1, 2, 3, 4, 5, 6], vec![0; 6], vec![1.0; 6]) + .expect("one column"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &one_column, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + ×[..5], + None, + &memberships, + &relations, + ) + .is_err() + ); + let mut missing_snapshot = CorpusSnapshot::new(); + missing_snapshot + .insert_if_eligible( + CorpusDocument::new( + document_ids[0], + AvailableTime::parse_rfc3339("2026-01-10T00:00:00Z").expect("available"), + ), + &KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"), + ) + .expect("eligible"); + assert!( + ReferenceTopicInput::new( + &missing_snapshot, + document_ids.clone(), + &counts, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + let mut outside_relations = RelationGraph::new(); + outside_relations + .insert(relation(Uuid::from_u128(999), document_ids[0], 1, 2)) + .expect("outside source"); + outside_relations + .insert(relation(document_ids[0], Uuid::from_u128(998), 2, 3)) + .expect("outside target"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + ×, + None, + &memberships, + &outside_relations, + ) + .is_err() + ); + + let empty_row = SparseMatrix::from_csr( + 6, + 2, + vec![0, 0, 1, 2, 3, 4, 5], + vec![0, 0, 1, 1, 1], + vec![1.0; 5], + ) + .expect("empty row"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &empty_row, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + let zero_row = SparseMatrix::from_csr( + 6, + 2, + vec![0, 1, 2, 3, 4, 5, 6], + vec![0, 0, 0, 1, 1, 1], + vec![0.0, 1.0, 1.0, 1.0, 1.0, 1.0], + ) + .expect("zero row"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &zero_row, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + + let mut duplicate_ids = document_ids.clone(); + duplicate_ids[1] = duplicate_ids[0]; + assert!( + ReferenceTopicInput::new( + &snapshot, + duplicate_ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + + let negative = SparseMatrix::from_csr( + 6, + 2, + vec![0, 1, 2, 3, 4, 5, 6], + vec![0, 0, 0, 1, 1, 1], + vec![-1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + ) + .expect("finite sparse values"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &negative, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + + let covariate = SparseMatrix::from_csc( + 6, + 1, + vec![0, 6], + vec![0, 1, 2, 3, 4, 5], + vec![-1.0, -0.5, 0.0, 0.0, 0.5, 1.0], + ) + .expect("covariate"); + let with_covariate = ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + ×, + Some(&covariate), + &memberships, + &relations, + ) + .expect("covariate input"); + assert!(matches!( + with_covariate.features()[2], + PrevalenceFeature::Covariate(0) + )); + let wrong_rows = + SparseMatrix::from_csr(2, 1, vec![0, 0, 0], vec![], vec![]).expect("covariate"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + ×, + Some(&wrong_rows), + &memberships, + &relations, + ) + .is_err() + ); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + ×, + None, + &MembershipNetwork::new(), + &relations, + ) + .is_err() + ); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + ×, + None, + &memberships, + &RelationGraph::new(), + ) + .is_err() + ); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + &[event_time(1); 6], + None, + &memberships, + &relations, + ) + .is_err() + ); + + let input = ReferenceTopicInput::new( + &snapshot, + document_ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("valid input"); + let exhausted = ReferenceTopicModelConfig::new(2, vec![1], 2, 1e-12).expect("exhausted"); + assert!(fit_reference_topic_model(&input, &exhausted).is_err()); + let quick = ReferenceTopicModelConfig::new(2, vec![1], 10, f64::MAX).expect("quick"); + assert!(fit_reference_topic_model(&input, &quick).is_ok()); + let unstable = ReferenceTopicModelConfig::new(2, vec![1], 10, 1e-6) + .expect("unstable") + .with_hyperparameters(1.0, 0.5, 0.01, 0.05, f64::MAX) + .expect("finite hyperparameters"); + assert!(fit_reference_topic_model(&input, &unstable).is_err()); +} diff --git a/docs/API_CONTRACT.md b/docs/API_CONTRACT.md index 59e16ffc..9a64e200 100644 --- a/docs/API_CONTRACT.md +++ b/docs/API_CONTRACT.md @@ -1,13 +1,13 @@ # TEPP API and Modular Integration Contract **Status:** Accepted target contract; exact endpoints are introduced only with executable services. -**Last reviewed:** 2026-08-19 +**Last reviewed:** 2026-08-21 ## 1. Authority boundary TEPP must work both as a standalone product and as a modular CWL component. Integrations with `naruon`, `contextual-orchestrator`, `.github`, or other repositories use explicit versioned API/artifact contracts. Cross-service direct table access is prohibited. -Current protected main exposes Rust library/domain contracts. The active stack adds a loopback HTTP/1.1 listener for naruon analysis-run and LineageWeave temporal-context POSTs, including `POST /v1/project-histories` on the `AnalysisRunLiveService` contract boundary. `tepp-loopback` runs the shared consumer listener on `127.0.0.1:18081` by default; a caller may pass another loopback socket address and an optional maximum request count as its two arguments. The container is intended for a trusted same-host or shared-network-namespace sidecar, checks readiness through a synthetic bounded temporal-context request, and deliberately cannot bind a public or bridge address. It is not a production TLS/`$PORT` service. Endpoint examples below that are not covered by `NaruonLiveService` or `AnalysisRunLiveService` remain target interface shapes; export retrieval stays a target shape until an executable export route ships. +Current protected main exposes Rust library/domain contracts. The active stack adds a loopback HTTP/1.1 listener for naruon analysis-run, LineageWeave temporal-context, and export POSTs, including `POST /v1/project-histories` on the `AnalysisRunLiveService` contract boundary. `tepp-loopback` runs the shared consumer listener on `127.0.0.1:18081` by default; a caller may pass another loopback socket address and an optional maximum request count as its two arguments. The container is intended for a trusted same-host or shared-network-namespace sidecar, checks readiness through a synthetic bounded temporal-context request, and deliberately cannot bind a public or bridge address. It is not a production TLS/`$PORT` service. Endpoint examples below that are not covered by `NaruonLiveService` or `AnalysisRunLiveService` remain target interface shapes; export retrieval stays a target shape until an executable export route ships. ## 2. Contract families @@ -20,9 +20,10 @@ Current protected main exposes Rust library/domain contracts. The active stack a | semantic/topic measurement API | future TEPP measurement service | naruon, batch jobs, visual analytics | accepted-target | | LLM interpretation provider port | `tepp_api` orchestration router + future HTTP gateway | contextual-orchestrator | partial | | model/artifact/export API | `tepp_api` export envelopes + future HTTP service | standalone UI/CWL consumers | partial | -| analysis-run request/accepted contracts | `tepp_api` v1 wire DTOs | naruon, orchestrator, UI | implemented-main | +| analysis-run request/accepted/status/terminal-result contracts | `tepp_api` v1 wire DTOs | naruon, orchestrator, UI | active product branch | | corpus-split leakage-audit manifest | `tepp_api` `CorpusSplitManifest` v1 | naruon, auditors, future UI | active-PR | | temporal-context ordering contract | `tepp_api` v1 wire DTOs | LineageWeave | active-PR | +| cutoff-safe analysis-run readiness execution | `analysis_engine` bounded Rust crate | `tepp_api`, future HTTP/service adapters | active product branch | | project-history projection contract | `tepp_api` v1 wire DTOs | LineageWeave | active-PR | ## 3. Versioning @@ -62,6 +63,24 @@ them by event time and opaque event ID, and emits adjacent forward temporal associations plus `candidate_not_causal` transition gaps. It does not infer causality, mutate TEPP state, or return a completed psychometric result. +The typed status/read contract returns `accepted`, `running`, `succeeded`, or +`failed`. Accepted and running statuses contain no measurement result. A +terminal status contains exactly one request-bound +`AnalysisRunTerminalResult`; consumers must validate its request, receipt, +snapshot, cutoff, model, profile, and idempotency bindings before treating the +run as measurement evidence. The Rust DTO is available before the future HTTP +service is deployed. + +The stacked `analysis_engine` slice provides the first executable service-side +path behind these DTOs. It consumes a bounded identity-free snapshot, excludes +evidence unavailable at the historical cutoff, preserves multiple-membership +counts, and emits a digest-bound terminal result or a redacted failure. For the +`trsl_topic_lineage_v1` profile it invokes the ADR-0012 `topic_measurement` +reference estimator and publishes validated fitted associations and counts in +`tepp.trsl_topic_lineage.v1`; it does not infer causality or replace production +`K` selection. This remains active product-branch evidence until its exact-head +checks and protected merge pass. + ## 5. Analysis request authority An analysis request cannot supply arbitrary facts that bypass validated domain state. The service resolves and validates: diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index a2baaffa..9f864156 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -46,9 +46,13 @@ The full APA 7th standards/literature register remains `docs/research/standards- | recovery metrics (RMSE, bias, coverage, graph, temporal order, Monte Carlo SE gates) | PRD; Test Strategy; ADR 0007/0014 | `validation_core` on protected main (PR #19); SE-aware Monte Carlo gates included | implemented-main | | PostgreSQL bitemporal/lineage persistence | ADR 0013; Architecture/ERD | `persistence_postgres` migration contracts, in-memory adapters, live SQL session/document SQL port, tenant RLS (`0002` + session GUC/role helpers), `DATABASE_URL` SQLx gate, optional session-affine `live-sqlx` `PgPool` driver, exact-head live PostgreSQL CI with isolation proof, append-only immutability triggers (`0004`), temporal interval ordering CHECKs (`0005`), typed membership assignment (`0006` implemented-main), event-relation/mention/instance SQL (#37–#39 implemented-main), source-artifact SQL (#40 implemented-main), audit-event SQL (#41 implemented-main), concurrent document-write stress (#43 implemented-main), backup/restore integrity revalidation (#44 implemented-main), `revision_order` later-revision system-time ordering implemented-main, entity/project target SQL on PR #131; remaining physical ERD constraints | partial | | known-truth temporal/event simulation manifests | PRD; TRD; Test Strategy | `tepp_simulation` on protected main; recovery metrics in `validation_core` | implemented-main | -| versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); HTTP service remaining accepted-target | partial | +| versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); LineageWeave loopback contracts and request-bound terminal result are composed on the active product branch; production TLS remaining | partial | +| executable cutoff-safe analysis runs | ADR 0012/0022; temporal research; API terminal-result contract | `analysis_engine` availability cutoff, snapshot binding, multiple-membership aggregation, digest-bound readiness artifact, and `tepp.trsl_topic_lineage.v1` execution through `topic_measurement`; synthetic recovery plus tamper/non-convergence tests and exact coverage on the active product branch | active-PR | | immutable split/run/reproducibility manifests | ADR 0013; ERD | `tepp_api` reproducibility manifest contract on protected main; `persistence_postgres` append-only SQL insert/lookup for `reproducibility_manifest`, `corpus_split_manifest`, `model_run`, and `model_artifact` (migration `0003`); full physical ERD constraints remaining | partial | | multilingual shared latent semantic space | PRD; ADR 0004; ADR 0020 | `semantic_core` span-grounded units (active-PR); concept dictionary and shared latent estimator remaining | active-PR | +| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR/ILR coordinates and bounded CPU `f64` reference estimator on the active product branch; calibrated posterior promotion, method effects, persistence, and accelerated backends remaining | partial | +| global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | `topic_lineage` activity/dormancy/reactivation identity on the active product branch; birth/split/merge remain later extensions | partial | +| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial | | TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | future `topic_measurement` | accepted-target | | global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | future topic lineage/activity state | accepted-target | | no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | future semantic/method-source model | accepted-target | @@ -58,7 +62,7 @@ The full APA 7th standards/literature register remains `docs/research/standards- | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; `modality_source` modality-versus-unique-content identity on the active PR; estimator-side method model remains future | partial | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; `corpus_background` background-versus-unique-content identity on the active PR; estimator-side method model remains future | partial | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; `prompt_source` prompt-versus-unique-content identity on the active PR; estimator-side method model remains future | partial | -| candidate K statistical/Pareto gates + blinded LLM review | ADR 0012; research | future `model_selection` | accepted-target | +| candidate K statistical/Pareto gates | ADR 0012; research | `model_selection` statistical/Pareto `K` gate on the active PR; candidate blinding, blinded LLM review, and backend comparison remain accepted-target | active-PR | | compositional topic correlation / stable clustering | ADR 0005/0012; research | future `network_analysis` | accepted-target | | posterior ESEM / longitudinal invariance / DSEM | ADR 0005 | `psychometric_core` construct/input gates, true-loading OLS recovery, posterior-draw point-estimate averaging, Rubin `T` on draw-level OLS loadings, CWC within/between OLS plus the contextual effect, event-time log-rate, constant- and time-varying-predictor discrete effects (Voelkle Eqs. 12 and 14), exact scalar discrete process noise (Driver et al., 2017, Eq. 3), lagged latent covariance and unconditional latent variance (Driver et al., 2017, Eq. 3–4), stationary within-subject variance (Driver et al., 2017, Eq. 4 as `Δt → ∞`; `asymDIFFUSION`), trait-plus-state variance (Driver et al., 2017, §4.3 `TRAITVAR`; not process noise), observed-indicator variance and lagged observed covariance (Driver et al., 2017, Eq. 5; Table 2 `MANIFESTVAR` is `Θ`, not `Var(y)`; `MANIFESTTRAITVAR` is not `MANIFESTVAR`; `Θ` does not enter lagged observed covariance; observed-indicator mean is `τ + λ μ`; `MANIFESTMEANS` is not `E(y)`; `CINT` is not `MANIFESTMEANS`; discrete latent mean is `exp(a Δt) μ_0 + (exp(a Δt) − 1)/a κ`; `T0MEANS` is not `μ_t`; evolved observed mean is `τ + λ μ_t`; `τ + λ μ_0` is not `E(y_t)`; contemporaneous `TDPREDEFFECT` impulse is `m x`, not `CINT`, not `TIPREDEFFECT`, and not Voelkle Eq. 14; Eq. 5 of that contemporaneous impulse is `τ + λ(μ_t + m x)`, and `τ + λ μ_t` is not that observed mean; time-independent `TIPREDEFFECT` increment is `A^{-1}[e^{A Δt} − I] B z`, not `CINT`, not `M x`, not Voelkle Eq. 14, and not the coefficient `B`; Eq. 5 of that increment is `τ + λ(μ_t + A^{-1}[e^{A Δt} − I] B z)`, and `τ + λ μ_t` is not that observed mean; `τ + λ(μ_t + m x)` is not that observed mean; `τ + λ(μ_t + e^{a(t−u)} m x)` is not that observed mean when `u ≠ t`; within-interval `TDPREDEFFECT` carry is `e^{A(t−u)} M x` for `t0 < u < t`, not the contemporaneous Dirac, not `CINT`, not `TIPREDEFFECT`, and not Voelkle Eq. 14; Eq. 5 of that carry is `τ + λ(μ_t + e^{a(t−u)} m x)`, and `τ + λ μ_t` is not that observed mean; `τ + λ(μ_t + m x)` is not that carried observed mean when `u ≠ t`; §7.2 level-change `CINT` is `κ = −a m x` (`a < 0`; not the dissipating Dirac, not a free `CINT`, not `TIPREDEFFECT`; Eq. 3 of that setting is `(1 − e^{a Δt}) m x`); §7.2 extra-process contribution is `a_{ηξ} x (e^{ε Δt} − e^{a Δt}) / (ε − a)` (not `κ = −a m x`, not `(1 − e^{a Δt}) m x`, not the dissipating Dirac; `ε ≥ 0` fails closed; Eq. 5 of that contribution is `τ + λ(μ_t + a_{ηξ} x (e^{ε Δt} − e^{a Δt}) / (ε − a)`; extra `LAMBDA` is 0; `τ + λ μ_t` is not that observed mean; after-t0 extra-process `TDPREDEFFECT` uses `t − u` with `t0 < u < t` while `μ_t` uses `Δt`; that after-t0 observed mean is not the first-occasion extra-process observed mean; §7.2 `asymTIPREDEFFECT` is `-B z / a` for `a < 0` and is not `B`, not `A^{-1}[e^{A Δt} − I] B z`, not `CINT`, and not `M x`; §7.2 `addedTIPREDVAR` is `(B / a)² v` and is not `TRAITVAR`, not `asymDIFFUSION`, and not `-B z / a`; Table 2 `asymCINT` is `-κ / a` for `a < 0` and is not `κ`, not `A^{-1}[e^{A Δt} − I] κ`, not `T0MEANS`, and not `-B z / a`; p. 16 stationary `T0MEANS` is `-κ / a + −B z / a` and is not free `T0MEANS`, not `asymCINT` alone, not `asymTIPREDEFFECT` alone, and not the finite-interval discrete latent mean; Eq. 5 of that constrained mean is `τ + λ(−κ / a + −B z / a)`; `τ + λ μ_0` is not that observed mean; `MANIFESTMEANS` is not `E(y_0)`; the constrained latent mean is not `E(y_0)`; stationary `T0VAR` is `trait + −q / (2 a) + (B / a)² v` (not free `T0VAR`, not `asymDIFFUSION` alone, not `TRAITVAR` alone, not `addedTIPREDVAR` alone, and not the finite-interval discrete latent variance. Eq. 5 of that constrained variance is `λ²(trait + −q / (2 a) + (B / a)² v) + θ + ψ` (JSS PDF re-opened 2026-08-22T03:20Z; form the stationary latent variance first, then `λ² p + θ + ψ`; `λ² p_0` is not that observed variance; `λ²(−q / (2 a)) + θ` is not that observed variance when `TRAITVAR` or `addedTIPREDVAR` is nonzero; `MANIFESTVAR` is not `Var(y_0)`; the constrained latent variance is not `Var(y_0)`)); lagged stationary `T0VAR` is `trait + e^{a Δt}(−q / (2 a)) + (B / a)² v` (trait and `addedTIPREDVAR` do not decay; contemporaneous `T0VAR` is not that lagged map; decaying the constrained total as if it were all state is not that lagged map; Eq. 5 of that lagged covariance is `λ²(trait + e^{a Δt}(−q / (2 a)) + (B / a)² v) + ψ`; `Θ` does not enter; contemporaneous `Var(y_0)` is not that lagged observed covariance; the lagged latent covariance is not that observed covariance); later-occasion stationary `T0VAR` is `trait + e^{2 a Δt}(−q / (2 a)) + Q_Δt + (B / a)² v` (trait and `addedTIPREDVAR` do not enter `Q_Δt`; under stationarity that composition equals contemporaneous `T0VAR`; evolving the constrained total as if it were all state is not that later map; the lagged covariance omits `Q_Δt`; `Q_Δt` is not that later map; Eq. 5 of that later-occasion variance is `λ²(trait + e^{2 a Δt}(−q / (2 a)) + Q_Δt + (B / a)² v) + θ + ψ`; lagged observed covariance omits `Q_Δt` and `θ`; `MANIFESTVAR` is not `Var(y_t)`; the later-occasion latent variance is not `Var(y_t)`)), irregular already-centered residual lag, and strong/strict-gated latent means on the stacked psychometric PR (two-observation residual variance is identically `0` and caps at strong/scalar; Putnick & Bornstein, 2016, PMC5145197 opened 2026-08-19T22:15Z); full ESEM/DSEM remaining | partial | | CPU bounded multithreading + GPU/VRAM streaming/parity | ADR 0001/0006 | future `compute_backend` | accepted-target | diff --git a/docs/adr/0010-adaptive-llm-orchestration.md b/docs/adr/0010-adaptive-llm-orchestration.md index e1e6b284..c4646686 100644 --- a/docs/adr/0010-adaptive-llm-orchestration.md +++ b/docs/adr/0010-adaptive-llm-orchestration.md @@ -1,10 +1,8 @@ # ADR 0010 — Adaptive LLM orchestration and test-time compute **Decision status:** Accepted -**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, and credential-free contextual-orchestrator binding are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target -**Date:** 2026-08-10 **Decision status:** Accepted -**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, credential-free contextual-orchestrator binding, and evidence-bounded `interpretation_gateway` are on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target +**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, credential-free contextual-orchestrator binding are implemented-main; evidence-bounded `interpretation_gateway` is on the active PR and is not implemented-main until exact-head checks, review, and protected-main integration complete; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target **Date:** 2026-08-10 **Supersedes:** The LLM orchestration-selection/ablation clauses previously co-located in ADR 0006. ADR 0006 remains authoritative for GPU/VRAM and model-credential separation; ADR 0015 governs autonomous repository-write/review/merge authority. diff --git a/docs/adr/0011-standalone-modular-msa-boundary.md b/docs/adr/0011-standalone-modular-msa-boundary.md index 9c64d304..d55aa8dd 100644 --- a/docs/adr/0011-standalone-modular-msa-boundary.md +++ b/docs/adr/0011-standalone-modular-msa-boundary.md @@ -1,10 +1,8 @@ # ADR 0011 — Standalone operation and modular CWL MSA boundary **Decision status:** Accepted -**Implementation maturity:** partial — Rust crates are independently usable; naruon HTTP interchange and loopback live listener (`POST /v1/analysis-runs` and `/v1/exports`, fail-closed table-access, NIM/proxy headers, RFC 3339 cutoff, stream deadline) is implemented-main; `service_tls` production rustls bind gates and orchestrator live-port refusal of loopback plaintext are on this active PR; remaining live HTTP listeners and production persistence integrations remain accepted-target +**Implementation maturity:** partial — Rust crates are independently usable; naruon HTTP interchange and loopback live listener (`POST /v1/analysis-runs` and `/v1/exports`, fail-closed table-access, NIM/proxy headers, RFC 3339 cutoff, stream deadline) is implemented-main; `service_tls` production rustls bind gates and orchestrator live-port refusal of loopback plaintext are on this active PR; the loopback consumer listener composition and terminal-result contract are composed on the active product branch; production TLS/`$PORT`, remaining live HTTP listeners, and remaining persistence integrations remain accepted-target **Date:** 2026-08-10 -**Decision status:** Accepted -**Implementation maturity:** partial — Rust crates are independently usable; naruon HTTP interchange and loopback live listener (`POST /v1/analysis-runs` and `/v1/exports`, fail-closed table-access, NIM/proxy headers, RFC 3339 cutoff, stream deadline) are on the active PR (not implemented-main); production TLS/`$PORT` and remaining persistence integrations remain accepted-target **Date:** 2026-08-10 **Implementation maturity:** partial — Rust crates are independently usable; naruon HTTP interchange and loopback live listener (`POST /v1/analysis-runs` and `/v1/exports`, fail-closed table-access, NIM/proxy headers, RFC 3339 cutoff, stream deadline) are on the active PR (not implemented-main); production TLS/`$PORT` and remaining persistence integrations remain accepted-target **Date:** 2026-08-10 diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index 1c36dac3..6f3fb9a6 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -1,6 +1,7 @@ # ADR 0012 — Temporal Relational Shared-Latent Topic Measurement **Decision status:** Accepted +**Implementation maturity:** partial — logistic-normal ALR/ILR coordinates, lexical-weight refusal, statistical/Pareto candidate-`K` gates, stable active/dormant/reactivated topic identity, and the bounded CPU `f64` reference estimator are implemented on the active product branch; `corpus_background`, `topic_lineage`, `network_analysis`, `model_selection`, `stopword_deletion`, `style_source`, `copied_text`, and `modality_source` implement bounded identity and recovery gates implemented-main; method effects, calibrated posterior acceptance, accelerated backends, and backend interchange remain accepted-target until implemented and protected-main integrated **Implementation maturity:** partial — `corpus_background`, `topic_lineage`, `network_analysis`, `model_selection`, `stopword_deletion`, `style_source`, `copied_text`, and `modality_source` implement bounded identity and recovery gates; the TRSL-TM estimator, method effects, global topic identity, and backend interchange remain accepted-target. **Date:** 2026-08-12 **Decision status:** Accepted @@ -44,9 +45,58 @@ For the first production line: - stopword deletion is not the default preprocessing rule; - TF-IDF and BM25 are not inferential weights for the statistical topic estimator; - topic proportions are compositional and downstream network/psychometric analysis uses logistic-normal coordinates or valid orthonormal log-ratio coordinates; +- ALR is a reference-dependent full-rank logistic-normal map, not an Aitchison-distance isometry; distance-based Euclidean Aitchison geometry uses an orthonormal ILR basis; - model selection uses statistical/recovery/stability/alignment/fairness gates and a Pareto-style comparison before any blinded LLM review; - the LLM may recommend among statistically admissible candidates but never defines the numerical optimum or bypasses diagnostics. +### CPU `f64` reference estimand and inference + +The bounded reference estimator uses sparse document-by-term counts `C`, one +global `K`-topic word matrix `β`, and document logistic-normal coordinates +`η_d`, with `θ_d = softmax([η_d, 0])`. Its prevalence mean is the PRD-owned +structural equation + +\[ +m_d = x_d\Gamma + \sum_{g \in G_d} w_{dg}u_g, +\qquad +\eta_d \sim N(m_d, \sigma^2 I), +\] + +where `x_d` includes an intercept, standardized event time, and admitted +prevalence covariates, while the second term retains every active weighted +cross-classified/multiple-membership assignment. This is the logistic-normal +prevalence boundary of correlated/structural topic models, not a raw-simplex +regression (Blei & Lafferty, 2007; Roberts et al., 2019). + +For explicit observed predecessor/successor relations only, the reference +objective adds the harmonic network penalty + +\[ +R(\Theta,G)=\frac{1}{2}\sum_{(d,e)\in E}a_{de} +\lVert\theta_d-\theta_e\rVert_2^2, +\] + +so absent relations remain unobserved rather than negative. This follows the +document-network regularization estimand of Mei et al. (2008); it is not a +causal edge, an event-identity promotion, or an RTM link-probability claim. +The full bounded MAP objective is + +\[ +\sum_{d,v} C_{dv}\log\!\left(\sum_k\theta_{dk}\beta_{kv}\right) +-\frac{1}{2\sigma^2}\sum_d\lVert\eta_d-m_d\rVert_2^2 +-\lambda R(\Theta,G)-\frac{\rho}{2}\lVert\Gamma,u\rVert_2^2. +\] + +Production inference uses deterministic generalized EM: normalized latent +term-topic responsibilities, smoothed multinomial `β` updates, bounded +gradient updates for `η` and structural coefficients, and a diagonal Laplace +curvature approximation for document-coordinate uncertainty. Multiple seeded +initializations retain the best finite converged objective. A non-finite +intermediate, invalid sparse matrix, missing cutoff-safe document, reverse +transition, or exhausted iteration budget returns a typed failure; it never +emits a partial topic artifact. Recovery gates remain caller-owned promotion +criteria over completed validation evidence. + ## Non-goals This ADR does not select one neural architecture forever, claim a unique true topic count for every corpus, or authorize topic labels as causal constructs. It does not allow a fitted backend to redefine TEPP's temporal/event/membership or evidence semantics. @@ -97,3 +147,12 @@ Mimno, D., Wallach, H. M., Naradowsky, J., Smith, D. A., & McCallum, A. (2009). Roberts, M. E., Stewart, B. M., & Tingley, D. (2019). stm: An R package for structural topic models. *Journal of Statistical Software, 91*(2), 1–40. https://doi.org/10.18637/jss.v091.i02 Roberts, M. E., Stewart, B. M., Tingley, D., Lucas, C., Leder-Luis, J., Gadarian, S. K., Albertson, B., & Rand, D. G. (2014). Structural topic models for open-ended survey responses. *American Journal of Political Science, 58*(4), 1064–1082. https://doi.org/10.1111/ajps.12103 + +Blei, D. M., & Lafferty, J. D. (2007). A correlated topic model of Science. +*The Annals of Applied Statistics, 1*(1), 17–35. +https://doi.org/10.1214/07-AOAS114 + +Mei, Q., Cai, D., Zhang, D., & Zhai, C. (2008). Topic modeling with network +regularization. In *Proceedings of the 17th International Conference on World +Wide Web* (pp. 101–110). Association for Computing Machinery. +https://doi.org/10.1145/1367497.1367512 diff --git a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md index 95172cf1..52610a7f 100644 --- a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md +++ b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md @@ -11,7 +11,6 @@ **Decision status:** Accepted **Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` driver with session-affine tenant GUC execution, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), typed `text_segment` SQL, retention/deletion/legal-hold (migration `0007`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, concurrent document-write stress, and backup/restore integrity revalidation implemented-main; entity/project target SQL is on PR #131 and is not implemented-main until exact-head checks, review, and protected-main integration complete; remaining physical ERD and DR-runbook depth accepted-target **Date:** 2026-08-12 - **Supersedes:** None; complements ADR 0002 (temporal semantics), ADR 0008 (evidence identity), and ADR 0011 (service ownership). ## Context diff --git a/docs/adr/0022-deterministic-analysis-run-execution.md b/docs/adr/0022-deterministic-analysis-run-execution.md new file mode 100644 index 00000000..cb7fe646 --- /dev/null +++ b/docs/adr/0022-deterministic-analysis-run-execution.md @@ -0,0 +1,99 @@ +# ADR 0022 — Deterministic cutoff-safe analysis-run execution + +**Decision status:** Accepted +**Implementation maturity:** active-PR — composed on the active product branch; not implemented-main +**Date:** 2026-08-21 +**Supersedes:** None; complements ADR 0002, ADR 0003, ADR 0011, ADR 0013, and the terminal-result contract. +**Figma File ID:** N/A — this increment changes a Rust service crate and has no user-interface surface. +**Storybook inventory:** N/A — no reusable web object or interaction changed. + +## Context + +TEPP already accepts an analysis request and can describe a completed result, +but a consumer needs a demonstrable path between those contracts. Without one +bounded execution slice, an accepted run is only a receipt and consumers cannot +verify cutoff safety, multiple-membership preservation, or artifact identity. + +## Decision + +Add the standalone `analysis_engine` Rust crate as the first executable vertical +slice. It consumes a request, an accepted receipt, and a bounded identity-free +evidence snapshot. It: + +- excludes evidence whose `available_time` is later than the request's + `knowledge_cutoff`; +- preserves multiple-membership assignments by summing their counts rather than + reducing an evidence unit to one group; +- binds the result to the accepted run and source snapshot; +- verifies request/receipt idempotency identity before scanning the corpus; +- emits a canonical SHA-256-digested `AnalysisArtifact` and the versioned + `AnalysisRunTerminalResult` from `tepp_api`; +- returns a content-redacted failed terminal result when no evidence is + eligible; and +- remains a readiness/counting slice, not latent-variable, topic, or + psychometric estimator authority. + +The engine is deterministic, synchronous, bounded to `100_000` evidence units, +and CPU-only. Scientific estimators and their Rust CPU `f64`/GPU parity +contracts remain separate boundaries under ADR 0001 and ADR 0006. + +For the `trsl_topic_lineage_v1` output profile, the engine may invoke the +ADR-0012 `topic_measurement` CPU `f64` reference estimator through its validated +`ReferenceTopicInput`. The engine does not reimplement or reinterpret the +estimator. It binds the request snapshot and cutoff, then emits a canonical +`tepp.trsl_topic_lineage.v1` artifact containing only the selected seed, +iteration/objective evidence, topic count, evidence count, fitted +predecessor/successor topic edges, connectable-post count, and lineage count. +The artifact is bounded, digest-bound, and self-validating; invalid or +non-converged estimation returns no partial artifact. Production selection of +`K` remains governed by ADR 0012 and `model_selection`, outside this executor. + +## Alternatives considered + +1. Keep the API as contracts only — rejected because an accepted run would not +produce a consumer-verifiable terminal outcome. +2. Put execution into `tepp_api` — rejected because transport contracts and + scientific execution would become one service boundary. +3. Add a bounded standalone engine behind the existing contracts — accepted + because it is independently testable and composable without shared tables. + +## Consequences + +Consumers can run a reproducible readiness check while seeing only opaque +identifiers, bounded counts, temporal extrema, and a digest. The engine does +not expose source text or identity mappings and does not claim a psychometric +measurement. The initial linear scan is intentionally simple; a production +large-corpus adapter must stream snapshots and preserve the same artifact +semantics before raising the bound. + +LineageWeave may consume the topic-lineage artifact as completed model evidence +beside, but never inside, the project-history temporal-association claim. The +two contracts keep separate schema identities and inference-status copy. + +## Verification + +The stacked PR includes Rust unit and integration tests for cutoff exclusion, +multiple-membership summation, snapshot binding, duplicate identities, empty +eligibility, receipt validation, and package identity. Run: + +```text +cargo fmt --all -- --check +cargo test -p analysis_engine +cargo clippy -p analysis_engine --all-targets -- -D warnings +``` + +The topic-lineage execution contract additionally verifies a synthetic +known-topic corpus, exact request/snapshot/cutoff binding, canonical artifact +round-trip and digest stability, predecessor/successor count consistency, and +fail-closed tamper/non-convergence paths. + +The supporting research and APA 7th citations are recorded in +`docs/doctoring/analysis-engine-v1.md` and the standards register. + +## Rollback and supersession + +Rollback removes the `analysis_engine` workspace member and stops publishing +the readiness artifact while preserving the request and terminal-result DTOs. +No persisted schema migration is introduced. Supersession requires a new ADR +if execution changes cutoff semantics, artifact authority, privacy fields, or +scientific estimands. diff --git a/docs/adr/README.md b/docs/adr/README.md index d1a9a120..c20b6c41 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -27,10 +27,15 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0019](0019-project-history-wire-size-symmetry.md) | Symmetric LineageWeave project-history wire-size enforcement | Accepted | active-PR | Request serialization and generated project-history projections share bounded size rules. | | [0020](0020-span-grounded-semantic-units.md) | Span-grounded semantic units; language tags are not identity | Accepted | active-PR | First ADR 0004 production slice; concept alignment, invariance, and topic estimation are not claimed. | | [0021](0021-lineageweave-project-history-boundary.md) | LineageWeave project-history service boundary | Accepted | active-PR | Credential-free bounded project-history API preserves LineageWeave authorization ownership. | +| [0022](0022-deterministic-analysis-run-execution.md) | Deterministic cutoff-safe analysis-run execution | Accepted | active-PR | Closes the first executable product path from accepted run to digest-bound terminal result without claiming estimator authority. | | [0001](0001-rust-first-modular-msa.md) | Rust-first numerical core and CPU `f64` reference | Accepted | partial | ADR 0011 owns cross-service/MSA authority; 0001 retains numerical/backend authority. | | [0002](0002-six-clock-temporal-semantics.md) | Six-clock temporal semantics and fail-closed historical leakage prevention | Accepted | partial | Typed clocks/intervals (merged PR #8), Allen/path-consistency (merged PR #9), the clock-identity/revision-order/document-completeness gates (`system_clock`, `event_clock`, `assertion_clock`, `cutoff_clock`, `available_clock`, `document_clocks`, `revision_order`), and the provenance/ordering gates (`citation_edge`, `support_edge`, `retrospective_edge`) are implemented-main; superseded PRs #5/#6 are historical lineage only; remaining graph/split enforcement stays accepted-target. | | [0003](0003-relational-event-multiple-membership.md) | Relational event ontology and time-varying cross-classified multiple membership | Accepted | partial | Membership network/roles, Kish ESS, nested ICC, subevent parent-window containment (`subevent_containment`), the forward-only relation graph, evidential-vs-transition identity (`support_edge`), inferred-versus-observed identity (`inferred_status`), retrospective-reporting identity (`retrospective_edge`), summary-versus-source identity (`summarizes_edge`), copy-versus-source identity (`copy_identity`), location-versus-entity/language identity (`location_membership`), and IPO event-time order (`outcome_order`) are implemented-main; typed target-kind identity in `membership_target` is on PR #131; full multilevel estimators and persistence remain accepted-target. ADR 0016 owns event-intelligence tasks. | | [0004](0004-shared-multilingual-latent-space.md) | One shared multilingual latent space with explicit invariance status | Accepted | accepted-target | ADR 0012 owns the full topic-estimator/backend/global-topic contract. | +| [0005](0005-posterior-esem-dsem.md) | Posterior-aware ESEM/DSEM and valid compositional coordinates | Accepted | active-PR | Within/between decomposition in `longitudinal_core` on the active PR; remaining ESEM/DSEM fit remains accepted-target. ADR 0012 owns the upstream topic/network contract. | +| [0006](0006-vram-gpu-nvidia-orchestration.md) | VRAM-adaptive GPU compute and model-credential boundary | Accepted | accepted-target | LLM orchestration policy superseded by ADR 0010; autonomous development authority governed by ADR 0015. | +| [0007](0007-rust-workspace-quality-gates.md) | Explicit Rust workspace, pinned toolchains, and exact quality gates | Accepted | implemented-main | ADR 0014 governs scientific/product claim promotion beyond repository-quality tooling. | +| [0008](0008-immutable-evidence-identities-digests-and-spans.md) | Immutable evidence identities, `SHA-256` digests, exact spans, and strict wire reconstruction | Accepted | implemented-main | ADR 0013 governs future persistence/reproducibility/split authority. | | [0005](0005-posterior-esem-dsem.md) | Posterior-aware ESEM/DSEM and valid compositional coordinates | Accepted | partial | Input gates, posterior-draw point estimates, Rubin `T` on OLS loadings, CWC within/between OLS plus the contextual effect, event-time log-rate, constant- and time-varying-predictor discrete effects, exact scalar discrete process noise, lagged latent covariance and unconditional latent variance, stationary within-subject variance (`asymDIFFUSION`), trait-plus-state variance (`TRAITVAR`; not process noise), observed-indicator variance and lagged observed covariance (Eq. 5; Table 2 `MANIFESTVAR` is `Θ`, not `Var(y)`; `MANIFESTTRAITVAR` is not `MANIFESTVAR`; `Θ` does not enter lagged observed covariance; observed-indicator mean is `τ + λ μ`; `MANIFESTMEANS` is not `E(y)`; `CINT` is not `MANIFESTMEANS`; discrete latent mean is `exp(a Δt) μ_0 + (exp(a Δt) − 1)/a κ`; `T0MEANS` is not `μ_t`; evolved observed mean is `τ + λ μ_t`; `τ + λ μ_0` is not `E(y_t)`; Eq. 5 of the contemporaneous impulse is `τ + λ(μ_t + m x)`; `τ + λ μ_t` is not that observed mean; Eq. 5 of the time-independent predictor is `τ + λ(μ_t + A^{-1}[e^{A Δt} − I] B z)`; `τ + λ μ_t` is not that observed mean; `τ + λ(μ_t + m x)` is not that observed mean; `τ + λ(μ_t + e^{a(t−u)} m x)` is not that observed mean when `u ≠ t`; Eq. 5 of the within-interval impulse carry is `τ + λ(μ_t + e^{a(t−u)} m x)`; `τ + λ μ_t` is not that observed mean; `τ + λ(μ_t + m x)` is not that carried observed mean when `u ≠ t`; Table 2 `asymCINT` is `-κ / a` for `a < 0` and is not `κ`, not the finite-interval increment, not `T0MEANS`, and not `-B z / a`; p. 16 stationary `T0MEANS` is `-κ / a + −B z / a` and is not free `T0MEANS`, not `asymCINT` alone, not `asymTIPREDEFFECT` alone, and not the finite-interval discrete latent mean; Eq. 5 of that constrained mean is `τ + λ(−κ / a + −B z / a)` (`τ + λ μ_0` is not that observed mean; `MANIFESTMEANS` is not `E(y_0)`); stationary `T0VAR` is `trait + −q / (2 a) + (B / a)² v` (not free `T0VAR`, not `asymDIFFUSION` alone, not `TRAITVAR` alone, not `addedTIPREDVAR` alone; Eq. 5 is `λ²(trait + −q / (2 a) + (B / a)² v) + θ + ψ` (`λ² p_0` is not `Var(y_0)`; `λ²(−q / (2 a)) + θ` is not `Var(y_0)` when trait or TI is nonzero; `MANIFESTVAR` is not `Var(y_0)`; the constrained latent variance is not `Var(y_0)`)); §7.2 `addedTIPREDVAR` is `(B / a)² v` and is not `TRAITVAR`, not `asymDIFFUSION`, and not `-B z / a`), irregular already-centered residual lag, and strong-gated latent means are on the stacked psychometric PR; full ESEM/DSEM estimator remains accepted-target. | | [0005](0005-posterior-esem-dsem.md) | Posterior-aware ESEM/DSEM and valid compositional coordinates | Accepted | active-PR | CPU `f64` ESEM/DSEM fit in `psychometric_fit` on the active PR; `psychometric_core` input gates remain #49; within/between decomposition in `longitudinal_core` is on the active PR; invariance/multilevel and remaining ESEM/DSEM fit remain accepted-target. ADR 0012 owns the upstream topic/network contract. | | [0006](0006-vram-gpu-nvidia-orchestration.md) | VRAM-adaptive GPU compute and model-credential boundary | Accepted | accepted-target | LLM orchestration policy superseded by ADR 0010; autonomous development authority governed by ADR 0015. | @@ -46,32 +51,20 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding on the active PR; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; `service_tls` production bind gates are on the active PR; no direct cross-service application-table coupling. | | [0009](0009-purpose-bound-pii-governance.md) | Purpose-bound PII governance without blanket masking | Accepted | partial | Persistence retention/deletion/legal-hold (`0007`) and provider-payload minimization implemented-main; deployment evidence remains accepted-target. | -| [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding implemented-main; evidence-bounded `interpretation_gateway` is on the active PR; live NIM execution and production ablation evidence remain accepted-target. | | [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding implemented-main; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; no direct cross-service application-table coupling. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | accepted-target | Prompt-versus-unique-content identity is `prompt_source` on the active PR; estimator-side method model, backend, and K gates remain accepted-target. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | accepted-target | Corpus-background-versus-unique-content identity is `corpus_background` on the active PR; estimator-side method model, backend, and K gates remain accepted-target. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | accepted-target | Modality-versus-unique-content identity is `modality_source` on the active PR; estimator-side method model, backend, and K gates remain accepted-target. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | accepted-target | Copied-versus-unique-content identity is `copied_text` on the active PR; estimator-side method model, backend, and K gates remain accepted-target. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | accepted-target | Style-versus-unique-content identity is `style_source` on the active PR; estimator-side method model, backend, and K gates remain accepted-target. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Default stopword-deletion refusal is `stopword_deletion` on the active PR; topic backend, global topic identity, method effects, K/model-selection, and TF-IDF/BM25 inferential-weight refusal remain accepted-target. | -| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold on the active PR; remaining physical ERD/backup accepted-target. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | accepted-target | Owns topic backend compatibility, global topic identity, method effects, K/model-selection prerequisites, and compositional topic coordinates. | -| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold implemented-main; document revision system-time order in `revision_order` is active on this PR; remaining physical ERD/backup accepted-target. | -| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, `0006` membership, `0007` retention/deletion/legal-hold, and backup/restore integrity revalidation implemented-main; remaining physical ERD/DR-runbook depth accepted-target. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | active-PR | Owns topic backend compatibility, global topic identity, method effects, K/model-selection prerequisites, and compositional topic coordinates; `topic_lineage` implements the active/dormant/reactivated identity slice on the active PR. | -| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, recovery identity, retention/deletion/legal-hold, backup/restore integrity, and concurrent writes; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; remaining physical ERD/backup evidence accepted-target. | -| [0014](0014-scientific-claim-promotion-and-release-evidence.md) | Scientific claim promotion and release evidence authority | Accepted | partial | Separates design, implementation, scientific/product claim, and release authority; repository SBOM/provenance generator implemented, full release bundle remaining. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | active-PR | Compositional cluster-pair gates in `network_analysis` on the active PR; remaining topic estimator/backend/global-K contract remains accepted-target. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | active-PR | Statistical/Pareto candidate-`K` gates in `model_selection` on the active PR; remaining topic estimator/backend/global-K contract remains accepted-target. | +| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR/ILR, lexical-weight refusal, statistical/Pareto candidate-`K` gates, and active/dormant/reactivated identity are on the active product branch; the estimator, method effects, and backend interchange remain accepted-target. | | [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold on the active PR; remaining physical ERD/backup accepted-target. | | [0014](0014-scientific-claim-promotion-and-release-evidence.md) | Scientific claim promotion and release evidence authority | Accepted | partial | Separates design, implementation, scientific/product claim, and release authority; repository SBOM/provenance generator implemented; checkpoint-versus-estimator refusal is `checkpoint_authority` on the active PR; full release bundle remaining. | | [0015](0015-autonomous-development-review-and-merge-authority.md) | Autonomous development, review, and merge authority separation | Accepted | active-PR | Separates model proposal, deterministic verification, publication, independent review, and merge/release authority. | +| [0016](0016-tdt-chronos-event-intelligence-boundary.md) | TDT, CHRONOS, and Event Ontology intelligence boundary | Accepted | accepted-target | Separates observed evidence, detection/tracking, prediction/schema inference, temporal consistency, and promoted transition authority. | | [0016](0016-tdt-chronos-event-intelligence-boundary.md) | TDT, CHRONOS, and Event Ontology intelligence boundary | Accepted | active-PR | First-story FAR/miss in existing `event_core`; remaining TDT/CHRONOS stack remains accepted-target. | | [0020](0020-span-grounded-semantic-units.md) | Span-grounded semantic units; language tags are not identity | Accepted | active-PR | First ADR 0004 production slice. Does not claim concept alignment, invariance, or a topic estimator. | | [0017](0017-hourly-contextual-orchestrator-gateway.md) | Hourly contextual-orchestrator gateway and all-provider model discovery | Accepted | active-PR | Keeps proposal-model execution behind a pinned loopback gateway while preserving independent verifier, publisher, reviewer, and merge authority. | | [0018](0018-consumer-scoped-analysis-run-ingress.md) | Consumer-scoped modular analysis-run ingress | Accepted | active-PR | Narrows ADR 0011 for the closed consumer registry, credential-free exchange, and consumer-qualified idempotency namespace; production TLS remains separate. | | [0019](0019-project-history-wire-size-symmetry.md) | Symmetric project-history wire-size enforcement | Accepted | active-PR | Narrows ADR 0008 for request serialization and generated LineageWeave project-history projections. | +| [0021](0021-lineageweave-project-history-boundary.md) | LineageWeave project-history service boundary | Accepted | active-PR | Narrows ADR 0011 for the credential-free bounded project-history API and preserves LineageWeave authorization ownership. | +| [0022](0022-deterministic-analysis-run-execution.md) | Deterministic cutoff-safe analysis-run execution | Accepted | active-PR | Closes the first executable product path from accepted run to digest-bound terminal result without claiming estimator authority. | ## Decision ownership summary @@ -97,6 +90,7 @@ Use the narrowest owning ADR when decisions overlap: - **modular consumer admission / replay identity:** ADR 0018. - **project-history wire-size symmetry:** ADR 0019. - **LineageWeave project-history service boundary:** ADR 0021. +- **accepted-run execution and terminal artifact production:** ADR 0022. ## Change and supersession rule diff --git a/docs/connectors/naruon-artifact-consumer.md b/docs/connectors/naruon-artifact-consumer.md index 136159fa..d49a86b4 100644 --- a/docs/connectors/naruon-artifact-consumer.md +++ b/docs/connectors/naruon-artifact-consumer.md @@ -1,6 +1,6 @@ # naruon modular consumer contract for TEPP artifacts -**Status:** Partial — versioned DTO, HTTP interchange, and loopback live listener on the active PR; production TLS/`$PORT` remaining +**Status:** Partial — versioned DTO and HTTP interchange are implemented-main at protected head `c45be17a9dbce95ef81cee230e9d128abc7160ac`; the loopback live listener and terminal-result contract are composed on the active product branch; production TLS/`$PORT` remaining **Last reviewed:** 2026-08-16 ## Boundary diff --git a/docs/doctoring/analysis-engine-gap-closure.md b/docs/doctoring/analysis-engine-gap-closure.md new file mode 100644 index 00000000..c9fef79d --- /dev/null +++ b/docs/doctoring/analysis-engine-gap-closure.md @@ -0,0 +1,46 @@ +# Analysis Engine v1 — Buyer Gap Closure + +**Review date:** 2026-08-21 +**Active slice:** PR #157 terminal-result contract → stacked analysis execution +engine for issue #166 + +The organization-wide buyer-gap register is maintained by a separate landing +vehicle. This document records only the analysis-engine slice so it can land +without competing with that register or requiring another PR to be present. + +## Buyer-visible gap + +An accepted analysis run previously had a durable receipt and a terminal-result +DTO, but no executable path that applied a historical availability cutoff and +returned a verifiable artifact. A buyer could submit work but could not yet +demonstrate that the result was complete, cutoff-safe, multiple-membership aware, +and unchanged after transport. + +## Bounded closure + +The stacked `analysis_engine` crate closes the first vertical slice with a +standalone Rust API. It accepts a bounded identity-free evidence snapshot, +rejects snapshot mismatches and duplicate opaque IDs, excludes future-available +evidence, preserves membership counts, and emits a digest-bound terminal result +or a redacted no-eligible-evidence failure. + +This is readiness evidence, not a psychometric estimate. Latent-variable, +multilingual, GPU, and HTTP service gaps remain separately governed by their +own ADRs and must not be implied by this slice. + +## Next leverage-ranked gaps + +1. Add a streaming snapshot adapter with the same digest and cutoff semantics. +2. Bind the engine to a versioned standalone HTTP port without cross-service + table access. +3. Add known-truth estimator execution with RMSE, bias, interval coverage, + multilevel/multiple-membership recovery, and CPU/GPU parity. +4. Add buyer-facing visual analytics only after the interaction contract is + stable; then create a Figma file and Storybook inventory and record its real + File ID in a new UI ADR. + +## Evidence boundary + +The current implementation is active-PR evidence only. Exact-head checks, +independent review, protected merge, release evidence, and deployment controls +are required before a capability is promoted to implemented-main. diff --git a/docs/doctoring/analysis-engine-v1.md b/docs/doctoring/analysis-engine-v1.md new file mode 100644 index 00000000..a54c5dbd --- /dev/null +++ b/docs/doctoring/analysis-engine-v1.md @@ -0,0 +1,55 @@ +# Analysis Engine v1 — Evidence Doctoring + +## Claim boundary + +The stacked PR proves one deterministic temporal-evidence-readiness execution +slice. It does not prove production psychometric estimation, topic validity, +GPU performance, HTTP deployment, certification, or customer-wide scale. + +## Decision-to-evidence mapping + +| Contract | Implementation evidence | Customer action enabled | +|---|---|---| +| Historical cutoff safety | `available_time <= knowledge_cutoff` filter in `analysis_engine` | Re-run a historical snapshot without future-availability leakage | +| Multiple membership | `membership_count` is summed for every eligible unit | Inspect inclusive counts without atomistic single-group collapse | +| Terminal completion | `AnalysisRunTerminalResult` is built from the accepted request and receipt | Poll one stable terminal contract instead of treating acceptance as completion | +| Artifact integrity | Canonical JSON and SHA-256 digest | Verify that a downloaded result matches the published artifact identity | +| Fitted topic lineage | `topic_measurement` reference fit projected as `tepp.trsl_topic_lineage.v1` | Read predecessor/successor-aware connectable-post and lineage counts without treating association as causation | +| Privacy boundary | Artifact contains opaque IDs, counts, and times only | Keep identity mapping in the authorized source boundary | + +## Scientific and standards basis + +The implementation preserves TEPP's distinct event and availability clocks and +does not infer an event time from availability time. The API payload is explicit +JSON, and the artifact digest is an integrity check rather than proof of origin +or scientific truth. These interpretations follow the existing temporal, +interchange, and hashing register entries (Bray, 2017; International +Organization for Standardization, 2012; National Institute of Standards and +Technology, 2015). + +## Verification record + +The local preflight for this slice passed with Rust 1.97.1: + +- `cargo fmt --all -- --check`; +- `cargo test -p analysis_engine` — 8 unit tests, 1 crate-contract test, 3 + readiness integration tests, 2 topic-lineage integration tests, and doctest + collection; +- `cargo clippy -p analysis_engine --all-targets -- -D warnings`. +- exact authored coverage — 148/148 lines and 74/74 branches. + +The protected-hosted exact-head checks and qualifying independent reviews are +still pending. This document must not be used as implemented-main or release +evidence before that merge. + +## APA 7th references + +Bray, T. (Ed.). (2017). *The JavaScript Object Notation (JSON) data interchange +format* (RFC 8259). RFC Editor. https://doi.org/10.17487/RFC8259 + +International Organization for Standardization. (2012). *Language resource +management—Semantic annotation framework (SemAF)—Part 1: Time and events +(SemAF-Time, ISO-TimeML)* (ISO Standard No. 24617-1:2012). + +National Institute of Standards and Technology. (2015). *Secure Hash Standard +(SHS)* (FIPS PUB 180-4). https://doi.org/10.6028/NIST.FIPS.180-4 diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index 2b39755d..5bee7ff8 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -102,7 +102,7 @@ Reynolds, L., & McDonell, K. (2021). Prompt programming for large language model Liu, P., Yuan, W., Fu, J., Jiang, Z., Hayashi, H., & Neubig, G. (2023). Pre-train, prompt, and predict: A systematic survey of prompting methods in natural language processing. *ACM Computing Surveys, 55*(9), Article 195. https://doi.org/10.1145/3560815 -TEPP retains a logistic-normal CPU reference while allowing adapter backends that satisfy shared-latent, posterior, temporal, relational, and measurement-invariance contracts. Brown et al. (2020) and Reynolds and McDonell (2021) provide primary research context for prompts as task-conditioning and prompt-programming mechanisms; they do not define TEPP's latent-content labels. As a normative ADR 0004/0012 contract, instruction and prompt boilerplate is therefore modeled as explicit method structure, not unique latent content and not a stopword deletion. Liu et al. (2023) is secondary survey background only and is not evidence for that repository-specific classification. +TEPP retains a logistic-normal CPU reference while allowing adapter backends that satisfy shared-latent, posterior, temporal, relational, and measurement-invariance contracts. Brown et al. (2020) and Reynolds and McDonell (2021) provide primary research context for prompts as task-conditioning and prompt-programming mechanisms; they do not define TEPP's latent-content labels. As a normative ADR 0004/0012 contract, instruction and prompt boilerplate is therefore modeled as explicit method structure, not unique latent content and not a stopword deletion. Liu et al. (2023) is secondary survey background only and is not evidence for that repository-specific classification. `topic_lineage` keeps one global topic identity when activity becomes dormant or reactivated. ## Topic-model evaluation and LLM judges @@ -114,12 +114,22 @@ Stammbach, D., Zouhar, V., Hoyle, A., Sachan, M., & Ash, E. (2023). Revisiting a Yang, X., Zhao, H., Phung, D., Buntine, W., & Du, L. (2025). LLM reading tea leaves: Automatically evaluating topic models with large language models. *Transactions of the Association for Computational Linguistics, 13*. -LLM evaluation complements but never replaces predictive, posterior, stability, alignment, fairness, recovery, and human-validation evidence. Candidates are blinded and statistically gated before LLM review. `interpretation_gateway` records those judgments as hypothetical proposals that must cite evidence spans and cannot become estimator results or observed facts. +Akaike, H. (1974). A new look at the statistical model identification. *IEEE Transactions on Automatic Control, 19*(6), 716–723. https://doi.org/10.1109/TAC.1974.1100705 + +Burnham, K. P., & Anderson, D. R. (2002). *Model selection and multimodel inference: A practical information-theoretic approach* (2nd ed.). Springer. + +Deb, K., Pratap, A., Agarwal, S., Meyarivan, T. (2002). A fast and elitist multiobjective genetic algorithm: NSGA-II. *IEEE Transactions on Evolutionary Computation, 6*(2), 182–197. https://doi.org/10.1109/4235.996017 + +LLM evaluation complements but never replaces predictive, posterior, stability, alignment, fairness, recovery, and human-validation evidence. The current `model_selection` crate performs statistical/Pareto gating; candidate blinding and blinded LLM review remain accepted-target extensions and are not executed by this crate. Pareto-filtered held-out log-likelihood and complexity admit a candidate `K`; an LLM vote cannot define the numerical optimum. `interpretation_gateway` records judgments as hypothetical proposals that must cite evidence spans and cannot become estimator results or observed facts. ## Compositional data, correlation, and clusters +Aitchison, J., & Shen, S. M. (1980). Logistic-normal distributions: Some properties and uses. *Biometrika, 67*(2), 261–272. https://doi.org/10.1093/biomet/67.2.261 + Aitchison, J. (1982). The statistical analysis of compositional data. *Journal of the Royal Statistical Society: Series B (Methodological), 44*(2), 139–177. https://doi.org/10.1111/j.2517-6161.1982.tb01195.x +Egozcue, J. J., Pawlowsky-Glahn, V., Mateu-Figueras, G., & Barceló-Vidal, C. (2003). Isometric logratio transformations for compositional data analysis. *Mathematical Geology, 35*(3), 279–300. https://doi.org/10.1023/A:1023818214614 + Friedman, J., Hastie, T., & Tibshirani, R. (2008). Sparse inverse covariance estimation with the graphical lasso. *Biostatistics, 9*(3), 432–441. https://doi.org/10.1093/biostatistics/kxm045 Traag, V. A., Waltman, L., & van Eck, N. J. (2019). From Louvain to Leiden: Guaranteeing well-connected communities. *Scientific Reports, 9*, Article 5233. https://doi.org/10.1038/s41598-019-41695-z diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md new file mode 100644 index 00000000..b24e706d --- /dev/null +++ b/docs/research/topic-logratio-coordinates.md @@ -0,0 +1,38 @@ +# Logistic-normal topic coordinates + +## Scope + +This note doctors the first `topic_measurement` production slice (ADR 0012): + +1. raw topic proportions are compositional rather than unconstrained Euclidean indicators (Aitchison, 1982); +2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models (Aitchison & Shen, 1980; Blei & Lafferty, 2007); +3. ALR is full rank but not an orthonormal Aitchison-distance isometry (Aitchison, 1982); +4. sequential Egozcue ILR supplies the orthonormal Aitchison-distance isometry when that Euclidean geometry is the estimand (Egozcue et al., 2003); +5. max-shifted inverses recover representable extreme coordinates without overflow (Aitchison & Shen, 1980); +6. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. + +The temporal STM backend, global topic identity, method-effect model, and K-selection remain accepted-target. No database migration is allocated. + +## Authoritative sources + +Aitchison, J., & Shen, S. M. (1980). Logistic-normal distributions: Some properties and uses. *Biometrika, 67*(2), 261–272. https://doi.org/10.1093/biomet/67.2.261 + +Aitchison, J. (1982). The statistical analysis of compositional data. *Journal of the Royal Statistical Society: Series B (Methodological), 44*(2), 139–177. https://doi.org/10.1111/j.2517-6161.1982.tb01195.x + +Blei, D. M., & Lafferty, J. D. (2007). A correlated topic model of Science. *The Annals of Applied Statistics, 1*(1), 17–35. https://doi.org/10.1214/07-AOAS114 + +Egozcue, J. J., Pawlowsky-Glahn, V., Mateu-Figueras, G., & Barceló-Vidal, C. (2003). Isometric logratio transformations for compositional data analysis. *Mathematical Geology, 35*(3), 279–300. https://doi.org/10.1023/A:1023818214614 + +## Application + +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore provides `additive_log_ratio` for future logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Egozcue et al. (2003) construct the sequential orthonormal ILR basis whose Euclidean distance between two coordinate vectors equals the corresponding Aitchison distance; a single vector's norm is its distance from the equal-share origin. `isometric_log_ratio` implements that basis and `from_isometric_log_ratio` inverts it through a max-shifted centered-log-ratio reconstruction. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR rather than ALR (Egozcue et al., 2003). `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation (Aitchison & Shen, 1980). The forward ALR map subtracts logarithms rather than forming a potentially overflowing ratio (Aitchison & Shen, 1980; Blei & Lafferty, 2007). Both inverses fail closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980). + +## Verification + +- closed-form simplex `(2,3,1)/6` maps to ALR `(ln 2, ln 3)` and sequential ILR `(√(2/3) ln(2√3/3), √(1/2) ln 3)` with computed RMSE below `1e-15`; +- representable ALR coordinates `(710, 709)` and representable ILR coordinates round-trip through max-shifted inverses without exponential overflow; +- extremes that would underflow a strictly positive `f64` simplex part fail closed; +- equal shares map to the ALR and ILR origins; +- two-part ILR preserves Aitchison distance `√(1/2) |ln(0.8/0.2) - ln(0.5/0.5)|` between `(0.8, 0.2)` and `(0.5, 0.5)`, and pairwise ILR Euclidean distance recovers CLR Aitchison distance between two non-origin compositions (Egozcue et al., 2003); +- zero, negative, non-unit-sum, non-finite, empty, and one-part vectors fail closed; +- `tfidf`, `bm25`, and `keyword` labels are refused. diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index a878a0ad..0e93dd24 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -76,6 +76,8 @@ This report tracks exact-head scientific and engineering evidence required befor | Subevent parent containment | `subevent_containment` | active-PR | this PR | containment-flag recovery vs accept-all | ADR 0003 | | Predicted-vs-observed contradiction | `prediction_contradiction` | active-PR | this PR | `refuse_promotion` requires observed coverage; `refuse_contradiction_or_adjacency` is not promotion authority; cutoff eligibility; label agreement is not RMSE recovery | ADR 0016 | | Provider-disclosure receipts | `provider_receipt` | active-PR | this PR | recovered field-code rate vs collapsed set | ADR 0009 | +| Purpose-bound provider payloads | `tepp_api` | implemented-main | provider-payload minimization | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | +| Adaptive orchestration router | `tepp_api` | partial | — | mode selection, document-control denial, ablation, credential-free bind; live NIM execution remains future | ADR 0010; `docs/research/adaptive-orchestration-router.md` | | Production TLS bind gates | `service_tls` | accepted-target | active PR | plaintext production, table-access host, mismatched PEM, and orchestrator loopback refusal plus recovery computed from `authorize_production_tls` / `authorize_orchestrator_live_port` | ADR 0011; rustls config is not a deployed listener | | Longitudinal within/between | `longitudinal_core` | active-PR | this PR | known-truth component recovery, computed component RMSE, grand-mean pooling baseline comparison, and between-as-within refusal | ADR 0005 | | Global topic activity identity | `topic_lineage` | active-PR | this PR | dormancy/reactivation identity recovery | ADR 0012; birth/split/merge remaining | @@ -99,8 +101,7 @@ This report tracks exact-head scientific and engineering evidence required befor | Evidence-bounded LLM interpretation | `interpretation_gateway` | active-PR | this PR | span citation + unsupported-claim rate | ADR 0010; live orchestration remaining | | TDT/CHRONOS evidence-status gates | `event_core` | active-PR | PR #50 | admission + first-story rates | known-stream miss/FA; full tracking/calibration/schema extraction remains future; ADR 0016; `docs/research/event-intelligence-status-gates.md` | | Encrypted identity mapping envelope | `encrypted_mapping` | active-PR | this PR | exact-head evidence with no unresolved security blocker; unauthorized purpose, wrong key identity/bytes, tampered ciphertext/tag, key redaction, empty input, persistence refusal, generated-nonce/reuse resistance, and recovered identity rate vs collapsed names | ADR 0009; persistence waits for later migration; promotion requires the complete fail-closed security matrix | -| Purpose-bound provider payloads | `tepp_api` | implemented-main | — | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | -| Adaptive orchestration router | `tepp_api` | partial | — | mode selection, document-control denial, ablation, credential-free bind; live NIM execution remains future | ADR 0010; `docs/research/adaptive-orchestration-router.md` | +| Logistic-normal topic coordinates and CPU reference estimator | `topic_measurement` | active-PR | stable ALR + sequential ILR + lexical refusal + bounded sparse TRSL-TM fit | known-simplex ALR/ILR RMSE, Aitchison-distance ILR isometry, known-topic RMSE, exact line/branch coverage | ADR 0012; calibrated posterior, method effects, persistence, and accelerated backends remaining | | CWL modular connectors | `docs/connectors/*` | implemented-main | — | contract docs + examples | PR #22; live HTTP ports remaining | | Release SBOM/provenance generator | `scripts/release_evidence.py` | partial | — | generate+validate in CI | Task 13 partial / PR #28 | | Default stopword deletion refusal | `stopword_deletion` | accepted-target | active PR | refuse default/global stopword lists + recovery vs stopword collapse | ADR 0004/0012 | diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..a635c5c0 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +pythonpath = . diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index bf2ff56c..b7230153 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -217,10 +217,10 @@ def is_executable_source_line( """Return whether *line_number* in *source_path* is an executable source line. LLVM LCOV sometimes emits zero-count DA records for documentation comments, - attributes, pure structural braces, multi-line signatures, literal - continuations, expression continuations, and in-file ``#[cfg(test)]`` - modules. Those records are not evidence of uncovered production behavior - and are excluded from the authored-line gate. + attributes, pure structural braces, multi-line signatures, Rust multiline + string continuations, literal continuations, expression continuations, and + in-file ``#[cfg(test)]`` modules. Those records are not evidence of + uncovered production behavior and are excluded from the authored-line gate. When *repository_root* is provided, *source_path* must resolve under that root (same fail-closed rule as LCOV ``SF:`` loading). @@ -242,6 +242,8 @@ def is_executable_source_line( return False if _line_in_cfg_not_feature_block(lines, line_number): return False + if _line_in_multiline_string_literal(lines, line_number): + return False if _line_in_multiline_string(lines, line_number): return False text = lines[line_number - 1].strip() @@ -265,6 +267,8 @@ def is_executable_source_line( "Ok(())", }: return False + if _is_standalone_string_literal(text) or text.startswith("} else"): + return False if text.endswith(" {"): type_name = text[:-2] if type_name and all(character.isalnum() or character in "_:" for character in type_name): @@ -291,6 +295,12 @@ def is_executable_source_line( return False body = text[text.find("{") + 1 : text.rfind("}")].strip() return bool(body) + # Keep guarded match arms in the authored-line denominator: the guard + # executes even though the arm label itself is structural. + if text.endswith("=> {") and " if " not in text: + if text.startswith("if ") or text.startswith("if("): + return True + return _is_multiline_match_guard(lines, line_number) if text.startswith("pub struct ") or text.startswith("struct "): return False if text.startswith("pub enum ") or text.startswith("enum "): @@ -309,6 +319,30 @@ def is_executable_source_line( return True +def _is_multiline_match_guard(lines: list[str], line_number: int) -> bool: + """Recognize a guard continued onto the lines immediately before an arm.""" + + target_prefix = lines[line_number - 1].strip().partition("=>")[0] + brace_depth = target_prefix.count("}") - target_prefix.count("{") + guard_found = False + boundary_candidate = False + for candidate in reversed(lines[: line_number - 1]): + stripped = candidate.strip() + if brace_depth == 0 and "=>" in stripped: + return guard_found and not boundary_candidate + if brace_depth == 1 and stripped.endswith("=> {"): + boundary_candidate = True + brace_depth += stripped.count("}") - stripped.count("{") + if ( + (stripped.startswith("if ") or stripped.startswith("if(")) + and not stripped.endswith(("}", ";")) + and brace_depth == 0 + ): + guard_found = True + if stripped.startswith("match "): + return guard_found and not boundary_candidate + return guard_found and not boundary_candidate + def _is_structural_comma_continuation( lines: list[str], line_number: int, text: str ) -> bool: @@ -522,6 +556,138 @@ def _line_in_cfg_not_feature_block(lines: list[str], line_number: int) -> bool: return False +def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> bool: + """Return whether a line is inside a Rust string continuation. + + The scanner tracks normal strings, raw strings, block comments, and character + literals so quotes in comments or literal contents cannot change the state of + a later source line. + """ + + in_string = False + raw_hashes: int | None = None + block_comment_depth = 0 + for index, raw in enumerate(lines, start=1): + target_continuation = (in_string or raw_hashes is not None) and index == line_number + target_closing_cursor: int | None = None + target_has_executable_suffix = False + cursor = 0 + while cursor < len(raw): + if block_comment_depth > 0: + if raw.startswith("/*", cursor): + block_comment_depth += 1 + cursor += 2 + elif raw.startswith("*/", cursor): + block_comment_depth -= 1 + cursor += 2 + else: + cursor += 1 + continue + if raw_hashes is not None: + delimiter = '"' + ("#" * raw_hashes) + closing = raw.find(delimiter, cursor) + if closing == -1: + cursor = len(raw) + else: + raw_hashes = None + cursor = closing + len(delimiter) + if target_continuation and target_closing_cursor is None: + target_closing_cursor = cursor + continue + if in_string: + character = raw[cursor] + if character == "\\": + cursor += 2 + elif character == '"': + in_string = False + cursor += 1 + if target_continuation and target_closing_cursor is None: + target_closing_cursor = cursor + else: + cursor += 1 + continue + if raw[cursor].isspace(): + cursor += 1 + continue + if raw.startswith("//", cursor): + break + if raw.startswith("/*", cursor): + block_comment_depth += 1 + cursor += 2 + continue + if target_continuation and target_closing_cursor is not None: + if raw[cursor] in ",;)]}": + cursor += 1 + continue + target_has_executable_suffix = True + raw_start = _raw_string_start(raw, cursor) + if raw_start is not None: + raw_hashes, cursor = raw_start + continue + if raw[cursor] == '"': + in_string = True + cursor += 1 + continue + if raw[cursor] == "'": + character_end = _character_literal_end(raw, cursor) + if character_end is not None: + cursor = character_end + continue + cursor += 1 + if target_continuation: + if target_closing_cursor is None: + return True + return not target_has_executable_suffix + return False + + +def _raw_string_start(line: str, cursor: int) -> tuple[int, int] | None: + """Return ``(hash_count, next_cursor)`` for a Rust raw-string opener.""" + + if line.startswith("br", cursor): + prefix_end = cursor + 2 + elif line.startswith("r", cursor): + prefix_end = cursor + 1 + else: + return None + hash_end = prefix_end + while hash_end < len(line) and line[hash_end] == "#": + hash_end += 1 + if hash_end < len(line) and line[hash_end] == '"': + return hash_end - prefix_end, hash_end + 1 + return None + + +def _character_literal_end(line: str, cursor: int) -> int | None: + """Return the cursor after a one-line Rust character literal, if present.""" + + candidate = cursor + 1 + if candidate >= len(line): + return None + if line[candidate] == "\\": + candidate += 2 + else: + candidate += 1 + if candidate < len(line) and line[candidate] == "'": + return candidate + 1 + return None + + +def _is_standalone_string_literal(text: str) -> bool: + """Return whether *text* is only a normal string literal and punctuation.""" + if not text.startswith('"'): + return False + escaped = False + for index, character in enumerate(text[1:], start=1): + if character == '"' and not escaped: + return text[index + 1 :].strip() in {"", ",", ";"} + if character == "\\": + escaped = not escaped + else: + escaped = False + return False + + def load_lcov_line_totals( path: Path, repository_root: Path | None = None ) -> Mapping[str, Any]: diff --git a/scripts/check_workspace_contract.py b/scripts/check_workspace_contract.py index c3fcfdb4..764bec93 100644 --- a/scripts/check_workspace_contract.py +++ b/scripts/check_workspace_contract.py @@ -65,6 +65,8 @@ "compute_backend", "episode_membership", "membership_target", + "topic_measurement", + "analysis_engine", "psychometric_core", ) diff --git a/scripts/validate_documentation.py b/scripts/validate_documentation.py index 968f0aaf..7b7364c9 100644 --- a/scripts/validate_documentation.py +++ b/scripts/validate_documentation.py @@ -44,7 +44,9 @@ "docs/adr/0017-hourly-contextual-orchestrator-gateway.md", "docs/adr/0018-consumer-scoped-analysis-run-ingress.md", "docs/adr/0019-project-history-wire-size-symmetry.md", + "docs/adr/0020-span-grounded-semantic-units.md", "docs/adr/0021-lineageweave-project-history-boundary.md", + "docs/adr/0022-deterministic-analysis-run-execution.md", "docs/product/prd-v0.4-approved.md", PRODUCT_TECHNICAL_GAP_BASELINE, "docs/roadmaps/2026-08-05-tepp-delivery-roadmap.md", diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index e9eb46d8..3e0edf72 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -532,33 +532,39 @@ def test_executable_source_line_filters_noise_records(self) -> None: " return value,", # 40 executable (return keeps it) " x + 1,", # 41 executable expression with trailing comma " record_uncovered(),", # 42 executable call with trailing comma - '#[cfg(feature = "live-sqlx")]', # 43 cfg attr - "fn live_path() {", # 44 fn - " live_body();", # 45 executable active feature body - "}", # 46 brace - '#[cfg(not(feature = "live-sqlx"))]', # 47 not-feature attr - "fn offline_path() {", # 48 inside not-feature - " offline_body();", # 49 inside not-feature - "}", # 50 inside not-feature close - "#[cfg(test)]", # 51 - "mod tests {", # 52 cfg(test) mod - " #[test]", # 53 inside test mod - " fn unit() {", # 54 inside test mod - " assert_eq!(1, 1);", # 55 inside test mod - " }", # 56 - "}", # 57 - " executable_statement();", # 58 executable - " append_value(", # 59 multiline call opener - " value,", # 60 trailing comma noise - " );", # 61 call close - " values", # 62 - " .iter()", # 63 method-chain continuation - " .collect::>()", # 64 method-chain continuation - " });", # 65 closure call close - "(", # 66 structural call opener - ")", # 67 structural call close - " Ok(())", # 68 structural unit result - " NaruonLiveResponse {", # 69 structural struct literal + '"standalone string literal",', # 43 standalone string noise + "} else {", # 44 structural branch noise + ")", # 45 structural close noise + '#[cfg(feature = "live-sqlx")]', # 46 cfg attr + "fn live_path() {", # 47 fn + " live_body();", # 48 executable active feature body + "}", # 49 brace + '#[cfg(not(feature = "live-sqlx"))]', # 50 not-feature attr + "fn offline_path() {", # 51 inside not-feature + " offline_body();", # 52 inside not-feature + "}", # 53 inside not-feature close + "#[cfg(test)]", # 54 + "mod tests {", # 55 cfg(test) mod + " #[test]", # 56 inside test mod + " fn unit() {", # 57 inside test mod + " assert_eq!(1, 1);", # 58 inside test mod + " }", # 59 + "}", # 60 + " executable_statement();", # 61 executable + " append_value(", # 62 multiline call opener + " value,", # 63 trailing comma noise + " );", # 64 call close + " values", # 65 receiver line stays executable + " .iter()", # 66 method-chain continuation + " .collect::>()", # 67 method-chain continuation + " });", # 68 closure call close + "(", # 69 structural call opener + ")", # 70 structural call close + " Ok(())", # 71 structural unit result + " NaruonLiveResponse {", # 72 structural struct literal + "pub(crate) fn crate_visible() {", # 73 visibility-qualified fn + "State::Accepted => {", # 74 match-arm structure + "State::Guarded(value) if valid(value) => {", # 75 guarded arm is executable ] source.write_text("\n".join(source_lines) + "\n", encoding="utf-8") path = str(source) @@ -573,7 +579,7 @@ def test_executable_source_line_filters_noise_records(self) -> None: coverage_contract.is_executable_source_line(path, len(source_lines) + 5) ) - expected_executable = {13, 40, 41, 42, 45, 58, 59, 62} + expected_executable = {13, 40, 41, 42, 48, 61, 62, 65, 75} for line_number in range(1, len(source_lines) + 1): is_exec = coverage_contract.is_executable_source_line(path, line_number) if line_number in expected_executable: @@ -592,13 +598,13 @@ def test_executable_source_line_filters_noise_records(self) -> None: "\n".join( [ f"SF:{path}", - "DA:58,1", - "DA:59,0", - "DA:63,0", - "DA:64,0", + "DA:61,1", + "DA:62,0", + "DA:66,0", + "DA:67,0", "DA:1,0", "DA:2,0", - "DA:48,0", + "DA:51,0", "end_of_record", "", ] @@ -643,6 +649,174 @@ def test_lcov_rejects_source_paths_outside_repository(self) -> None: if outside.exists(): outside.unlink() + def test_multiline_guard_arm_is_executable(self) -> None: + """Retain the final expression of a multiline Rust match guard.""" + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "multiline_guard.rs" + source.write_text( + "match state {\n" + " State::Ready(value)\n" + " if value.is_valid()\n" + " && value.is_fresh() => {\n" + " consume(value);\n" + " }\n" + " _ => {\n" + " ignore(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + + self.assertTrue(coverage_contract.is_executable_source_line(str(source), 4)) + self.assertFalse(coverage_contract.is_executable_source_line(str(source), 7)) + + def test_guard_after_brace_closing_pattern_is_executable(self) -> None: + """Count a guard after a destructuring pattern that closes with a brace.""" + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "destructured_guard.rs" + source.write_text( + "match state {\n" + " State::Ready { value }\n" + " if value.is_valid() => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + + self.assertTrue(coverage_contract.is_executable_source_line(str(source), 3)) + + def test_long_and_nested_match_guards_are_executable(self) -> None: + """Track guard boundaries beyond the old scan window and nested arms.""" + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "complex_guard.rs" + long_guard = [ + "match state {", + " State::Ready(value)", + " if value.is_valid()", + *[f" && value.part_{index}()" for index in range(40)], + " && value.is_fresh() => {", + " consume(value);", + " }", + "}", + ] + source.write_text("\n".join(long_guard) + "\n", encoding="utf-8") + self.assertTrue( + coverage_contract.is_executable_source_line( + str(source), len(long_guard) - 3 + ) + ) + + source.write_text( + "match state {\n" + " State::Ready(value)\n" + " if match value {\n" + " 0 => true,\n" + " _ => false,\n" + " } && value.is_fresh() => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + self.assertTrue(coverage_contract.is_executable_source_line(str(source), 6)) + + source.write_text( + "match state {\n" + " State::Ready(value)\n" + " if match value {\n" + " 0 => true,\n" + " _ => false,\n" + " }\n" + " && value.is_fresh() => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + self.assertTrue(coverage_contract.is_executable_source_line(str(source), 7)) + + def test_previous_arm_body_does_not_make_next_label_executable(self) -> None: + """Do not treat an ``if`` inside the preceding arm as a guard.""" + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "previous_arm.rs" + source.write_text( + "match state {\n" + " State::Previous => {\n" + " if value.is_valid() {\n" + " consume(value);\n" + " }\n" + " }\n" + " State::Current => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + + self.assertFalse(coverage_contract.is_executable_source_line(str(source), 7)) + + one_line_previous = Path(temporary) / "one_line_previous.rs" + one_line_previous.write_text( + "match state {\n" + " State::Previous => value,\n" + " State::Current => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + self.assertFalse( + coverage_contract.is_executable_source_line( + str(one_line_previous), 3 + ) + ) + + second_guard = Path(temporary) / "second_guard.rs" + second_guard.write_text( + "match state {\n" + " State::First => value,\n" + " State::Ready(value)\n" + " if value.is_valid()\n" + " && value.is_fresh() => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + self.assertTrue( + coverage_contract.is_executable_source_line(str(second_guard), 5) + ) + + first_arm = Path(temporary) / "first_arm.rs" + first_arm.write_text( + "match state {\n" + " State::Current => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + self.assertFalse( + coverage_contract.is_executable_source_line(str(first_arm), 2) + ) + self.assertFalse( + coverage_contract._is_multiline_match_guard( # noqa: SLF001 + [" if value.is_valid() { consume(value); }", "State::Current => {"], + 2, + ) + ) + self.assertFalse( + coverage_contract._is_multiline_match_guard( + ["State::Current", "State::Current => {"], + 2, + ) + ) + def test_cfg_test_and_not_feature_block_helpers(self) -> None: """cfg(test) modules and cfg(not(feature)) blocks are fully recognized.""" @@ -668,6 +842,7 @@ def test_cfg_test_and_not_feature_block_helpers(self) -> None: self.assertTrue(coverage_contract._line_in_cfg_not_feature_block(lines, 9)) self.assertFalse(coverage_contract._line_in_cfg_not_feature_block(lines, 1)) self.assertFalse(coverage_contract._line_in_cfg_not_feature_block(lines, 11)) + open_only = [ '#[cfg(not(feature = "x"))]', "fn unfinished()", @@ -715,6 +890,157 @@ def test_cfg_test_and_not_feature_block_helpers(self) -> None: coverage_contract._line_in_cfg_not_feature_block(unclosed_not_feature, 99) ) + def test_multiline_string_continuations_are_not_authored_lines(self) -> None: + """Rust multiline string fragments are excluded from authored coverage.""" + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "query.rs" + source.write_text( + 'fn query() {\n' + ' let sql = format!("SELECT id \\\n' + ' FROM document_record \\\n' + ' WHERE tenant_record_id = \'x\'");\n' + ' execute(sql);\n' + '}\n', + encoding="utf-8", + ) + self.assertTrue( + coverage_contract.is_executable_source_line(str(source), 2) + ) + self.assertFalse( + coverage_contract.is_executable_source_line(str(source), 3) + ) + self.assertFalse( + coverage_contract.is_executable_source_line(str(source), 4) + ) + self.assertTrue( + coverage_contract.is_executable_source_line(str(source), 5) + ) + + def test_string_scanner_handles_comments_backslash_parity_and_methods(self) -> None: + """Quoted comments and escaped delimiters do not corrupt source classification.""" + + with tempfile.TemporaryDirectory() as temporary: + backslash = "\\" + source = Path(temporary) / "scanner.rs" + source_lines = [ + "fn query() {", + f' let sql = "SELECT id {backslash}', + ' FROM document";', + r' // comment contains one " quote', + " execute(sql);", + f' let even = "ends with two slashes {backslash * 2}";', + " execute(even);", + r' "literal".to_string();', + "}", + ] + source.write_text("\n".join(source_lines) + "\n", encoding="utf-8") + path = str(source) + + self.assertFalse(coverage_contract.is_executable_source_line(path, 3)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 5)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 7)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 8)) + + block_comment = [ + "/* comment starts", + r' comment has a " quote', + " still comment", + "*/", + "execute();", + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(block_comment, 5) + ) + nested_block = [ + "/* outer", + " /* inner */", + r' still outer with " quote', + "*/", + "execute();", + ] + nested_path = Path(temporary) / "nested.rs" + nested_path.write_text("\n".join(nested_block) + "\n", encoding="utf-8") + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(nested_block, 5) + ) + self.assertTrue( + coverage_contract.is_executable_source_line(str(nested_path), 5) + ) + self.assertTrue( + coverage_contract._is_standalone_string_literal(r'"escaped\\",') + ) + self.assertFalse( + coverage_contract._is_standalone_string_literal('"unfinished') + ) + + raw_string = [ + ' let text = r##"', + ' a " quote in raw text', + ' "##;', + ' execute(text);', + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(raw_string, 1) + ) + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(raw_string, 2) + ) + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(raw_string, 3) + ) + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(raw_string, 4) + ) + + byte_raw_string = [ + ' let bytes = br#"', + ' raw bytes', + ' "#;', + ] + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(byte_raw_string, 2) + ) + + closing_with_code = [ + ' let text = "first', + ' second"; execute(text);', + ] + closing_with_comment = [ + ' let text = "first', + ' second"; // no executable suffix', + ] + closing_with_block_comment_and_code = [ + ' let text = "first', + ' second"; /* note */ execute(text);', + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(closing_with_code, 2) + ) + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(closing_with_comment, 2) + ) + self.assertFalse( + coverage_contract._line_in_multiline_string_literal( + closing_with_block_comment_and_code, 2 + ) + ) + + character_and_lifetime = [ + "fn query<'a>() {", + " let quote: char = '\"';", + " execute();", + "}", + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal( + character_and_lifetime, 3 + ) + ) + self.assertIsNone(coverage_contract._character_literal_end("'", 0)) + self.assertEqual( + coverage_contract._character_literal_end(r"'\''", 0), 4 + ) def test_line_filter_excludes_literal_and_structural_continuations(self) -> None: """LCOV-only literal fragments and branch continuations are filtered.""" @@ -787,6 +1113,74 @@ def test_multiline_scanner_ignores_comments_char_literals_and_raw_strings(self) self.assertTrue(coverage_contract.is_executable_source_line(path, 8)) self.assertFalse(coverage_contract.is_executable_source_line(path, 7)) + def test_structural_comma_skips_blank_predecessors(self) -> None: + """Blank predecessors do not invent a call opener for a trailing comma.""" + + lines = [ + "", + " ", + " field_name,", + ] + self.assertFalse( + coverage_contract._is_structural_comma_continuation( + lines, 3, "field_name," + ) + ) + self.assertFalse( + coverage_contract._is_structural_comma_continuation( + ["field_name,"], 1, "field_name," + ) + ) + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "blank_predecessor.rs" + source.write_text("\n".join(lines) + "\n", encoding="utf-8") + self.assertTrue( + coverage_contract.is_executable_source_line(str(source), 3) + ) + + def test_structural_comma_after_blank_lines_still_sees_call_opener(self) -> None: + """Empty lines between a call opener and an argument remain structural.""" + + lines = [ + "record_value(", + "", + " field_name,", + ")", + ] + self.assertTrue( + coverage_contract._is_structural_comma_continuation( + lines, 3, "field_name," + ) + ) + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "call_opener.rs" + source.write_text("\n".join(lines) + "\n", encoding="utf-8") + self.assertFalse( + coverage_contract.is_executable_source_line(str(source), 3) + ) + + def test_multiline_string_scanner_handles_escaped_char_and_past_eof(self) -> None: + """Escaped char literals keep later lines classified; past-EOF is closed.""" + + lines = [ + "fn query() {", + r" let quote = '\'';", + r" let slash = '\\';", + r" let newline = '\n';", + " execute();", + "}", + ] + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "escaped_char.rs" + source.write_text("\n".join(lines) + "\n", encoding="utf-8") + path = str(source) + self.assertFalse(coverage_contract._line_in_multiline_string(lines, 2)) + self.assertFalse(coverage_contract._line_in_multiline_string(lines, 5)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 5)) + self.assertFalse( + coverage_contract._line_in_multiline_string(lines, len(lines) + 1) + ) + diff --git a/tests/quality/test_check_workspace_contract.py b/tests/quality/test_check_workspace_contract.py index 906d8a27..352058bf 100644 --- a/tests/quality/test_check_workspace_contract.py +++ b/tests/quality/test_check_workspace_contract.py @@ -47,6 +47,18 @@ def test_standards_register_cites_rfc_5646_once(self) -> None: ).read_text(encoding="utf-8") self.assertEqual(text.count("RFC 5646"), 1) + def test_liu_2023_register_cites_graham_neubig(self) -> None: + """The Liu et al. (2023) survey must keep Graham Neubig's initial.""" + + text = ( + REPOSITORY_ROOT / "docs" / "research" / "standards-and-literature.md" + ).read_text(encoding="utf-8") + self.assertIn( + "Liu, P., Yuan, W., Fu, J., Jiang, Z., Hayashi, H., & Neubig, G. (2023).", + text, + ) + self.assertNotIn("Neubig, P.", text) + def test_member_paths_match_expected_crates(self) -> None: """Workspace members resolve to the approved crate roots by name."""