From 9701b3d0363d765a974ae6f07b14b3bfefec9865 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 18:35:21 +0900 Subject: [PATCH 01/38] feat(topic): logistic-normal ALR coordinates with true-parameter RMSE ADR 0012 first production slice: additive log-ratio maps on the unit simplex, fail-closed invalid compositions, and refusal of TF-IDF/BM25 keyword scores as inferential coordinates. No new migration. --- ARCHITECTURE.md | 1 + CHANGELOG.md | 1 + Cargo.lock | 4 + Cargo.toml | 2 + DOCUMENTATION.md | 1 + README.md | 6 +- crates/topic_measurement/Cargo.toml | 17 ++++ crates/topic_measurement/src/coordinates.rs | 92 +++++++++++++++++++ crates/topic_measurement/src/error.rs | 50 ++++++++++ crates/topic_measurement/src/lexical.rs | 36 ++++++++ crates/topic_measurement/src/lib.rs | 20 ++++ .../topic_measurement/tests/crate_contract.rs | 7 ++ .../tests/logratio_recovery_contract.rs | 92 +++++++++++++++++++ docs/TRACEABILITY.md | 4 +- ...ational-shared-latent-topic-measurement.md | 2 +- docs/adr/README.md | 6 +- docs/research/standards-and-literature.md | 2 + docs/research/topic-logratio-coordinates.md | 31 +++++++ docs/validation/temporal-event-foundation.md | 3 +- scripts/check_workspace_contract.py | 1 + 20 files changed, 368 insertions(+), 10 deletions(-) create mode 100644 crates/topic_measurement/Cargo.toml create mode 100644 crates/topic_measurement/src/coordinates.rs create mode 100644 crates/topic_measurement/src/error.rs create mode 100644 crates/topic_measurement/src/lexical.rs create mode 100644 crates/topic_measurement/src/lib.rs create mode 100644 crates/topic_measurement/tests/crate_contract.rs create mode 100644 crates/topic_measurement/tests/logratio_recovery_contract.rs create mode 100644 docs/research/topic-logratio-coordinates.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ffe514db..e3212df5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,6 +61,7 @@ boundaries above remain the target modular MSA architecture. | `tepp_simulation` | known-truth temporal/event data generation | | `validation_core` | RMSE, bias, coverage, graph, and Monte Carlo metrics | | `tepp_api` | versioned DTO, schema, and export contracts | +| `topic_measurement` | logistic-normal / additive log-ratio topic coordinates | No crate exposes placeholder production behavior in Task 1. This prevents an empty façade from becoming a de facto public API before its invariants and tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 93891a27..e97ec769 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added +- `topic_measurement` logistic-normal additive log-ratio coordinates: fail-closed simplex validation, ALR/inverse maps with true-parameter round-trip RMSE, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). diff --git a/Cargo.lock b/Cargo.lock index 616bfd78..1068d99c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1372,6 +1372,10 @@ dependencies = [ "tokio", ] +[[package]] +name = "topic_measurement" +version = "0.1.0" + [[package]] name = "tracing" version = "0.1.44" diff --git a/Cargo.toml b/Cargo.toml index 92565940..35ba7a1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "crates/tepp_simulation", "crates/validation_core", "crates/tepp_api", + "crates/topic_measurement", ] default-members = [ "crates/evidence_core", @@ -23,6 +24,7 @@ default-members = [ "crates/tepp_simulation", "crates/validation_core", "crates/tepp_api", + "crates/topic_measurement", ] [workspace.package] diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 3f094947..9e93a508 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -36,6 +36,7 @@ TEPP's approved PRD v0.4 and implementation plan are the primary product baselin | Retention/deletion/legal-hold doctoring | [`docs/research/retention-deletion-legal-hold.md`](docs/research/retention-deletion-legal-hold.md) | | Provider-payload minimization doctoring | [`docs/research/provider-payload-minimization.md`](docs/research/provider-payload-minimization.md) | | Adaptive orchestration router doctoring | [`docs/research/adaptive-orchestration-router.md`](docs/research/adaptive-orchestration-router.md) | +| Topic log-ratio coordinate doctoring | [`docs/research/topic-logratio-coordinates.md`](docs/research/topic-logratio-coordinates.md) | | Hourly NIM OpenCode doctoring | [`docs/doctoring/hourly-nim-opencode-development.md`](docs/doctoring/hourly-nim-opencode-development.md) | | Change history | [`CHANGELOG.md`](CHANGELOG.md) | diff --git a/README.md b/README.md index ae74015d..ed7a2d74 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,8 @@ implemented in Rust. ## Current implementation state This branch establishes the Task 1 Rust workspace and quality-gate foundation. -The ten bounded crates compile independently but intentionally expose no -placeholder production APIs. Domain behavior begins in Task 2 with immutable -evidence identifiers and source records. +The eleven bounded crates compile independently. Domain crates expose only +validated production APIs; placeholder surfaces are prohibited. ```text crates/evidence_core @@ -22,6 +21,7 @@ crates/corpus_split crates/tepp_simulation crates/validation_core crates/tepp_api +crates/topic_measurement ``` ## Local verification diff --git a/crates/topic_measurement/Cargo.toml b/crates/topic_measurement/Cargo.toml new file mode 100644 index 00000000..299f03c2 --- /dev/null +++ b/crates/topic_measurement/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "topic_measurement" +description = "Logistic-normal and log-ratio coordinates for compositional topics." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[lints] +workspace = true diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs new file mode 100644 index 00000000..cde99ce3 --- /dev/null +++ b/crates/topic_measurement/src/coordinates.rs @@ -0,0 +1,92 @@ +//! Additive log-ratio maps for logistic-normal topic coordinates. + +use crate::error::TopicMeasurementError; + +const UNIT_SUM_TOLERANCE: f64 = 1e-12; + +/// Map a strictly positive unit simplex vector to additive log-ratio coordinates. +/// +/// For a `K`-part composition `θ` the image is the `K-1` vector +/// `y_k = ln(θ_k / θ_K)`. This is the logistic-normal coordinate system used +/// by correlated topic models and required before Euclidean or ESEM/DSEM work. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidComposition`] when the vector is +/// empty, has fewer than two parts, contains a non-finite or non-positive +/// entry, or does not sum to one within a tight absolute tolerance. +pub fn additive_log_ratio(proportions: &[f64]) -> Result, TopicMeasurementError> { + let last = require_composition(proportions)?; + Ok(proportions[..proportions.len() - 1] + .iter() + .map(|part| (part / last).ln()) + .collect()) +} + +/// Invert additive log-ratio coordinates back to the unit simplex. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the +/// coordinate vector is empty or contains a non-finite value. +pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { + if coordinates.is_empty() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut exponentiated = Vec::with_capacity(coordinates.len()); + let mut denom = 1.0_f64; + for &value in coordinates { + if !value.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let exp = value.exp(); + if !exp.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + denom += exp; + exponentiated.push(exp); + } + if !denom.is_finite() || denom <= 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut simplex = Vec::with_capacity(coordinates.len() + 1); + for exp in exponentiated { + simplex.push(exp / denom); + } + simplex.push(1.0 / denom); + Ok(simplex) +} + +fn require_composition(proportions: &[f64]) -> Result { + if proportions.len() < 2 { + return Err(TopicMeasurementError::InvalidComposition); + } + let mut sum = 0.0_f64; + for &part in proportions { + if !part.is_finite() || part <= 0.0 { + return Err(TopicMeasurementError::InvalidComposition); + } + sum += part; + } + if !sum.is_finite() || (sum - 1.0).abs() > UNIT_SUM_TOLERANCE { + return Err(TopicMeasurementError::InvalidComposition); + } + Ok(proportions[proportions.len() - 1]) +} + +#[cfg(test)] +mod tests { + use super::{additive_log_ratio, from_additive_log_ratio}; + use crate::error::TopicMeasurementError; + + #[test] + fn two_part_equal_shares_are_zero_and_overflow_fails_closed() { + let pair = additive_log_ratio(&[0.5, 0.5]).expect("pair"); + assert_eq!(pair.len(), 1); + assert!(pair[0].abs() < 1e-15); + assert_eq!( + from_additive_log_ratio(&[1.0e9]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + } +} diff --git a/crates/topic_measurement/src/error.rs b/crates/topic_measurement/src/error.rs new file mode 100644 index 00000000..f2922574 --- /dev/null +++ b/crates/topic_measurement/src/error.rs @@ -0,0 +1,50 @@ +//! Fail-closed topic-coordinate errors. + +use std::fmt; + +/// A fail-closed topic-measurement error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum TopicMeasurementError { + /// Composition is empty, has fewer than two parts, is non-positive, + /// non-finite, or does not sum to one. + InvalidComposition, + /// Log-ratio vector is empty or non-finite. + InvalidLogRatioDimension, + /// TF-IDF, BM25, or keyword scores were offered as inferential coordinates. + LexicalWeightForbidden, +} + +impl fmt::Display for TopicMeasurementError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::InvalidComposition => "invalid compositional topic vector", + Self::InvalidLogRatioDimension => "invalid log-ratio dimension", + Self::LexicalWeightForbidden => "lexical inferential weights are forbidden", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for TopicMeasurementError {} + +#[cfg(test)] +mod tests { + use super::TopicMeasurementError; + + #[test] + fn messages_are_stable() { + assert_eq!( + TopicMeasurementError::InvalidComposition.to_string(), + "invalid compositional topic vector" + ); + assert_eq!( + TopicMeasurementError::InvalidLogRatioDimension.to_string(), + "invalid log-ratio dimension" + ); + assert_eq!( + TopicMeasurementError::LexicalWeightForbidden.to_string(), + "lexical inferential weights are forbidden" + ); + } +} diff --git a/crates/topic_measurement/src/lexical.rs b/crates/topic_measurement/src/lexical.rs new file mode 100644 index 00000000..588f60f9 --- /dev/null +++ b/crates/topic_measurement/src/lexical.rs @@ -0,0 +1,36 @@ +//! Refusal of lexical heuristics as inferential topic coordinates. + +use crate::error::TopicMeasurementError; + +/// Refuse TF-IDF, BM25, and keyword scores as topic-estimator coordinates. +/// +/// ADR 0012 forbids treating lexical retrieval weights as inferential topic +/// coordinates. A recognized statistical method name is accepted so callers +/// can share one vocabulary gate. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::LexicalWeightForbidden`] for empty labels +/// and for `tfidf`, `bm25`, and `keyword` after alphanumeric folding. +pub fn refuse_lexical_inferential_weight(method: &str) -> Result<(), TopicMeasurementError> { + let folded: String = method + .chars() + .filter(char::is_ascii_alphanumeric) + .flat_map(char::to_lowercase) + .collect(); + if folded.is_empty() || matches!(folded.as_str(), "tfidf" | "bm25" | "keyword") { + return Err(TopicMeasurementError::LexicalWeightForbidden); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::refuse_lexical_inferential_weight; + + #[test] + fn statistical_method_names_are_allowed() { + refuse_lexical_inferential_weight("tepp_topic_measurement").expect("allowed"); + refuse_lexical_inferential_weight("logistic_normal").expect("allowed"); + } +} diff --git a/crates/topic_measurement/src/lib.rs b/crates/topic_measurement/src/lib.rs new file mode 100644 index 00000000..013732ac --- /dev/null +++ b/crates/topic_measurement/src/lib.rs @@ -0,0 +1,20 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +//! Logistic-normal and log-ratio coordinates for compositional topic proportions. +//! +//! Raw topic proportions are not Euclidean indicators. Downstream network and +//! psychometric analysis must use additive log-ratio (logistic-normal) maps +//! rather than TF-IDF, BM25, or keyword scores as inferential coordinates. + +mod coordinates; +mod error; +mod lexical; + +/// Additive log-ratio map from a simplex vector. +pub use coordinates::additive_log_ratio; +/// Inverse additive log-ratio map back to the simplex. +pub use coordinates::from_additive_log_ratio; +/// Fail-closed topic-coordinate errors. +pub use error::TopicMeasurementError; +/// Refuse lexical retrieval weights as inferential coordinates. +pub use lexical::refuse_lexical_inferential_weight; diff --git a/crates/topic_measurement/tests/crate_contract.rs b/crates/topic_measurement/tests/crate_contract.rs new file mode 100644 index 00000000..8f9cfd6a --- /dev/null +++ b/crates/topic_measurement/tests/crate_contract.rs @@ -0,0 +1,7 @@ +//! Integration contract for the `topic_measurement` package identity. + +#[test] +fn package_identity_is_stable() { + let observed = std::hint::black_box(env!("CARGO_PKG_NAME")); + assert_eq!(observed, "topic_measurement"); +} diff --git a/crates/topic_measurement/tests/logratio_recovery_contract.rs b/crates/topic_measurement/tests/logratio_recovery_contract.rs new file mode 100644 index 00000000..3c0dcdd4 --- /dev/null +++ b/crates/topic_measurement/tests/logratio_recovery_contract.rs @@ -0,0 +1,92 @@ +//! True-parameter recovery of logistic-normal topic coordinates. +#![allow(clippy::cast_precision_loss)] + +use topic_measurement::{ + TopicMeasurementError, additive_log_ratio, from_additive_log_ratio, + refuse_lexical_inferential_weight, +}; + +fn rmse(truth: &[f64], recovered: &[f64]) -> f64 { + let n = truth.len() as f64; + let sum_sq: f64 = truth + .iter() + .zip(recovered) + .map(|(left, right)| { + let residual = left - right; + residual * residual + }) + .sum(); + (sum_sq / n).sqrt() +} + +#[test] +fn known_simplex_recovers_through_alr_with_computed_rmse() { + // Closed-form simplex: (2, 3, 1) / 6. ALR is (ln 2, ln 3). + let truth = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; + let coordinates = additive_log_ratio(&truth).expect("alr"); + assert_eq!(coordinates.len(), 2); + assert!((coordinates[0] - 2.0_f64.ln()).abs() < 1e-15); + assert!((coordinates[1] - 3.0_f64.ln()).abs() < 1e-15); + + let recovered = from_additive_log_ratio(&coordinates).expect("inverse"); + let error = rmse(&truth, &recovered); + assert!( + error < 1e-15, + "ALR round-trip RMSE {error} exceeded machine-scale bound" + ); + let sum: f64 = recovered.iter().sum(); + assert!((sum - 1.0).abs() < 1e-15); +} + +#[test] +fn equal_shares_map_to_zero_alr_and_refuse_raw_euclidean_use() { + let thirds = [1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0]; + let coordinates = additive_log_ratio(&thirds).expect("equal"); + assert!(coordinates.iter().all(|value| value.abs() < 1e-15)); + let recovered = from_additive_log_ratio(&[0.0, 0.0]).expect("zeros"); + assert!(rmse(&thirds, &recovered) < 1e-15); +} + +#[test] +fn invalid_compositions_and_lexical_weights_fail_closed() { + // K=2 is valid; zero/negative/non-unit-sum/non-finite/K<2 are not. + assert_eq!( + additive_log_ratio(&[0.0, 1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[-0.1, 1.1]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[0.2, 0.2, 0.2]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[f64::NAN, 1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + additive_log_ratio(&[1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + from_additive_log_ratio(&[]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_additive_log_ratio(&[f64::INFINITY]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + + for method in ["tfidf", "bm25", "keyword", "TF-IDF", ""] { + assert_eq!( + refuse_lexical_inferential_weight(method), + Err(TopicMeasurementError::LexicalWeightForbidden) + ); + } +} diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index f6739641..9bd65faa 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -22,9 +22,9 @@ The full APA 7th standards/literature register remains `docs/research/standards- | versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); HTTP service remaining accepted-target | partial | | immutable split/run/reproducibility manifests | ADR 0013; ERD | `tepp_api` reproducibility manifest contract on protected main; `persistence_postgres` append-only SQL insert/lookup for `reproducibility_manifest`, `corpus_split_manifest`, `model_run`, and `model_artifact` (migration `0003`); full physical ERD constraints remaining | partial | | multilingual shared latent semantic space | PRD; ADR 0004 | future semantic/concept/topic crates | accepted-target | -| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | future `topic_measurement` | accepted-target | +| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` ALR coordinates on the active PR; STM backend remaining | partial | | global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | future topic lineage/activity state | accepted-target | -| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | future semantic/method-source model | accepted-target | +| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; estimator-side method model remains future | partial | | candidate K statistical/Pareto gates + blinded LLM review | ADR 0012; research | future `model_selection` | accepted-target | | compositional topic correlation / stable clustering | ADR 0005/0012; research | future `network_analysis` | accepted-target | diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index c3d5085f..c2262965 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -1,7 +1,7 @@ # ADR 0012 — Temporal Relational Shared-Latent Topic Measurement **Decision status:** Accepted -**Implementation maturity:** accepted-target +**Implementation maturity:** partial — logistic-normal additive log-ratio coordinates and lexical-weight refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; temporal topic identity, backend STM estimator, method-effect model, and K-selection remain accepted-target **Date:** 2026-08-12 **Supersedes:** None; refines ADR 0004 and ADR 0005 without replacing their multilingual and psychometric authorities. diff --git a/docs/adr/README.md b/docs/adr/README.md index 258eb7f3..4f7fe4bc 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -15,10 +15,10 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0007](0007-rust-workspace-quality-gates.md) | Explicit Rust workspace, pinned toolchains, and exact quality gates | Accepted | implemented-main | ADR 0014 governs scientific/product claim promotion beyond repository-quality tooling. | | [0008](0008-immutable-evidence-identities-digests-and-spans.md) | Immutable evidence identities, `SHA-256` digests, exact spans, and strict wire reconstruction | Accepted | implemented-main | ADR 0013 governs future persistence/reproducibility/split authority. | | [0009](0009-purpose-bound-pii-governance.md) | Purpose-bound PII governance without blanket masking | Accepted | partial | Persistence retention/deletion/legal-hold (`0007`) and provider-payload minimization implemented-main; deployment evidence remains accepted-target. | -| [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding on the active PR; live NIM execution and production ablation evidence remain accepted-target. | +| [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding implemented-main; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; no direct cross-service application-table coupling. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | accepted-target | Owns topic backend compatibility, global topic identity, method effects, K/model-selection prerequisites, and compositional topic coordinates. | -| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold on the active PR; remaining physical ERD/backup accepted-target. | +| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR coordinates and lexical-weight refusal are on the active PR; temporal topic identity, STM backend, method effects, and K-selection remain accepted-target. | +| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold implemented-main; remaining physical ERD/backup accepted-target. | | [0014](0014-scientific-claim-promotion-and-release-evidence.md) | Scientific claim promotion and release evidence authority | Accepted | partial | Separates design, implementation, scientific/product claim, and release authority; repository SBOM/provenance generator implemented, full release bundle remaining. | | [0015](0015-autonomous-development-review-and-merge-authority.md) | Autonomous development, review, and merge authority separation | Accepted | active-PR | Separates model proposal, deterministic verification, publication, independent review, and merge/release authority. | | [0016](0016-tdt-chronos-event-intelligence-boundary.md) | TDT, CHRONOS, and Event Ontology intelligence boundary | Accepted | accepted-target | Separates observed evidence, detection/tracking, prediction/schema inference, temporal consistency, and promoted transition authority. | diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index bfda7a79..d793954a 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -48,6 +48,8 @@ LLM evaluation complements but never replaces predictive, posterior, stability, ## Compositional data, correlation, and clusters +Aitchison, J., & Shen, S. M. (1980). Logistic-normal distributions: Some properties and uses. *Biometrika, 67*(2), 261–272. https://doi.org/10.1093/biomet/67.2.261 + Aitchison, J. (1982). The statistical analysis of compositional data. *Journal of the Royal Statistical Society: Series B (Methodological), 44*(2), 139–177. https://doi.org/10.1111/j.2517-6161.1982.tb01195.x Friedman, J., Hastie, T., & Tibshirani, R. (2008). Sparse inverse covariance estimation with the graphical lasso. *Biostatistics, 9*(3), 432–441. https://doi.org/10.1093/biostatistics/kxm045 diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md new file mode 100644 index 00000000..a3514cd9 --- /dev/null +++ b/docs/research/topic-logratio-coordinates.md @@ -0,0 +1,31 @@ +# Logistic-normal topic coordinates + +## Scope + +This note doctors the first `topic_measurement` production slice (ADR 0012): + +1. raw topic proportions are not Euclidean indicators; +2. additive log-ratio coordinates implement the logistic-normal map used by correlated topic models; +3. inverse ALR recovers a known simplex with a computed RMSE; +4. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. + +The temporal STM backend, global topic identity, method-effect model, and K-selection remain accepted-target. No database migration is allocated. + +## Authoritative sources + +Aitchison, J., & Shen, S. M. (1980). Logistic-normal distributions: Some properties and uses. *Biometrika, 67*(2), 261–272. https://doi.org/10.1093/biomet/67.2.261 + +Aitchison, J. (1982). The statistical analysis of compositional data. *Journal of the Royal Statistical Society: Series B (Methodological), 44*(2), 139–177. https://doi.org/10.1111/j.2517-6161.1982.tb01195.x + +Blei, D. M., & Lafferty, J. D. (2007). A correlated topic model of Science. *The Annals of Applied Statistics, 1*(1), 17–35. https://doi.org/10.1214/07-AOAS114 + +## Application + +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same map for correlated topic models. TEPP therefore converts a strictly positive unit simplex through `additive_log_ratio` before any Euclidean or psychometric operation, and recovers the simplex with `from_additive_log_ratio` (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). + +## Verification + +- closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; +- equal shares map to a zero ALR vector; +- zero, negative, non-unit-sum, non-finite, empty, and one-part vectors fail closed; +- `tfidf`, `bm25`, and `keyword` labels are refused. diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index aae1a06e..307624d7 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -24,7 +24,8 @@ This report tracks exact-head scientific and engineering evidence required befor | Recovery metrics | `validation_core` | implemented-main | — | RMSE/bias/coverage/MC gates | Task 11 / PR #19 | | Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining | | Purpose-bound provider payloads | `tepp_api` | implemented-main | provider-payload minimization | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | -| Adaptive orchestration router | `tepp_api` | accepted-target | active PR | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | +| Adaptive orchestration router | `tepp_api` | implemented-main | router + ablation | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | +| Logistic-normal topic coordinates | `topic_measurement` | active-PR | ALR + lexical refusal | true-parameter ALR RMSE | ADR 0012; `docs/research/topic-logratio-coordinates.md` | | CWL modular connectors | `docs/connectors/*` | implemented-main | — | contract docs + examples | PR #22; live HTTP ports remaining | | Release SBOM/provenance generator | `scripts/release_evidence.py` | partial | — | generate+validate in CI | Task 13 partial / PR #28 | diff --git a/scripts/check_workspace_contract.py b/scripts/check_workspace_contract.py index c7b1ecf5..616f8d07 100644 --- a/scripts/check_workspace_contract.py +++ b/scripts/check_workspace_contract.py @@ -23,6 +23,7 @@ "tepp_simulation", "validation_core", "tepp_api", + "topic_measurement", ) REQUIRED_CI_SNIPPETS: tuple[str, ...] = ( From 130bc714735e09d4384090892a92dd605db07b44 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 18:24:16 +0900 Subject: [PATCH 02/38] test(topic): expose unstable ALR overflow path --- .../tests/logratio_recovery_contract.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/topic_measurement/tests/logratio_recovery_contract.rs b/crates/topic_measurement/tests/logratio_recovery_contract.rs index 3c0dcdd4..d0f133f9 100644 --- a/crates/topic_measurement/tests/logratio_recovery_contract.rs +++ b/crates/topic_measurement/tests/logratio_recovery_contract.rs @@ -38,6 +38,22 @@ fn known_simplex_recovers_through_alr_with_computed_rmse() { assert!((sum - 1.0).abs() < 1e-15); } +#[test] +fn large_finite_coordinates_round_trip_without_exponential_overflow() { + let truth = [710.0, 709.0]; + let simplex = from_additive_log_ratio(&truth) + .expect("finite representable ALR coordinates must use a stable inverse"); + assert!(simplex.iter().all(|part| part.is_finite() && *part > 0.0)); + assert!((simplex.iter().sum::() - 1.0).abs() < 1e-15); + + let recovered = additive_log_ratio(&simplex) + .expect("forward ALR must subtract logs instead of overflowing the ratio"); + assert!( + rmse(&truth, &recovered) < 1e-10, + "large-coordinate round trip must retain the true parameters" + ); +} + #[test] fn equal_shares_map_to_zero_alr_and_refuse_raw_euclidean_use() { let thirds = [1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0]; From e76a0deb5f31828f587a7802c8987f31bc99214f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 18:27:42 +0900 Subject: [PATCH 03/38] test(topic): reject inverse underflow to a zero simplex part --- crates/topic_measurement/tests/logratio_recovery_contract.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/topic_measurement/tests/logratio_recovery_contract.rs b/crates/topic_measurement/tests/logratio_recovery_contract.rs index d0f133f9..e6af5f8d 100644 --- a/crates/topic_measurement/tests/logratio_recovery_contract.rs +++ b/crates/topic_measurement/tests/logratio_recovery_contract.rs @@ -98,6 +98,11 @@ fn invalid_compositions_and_lexical_weights_fail_closed() { from_additive_log_ratio(&[f64::INFINITY]), Err(TopicMeasurementError::InvalidLogRatioDimension) ); + assert_eq!( + from_additive_log_ratio(&[-1.0e9]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "inverse must not return a zero simplex part after underflow" + ); for method in ["tfidf", "bm25", "keyword", "TF-IDF", ""] { assert_eq!( From 30ae9e0c59cfc63dd9d61e1413faba4d06aa7dbd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 18:29:45 +0900 Subject: [PATCH 04/38] fix(topic): script stable ALR and preserve shared ledgers --- scripts/repair_pr48_logratio_stability.py | 258 ++++++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 scripts/repair_pr48_logratio_stability.py diff --git a/scripts/repair_pr48_logratio_stability.py b/scripts/repair_pr48_logratio_stability.py new file mode 100644 index 00000000..cc6f4dae --- /dev/null +++ b/scripts/repair_pr48_logratio_stability.py @@ -0,0 +1,258 @@ +"""Apply PR 48 stable log-ratio arithmetic and documentation repairs.""" + +from pathlib import Path + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + """Replace exactly one fragment or fail closed.""" + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected one target, found {count}") + return text.replace(old, new, 1) + + +def update_coordinates() -> None: + """Use stable log differences and max-shifted inverse softmax.""" + path = Path("crates/topic_measurement/src/coordinates.rs") + text = path.read_text(encoding="utf-8") + text = replace_once( + text, + """/// For a `K`-part composition `θ` the image is the `K-1` vector +/// `y_k = ln(θ_k / θ_K)`. This is the logistic-normal coordinate system used +/// by correlated topic models and required before Euclidean or ESEM/DSEM work. +""", + """/// For a `K`-part composition `θ` the image is the `K-1` vector +/// `y_k = ln(θ_k / θ_K)`. This reference-dependent, full-rank coordinate map +/// supports logistic-normal regression and ESEM/DSEM interfaces. It is not an +/// orthonormal isometry for Aitchison distance; use ILR coordinates when that +/// Euclidean geometry is the estimand. +""", + "coordinate documentation", + ) + text = replace_once( + text, + ".map(|part| (part / last).ln())", + ".map(|part| part.ln() - last.ln())", + "stable forward ALR", + ) + old_inverse = """pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { + if coordinates.is_empty() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut exponentiated = Vec::with_capacity(coordinates.len()); + let mut denom = 1.0_f64; + for &value in coordinates { + if !value.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let exp = value.exp(); + if !exp.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + denom += exp; + exponentiated.push(exp); + } + if !denom.is_finite() || denom <= 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut simplex = Vec::with_capacity(coordinates.len() + 1); + for exp in exponentiated { + simplex.push(exp / denom); + } + simplex.push(1.0 / denom); + Ok(simplex) +} +""" + new_inverse = """pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { + if coordinates.is_empty() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut maximum = 0.0_f64; + for &value in coordinates { + if !value.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + maximum = maximum.max(value); + } + + let reference_weight = (-maximum).exp(); + if reference_weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + let mut shifted_weights = Vec::with_capacity(coordinates.len()); + let mut denominator = reference_weight; + for &value in coordinates { + let weight = (value - maximum).exp(); + if weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + denominator += weight; + shifted_weights.push(weight); + } + + let mut simplex = Vec::with_capacity(coordinates.len() + 1); + for weight in shifted_weights { + simplex.push(weight / denominator); + } + simplex.push(reference_weight / denominator); + Ok(simplex) +} +""" + text = replace_once(text, old_inverse, new_inverse, "stable inverse ALR") + text = replace_once( + text, + """/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the +/// coordinate vector is empty or contains a non-finite value. +""", + """/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the +/// coordinate vector is empty, non-finite, or would underflow a part to zero +/// in the strictly positive `f64` simplex representation. +""", + "inverse error documentation", + ) + text = replace_once( + text, + "fn two_part_equal_shares_are_zero_and_overflow_fails_closed()", + "fn two_part_equal_shares_are_zero_and_unrepresentable_extremes_fail_closed()", + "internal test name", + ) + path.write_text(text, encoding="utf-8") + + +def update_error_and_crate_docs() -> None: + """Describe representability and ALR geometry without overclaiming isometry.""" + error_path = Path("crates/topic_measurement/src/error.rs") + error_text = error_path.read_text(encoding="utf-8") + error_text = replace_once( + error_text, + " /// Log-ratio vector is empty or non-finite.\n", + " /// Log-ratio vector is empty, non-finite, or not representable as a strictly positive `f64` simplex.\n", + "error variant documentation", + ) + error_path.write_text(error_text, encoding="utf-8") + + lib_path = Path("crates/topic_measurement/src/lib.rs") + lib_text = lib_path.read_text(encoding="utf-8") + lib_text = replace_once( + lib_text, + """//! Raw topic proportions are not Euclidean indicators. Downstream network and +//! psychometric analysis must use additive log-ratio (logistic-normal) maps +//! rather than TF-IDF, BM25, or keyword scores as inferential coordinates. +""", + """//! Raw topic proportions are compositional rather than unconstrained Euclidean +//! indicators. ALR supplies a reference-dependent full-rank logistic-normal map +//! for regression and psychometric interfaces; it is not an orthonormal +//! Aitchison-distance isometry. Distance-based Aitchison geometry requires ILR. +//! TF-IDF, BM25, and keyword scores remain forbidden inferential coordinates. +""", + "crate geometry documentation", + ) + lib_path.write_text(lib_text, encoding="utf-8") + + +def update_research_and_adr() -> None: + """Clarify ALR versus ILR and record stable arithmetic evidence.""" + research_path = Path("docs/research/topic-logratio-coordinates.md") + research = research_path.read_text(encoding="utf-8") + research = replace_once( + research, + """1. raw topic proportions are not Euclidean indicators; +2. additive log-ratio coordinates implement the logistic-normal map used by correlated topic models; +3. inverse ALR recovers a known simplex with a computed RMSE; +4. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. +""", + """1. raw topic proportions are compositional rather than unconstrained Euclidean indicators; +2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models; +3. ALR is full rank but not an orthonormal Aitchison-distance isometry; ILR is required when that Euclidean geometry is the estimand; +4. max-shifted inverse ALR and log-difference forward ALR recover representable extreme coordinates without overflow; +5. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. +""", + "research scope", + ) + research = replace_once( + research, + """Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same map for correlated topic models. TEPP therefore converts a strictly positive unit simplex through `additive_log_ratio` before any Euclidean or psychometric operation, and recovers the simplex with `from_additive_log_ratio` (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). +""", + """Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR. `from_additive_log_ratio` uses a max-shifted inverse softmax and the forward map subtracts logarithms, avoiding avoidable exponential and ratio overflow while failing closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). +""", + "research application", + ) + research = replace_once( + research, + """- closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; +- equal shares map to a zero ALR vector; +""", + """- closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; +- representable coordinates `(710, 709)` round-trip through the max-shifted inverse without exponential overflow; +- extremes that would underflow a strictly positive `f64` simplex part fail closed; +- equal shares map to a zero ALR vector; +""", + "research verification", + ) + research_path.write_text(research, encoding="utf-8") + + adr_path = Path("docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md") + adr = adr_path.read_text(encoding="utf-8") + adr = replace_once( + adr, + """- topic proportions are compositional and downstream network/psychometric analysis uses logistic-normal coordinates or valid orthonormal log-ratio coordinates; +""", + """- topic proportions are compositional and downstream network/psychometric analysis uses logistic-normal coordinates or valid orthonormal log-ratio coordinates; +- ALR is a reference-dependent full-rank logistic-normal map, not an Aitchison-distance isometry; distance-based Euclidean Aitchison geometry uses an orthonormal ILR basis; +""", + "ADR ALR/ILR boundary", + ) + adr_path.write_text(adr, encoding="utf-8") + + +def restore_conflict_resolved_ledgers() -> None: + """Reapply the topic slice to main-owned shared ledgers after the merge.""" + changelog_path = Path("CHANGELOG.md") + changelog = changelog_path.read_text(encoding="utf-8") + item = ( + "- `topic_measurement` logistic-normal additive log-ratio coordinates: " + "fail-closed simplex validation, max-shifted stable ALR/inverse maps with " + "true-parameter round-trip RMSE, explicit ALR-versus-ILR geometry boundary, " + "and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates " + "(ADR 0012 first production slice; no new migration).\n" + ) + if item not in changelog: + changelog = replace_once(changelog, "### Added\n\n", "### Added\n\n" + item, "changelog marker") + changelog_path.write_text(changelog, encoding="utf-8") + + trace_path = Path("docs/TRACEABILITY.md") + trace = trace_path.read_text(encoding="utf-8") + trace = replace_once( + trace, + "| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | future `topic_measurement` | accepted-target |", + "| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR coordinates on the active PR; temporal STM backend remaining | partial |", + "trace topic backend", + ) + trace = replace_once( + trace, + "| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | future semantic/method-source model | accepted-target |", + "| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial |", + "trace lexical refusal", + ) + trace_path.write_text(trace, encoding="utf-8") + + validation_path = Path("docs/validation/temporal-event-foundation.md") + validation = validation_path.read_text(encoding="utf-8") + row = ( + "| Logistic-normal topic coordinates | `topic_measurement` | active-PR | stable ALR + lexical refusal | " + "known-simplex and extreme-coordinate RMSE, ALR/ILR boundary | ADR 0012; " + "`docs/research/topic-logratio-coordinates.md` |\n" + ) + if row not in validation: + marker = ( + "| Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | " + "unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining |\n" + ) + validation = replace_once(validation, marker, marker + row, "validation API row") + validation_path.write_text(validation, encoding="utf-8") + + +update_coordinates() +update_error_and_crate_docs() +update_research_and_adr() +restore_conflict_resolved_ledgers() From bce4ac69fce26fb9933dbcd3ad9148c8e46e9867 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 18:30:24 +0900 Subject: [PATCH 05/38] chore(ci): verify PR 48 stable log-ratio repair --- .../repair-pr48-logratio-stability.yml | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/repair-pr48-logratio-stability.yml diff --git a/.github/workflows/repair-pr48-logratio-stability.yml b/.github/workflows/repair-pr48-logratio-stability.yml new file mode 100644 index 00000000..dc1af6c2 --- /dev/null +++ b/.github/workflows/repair-pr48-logratio-stability.yml @@ -0,0 +1,75 @@ +name: Repair PR 48 log-ratio numerical stability + +on: + pull_request: + types: + - synchronize + - reopened + - ready_for_review + +permissions: + contents: read + +concurrency: + group: repair-tepp-pr-48-logratio-stability + cancel-in-progress: false + +jobs: + repair: + if: >- + github.event.pull_request.number == 48 && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.head.ref == 'agent/topic-logratio-coordinates' + runs-on: ubuntu-latest + timeout-minutes: 35 + permissions: + contents: write + steps: + - name: Checkout exact PR branch + uses: actions/checkout@631c942040754b6e095e929c1677c07e10ed4f87 + with: + ref: agent/topic-logratio-coordinates + fetch-depth: 0 + persist-credentials: true + + - name: Install pinned Rust toolchain + run: rustup toolchain install 1.97.1 --profile minimal --component clippy --component rustfmt + + - name: Prove stable extreme-coordinate contracts are RED + run: | + set +e + output=$(cargo +1.97.1 test -p topic_measurement --test logratio_recovery_contract 2>&1) + status=$? + set -e + printf '%s\n' "$output" + if [ "$status" -eq 0 ]; then + echo "Expected direct exponentiation or ratio arithmetic to fail the new stability contracts" >&2 + exit 1 + fi + grep -E "large_finite_coordinates_round_trip|inverse must not return a zero simplex part" <<<"$output" + + - name: Apply stable ALR arithmetic and documentation repair + run: | + python3 scripts/repair_pr48_logratio_stability.py + cargo +1.97.1 fmt --all + + - name: Verify focused and workspace contracts + run: | + cargo +1.97.1 fmt --all --check + cargo +1.97.1 test -p topic_measurement --all-features + cargo +1.97.1 clippy -p topic_measurement --all-targets --all-features -- -D warnings + cargo +1.97.1 test --workspace --all-features + python3 scripts/check_workspace_contract.py + python3 scripts/check_docstrings.py + python3 scripts/validate_documentation.py + + - name: Commit verified repair and remove one-shot files + run: | + rm -f .github/workflows/repair-pr48-logratio-stability.yml + rm -f scripts/repair_pr48_logratio_stability.py + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git diff --cached --check + git commit -m "fix(topic): stabilize extreme log-ratio coordinates" + git push origin HEAD:agent/topic-logratio-coordinates From c95ebe1104d08a945276c02ac4595c212e4bf404 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 10:11:11 +0000 Subject: [PATCH 06/38] fix(topic): stabilize extreme log-ratio coordinates --- .../repair-pr48-logratio-stability.yml | 75 ----- CHANGELOG.md | 2 +- crates/topic_measurement/src/coordinates.rs | 44 +-- crates/topic_measurement/src/error.rs | 2 +- crates/topic_measurement/src/lib.rs | 8 +- docs/TRACEABILITY.md | 2 +- ...ational-shared-latent-topic-measurement.md | 1 + docs/research/topic-logratio-coordinates.md | 13 +- docs/validation/temporal-event-foundation.md | 2 +- scripts/repair_pr48_logratio_stability.py | 258 ------------------ 10 files changed, 45 insertions(+), 362 deletions(-) delete mode 100644 .github/workflows/repair-pr48-logratio-stability.yml delete mode 100644 scripts/repair_pr48_logratio_stability.py diff --git a/.github/workflows/repair-pr48-logratio-stability.yml b/.github/workflows/repair-pr48-logratio-stability.yml deleted file mode 100644 index dc1af6c2..00000000 --- a/.github/workflows/repair-pr48-logratio-stability.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Repair PR 48 log-ratio numerical stability - -on: - pull_request: - types: - - synchronize - - reopened - - ready_for_review - -permissions: - contents: read - -concurrency: - group: repair-tepp-pr-48-logratio-stability - cancel-in-progress: false - -jobs: - repair: - if: >- - github.event.pull_request.number == 48 && - github.event.pull_request.head.repo.full_name == github.repository && - github.event.pull_request.head.ref == 'agent/topic-logratio-coordinates' - runs-on: ubuntu-latest - timeout-minutes: 35 - permissions: - contents: write - steps: - - name: Checkout exact PR branch - uses: actions/checkout@631c942040754b6e095e929c1677c07e10ed4f87 - with: - ref: agent/topic-logratio-coordinates - fetch-depth: 0 - persist-credentials: true - - - name: Install pinned Rust toolchain - run: rustup toolchain install 1.97.1 --profile minimal --component clippy --component rustfmt - - - name: Prove stable extreme-coordinate contracts are RED - run: | - set +e - output=$(cargo +1.97.1 test -p topic_measurement --test logratio_recovery_contract 2>&1) - status=$? - set -e - printf '%s\n' "$output" - if [ "$status" -eq 0 ]; then - echo "Expected direct exponentiation or ratio arithmetic to fail the new stability contracts" >&2 - exit 1 - fi - grep -E "large_finite_coordinates_round_trip|inverse must not return a zero simplex part" <<<"$output" - - - name: Apply stable ALR arithmetic and documentation repair - run: | - python3 scripts/repair_pr48_logratio_stability.py - cargo +1.97.1 fmt --all - - - name: Verify focused and workspace contracts - run: | - cargo +1.97.1 fmt --all --check - cargo +1.97.1 test -p topic_measurement --all-features - cargo +1.97.1 clippy -p topic_measurement --all-targets --all-features -- -D warnings - cargo +1.97.1 test --workspace --all-features - python3 scripts/check_workspace_contract.py - python3 scripts/check_docstrings.py - python3 scripts/validate_documentation.py - - - name: Commit verified repair and remove one-shot files - run: | - rm -f .github/workflows/repair-pr48-logratio-stability.yml - rm -f scripts/repair_pr48_logratio_stability.py - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git diff --cached --check - git commit -m "fix(topic): stabilize extreme log-ratio coordinates" - git push origin HEAD:agent/topic-logratio-coordinates diff --git a/CHANGELOG.md b/CHANGELOG.md index e97ec769..8c43f24d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added -- `topic_measurement` logistic-normal additive log-ratio coordinates: fail-closed simplex validation, ALR/inverse maps with true-parameter round-trip RMSE, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). +- `topic_measurement` logistic-normal additive log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/inverse maps with true-parameter round-trip RMSE, explicit ALR-versus-ILR geometry boundary, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index cde99ce3..1909df2f 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -7,8 +7,10 @@ const UNIT_SUM_TOLERANCE: f64 = 1e-12; /// Map a strictly positive unit simplex vector to additive log-ratio coordinates. /// /// For a `K`-part composition `θ` the image is the `K-1` vector -/// `y_k = ln(θ_k / θ_K)`. This is the logistic-normal coordinate system used -/// by correlated topic models and required before Euclidean or ESEM/DSEM work. +/// `y_k = ln(θ_k / θ_K)`. This reference-dependent, full-rank coordinate map +/// supports logistic-normal regression and ESEM/DSEM interfaces. It is not an +/// orthonormal isometry for Aitchison distance; use ILR coordinates when that +/// Euclidean geometry is the estimand. /// /// # Errors /// @@ -19,7 +21,7 @@ pub fn additive_log_ratio(proportions: &[f64]) -> Result, TopicMeasurem let last = require_composition(proportions)?; Ok(proportions[..proportions.len() - 1] .iter() - .map(|part| (part / last).ln()) + .map(|part| part.ln() - last.ln()) .collect()) } @@ -28,32 +30,40 @@ pub fn additive_log_ratio(proportions: &[f64]) -> Result, TopicMeasurem /// # Errors /// /// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the -/// coordinate vector is empty or contains a non-finite value. +/// coordinate vector is empty, non-finite, or would underflow a part to zero +/// in the strictly positive `f64` simplex representation. pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { if coordinates.is_empty() { return Err(TopicMeasurementError::InvalidLogRatioDimension); } - let mut exponentiated = Vec::with_capacity(coordinates.len()); - let mut denom = 1.0_f64; + let mut maximum = 0.0_f64; for &value in coordinates { if !value.is_finite() { return Err(TopicMeasurementError::InvalidLogRatioDimension); } - let exp = value.exp(); - if !exp.is_finite() { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - denom += exp; - exponentiated.push(exp); + maximum = maximum.max(value); } - if !denom.is_finite() || denom <= 0.0 { + + let reference_weight = (-maximum).exp(); + if reference_weight == 0.0 { return Err(TopicMeasurementError::InvalidLogRatioDimension); } + let mut shifted_weights = Vec::with_capacity(coordinates.len()); + let mut denominator = reference_weight; + for &value in coordinates { + let weight = (value - maximum).exp(); + if weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + denominator += weight; + shifted_weights.push(weight); + } + let mut simplex = Vec::with_capacity(coordinates.len() + 1); - for exp in exponentiated { - simplex.push(exp / denom); + for weight in shifted_weights { + simplex.push(weight / denominator); } - simplex.push(1.0 / denom); + simplex.push(reference_weight / denominator); Ok(simplex) } @@ -80,7 +90,7 @@ mod tests { use crate::error::TopicMeasurementError; #[test] - fn two_part_equal_shares_are_zero_and_overflow_fails_closed() { + fn two_part_equal_shares_are_zero_and_unrepresentable_extremes_fail_closed() { let pair = additive_log_ratio(&[0.5, 0.5]).expect("pair"); assert_eq!(pair.len(), 1); assert!(pair[0].abs() < 1e-15); diff --git a/crates/topic_measurement/src/error.rs b/crates/topic_measurement/src/error.rs index f2922574..ea934e52 100644 --- a/crates/topic_measurement/src/error.rs +++ b/crates/topic_measurement/src/error.rs @@ -9,7 +9,7 @@ pub enum TopicMeasurementError { /// Composition is empty, has fewer than two parts, is non-positive, /// non-finite, or does not sum to one. InvalidComposition, - /// Log-ratio vector is empty or non-finite. + /// Log-ratio vector is empty, non-finite, or not representable as a strictly positive `f64` simplex. InvalidLogRatioDimension, /// TF-IDF, BM25, or keyword scores were offered as inferential coordinates. LexicalWeightForbidden, diff --git a/crates/topic_measurement/src/lib.rs b/crates/topic_measurement/src/lib.rs index 013732ac..45d19f84 100644 --- a/crates/topic_measurement/src/lib.rs +++ b/crates/topic_measurement/src/lib.rs @@ -2,9 +2,11 @@ #![deny(missing_docs)] //! Logistic-normal and log-ratio coordinates for compositional topic proportions. //! -//! Raw topic proportions are not Euclidean indicators. Downstream network and -//! psychometric analysis must use additive log-ratio (logistic-normal) maps -//! rather than TF-IDF, BM25, or keyword scores as inferential coordinates. +//! Raw topic proportions are compositional rather than unconstrained Euclidean +//! indicators. ALR supplies a reference-dependent full-rank logistic-normal map +//! for regression and psychometric interfaces; it is not an orthonormal +//! Aitchison-distance isometry. Distance-based Aitchison geometry requires ILR. +//! TF-IDF, BM25, and keyword scores remain forbidden inferential coordinates. mod coordinates; mod error; diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index 9bd65faa..b18b296e 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -22,7 +22,7 @@ The full APA 7th standards/literature register remains `docs/research/standards- | versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); HTTP service remaining accepted-target | partial | | immutable split/run/reproducibility manifests | ADR 0013; ERD | `tepp_api` reproducibility manifest contract on protected main; `persistence_postgres` append-only SQL insert/lookup for `reproducibility_manifest`, `corpus_split_manifest`, `model_run`, and `model_artifact` (migration `0003`); full physical ERD constraints remaining | partial | | multilingual shared latent semantic space | PRD; ADR 0004 | future semantic/concept/topic crates | accepted-target | -| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` ALR coordinates on the active PR; STM backend remaining | partial | +| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR coordinates on the active PR; temporal STM backend remaining | partial | | global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | future topic lineage/activity state | accepted-target | | no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; estimator-side method model remains future | partial | diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index c2262965..b938a730 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -25,6 +25,7 @@ For the first production line: - stopword deletion is not the default preprocessing rule; - TF-IDF and BM25 are not inferential weights for the statistical topic estimator; - topic proportions are compositional and downstream network/psychometric analysis uses logistic-normal coordinates or valid orthonormal log-ratio coordinates; +- ALR is a reference-dependent full-rank logistic-normal map, not an Aitchison-distance isometry; distance-based Euclidean Aitchison geometry uses an orthonormal ILR basis; - model selection uses statistical/recovery/stability/alignment/fairness gates and a Pareto-style comparison before any blinded LLM review; - the LLM may recommend among statistically admissible candidates but never defines the numerical optimum or bypasses diagnostics. diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index a3514cd9..cc79837a 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -4,10 +4,11 @@ This note doctors the first `topic_measurement` production slice (ADR 0012): -1. raw topic proportions are not Euclidean indicators; -2. additive log-ratio coordinates implement the logistic-normal map used by correlated topic models; -3. inverse ALR recovers a known simplex with a computed RMSE; -4. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. +1. raw topic proportions are compositional rather than unconstrained Euclidean indicators; +2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models; +3. ALR is full rank but not an orthonormal Aitchison-distance isometry; ILR is required when that Euclidean geometry is the estimand; +4. max-shifted inverse ALR and log-difference forward ALR recover representable extreme coordinates without overflow; +5. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. The temporal STM backend, global topic identity, method-effect model, and K-selection remain accepted-target. No database migration is allocated. @@ -21,11 +22,13 @@ Blei, D. M., & Lafferty, J. D. (2007). A correlated topic model of Science. *The ## Application -Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same map for correlated topic models. TEPP therefore converts a strictly positive unit simplex through `additive_log_ratio` before any Euclidean or psychometric operation, and recovers the simplex with `from_additive_log_ratio` (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR. `from_additive_log_ratio` uses a max-shifted inverse softmax and the forward map subtracts logarithms, avoiding avoidable exponential and ratio overflow while failing closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). ## Verification - closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; +- representable coordinates `(710, 709)` round-trip through the max-shifted inverse without exponential overflow; +- extremes that would underflow a strictly positive `f64` simplex part fail closed; - equal shares map to a zero ALR vector; - zero, negative, non-unit-sum, non-finite, empty, and one-part vectors fail closed; - `tfidf`, `bm25`, and `keyword` labels are refused. diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index 307624d7..9749f8af 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -25,7 +25,7 @@ This report tracks exact-head scientific and engineering evidence required befor | Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining | | Purpose-bound provider payloads | `tepp_api` | implemented-main | provider-payload minimization | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | | Adaptive orchestration router | `tepp_api` | implemented-main | router + ablation | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | -| Logistic-normal topic coordinates | `topic_measurement` | active-PR | ALR + lexical refusal | true-parameter ALR RMSE | ADR 0012; `docs/research/topic-logratio-coordinates.md` | +| Logistic-normal topic coordinates | `topic_measurement` | active-PR | stable ALR + lexical refusal | known-simplex and extreme-coordinate RMSE, ALR/ILR boundary | ADR 0012; `docs/research/topic-logratio-coordinates.md` | | CWL modular connectors | `docs/connectors/*` | implemented-main | — | contract docs + examples | PR #22; live HTTP ports remaining | | Release SBOM/provenance generator | `scripts/release_evidence.py` | partial | — | generate+validate in CI | Task 13 partial / PR #28 | diff --git a/scripts/repair_pr48_logratio_stability.py b/scripts/repair_pr48_logratio_stability.py deleted file mode 100644 index cc6f4dae..00000000 --- a/scripts/repair_pr48_logratio_stability.py +++ /dev/null @@ -1,258 +0,0 @@ -"""Apply PR 48 stable log-ratio arithmetic and documentation repairs.""" - -from pathlib import Path - - -def replace_once(text: str, old: str, new: str, label: str) -> str: - """Replace exactly one fragment or fail closed.""" - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected one target, found {count}") - return text.replace(old, new, 1) - - -def update_coordinates() -> None: - """Use stable log differences and max-shifted inverse softmax.""" - path = Path("crates/topic_measurement/src/coordinates.rs") - text = path.read_text(encoding="utf-8") - text = replace_once( - text, - """/// For a `K`-part composition `θ` the image is the `K-1` vector -/// `y_k = ln(θ_k / θ_K)`. This is the logistic-normal coordinate system used -/// by correlated topic models and required before Euclidean or ESEM/DSEM work. -""", - """/// For a `K`-part composition `θ` the image is the `K-1` vector -/// `y_k = ln(θ_k / θ_K)`. This reference-dependent, full-rank coordinate map -/// supports logistic-normal regression and ESEM/DSEM interfaces. It is not an -/// orthonormal isometry for Aitchison distance; use ILR coordinates when that -/// Euclidean geometry is the estimand. -""", - "coordinate documentation", - ) - text = replace_once( - text, - ".map(|part| (part / last).ln())", - ".map(|part| part.ln() - last.ln())", - "stable forward ALR", - ) - old_inverse = """pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { - if coordinates.is_empty() { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - let mut exponentiated = Vec::with_capacity(coordinates.len()); - let mut denom = 1.0_f64; - for &value in coordinates { - if !value.is_finite() { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - let exp = value.exp(); - if !exp.is_finite() { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - denom += exp; - exponentiated.push(exp); - } - if !denom.is_finite() || denom <= 0.0 { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - let mut simplex = Vec::with_capacity(coordinates.len() + 1); - for exp in exponentiated { - simplex.push(exp / denom); - } - simplex.push(1.0 / denom); - Ok(simplex) -} -""" - new_inverse = """pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { - if coordinates.is_empty() { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - let mut maximum = 0.0_f64; - for &value in coordinates { - if !value.is_finite() { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - maximum = maximum.max(value); - } - - let reference_weight = (-maximum).exp(); - if reference_weight == 0.0 { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - let mut shifted_weights = Vec::with_capacity(coordinates.len()); - let mut denominator = reference_weight; - for &value in coordinates { - let weight = (value - maximum).exp(); - if weight == 0.0 { - return Err(TopicMeasurementError::InvalidLogRatioDimension); - } - denominator += weight; - shifted_weights.push(weight); - } - - let mut simplex = Vec::with_capacity(coordinates.len() + 1); - for weight in shifted_weights { - simplex.push(weight / denominator); - } - simplex.push(reference_weight / denominator); - Ok(simplex) -} -""" - text = replace_once(text, old_inverse, new_inverse, "stable inverse ALR") - text = replace_once( - text, - """/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the -/// coordinate vector is empty or contains a non-finite value. -""", - """/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the -/// coordinate vector is empty, non-finite, or would underflow a part to zero -/// in the strictly positive `f64` simplex representation. -""", - "inverse error documentation", - ) - text = replace_once( - text, - "fn two_part_equal_shares_are_zero_and_overflow_fails_closed()", - "fn two_part_equal_shares_are_zero_and_unrepresentable_extremes_fail_closed()", - "internal test name", - ) - path.write_text(text, encoding="utf-8") - - -def update_error_and_crate_docs() -> None: - """Describe representability and ALR geometry without overclaiming isometry.""" - error_path = Path("crates/topic_measurement/src/error.rs") - error_text = error_path.read_text(encoding="utf-8") - error_text = replace_once( - error_text, - " /// Log-ratio vector is empty or non-finite.\n", - " /// Log-ratio vector is empty, non-finite, or not representable as a strictly positive `f64` simplex.\n", - "error variant documentation", - ) - error_path.write_text(error_text, encoding="utf-8") - - lib_path = Path("crates/topic_measurement/src/lib.rs") - lib_text = lib_path.read_text(encoding="utf-8") - lib_text = replace_once( - lib_text, - """//! Raw topic proportions are not Euclidean indicators. Downstream network and -//! psychometric analysis must use additive log-ratio (logistic-normal) maps -//! rather than TF-IDF, BM25, or keyword scores as inferential coordinates. -""", - """//! Raw topic proportions are compositional rather than unconstrained Euclidean -//! indicators. ALR supplies a reference-dependent full-rank logistic-normal map -//! for regression and psychometric interfaces; it is not an orthonormal -//! Aitchison-distance isometry. Distance-based Aitchison geometry requires ILR. -//! TF-IDF, BM25, and keyword scores remain forbidden inferential coordinates. -""", - "crate geometry documentation", - ) - lib_path.write_text(lib_text, encoding="utf-8") - - -def update_research_and_adr() -> None: - """Clarify ALR versus ILR and record stable arithmetic evidence.""" - research_path = Path("docs/research/topic-logratio-coordinates.md") - research = research_path.read_text(encoding="utf-8") - research = replace_once( - research, - """1. raw topic proportions are not Euclidean indicators; -2. additive log-ratio coordinates implement the logistic-normal map used by correlated topic models; -3. inverse ALR recovers a known simplex with a computed RMSE; -4. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. -""", - """1. raw topic proportions are compositional rather than unconstrained Euclidean indicators; -2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models; -3. ALR is full rank but not an orthonormal Aitchison-distance isometry; ILR is required when that Euclidean geometry is the estimand; -4. max-shifted inverse ALR and log-difference forward ALR recover representable extreme coordinates without overflow; -5. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. -""", - "research scope", - ) - research = replace_once( - research, - """Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same map for correlated topic models. TEPP therefore converts a strictly positive unit simplex through `additive_log_ratio` before any Euclidean or psychometric operation, and recovers the simplex with `from_additive_log_ratio` (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). -""", - """Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR. `from_additive_log_ratio` uses a max-shifted inverse softmax and the forward map subtracts logarithms, avoiding avoidable exponential and ratio overflow while failing closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). -""", - "research application", - ) - research = replace_once( - research, - """- closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; -- equal shares map to a zero ALR vector; -""", - """- closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; -- representable coordinates `(710, 709)` round-trip through the max-shifted inverse without exponential overflow; -- extremes that would underflow a strictly positive `f64` simplex part fail closed; -- equal shares map to a zero ALR vector; -""", - "research verification", - ) - research_path.write_text(research, encoding="utf-8") - - adr_path = Path("docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md") - adr = adr_path.read_text(encoding="utf-8") - adr = replace_once( - adr, - """- topic proportions are compositional and downstream network/psychometric analysis uses logistic-normal coordinates or valid orthonormal log-ratio coordinates; -""", - """- topic proportions are compositional and downstream network/psychometric analysis uses logistic-normal coordinates or valid orthonormal log-ratio coordinates; -- ALR is a reference-dependent full-rank logistic-normal map, not an Aitchison-distance isometry; distance-based Euclidean Aitchison geometry uses an orthonormal ILR basis; -""", - "ADR ALR/ILR boundary", - ) - adr_path.write_text(adr, encoding="utf-8") - - -def restore_conflict_resolved_ledgers() -> None: - """Reapply the topic slice to main-owned shared ledgers after the merge.""" - changelog_path = Path("CHANGELOG.md") - changelog = changelog_path.read_text(encoding="utf-8") - item = ( - "- `topic_measurement` logistic-normal additive log-ratio coordinates: " - "fail-closed simplex validation, max-shifted stable ALR/inverse maps with " - "true-parameter round-trip RMSE, explicit ALR-versus-ILR geometry boundary, " - "and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates " - "(ADR 0012 first production slice; no new migration).\n" - ) - if item not in changelog: - changelog = replace_once(changelog, "### Added\n\n", "### Added\n\n" + item, "changelog marker") - changelog_path.write_text(changelog, encoding="utf-8") - - trace_path = Path("docs/TRACEABILITY.md") - trace = trace_path.read_text(encoding="utf-8") - trace = replace_once( - trace, - "| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | future `topic_measurement` | accepted-target |", - "| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR coordinates on the active PR; temporal STM backend remaining | partial |", - "trace topic backend", - ) - trace = replace_once( - trace, - "| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | future semantic/method-source model | accepted-target |", - "| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial |", - "trace lexical refusal", - ) - trace_path.write_text(trace, encoding="utf-8") - - validation_path = Path("docs/validation/temporal-event-foundation.md") - validation = validation_path.read_text(encoding="utf-8") - row = ( - "| Logistic-normal topic coordinates | `topic_measurement` | active-PR | stable ALR + lexical refusal | " - "known-simplex and extreme-coordinate RMSE, ALR/ILR boundary | ADR 0012; " - "`docs/research/topic-logratio-coordinates.md` |\n" - ) - if row not in validation: - marker = ( - "| Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | " - "unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining |\n" - ) - validation = replace_once(validation, marker, marker + row, "validation API row") - validation_path.write_text(validation, encoding="utf-8") - - -update_coordinates() -update_error_and_crate_docs() -update_research_and_adr() -restore_conflict_resolved_ledgers() From 5106f4bd451daca21b9e5fd3b57105a932e19b0b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:02:30 +0900 Subject: [PATCH 07/38] docs(topic): specify stable inverse reference coordinate --- docs/research/topic-logratio-coordinates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index cc79837a..a1315a16 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -22,7 +22,7 @@ Blei, D. M., & Lafferty, J. D. (2007). A correlated topic model of Science. *The ## Application -Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR. `from_additive_log_ratio` uses a max-shifted inverse softmax and the forward map subtracts logarithms, avoiding avoidable exponential and ratio overflow while failing closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR. `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation. The forward map subtracts logarithms rather than forming a potentially overflowing ratio. This avoids avoidable exponential and ratio overflow while failing closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). ## Verification From 0269df004a69673476cebf219c40c1303124b059 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:15:34 +0900 Subject: [PATCH 08/38] test(topic): detect hidden simplex mass with compensated summation --- .../composition_sum_precision_contract.rs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 crates/topic_measurement/tests/composition_sum_precision_contract.rs diff --git a/crates/topic_measurement/tests/composition_sum_precision_contract.rs b/crates/topic_measurement/tests/composition_sum_precision_contract.rs new file mode 100644 index 00000000..2fa8874c --- /dev/null +++ b/crates/topic_measurement/tests/composition_sum_precision_contract.rs @@ -0,0 +1,30 @@ +//! Composition validation must not lose tiny positive mass after a dominant part. + +use topic_measurement::{TopicMeasurementError, additive_log_ratio}; + +#[test] +fn compensated_sum_rejects_mass_hidden_by_naive_floating_point_addition() { + let mut composition = Vec::with_capacity(20_001); + composition.push(1.0); + composition.extend(std::iter::repeat_n(1.0e-16, 20_000)); + + assert_eq!( + additive_log_ratio(&composition), + Err(TopicMeasurementError::InvalidComposition), + "the true mass exceeds one by 2e-12 even though naive ordered addition rounds to one" + ); +} + +#[test] +fn compensated_sum_accepts_a_valid_many_part_composition() { + let tiny_mass = 1.0e-16; + let tiny_parts = 10_000_usize; + let dominant = 1.0 - tiny_mass * tiny_parts as f64; + let mut composition = Vec::with_capacity(tiny_parts + 1); + composition.push(dominant); + composition.extend(std::iter::repeat_n(tiny_mass, tiny_parts)); + + let coordinates = additive_log_ratio(&composition).expect("valid unit simplex"); + assert_eq!(coordinates.len(), tiny_parts); + assert!(coordinates.iter().all(|value| value.is_finite())); +} From d70f5c73972a32019eefa9595e786ecbc24349a1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:16:09 +0900 Subject: [PATCH 09/38] fix(topic): validate simplex mass with compensated summation --- crates/topic_measurement/src/coordinates.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 1909df2f..94fa810f 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -19,9 +19,10 @@ const UNIT_SUM_TOLERANCE: f64 = 1e-12; /// entry, or does not sum to one within a tight absolute tolerance. pub fn additive_log_ratio(proportions: &[f64]) -> Result, TopicMeasurementError> { let last = require_composition(proportions)?; + let reference_log = last.ln(); Ok(proportions[..proportions.len() - 1] .iter() - .map(|part| part.ln() - last.ln()) + .map(|part| part.ln() - reference_log) .collect()) } @@ -72,13 +73,23 @@ fn require_composition(proportions: &[f64]) -> Result= part.abs() { + (sum - next) + part + } else { + (part - next) + sum + }; + sum = next; } - if !sum.is_finite() || (sum - 1.0).abs() > UNIT_SUM_TOLERANCE { + let compensated_sum = sum + compensation; + if !compensated_sum.is_finite() + || (compensated_sum - 1.0).abs() > UNIT_SUM_TOLERANCE + { return Err(TopicMeasurementError::InvalidComposition); } Ok(proportions[proportions.len() - 1]) From 0c3d88d42f5386a2f2444649453b4698235ba5a1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:16:46 +0900 Subject: [PATCH 10/38] test(topic): keep compensated-sum fixture clippy-clean --- .../tests/composition_sum_precision_contract.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/topic_measurement/tests/composition_sum_precision_contract.rs b/crates/topic_measurement/tests/composition_sum_precision_contract.rs index 2fa8874c..9cdd1cfe 100644 --- a/crates/topic_measurement/tests/composition_sum_precision_contract.rs +++ b/crates/topic_measurement/tests/composition_sum_precision_contract.rs @@ -19,7 +19,7 @@ fn compensated_sum_rejects_mass_hidden_by_naive_floating_point_addition() { fn compensated_sum_accepts_a_valid_many_part_composition() { let tiny_mass = 1.0e-16; let tiny_parts = 10_000_usize; - let dominant = 1.0 - tiny_mass * tiny_parts as f64; + let dominant = 1.0 - tiny_mass * 10_000.0; let mut composition = Vec::with_capacity(tiny_parts + 1); composition.push(dominant); composition.extend(std::iter::repeat_n(tiny_mass, tiny_parts)); From 0d7c65fe6c76f32b41229a373e586a058db04fe4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:45:35 +0900 Subject: [PATCH 11/38] style(topic): apply pinned rustfmt output --- crates/topic_measurement/src/coordinates.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 94fa810f..230b9912 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -87,9 +87,7 @@ fn require_composition(proportions: &[f64]) -> Result UNIT_SUM_TOLERANCE - { + if !compensated_sum.is_finite() || (compensated_sum - 1.0).abs() > UNIT_SUM_TOLERANCE { return Err(TopicMeasurementError::InvalidComposition); } Ok(proportions[proportions.len() - 1]) From f3540faa80e788a82fbe7950e3f4b0bdf700c89f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:46:24 +0900 Subject: [PATCH 12/38] test(quality): include topic measurement in docstring inventory --- tests/quality/test_check_docstrings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/quality/test_check_docstrings.py b/tests/quality/test_check_docstrings.py index 2c11f7a5..b99537c5 100644 --- a/tests/quality/test_check_docstrings.py +++ b/tests/quality/test_check_docstrings.py @@ -24,7 +24,7 @@ def test_live_repository_is_documented(self) -> None: sources = docstrings.rust_sources(REPOSITORY_ROOT) crate_roots = sorted(REPOSITORY_ROOT.glob("crates/*/src/lib.rs")) - self.assertEqual(len(crate_roots), 10) + self.assertEqual(len(crate_roots), 11) self.assertTrue(set(crate_roots).issubset(sources)) self.assertGreaterEqual(len(sources), len(crate_roots)) self.assertEqual(docstrings.validate_repository(REPOSITORY_ROOT), []) From ab29247a3cd138a140daa9f9e8d4b7b2302fe471 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 13:06:45 +0000 Subject: [PATCH 13/38] test(topic): cover non-finite compensated simplex mass Exercise overflowing finite parts so the nightly branch gate hits !compensated_sum.is_finite(), and assert true-parameter ALR RMSE against the closed-form (ln 2, ln 3) coordinates. Co-authored-by: Seongho Bae --- crates/topic_measurement/src/coordinates.rs | 8 +++++++ .../composition_sum_precision_contract.rs | 9 +++++++ .../tests/logratio_recovery_contract.rs | 24 +++++++++++++++---- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 230b9912..993e11d2 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -103,9 +103,17 @@ mod tests { let pair = additive_log_ratio(&[0.5, 0.5]).expect("pair"); assert_eq!(pair.len(), 1); assert!(pair[0].abs() < 1e-15); + let recovered = from_additive_log_ratio(&pair).expect("inverse"); + assert!((recovered[0] - 0.5).abs() < 1e-15); + assert!((recovered[1] - 0.5).abs() < 1e-15); assert_eq!( from_additive_log_ratio(&[1.0e9]), Err(TopicMeasurementError::InvalidLogRatioDimension) ); + assert_eq!( + additive_log_ratio(&[f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidComposition), + "overflowing finite parts must fail closed because compensated mass is non-finite" + ); } } diff --git a/crates/topic_measurement/tests/composition_sum_precision_contract.rs b/crates/topic_measurement/tests/composition_sum_precision_contract.rs index 9cdd1cfe..f76cb763 100644 --- a/crates/topic_measurement/tests/composition_sum_precision_contract.rs +++ b/crates/topic_measurement/tests/composition_sum_precision_contract.rs @@ -15,6 +15,15 @@ fn compensated_sum_rejects_mass_hidden_by_naive_floating_point_addition() { ); } +#[test] +fn overflowing_finite_parts_fail_closed_as_non_finite_mass() { + assert_eq!( + additive_log_ratio(&[f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidComposition), + "Kahan-compensated MAX+MAX is non-finite; NaN cannot pass a unit-sum tolerance comparison" + ); +} + #[test] fn compensated_sum_accepts_a_valid_many_part_composition() { let tiny_mass = 1.0e-16; diff --git a/crates/topic_measurement/tests/logratio_recovery_contract.rs b/crates/topic_measurement/tests/logratio_recovery_contract.rs index e6af5f8d..2b50250d 100644 --- a/crates/topic_measurement/tests/logratio_recovery_contract.rs +++ b/crates/topic_measurement/tests/logratio_recovery_contract.rs @@ -24,15 +24,19 @@ fn known_simplex_recovers_through_alr_with_computed_rmse() { // Closed-form simplex: (2, 3, 1) / 6. ALR is (ln 2, ln 3). let truth = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; let coordinates = additive_log_ratio(&truth).expect("alr"); + let true_parameters = [2.0_f64.ln(), 3.0_f64.ln()]; assert_eq!(coordinates.len(), 2); - assert!((coordinates[0] - 2.0_f64.ln()).abs() < 1e-15); - assert!((coordinates[1] - 3.0_f64.ln()).abs() < 1e-15); + let parameter_rmse = rmse(&true_parameters, &coordinates); + assert!( + parameter_rmse < 1e-15, + "true-parameter ALR RMSE {parameter_rmse} exceeded machine-scale bound" + ); let recovered = from_additive_log_ratio(&coordinates).expect("inverse"); - let error = rmse(&truth, &recovered); + let simplex_rmse = rmse(&truth, &recovered); assert!( - error < 1e-15, - "ALR round-trip RMSE {error} exceeded machine-scale bound" + simplex_rmse < 1e-15, + "ALR round-trip RMSE {simplex_rmse} exceeded machine-scale bound" ); let sum: f64 = recovered.iter().sum(); assert!((sum - 1.0).abs() < 1e-15); @@ -98,11 +102,21 @@ fn invalid_compositions_and_lexical_weights_fail_closed() { from_additive_log_ratio(&[f64::INFINITY]), Err(TopicMeasurementError::InvalidLogRatioDimension) ); + assert_eq!( + from_additive_log_ratio(&[1.0e9]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "max-shifted reference weight must fail closed when it underflows to zero" + ); assert_eq!( from_additive_log_ratio(&[-1.0e9]), Err(TopicMeasurementError::InvalidLogRatioDimension), "inverse must not return a zero simplex part after underflow" ); + assert_eq!( + additive_log_ratio(&[f64::MAX, f64::MAX]), + Err(TopicMeasurementError::InvalidComposition), + "overflowing finite parts must fail closed because compensated mass is non-finite" + ); for method in ["tfidf", "bm25", "keyword", "TF-IDF", ""] { assert_eq!( From 4ae9b5df51beea9f8c8bf8aa13f5ad511ad5f800 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 17:20:14 +0000 Subject: [PATCH 14/38] feat(topic): add sequential Egozcue ILR coordinates Implement isometric_log_ratio and its max-shifted inverse on the Egozcue sequential orthonormal basis, with true-parameter RMSE and Aitchison-distance recovery. ALR remains the logistic-normal map. Co-authored-by: Seongho Bae --- ARCHITECTURE.md | 2 +- CHANGELOG.md | 2 +- crates/topic_measurement/src/coordinates.rs | 102 +++++++++++++++- crates/topic_measurement/src/lib.rs | 9 +- .../tests/ilr_recovery_contract.rs | 111 ++++++++++++++++++ docs/TRACEABILITY.md | 2 +- ...ational-shared-latent-topic-measurement.md | 2 +- docs/adr/README.md | 2 +- docs/research/standards-and-literature.md | 2 + docs/research/topic-logratio-coordinates.md | 18 +-- docs/validation/temporal-event-foundation.md | 2 +- 11 files changed, 237 insertions(+), 17 deletions(-) create mode 100644 crates/topic_measurement/tests/ilr_recovery_contract.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e3212df5..f10bf73a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,7 +61,7 @@ boundaries above remain the target modular MSA architecture. | `tepp_simulation` | known-truth temporal/event data generation | | `validation_core` | RMSE, bias, coverage, graph, and Monte Carlo metrics | | `tepp_api` | versioned DTO, schema, and export contracts | -| `topic_measurement` | logistic-normal / additive log-ratio topic coordinates | +| `topic_measurement` | logistic-normal ALR and sequential Egozcue ILR topic coordinates | No crate exposes placeholder production behavior in Task 1. This prevents an empty façade from becoming a de facto public API before its invariants and tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c43f24d..0ad9e324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added -- `topic_measurement` logistic-normal additive log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/inverse maps with true-parameter round-trip RMSE, explicit ALR-versus-ILR geometry boundary, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). +- `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, Aitchison-distance ILR isometry, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 993e11d2..1ab1306f 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -1,4 +1,4 @@ -//! Additive log-ratio maps for logistic-normal topic coordinates. +//! Additive and isometric log-ratio maps for compositional topic coordinates. use crate::error::TopicMeasurementError; @@ -68,6 +68,90 @@ pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMea Ok(simplex) } +/// Map a strictly positive unit simplex vector to isometric log-ratio coordinates. +/// +/// The sequential Egozcue orthonormal basis sends a `K`-part composition to +/// the `K-1` vector whose Euclidean norm equals Aitchison distance. This is +/// the coordinate system for distance-based topic geometry. It is not the +/// reference-dependent logistic-normal map; use [`additive_log_ratio`] when +/// that regression interface is the estimand. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidComposition`] when the vector is +/// empty, has fewer than two parts, contains a non-finite or non-positive +/// entry, or does not sum to one within a tight absolute tolerance. +pub fn isometric_log_ratio(proportions: &[f64]) -> Result, TopicMeasurementError> { + require_composition(proportions)?; + let dimension = proportions.len(); + let logs: Vec = proportions.iter().map(|part| part.ln()).collect(); + let mut coordinates = Vec::with_capacity(dimension - 1); + for index in 0..(dimension - 1) { + let remaining = dimension - index - 1; + #[allow(clippy::cast_precision_loss)] + let remaining_f = remaining as f64; + let scale = (remaining_f / (remaining_f + 1.0)).sqrt(); + let mut rest_sum = 0.0_f64; + for log_part in &logs[index + 1..] { + rest_sum += *log_part; + } + coordinates.push(scale * (logs[index] - rest_sum / remaining_f)); + } + Ok(coordinates) +} + +/// Invert isometric log-ratio coordinates back to the unit simplex. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidLogRatioDimension`] when the +/// coordinate vector is empty, non-finite, or would underflow a part to zero +/// in the strictly positive `f64` simplex representation. +pub fn from_isometric_log_ratio(coordinates: &[f64]) -> Result, TopicMeasurementError> { + if coordinates.is_empty() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + for &value in coordinates { + if !value.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + } + + let dimension = coordinates.len() + 1; + let mut centered_logs = vec![0.0_f64; dimension]; + for (index, &coordinate) in coordinates.iter().enumerate() { + let remaining = dimension - index - 1; + #[allow(clippy::cast_precision_loss)] + let remaining_f = remaining as f64; + let scale = (remaining_f / (remaining_f + 1.0)).sqrt(); + let negative = -1.0 / (remaining_f * (remaining_f + 1.0)).sqrt(); + centered_logs[index] += scale * coordinate; + for centered in &mut centered_logs[index + 1..] { + *centered += negative * coordinate; + } + } + + let mut maximum = centered_logs[0]; + for &value in ¢ered_logs[1..] { + maximum = maximum.max(value); + } + if !maximum.is_finite() { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + + let mut weights = Vec::with_capacity(dimension); + let mut denominator = 0.0_f64; + for &value in ¢ered_logs { + let weight = (value - maximum).exp(); + if weight == 0.0 { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + denominator += weight; + weights.push(weight); + } + Ok(weights.iter().map(|weight| weight / denominator).collect()) +} + fn require_composition(proportions: &[f64]) -> Result { if proportions.len() < 2 { return Err(TopicMeasurementError::InvalidComposition); @@ -95,7 +179,9 @@ fn require_composition(proportions: &[f64]) -> Result f64 { + let n = truth.len() as f64; + let sum_sq: f64 = truth + .iter() + .zip(recovered) + .map(|(left, right)| { + let residual = left - right; + residual * residual + }) + .sum(); + (sum_sq / n).sqrt() +} + +#[test] +fn known_simplex_recovers_through_ilr_with_computed_rmse() { + // Closed-form simplex: (2, 3, 1) / 6. + // Sequential Egozcue ILR: y1 = √(2/3) ln(2√3 / 3), y2 = √(1/2) ln 3. + let truth = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; + let true_parameters = [ + (2.0_f64 / 3.0).sqrt() * (2.0 * 3.0_f64.sqrt() / 3.0).ln(), + (1.0_f64 / 2.0).sqrt() * 3.0_f64.ln(), + ]; + let coordinates = isometric_log_ratio(&truth).expect("ilr"); + assert_eq!(coordinates.len(), 2); + let parameter_rmse = rmse(&true_parameters, &coordinates); + assert!( + parameter_rmse < 1e-15, + "true-parameter ILR RMSE {parameter_rmse} exceeded machine-scale bound" + ); + + let recovered = from_isometric_log_ratio(&coordinates).expect("inverse"); + let simplex_rmse = rmse(&truth, &recovered); + assert!( + simplex_rmse < 1e-15, + "ILR round-trip RMSE {simplex_rmse} exceeded machine-scale bound" + ); + let sum: f64 = recovered.iter().sum(); + assert!((sum - 1.0).abs() < 1e-15); + + let alr = additive_log_ratio(&truth).expect("alr"); + assert!( + (alr[0] - coordinates[0]).abs() > 1e-6, + "ILR must not collapse to the reference-dependent ALR map" + ); +} + +#[test] +fn equal_shares_are_the_ilr_origin_and_preserve_aitchison_distance() { + let halves = [0.5, 0.5]; + let origin = isometric_log_ratio(&halves).expect("origin"); + assert_eq!(origin.len(), 1); + assert!(origin[0].abs() < 1e-15); + + let unbalanced = [0.8, 0.2]; + let coordinates = isometric_log_ratio(&unbalanced).expect("pair"); + let expected = (0.5_f64).sqrt() * 4.0_f64.ln(); + assert!((coordinates[0] - expected).abs() < 1e-15); + + let recovered = from_isometric_log_ratio(&coordinates).expect("inverse"); + assert!(rmse(&unbalanced, &recovered) < 1e-15); +} + +#[test] +fn large_finite_ilr_coordinates_round_trip_or_fail_closed() { + let representable = from_isometric_log_ratio(&[40.0]).expect("representable"); + assert!( + representable + .iter() + .all(|part| part.is_finite() && *part > 0.0) + ); + let recovered = isometric_log_ratio(&representable).expect("forward"); + assert!(rmse(&[40.0], &recovered) < 1e-12); + + assert_eq!( + from_isometric_log_ratio(&[1000.0]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "ILR inverse must not return a zero simplex part after underflow" + ); + assert_eq!( + from_isometric_log_ratio(&[f64::MAX]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "overflowing CLR reconstruction must fail closed" + ); +} + +#[test] +fn invalid_ilr_inputs_fail_closed() { + assert_eq!( + isometric_log_ratio(&[]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + from_isometric_log_ratio(&[]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_isometric_log_ratio(&[f64::NAN]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); + assert_eq!( + from_isometric_log_ratio(&[f64::INFINITY]), + Err(TopicMeasurementError::InvalidLogRatioDimension) + ); +} diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index b18b296e..7f6d90de 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -22,7 +22,7 @@ The full APA 7th standards/literature register remains `docs/research/standards- | versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); HTTP service remaining accepted-target | partial | | immutable split/run/reproducibility manifests | ADR 0013; ERD | `tepp_api` reproducibility manifest contract on protected main; `persistence_postgres` append-only SQL insert/lookup for `reproducibility_manifest`, `corpus_split_manifest`, `model_run`, and `model_artifact` (migration `0003`); full physical ERD constraints remaining | partial | | multilingual shared latent semantic space | PRD; ADR 0004 | future semantic/concept/topic crates | accepted-target | -| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR coordinates on the active PR; temporal STM backend remaining | partial | +| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR and sequential Egozcue ILR coordinates on the active PR; temporal STM backend remaining | partial | | global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | future topic lineage/activity state | accepted-target | | no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; estimator-side method model remains future | partial | diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index b938a730..23cd4162 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -1,7 +1,7 @@ # ADR 0012 — Temporal Relational Shared-Latent Topic Measurement **Decision status:** Accepted -**Implementation maturity:** partial — logistic-normal additive log-ratio coordinates and lexical-weight refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; temporal topic identity, backend STM estimator, method-effect model, and K-selection remain accepted-target +**Implementation maturity:** partial — logistic-normal additive log-ratio coordinates, sequential Egozcue isometric log-ratio coordinates, and lexical-weight refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; temporal topic identity, backend STM estimator, method-effect model, and K-selection remain accepted-target **Date:** 2026-08-12 **Supersedes:** None; refines ADR 0004 and ADR 0005 without replacing their multilingual and psychometric authorities. diff --git a/docs/adr/README.md b/docs/adr/README.md index 4f7fe4bc..9d549d0d 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -17,7 +17,7 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0009](0009-purpose-bound-pii-governance.md) | Purpose-bound PII governance without blanket masking | Accepted | partial | Persistence retention/deletion/legal-hold (`0007`) and provider-payload minimization implemented-main; deployment evidence remains accepted-target. | | [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding implemented-main; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; no direct cross-service application-table coupling. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR coordinates and lexical-weight refusal are on the active PR; temporal topic identity, STM backend, method effects, and K-selection remain accepted-target. | +| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR, sequential Egozcue ILR, and lexical-weight refusal are on the active PR; temporal topic identity, STM backend, method effects, and K-selection remain accepted-target. | | [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold implemented-main; remaining physical ERD/backup accepted-target. | | [0014](0014-scientific-claim-promotion-and-release-evidence.md) | Scientific claim promotion and release evidence authority | Accepted | partial | Separates design, implementation, scientific/product claim, and release authority; repository SBOM/provenance generator implemented, full release bundle remaining. | | [0015](0015-autonomous-development-review-and-merge-authority.md) | Autonomous development, review, and merge authority separation | Accepted | active-PR | Separates model proposal, deterministic verification, publication, independent review, and merge/release authority. | diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index d793954a..54072f27 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -52,6 +52,8 @@ Aitchison, J., & Shen, S. M. (1980). Logistic-normal distributions: Some propert Aitchison, J. (1982). The statistical analysis of compositional data. *Journal of the Royal Statistical Society: Series B (Methodological), 44*(2), 139–177. https://doi.org/10.1111/j.2517-6161.1982.tb01195.x +Egozcue, J. J., Pawlowsky-Glahn, V., Mateu-Figueras, G., & Barceló-Vidal, C. (2003). Isometric logratio transformations for compositional data analysis. *Mathematical Geology, 35*(3), 279–300. https://doi.org/10.1023/A:1023818214614 + Friedman, J., Hastie, T., & Tibshirani, R. (2008). Sparse inverse covariance estimation with the graphical lasso. *Biostatistics, 9*(3), 432–441. https://doi.org/10.1093/biostatistics/kxm045 Traag, V. A., Waltman, L., & van Eck, N. J. (2019). From Louvain to Leiden: Guaranteeing well-connected communities. *Scientific Reports, 9*, Article 5233. https://doi.org/10.1038/s41598-019-41695-z diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index a1315a16..ac5c242d 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -6,9 +6,10 @@ This note doctors the first `topic_measurement` production slice (ADR 0012): 1. raw topic proportions are compositional rather than unconstrained Euclidean indicators; 2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models; -3. ALR is full rank but not an orthonormal Aitchison-distance isometry; ILR is required when that Euclidean geometry is the estimand; -4. max-shifted inverse ALR and log-difference forward ALR recover representable extreme coordinates without overflow; -5. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. +3. ALR is full rank but not an orthonormal Aitchison-distance isometry; +4. sequential Egozcue ILR supplies the orthonormal Aitchison-distance isometry when that Euclidean geometry is the estimand; +5. max-shifted inverses recover representable extreme coordinates without overflow; +6. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. The temporal STM backend, global topic identity, method-effect model, and K-selection remain accepted-target. No database migration is allocated. @@ -20,15 +21,18 @@ Aitchison, J. (1982). The statistical analysis of compositional data. *Journal o Blei, D. M., & Lafferty, J. D. (2007). A correlated topic model of Science. *The Annals of Applied Statistics, 1*(1), 17–35. https://doi.org/10.1214/07-AOAS114 +Egozcue, J. J., Pawlowsky-Glahn, V., Mateu-Figueras, G., & Barceló-Vidal, C. (2003). Isometric logratio transformations for compositional data analysis. *Mathematical Geology, 35*(3), 279–300. https://doi.org/10.1023/A:1023818214614 + ## Application -Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR. `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation. The forward map subtracts logarithms rather than forming a potentially overflowing ratio. This avoids avoidable exponential and ratio overflow while failing closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007). +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Egozcue et al. (2003) construct the sequential orthonormal ILR basis whose Euclidean norm equals Aitchison distance; `isometric_log_ratio` implements that basis and `from_isometric_log_ratio` inverts it through a max-shifted centered-log-ratio reconstruction. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR rather than ALR. `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation. The forward ALR map subtracts logarithms rather than forming a potentially overflowing ratio. Both inverses fail closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007; Egozcue et al., 2003). ## Verification -- closed-form simplex `(2,3,1)/6` maps to `(ln 2, ln 3)` and inverts with computed RMSE below `1e-15`; -- representable coordinates `(710, 709)` round-trip through the max-shifted inverse without exponential overflow; +- closed-form simplex `(2,3,1)/6` maps to ALR `(ln 2, ln 3)` and sequential ILR `(√(2/3) ln(2√3/3), √(1/2) ln 3)` with computed RMSE below `1e-15`; +- representable ALR coordinates `(710, 709)` and representable ILR coordinates round-trip through max-shifted inverses without exponential overflow; - extremes that would underflow a strictly positive `f64` simplex part fail closed; -- equal shares map to a zero ALR vector; +- equal shares map to the ALR and ILR origins; +- two-part ILR preserves Aitchison distance `√(1/2) ln(0.8/0.2)` for `(0.8, 0.2)`; - zero, negative, non-unit-sum, non-finite, empty, and one-part vectors fail closed; - `tfidf`, `bm25`, and `keyword` labels are refused. diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index 9749f8af..c3936755 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -25,7 +25,7 @@ This report tracks exact-head scientific and engineering evidence required befor | Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining | | Purpose-bound provider payloads | `tepp_api` | implemented-main | provider-payload minimization | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | | Adaptive orchestration router | `tepp_api` | implemented-main | router + ablation | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | -| Logistic-normal topic coordinates | `topic_measurement` | active-PR | stable ALR + lexical refusal | known-simplex and extreme-coordinate RMSE, ALR/ILR boundary | ADR 0012; `docs/research/topic-logratio-coordinates.md` | +| Logistic-normal topic coordinates | `topic_measurement` | active-PR | stable ALR + sequential ILR + lexical refusal | known-simplex ALR/ILR RMSE, Aitchison-distance ILR isometry | ADR 0012; `docs/research/topic-logratio-coordinates.md` | | CWL modular connectors | `docs/connectors/*` | implemented-main | — | contract docs + examples | PR #22; live HTTP ports remaining | | Release SBOM/provenance generator | `scripts/release_evidence.py` | partial | — | generate+validate in CI | Task 13 partial / PR #28 | From a52dd5597cc53943a9c3c5a339dd9e10fb216524 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 17:21:12 +0000 Subject: [PATCH 15/38] test(topic): hit non-finite ILR centered-log reconstruction A single MAX coordinate underflows a simplex weight. Opposite-signed MAX coordinates overflow a CLR entry so the nightly branch gate covers !maximum.is_finite(). Co-authored-by: Seongho Bae --- crates/topic_measurement/src/coordinates.rs | 2 +- crates/topic_measurement/tests/ilr_recovery_contract.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 1ab1306f..bcc74ce7 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -210,7 +210,7 @@ mod tests { Err(TopicMeasurementError::InvalidLogRatioDimension) ); assert_eq!( - from_isometric_log_ratio(&[f64::MAX]), + from_isometric_log_ratio(&[-f64::MAX, f64::MAX]), Err(TopicMeasurementError::InvalidLogRatioDimension) ); } diff --git a/crates/topic_measurement/tests/ilr_recovery_contract.rs b/crates/topic_measurement/tests/ilr_recovery_contract.rs index b8606be9..cc2815ca 100644 --- a/crates/topic_measurement/tests/ilr_recovery_contract.rs +++ b/crates/topic_measurement/tests/ilr_recovery_contract.rs @@ -84,7 +84,7 @@ fn large_finite_ilr_coordinates_round_trip_or_fail_closed() { "ILR inverse must not return a zero simplex part after underflow" ); assert_eq!( - from_isometric_log_ratio(&[f64::MAX]), + from_isometric_log_ratio(&[-f64::MAX, f64::MAX]), Err(TopicMeasurementError::InvalidLogRatioDimension), "overflowing CLR reconstruction must fail closed" ); From 30c1d1f7a58061c8ed0af58deaf72e2ac6a84337 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 00:30:55 +0000 Subject: [PATCH 16/38] test(topic): recover three-part sequential ILR in unit tests Cover the remaining-parts>1 Egozcue step inside the crate unit instantiation so the same closed-form (2,3,1)/6 ILR is recovered without relying only on the integration crate. Co-authored-by: Seongho Bae --- crates/topic_measurement/src/coordinates.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index bcc74ce7..3c8d35ff 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -213,5 +213,9 @@ mod tests { from_isometric_log_ratio(&[-f64::MAX, f64::MAX]), Err(TopicMeasurementError::InvalidLogRatioDimension) ); + let three = isometric_log_ratio(&[2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]).expect("ilr three"); + assert!((three[1] - (0.5_f64).sqrt() * 3.0_f64.ln()).abs() < 1e-15); + let recovered_three = from_isometric_log_ratio(&three).expect("ilr three inverse"); + assert!((recovered_three.iter().sum::() - 1.0).abs() < 1e-15); } } From 2f08cdd235fa0d48b34bce9e1c3f5b8fa5e2ba83 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 01:48:53 +0000 Subject: [PATCH 17/38] docs(adr): align 0010 and 0013 maturity with merged main Match the ADR index to each source Implementation maturity after #47 and #44 landed: tepp_api orchestration is implemented-main, restore integrity revalidation is implemented-main, and remaining physical ERD/backup stays accepted-target. Co-authored-by: Seongho Bae --- docs/adr/0010-adaptive-llm-orchestration.md | 2 +- ...itemporal-persistence-reproducibility-and-split-authority.md | 2 +- docs/adr/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/adr/0010-adaptive-llm-orchestration.md b/docs/adr/0010-adaptive-llm-orchestration.md index d04bd6bc..0f367a19 100644 --- a/docs/adr/0010-adaptive-llm-orchestration.md +++ b/docs/adr/0010-adaptive-llm-orchestration.md @@ -1,7 +1,7 @@ # ADR 0010 — Adaptive LLM orchestration and test-time compute **Decision status:** Accepted -**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, and credential-free contextual-orchestrator binding are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target +**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, and credential-free contextual-orchestrator binding are implemented-main; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target **Date:** 2026-08-10 **Supersedes:** The LLM orchestration-selection/ablation clauses previously co-located in ADR 0006. ADR 0006 remains authoritative for GPU/VRAM and model-credential separation; ADR 0015 governs autonomous repository-write/review/merge authority. diff --git a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md index a593ddb1..92ae801f 100644 --- a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md +++ b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md @@ -1,7 +1,7 @@ # ADR 0013 — Bitemporal persistence, reproducibility manifests, and split authority **Decision status:** Accepted -**Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` open/execute driver, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, and concurrent document-write stress implemented-main; backup/restore integrity revalidation on the active PR +**Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` open/execute driver, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, and concurrent document-write stress implemented-main; backup/restore integrity revalidation implemented-main; remaining physical ERD/backup accepted-target **Date:** 2026-08-12 **Supersedes:** None; complements ADR 0002 (temporal semantics), ADR 0008 (evidence identity), and ADR 0011 (service ownership). diff --git a/docs/adr/README.md b/docs/adr/README.md index 9d549d0d..ba45bcd6 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -18,7 +18,7 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding implemented-main; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; no direct cross-service application-table coupling. | | [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR, sequential Egozcue ILR, and lexical-weight refusal are on the active PR; temporal topic identity, STM backend, method effects, and K-selection remain accepted-target. | -| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold implemented-main; remaining physical ERD/backup accepted-target. | +| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; backup/restore integrity revalidation implemented-main; `0007` retention/deletion/legal-hold implemented-main; remaining physical ERD/backup accepted-target. | | [0014](0014-scientific-claim-promotion-and-release-evidence.md) | Scientific claim promotion and release evidence authority | Accepted | partial | Separates design, implementation, scientific/product claim, and release authority; repository SBOM/provenance generator implemented, full release bundle remaining. | | [0015](0015-autonomous-development-review-and-merge-authority.md) | Autonomous development, review, and merge authority separation | Accepted | active-PR | Separates model proposal, deterministic verification, publication, independent review, and merge/release authority. | | [0016](0016-tdt-chronos-event-intelligence-boundary.md) | TDT, CHRONOS, and Event Ontology intelligence boundary | Accepted | accepted-target | Separates observed evidence, detection/tracking, prediction/schema inference, temporal consistency, and promoted transition authority. | From 6fe6a7853f06d8418701708ca3bf96f3f56c9b15 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 19:39:49 +0900 Subject: [PATCH 18/38] chore(docs): remove trailing whitespace from ADR maturity lines --- docs/adr/0010-adaptive-llm-orchestration.md | 2 +- .../0012-temporal-relational-shared-latent-topic-measurement.md | 2 +- ...itemporal-persistence-reproducibility-and-split-authority.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/adr/0010-adaptive-llm-orchestration.md b/docs/adr/0010-adaptive-llm-orchestration.md index 0f367a19..b8a0e17f 100644 --- a/docs/adr/0010-adaptive-llm-orchestration.md +++ b/docs/adr/0010-adaptive-llm-orchestration.md @@ -1,7 +1,7 @@ # ADR 0010 — Adaptive LLM orchestration and test-time compute **Decision status:** Accepted -**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, and credential-free contextual-orchestrator binding are implemented-main; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target +**Implementation maturity:** partial — `tepp_api` governed router, comparable-budget ablation record, and credential-free contextual-orchestrator binding are implemented-main; live NIM execution, learned conductor calibration, and production ablation evidence remain accepted-target **Date:** 2026-08-10 **Supersedes:** The LLM orchestration-selection/ablation clauses previously co-located in ADR 0006. ADR 0006 remains authoritative for GPU/VRAM and model-credential separation; ADR 0015 governs autonomous repository-write/review/merge authority. diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index 23cd4162..40f338ba 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -1,7 +1,7 @@ # ADR 0012 — Temporal Relational Shared-Latent Topic Measurement **Decision status:** Accepted -**Implementation maturity:** partial — logistic-normal additive log-ratio coordinates, sequential Egozcue isometric log-ratio coordinates, and lexical-weight refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; temporal topic identity, backend STM estimator, method-effect model, and K-selection remain accepted-target +**Implementation maturity:** partial — logistic-normal additive log-ratio coordinates, sequential Egozcue isometric log-ratio coordinates, and lexical-weight refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; temporal topic identity, backend STM estimator, method-effect model, and K-selection remain accepted-target **Date:** 2026-08-12 **Supersedes:** None; refines ADR 0004 and ADR 0005 without replacing their multilingual and psychometric authorities. diff --git a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md index 92ae801f..7f3706e5 100644 --- a/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md +++ b/docs/adr/0013-bitemporal-persistence-reproducibility-and-split-authority.md @@ -1,7 +1,7 @@ # ADR 0013 — Bitemporal persistence, reproducibility manifests, and split authority **Decision status:** Accepted -**Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` open/execute driver, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, and concurrent document-write stress implemented-main; backup/restore integrity revalidation implemented-main; remaining physical ERD/backup accepted-target +**Implementation maturity:** partial — migration contracts, cutoff eligibility, in-memory bitemporal adapters, live SQL session/migration port, document SQL contracts, `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` open/execute driver, exact-head live PostgreSQL CI, tenant RLS (`tepp_app_runtime` + session GUC), append-only reproducibility-manifest SQL insert/lookup, model-run / model-artifact / corpus-split-manifest chain (migration `0003`), append-only immutability triggers (migration `0004`), temporal interval ordering CHECK constraints (migration `0005`), typed membership-assignment storage (migration `0006`), event-relation/mention/instance SQL, source-artifact SQL, audit-event action-code validation, and concurrent document-write stress implemented-main; backup/restore integrity revalidation implemented-main; remaining physical ERD/backup accepted-target **Date:** 2026-08-12 **Supersedes:** None; complements ADR 0002 (temporal semantics), ADR 0008 (evidence identity), and ADR 0011 (service ownership). From d1f3dc3d52b09452b5799281e54f88f723be74c1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 19:53:26 +0900 Subject: [PATCH 19/38] fix(coverage): ignore Rust multiline string fragments --- CHANGELOG.md | 1 + scripts/check_coverage.py | 25 ++++++++-- tests/quality/test_check_coverage.py | 69 ++++++++++++++++++++-------- 3 files changed, 72 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ad9e324..da1fc8a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added - `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, Aitchison-distance ILR isometry, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). +- Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index 50234635..fb7ecc61 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -51,9 +51,10 @@ def is_executable_source_line( """Return whether *line_number* in *source_path* is an executable source line. LLVM LCOV sometimes emits zero-count DA records for documentation comments, - attributes, pure structural braces, multi-line signatures, and in-file - ``#[cfg(test)]`` modules. Those records are not evidence of uncovered - production behavior and are excluded from the authored-line gate. + attributes, pure structural braces, multi-line signatures, Rust multiline + string continuations, and in-file ``#[cfg(test)]`` modules. Those records + are not evidence of uncovered production behavior and are excluded from the + authored-line gate. When *repository_root* is provided, *source_path* must resolve under that root (same fail-closed rule as LCOV ``SF:`` loading). @@ -75,6 +76,8 @@ def is_executable_source_line( return False if _line_in_cfg_not_feature_block(lines, line_number): return False + if _line_in_multiline_string_literal(lines, line_number): + return False text = lines[line_number - 1].strip() if not text: return False @@ -82,7 +85,9 @@ def is_executable_source_line( return False if text.startswith("#[") or text.startswith("#!["): return False - if text in {"{", "}", "},", ");", "];", "();", "};"}: + if text in {"{", "}", "},", ")", ");", "];", "();", "};"}: + return False + if text.startswith('"') or text.startswith("} else"): return False if text.startswith("use ") or text.startswith("pub use "): return False @@ -164,6 +169,18 @@ def _line_in_cfg_not_feature_block(lines: list[str], line_number: int) -> bool: return False +def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> bool: + """Return whether a line is inside a Rust normal-string continuation.""" + in_string = False + for index, raw in enumerate(lines, start=1): + if in_string and index == line_number: + return True + quote_count = raw.replace('\\"', "").count('"') + if quote_count % 2: + in_string = not in_string + return False + + def load_lcov_line_totals( path: Path, repository_root: Path | None = None ) -> Mapping[str, Any]: diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index a4337397..2c122e2f 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -305,22 +305,25 @@ def test_executable_source_line_filters_noise_records(self) -> None: "})", # 39 " return value,", # 40 executable (return keeps it) " x + 1,", # 41 trailing comma noise - '#[cfg(feature = "live-sqlx")]', # 42 cfg attr - "fn live_path() {", # 43 fn - " live_body();", # 44 executable active feature body - "}", # 45 brace - '#[cfg(not(feature = "live-sqlx"))]', # 46 not-feature attr - "fn offline_path() {", # 47 inside not-feature - " offline_body();", # 48 inside not-feature - "}", # 49 inside not-feature close - "#[cfg(test)]", # 50 - "mod tests {", # 51 cfg(test) mod - " #[test]", # 52 inside test mod - " fn unit() {", # 53 inside test mod - " assert_eq!(1, 1);", # 54 inside test mod - " }", # 55 - "}", # 56 - " executable_statement();", # 57 executable + '"standalone string literal",', # 42 string noise + "} else {", # 43 structural branch noise + ")", # 44 structural close noise + '#[cfg(feature = "live-sqlx")]', # 45 cfg attr + "fn live_path() {", # 46 fn + " live_body();", # 47 executable active feature body + "}", # 48 brace + '#[cfg(not(feature = "live-sqlx"))]', # 49 not-feature attr + "fn offline_path() {", # 50 inside not-feature + " offline_body();", # 51 inside not-feature + "}", # 52 inside not-feature close + "#[cfg(test)]", # 53 + "mod tests {", # 54 cfg(test) mod + " #[test]", # 55 inside test mod + " fn unit() {", # 56 inside test mod + " assert_eq!(1, 1);", # 57 inside test mod + " }", # 58 + "}", # 59 + " executable_statement();", # 60 executable ] source.write_text("\n".join(source_lines) + "\n", encoding="utf-8") path = str(source) @@ -335,7 +338,7 @@ def test_executable_source_line_filters_noise_records(self) -> None: coverage_contract.is_executable_source_line(path, len(source_lines) + 5) ) - expected_executable = {13, 40, 44, 57} + expected_executable = {13, 40, 47, 60} for line_number in range(1, len(source_lines) + 1): is_exec = coverage_contract.is_executable_source_line(path, line_number) if line_number in expected_executable: @@ -354,10 +357,10 @@ def test_executable_source_line_filters_noise_records(self) -> None: "\n".join( [ f"SF:{path}", - "DA:57,1", + "DA:60,1", "DA:1,0", "DA:2,0", - "DA:48,0", + "DA:51,0", "end_of_record", "", ] @@ -427,6 +430,7 @@ def test_cfg_test_and_not_feature_block_helpers(self) -> None: self.assertTrue(coverage_contract._line_in_cfg_not_feature_block(lines, 9)) self.assertFalse(coverage_contract._line_in_cfg_not_feature_block(lines, 1)) self.assertFalse(coverage_contract._line_in_cfg_not_feature_block(lines, 11)) + open_only = [ '#[cfg(not(feature = "x"))]', "fn unfinished()", @@ -474,6 +478,33 @@ def test_cfg_test_and_not_feature_block_helpers(self) -> None: coverage_contract._line_in_cfg_not_feature_block(unclosed_not_feature, 99) ) + def test_multiline_string_continuations_are_not_authored_lines(self) -> None: + """Rust multiline string fragments are excluded from authored coverage.""" + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "query.rs" + source.write_text( + 'fn query() {\n' + ' let sql = format!("SELECT id \\\n' + ' FROM document_record \\\n' + ' WHERE tenant_record_id = \'x\'");\n' + ' execute(sql);\n' + '}\n', + encoding="utf-8", + ) + self.assertTrue( + coverage_contract.is_executable_source_line(str(source), 2) + ) + self.assertFalse( + coverage_contract.is_executable_source_line(str(source), 3) + ) + self.assertFalse( + coverage_contract.is_executable_source_line(str(source), 4) + ) + self.assertTrue( + coverage_contract.is_executable_source_line(str(source), 5) + ) + if __name__ == "__main__": # pragma: no cover unittest.main() From 4b54dc82b1685bfc40f2c563f836e5228976870c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 13:15:56 +0900 Subject: [PATCH 20/38] test(topic): assert direct Aitchison distance recovery --- .../topic_measurement/tests/ilr_recovery_contract.rs | 11 +++++++++-- docs/research/topic-logratio-coordinates.md | 12 ++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/topic_measurement/tests/ilr_recovery_contract.rs b/crates/topic_measurement/tests/ilr_recovery_contract.rs index cc2815ca..12817f6e 100644 --- a/crates/topic_measurement/tests/ilr_recovery_contract.rs +++ b/crates/topic_measurement/tests/ilr_recovery_contract.rs @@ -60,8 +60,15 @@ fn equal_shares_are_the_ilr_origin_and_preserve_aitchison_distance() { let unbalanced = [0.8, 0.2]; let coordinates = isometric_log_ratio(&unbalanced).expect("pair"); - let expected = (0.5_f64).sqrt() * 4.0_f64.ln(); - assert!((coordinates[0] - expected).abs() < 1e-15); + let direct_aitchison_distance = (0.5_f64).sqrt() + * ((unbalanced[0] / unbalanced[1]).ln() - (halves[0] / halves[1]).ln()).abs(); + let ilr_euclidean_distance = coordinates + .iter() + .zip(&origin) + .map(|(left, right)| (left - right).powi(2)) + .sum::() + .sqrt(); + assert!((ilr_euclidean_distance - direct_aitchison_distance).abs() < 1e-15); let recovered = from_isometric_log_ratio(&coordinates).expect("inverse"); assert!(rmse(&unbalanced, &recovered) < 1e-15); diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index ac5c242d..8ea997a5 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -4,11 +4,11 @@ This note doctors the first `topic_measurement` production slice (ADR 0012): -1. raw topic proportions are compositional rather than unconstrained Euclidean indicators; -2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models; -3. ALR is full rank but not an orthonormal Aitchison-distance isometry; -4. sequential Egozcue ILR supplies the orthonormal Aitchison-distance isometry when that Euclidean geometry is the estimand; -5. max-shifted inverses recover representable extreme coordinates without overflow; +1. raw topic proportions are compositional rather than unconstrained Euclidean indicators (Aitchison, 1982); +2. additive log-ratio coordinates implement the reference-dependent logistic-normal map used by correlated topic models (Aitchison & Shen, 1980; Blei & Lafferty, 2007); +3. ALR is full rank but not an orthonormal Aitchison-distance isometry (Aitchison, 1982); +4. sequential Egozcue ILR supplies the orthonormal Aitchison-distance isometry when that Euclidean geometry is the estimand (Egozcue et al., 2003); +5. max-shifted inverses recover representable extreme coordinates without overflow (Aitchison & Shen, 1980); 6. TF-IDF, BM25, and keyword scores are refused as inferential coordinates. The temporal STM backend, global topic identity, method-effect model, and K-selection remain accepted-target. No database migration is allocated. @@ -33,6 +33,6 @@ Aitchison and Shen (1980) define the logistic-normal family via the additive log - representable ALR coordinates `(710, 709)` and representable ILR coordinates round-trip through max-shifted inverses without exponential overflow; - extremes that would underflow a strictly positive `f64` simplex part fail closed; - equal shares map to the ALR and ILR origins; -- two-part ILR preserves Aitchison distance `√(1/2) ln(0.8/0.2)` for `(0.8, 0.2)`; +- two-part ILR preserves Aitchison distance `√(1/2) |ln(0.8/0.2) - ln(0.5/0.5)|` between `(0.8, 0.2)` and `(0.5, 0.5)` (Egozcue et al., 2003); - zero, negative, non-unit-sum, non-finite, empty, and one-part vectors fail closed; - `tfidf`, `bm25`, and `keyword` labels are refused. From 543982cec92585dbbb601b33f4886432b8e1eacd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 13:24:52 +0900 Subject: [PATCH 21/38] fix(coverage): parse Rust strings and comments statefully --- scripts/check_coverage.py | 50 +++++++++++++++++++++++++--- tests/quality/test_check_coverage.py | 42 +++++++++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index fb7ecc61..87e27755 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -87,7 +87,7 @@ def is_executable_source_line( return False if text in {"{", "}", "},", ")", ");", "];", "();", "};"}: return False - if text.startswith('"') or text.startswith("} else"): + if _is_standalone_string_literal(text) or text.startswith("} else"): return False if text.startswith("use ") or text.startswith("pub use "): return False @@ -172,12 +172,54 @@ def _line_in_cfg_not_feature_block(lines: list[str], line_number: int) -> bool: def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> bool: """Return whether a line is inside a Rust normal-string continuation.""" in_string = False + in_block_comment = False for index, raw in enumerate(lines, start=1): if in_string and index == line_number: return True - quote_count = raw.replace('\\"', "").count('"') - if quote_count % 2: - in_string = not in_string + escaped = False + cursor = 0 + while cursor < len(raw): + if in_block_comment: + if raw.startswith("*/", cursor): + in_block_comment = False + cursor += 2 + else: + cursor += 1 + continue + if in_string: + character = raw[cursor] + if character == '"' and not escaped: + in_string = False + if character == "\\": + escaped = not escaped + else: + escaped = False + cursor += 1 + continue + if raw.startswith("//", cursor): + break + if raw.startswith("/*", cursor): + in_block_comment = True + cursor += 2 + continue + if raw[cursor] == '"': + in_string = True + cursor += 1 + return False + + +def _is_standalone_string_literal(text: str) -> bool: + """Return whether *text* is only a normal string literal and punctuation.""" + if not text.startswith('"'): + return False + escaped = False + for index, character in enumerate(text[1:], start=1): + if character == '"' and not escaped: + return text[index + 1 :].strip() in {"", ",", ";"} + if character == "\\": + escaped = not escaped + else: + escaped = False return False diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index 2c122e2f..831c44ee 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -505,6 +505,48 @@ def test_multiline_string_continuations_are_not_authored_lines(self) -> None: coverage_contract.is_executable_source_line(str(source), 5) ) + def test_string_scanner_handles_comments_backslash_parity_and_methods(self) -> None: + """Quoted comments and escaped delimiters do not corrupt source classification.""" + + with tempfile.TemporaryDirectory() as temporary: + backslash = "\\" + source = Path(temporary) / "scanner.rs" + source_lines = [ + "fn query() {", + f' let sql = "SELECT id {backslash}', + ' FROM document";', + r' // comment contains one " quote', + " execute(sql);", + f' let even = "ends with two slashes {backslash * 2}";', + " execute(even);", + r' "literal".to_string();', + "}", + ] + source.write_text("\n".join(source_lines) + "\n", encoding="utf-8") + path = str(source) + + self.assertFalse(coverage_contract.is_executable_source_line(path, 3)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 5)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 7)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 8)) + + block_comment = [ + "/* comment starts", + r' comment has a " quote', + " still comment", + "*/", + "execute();", + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(block_comment, 5) + ) + self.assertTrue( + coverage_contract._is_standalone_string_literal(r'"escaped\\",') + ) + self.assertFalse( + coverage_contract._is_standalone_string_literal('"unfinished') + ) + if __name__ == "__main__": # pragma: no cover unittest.main() From 621cae0787b6fcfcd58c192a120b2579d1caf1cf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 13:25:15 +0900 Subject: [PATCH 22/38] docs: record stateful coverage parser repair --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index da1fc8a4..312822f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang - `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, Aitchison-distance ILR isometry, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. +- Coverage source classification now scans Rust strings and comments with escape-parity state, preserving executable string method calls and ignoring quoted comments. - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). From 8e88b337e0bb1a44749b0fe48c635e3e3e7535da Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 13:38:42 +0900 Subject: [PATCH 23/38] docs(topic): define ILR pairwise distance --- crates/topic_measurement/src/coordinates.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 3c8d35ff..0ab392d9 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -71,10 +71,11 @@ pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMea /// Map a strictly positive unit simplex vector to isometric log-ratio coordinates. /// /// The sequential Egozcue orthonormal basis sends a `K`-part composition to -/// the `K-1` vector whose Euclidean norm equals Aitchison distance. This is -/// the coordinate system for distance-based topic geometry. It is not the -/// reference-dependent logistic-normal map; use [`additive_log_ratio`] when -/// that regression interface is the estimand. +/// the `K-1` vector whose Euclidean distance from another composition's ILR +/// vector equals their Aitchison distance. A vector norm is only the distance +/// from the equal-share origin. This is the coordinate system for distance-based +/// topic geometry. It is not the reference-dependent logistic-normal map; use +/// [`additive_log_ratio`] when that regression interface is the estimand. /// /// # Errors /// From f352e15d5fb240d700decd1bf8959c891811caeb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 01:27:41 +0900 Subject: [PATCH 24/38] docs: trace logratio claims to sources --- docs/research/topic-logratio-coordinates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index 8ea997a5..4faf429b 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -25,7 +25,7 @@ Egozcue, J. J., Pawlowsky-Glahn, V., Mateu-Figueras, G., & Barceló-Vidal, C. (2 ## Application -Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Egozcue et al. (2003) construct the sequential orthonormal ILR basis whose Euclidean norm equals Aitchison distance; `isometric_log_ratio` implements that basis and `from_isometric_log_ratio` inverts it through a max-shifted centered-log-ratio reconstruction. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR rather than ALR. `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation. The forward ALR map subtracts logarithms rather than forming a potentially overflowing ratio. Both inverses fail closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980; Aitchison, 1982; Blei & Lafferty, 2007; Egozcue et al., 2003). +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Egozcue et al. (2003) construct the sequential orthonormal ILR basis whose Euclidean norm equals Aitchison distance; `isometric_log_ratio` implements that basis and `from_isometric_log_ratio` inverts it through a max-shifted centered-log-ratio reconstruction. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR rather than ALR (Egozcue et al., 2003). `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation (Aitchison & Shen, 1980). The forward ALR map subtracts logarithms rather than forming a potentially overflowing ratio (Aitchison & Shen, 1980; Blei & Lafferty, 2007). Both inverses fail closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980). ## Verification From ff417148769fa2980ed6f43d767db837d731fda5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 01:42:54 +0900 Subject: [PATCH 25/38] fix(quality): parse Rust literal state in coverage gate --- scripts/check_coverage.py | 74 +++++++++++++++++++++++++--- tests/quality/test_check_coverage.py | 44 +++++++++++++++++ 2 files changed, 110 insertions(+), 8 deletions(-) diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index 87e27755..6f30a8ab 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -170,13 +170,19 @@ def _line_in_cfg_not_feature_block(lines: list[str], line_number: int) -> bool: def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> bool: - """Return whether a line is inside a Rust normal-string continuation.""" + """Return whether a line is inside a Rust string continuation. + + The scanner tracks normal strings, raw strings, block comments, and character + literals so quotes in comments or literal contents cannot change the state of + a later source line. + """ + in_string = False + raw_hashes: int | None = None in_block_comment = False for index, raw in enumerate(lines, start=1): - if in_string and index == line_number: + if (in_string or raw_hashes is not None) and index == line_number: return True - escaped = False cursor = 0 while cursor < len(raw): if in_block_comment: @@ -186,15 +192,24 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo else: cursor += 1 continue + if raw_hashes is not None: + delimiter = '"' + ("#" * raw_hashes) + closing = raw.find(delimiter, cursor) + if closing == -1: + cursor = len(raw) + else: + raw_hashes = None + cursor = closing + len(delimiter) + continue if in_string: character = raw[cursor] - if character == '"' and not escaped: - in_string = False if character == "\\": - escaped = not escaped + cursor += 2 + elif character == '"': + in_string = False + cursor += 1 else: - escaped = False - cursor += 1 + cursor += 1 continue if raw.startswith("//", cursor): break @@ -202,12 +217,55 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo in_block_comment = True cursor += 2 continue + raw_start = _raw_string_start(raw, cursor) + if raw_start is not None: + raw_hashes, cursor = raw_start + continue if raw[cursor] == '"': in_string = True + cursor += 1 + continue + if raw[cursor] == "'": + character_end = _character_literal_end(raw, cursor) + if character_end is not None: + cursor = character_end + continue cursor += 1 return False +def _raw_string_start(line: str, cursor: int) -> tuple[int, int] | None: + """Return ``(hash_count, next_cursor)`` for a Rust raw-string opener.""" + + if line.startswith("br", cursor): + prefix_end = cursor + 2 + elif line.startswith("r", cursor): + prefix_end = cursor + 1 + else: + return None + hash_end = prefix_end + while hash_end < len(line) and line[hash_end] == "#": + hash_end += 1 + if hash_end < len(line) and line[hash_end] == '"': + return hash_end - prefix_end, hash_end + 1 + return None + + +def _character_literal_end(line: str, cursor: int) -> int | None: + """Return the cursor after a one-line Rust character literal, if present.""" + + candidate = cursor + 1 + if candidate >= len(line): + return None + if line[candidate] == "\\": + candidate += 2 + else: + candidate += 1 + if candidate < len(line) and line[candidate] == "'": + return candidate + 1 + return None + + def _is_standalone_string_literal(text: str) -> bool: """Return whether *text* is only a normal string literal and punctuation.""" if not text.startswith('"'): diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index 831c44ee..4a62b2c9 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -547,6 +547,50 @@ def test_string_scanner_handles_comments_backslash_parity_and_methods(self) -> N coverage_contract._is_standalone_string_literal('"unfinished') ) + raw_string = [ + ' let text = r##"', + ' a " quote in raw text', + ' "##;', + ' execute(text);', + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(raw_string, 1) + ) + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(raw_string, 2) + ) + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(raw_string, 3) + ) + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(raw_string, 4) + ) + + byte_raw_string = [ + ' let bytes = br#"', + ' raw bytes', + ' "#;', + ] + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(byte_raw_string, 2) + ) + + character_and_lifetime = [ + "fn query<'a>() {", + " let quote: char = '\"';", + " execute();", + "}", + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal( + character_and_lifetime, 3 + ) + ) + self.assertIsNone(coverage_contract._character_literal_end("'", 0)) + self.assertEqual( + coverage_contract._character_literal_end(r"'\''", 0), 4 + ) + if __name__ == "__main__": # pragma: no cover unittest.main() From 45224e55792165fd2d752d9a3bd62b016ec7c970 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 01:45:37 +0900 Subject: [PATCH 26/38] docs: record Rust literal coverage hardening --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 312822f9..ff44779f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang - `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, Aitchison-distance ILR isometry, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. -- Coverage source classification now scans Rust strings and comments with escape-parity state, preserving executable string method calls and ignoring quoted comments. +- Coverage source classification now scans Rust normal/raw/byte strings, comments, and character literals with escape-aware state, preserving executable string method calls and ignoring quoted comments. - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). From 73403994a3fb386126e8276b7ab5160355063341 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 13:15:02 +0900 Subject: [PATCH 27/38] docs(adr): align maturity index with source decisions --- docs/adr/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/adr/README.md b/docs/adr/README.md index ba45bcd6..63c0d033 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -14,8 +14,8 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0006](0006-vram-gpu-nvidia-orchestration.md) | VRAM-adaptive GPU compute and model-credential boundary | Accepted | accepted-target | LLM orchestration policy superseded by ADR 0010; autonomous development authority governed by ADR 0015. | | [0007](0007-rust-workspace-quality-gates.md) | Explicit Rust workspace, pinned toolchains, and exact quality gates | Accepted | implemented-main | ADR 0014 governs scientific/product claim promotion beyond repository-quality tooling. | | [0008](0008-immutable-evidence-identities-digests-and-spans.md) | Immutable evidence identities, `SHA-256` digests, exact spans, and strict wire reconstruction | Accepted | implemented-main | ADR 0013 governs future persistence/reproducibility/split authority. | -| [0009](0009-purpose-bound-pii-governance.md) | Purpose-bound PII governance without blanket masking | Accepted | partial | Persistence retention/deletion/legal-hold (`0007`) and provider-payload minimization implemented-main; deployment evidence remains accepted-target. | -| [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding implemented-main; live NIM execution and production ablation evidence remain accepted-target. | +| [0009](0009-purpose-bound-pii-governance.md) | Purpose-bound PII governance without blanket masking | Accepted | partial | Persistence retention/deletion/legal-hold (`0007`) is implemented-main; provider-payload minimization remains on the active PR until exact-head checks, review, and protected-main integration; deployment evidence remains accepted-target. | +| [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding remain on the active PR until exact-head checks, review, and protected-main integration; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; no direct cross-service application-table coupling. | | [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR, sequential Egozcue ILR, and lexical-weight refusal are on the active PR; temporal topic identity, STM backend, method effects, and K-selection remain accepted-target. | | [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; backup/restore integrity revalidation implemented-main; `0007` retention/deletion/legal-hold implemented-main; remaining physical ERD/backup accepted-target. | From 26a9fde204a1160e4bd5b9e5639c5df77bdd3bf5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 21:49:23 +0900 Subject: [PATCH 28/38] feat(topic): recover pairwise Aitchison distance from ILR Add a shipped CLR Aitchison distance and prove ILR Euclidean isometry on two non-origin compositions. ALR Euclidean is not accepted as that distance. True-parameter ALR RMSE remains a separate recovery check. --- CHANGELOG.md | 2 +- crates/topic_measurement/src/coordinates.rs | 51 +++++++++++++++- crates/topic_measurement/src/lib.rs | 5 +- .../tests/ilr_recovery_contract.rs | 61 +++++++++++++++++-- docs/research/topic-logratio-coordinates.md | 2 +- 5 files changed, 111 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff79e6fc..47459edc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added -- `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, Aitchison-distance ILR isometry, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). +- `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, pairwise CLR Aitchison distance recovered by ILR Euclidean isometry away from the equal-share origin, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. - Coverage source classification now scans Rust normal/raw/byte strings, comments, and character literals with escape-aware state, preserving executable string method calls and ignoring quoted comments. - `tepp_api` naruon live loopback HTTP/1.1 listener: `serve_one` installs a read/write deadline, requires a loopback `Host`, refuses `Transfer-Encoding` and NIM/proxy credential headers, parses `knowledge_cutoff` as RFC 3339 and refuses a future cutoff, keys analysis-run idempotency by tenant plus key, and proves both analysis-run and export POSTs over a real `TcpStream`. Not a production TLS/`$PORT` service (ADR 0011). diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index 0ab392d9..ba1d6a99 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -153,6 +153,41 @@ pub fn from_isometric_log_ratio(coordinates: &[f64]) -> Result, TopicMe Ok(weights.iter().map(|weight| weight / denominator).collect()) } +/// Aitchison distance between two strictly positive unit simplex vectors. +/// +/// The distance is the Euclidean norm of the clr residual +/// `clr(x) − clr(y)`. Sequential ILR is an isometry for this distance, so +/// `‖ilr(x) − ilr(y)‖` must recover the same value. A single ILR vector +/// norm is only the distance from the equal-share origin. +/// +/// # Errors +/// +/// Returns [`TopicMeasurementError::InvalidComposition`] when the vectors +/// have unequal length or fail simplex validation. +pub fn aitchison_distance(left: &[f64], right: &[f64]) -> Result { + if left.len() != right.len() { + return Err(TopicMeasurementError::InvalidComposition); + } + require_composition(left)?; + require_composition(right)?; + #[allow(clippy::cast_precision_loss)] + let parts = left.len() as f64; + let mut left_log_sum = 0.0_f64; + let mut right_log_sum = 0.0_f64; + for index in 0..left.len() { + left_log_sum += left[index].ln(); + right_log_sum += right[index].ln(); + } + let left_mean = left_log_sum / parts; + let right_mean = right_log_sum / parts; + let mut square_sum = 0.0_f64; + for index in 0..left.len() { + let residual = (left[index].ln() - left_mean) - (right[index].ln() - right_mean); + square_sum += residual * residual; + } + Ok(square_sum.sqrt()) +} + fn require_composition(proportions: &[f64]) -> Result { if proportions.len() < 2 { return Err(TopicMeasurementError::InvalidComposition); @@ -181,7 +216,8 @@ fn require_composition(proportions: &[f64]) -> Result() - 1.0).abs() < 1e-15); + assert!(aitchison_distance(&[0.5, 0.5], &[0.5, 0.5]).expect("self") < 1e-15); + assert_eq!( + aitchison_distance(&[0.5, 0.5], &[1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + aitchison_distance(&[0.0, 1.0], &[0.5, 0.5]), + Err(TopicMeasurementError::InvalidComposition) + ); + assert_eq!( + aitchison_distance(&[0.5, 0.5], &[0.0, 1.0]), + Err(TopicMeasurementError::InvalidComposition) + ); } } diff --git a/crates/topic_measurement/src/lib.rs b/crates/topic_measurement/src/lib.rs index 0fff8c73..26c8d82b 100644 --- a/crates/topic_measurement/src/lib.rs +++ b/crates/topic_measurement/src/lib.rs @@ -6,7 +6,8 @@ //! indicators. ALR supplies a reference-dependent full-rank logistic-normal map //! for regression and psychometric interfaces; it is not an orthonormal //! Aitchison-distance isometry. Distance-based Aitchison geometry uses the -//! sequential Egozcue ILR basis. TF-IDF, BM25, and keyword scores remain +//! sequential Egozcue ILR basis, whose pairwise Euclidean distance recovers +//! CLR Aitchison distance. TF-IDF, BM25, and keyword scores remain //! forbidden inferential coordinates. mod coordinates; @@ -15,6 +16,8 @@ mod lexical; /// Additive log-ratio map from a simplex vector. pub use coordinates::additive_log_ratio; +/// Aitchison distance between two simplex vectors. +pub use coordinates::aitchison_distance; /// Inverse additive log-ratio map back to the simplex. pub use coordinates::from_additive_log_ratio; /// Inverse isometric log-ratio map back to the simplex. diff --git a/crates/topic_measurement/tests/ilr_recovery_contract.rs b/crates/topic_measurement/tests/ilr_recovery_contract.rs index 12817f6e..9e1b58ca 100644 --- a/crates/topic_measurement/tests/ilr_recovery_contract.rs +++ b/crates/topic_measurement/tests/ilr_recovery_contract.rs @@ -2,9 +2,21 @@ #![allow(clippy::cast_precision_loss)] use topic_measurement::{ - TopicMeasurementError, additive_log_ratio, from_isometric_log_ratio, isometric_log_ratio, + additive_log_ratio, aitchison_distance, from_isometric_log_ratio, isometric_log_ratio, + TopicMeasurementError, }; +fn euclidean(left: &[f64], right: &[f64]) -> f64 { + left.iter() + .zip(right) + .map(|(left_value, right_value)| { + let residual = left_value - right_value; + residual * residual + }) + .sum::() + .sqrt() +} + fn rmse(truth: &[f64], recovered: &[f64]) -> f64 { let n = truth.len() as f64; let sum_sq: f64 = truth @@ -72,16 +84,53 @@ fn equal_shares_are_the_ilr_origin_and_preserve_aitchison_distance() { let recovered = from_isometric_log_ratio(&coordinates).expect("inverse"); assert!(rmse(&unbalanced, &recovered) < 1e-15); + assert!( + (aitchison_distance(&unbalanced, &halves).expect("clr") - direct_aitchison_distance).abs() + < 1e-15 + ); } #[test] -fn large_finite_ilr_coordinates_round_trip_or_fail_closed() { - let representable = from_isometric_log_ratio(&[40.0]).expect("representable"); +fn pairwise_ilr_euclidean_recovers_aitchison_distance_away_from_the_origin() { + let left = [0.70, 0.30]; + let right = [0.20, 0.80]; + let left_ilr = isometric_log_ratio(&left).expect("left"); + let right_ilr = isometric_log_ratio(&right).expect("right"); + let ilr_euclidean = euclidean(&left_ilr, &right_ilr); + let distance = aitchison_distance(&left, &right).expect("aitchison"); assert!( - representable - .iter() - .all(|part| part.is_finite() && *part > 0.0) + (ilr_euclidean - distance).abs() < 1e-15, + "two-part ILR Euclidean {ilr_euclidean} must equal Aitchison {distance}" ); + assert!((distance - aitchison_distance(&right, &left).expect("symmetric")).abs() < 1e-15); + assert!(aitchison_distance(&left, &left).expect("self") < 1e-15); + + let three_left = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; + let three_right = [1.0 / 6.0, 1.0 / 6.0, 4.0 / 6.0]; + let three_left_ilr = isometric_log_ratio(&three_left).expect("three left"); + let three_right_ilr = isometric_log_ratio(&three_right).expect("three right"); + let three_ilr = euclidean(&three_left_ilr, &three_right_ilr); + let three_distance = aitchison_distance(&three_left, &three_right).expect("three"); + assert!( + (three_ilr - three_distance).abs() < 1e-15, + "three-part ILR Euclidean {three_ilr} must equal Aitchison {three_distance}" + ); + + let three_left_alr = additive_log_ratio(&three_left).expect("alr left"); + let three_right_alr = additive_log_ratio(&three_right).expect("alr right"); + let alr_euclidean = euclidean(&three_left_alr, &three_right_alr); + assert!( + (alr_euclidean - three_distance).abs() > 1e-6, + "ALR Euclidean must not be treated as Aitchison distance" + ); +} + +#[test] +fn large_finite_ilr_coordinates_round_trip_or_fail_closed() { + let representable = from_isometric_log_ratio(&[40.0]).expect("representable"); + assert!(representable + .iter() + .all(|part| part.is_finite() && *part > 0.0)); let recovered = isometric_log_ratio(&representable).expect("forward"); assert!(rmse(&[40.0], &recovered) < 1e-12); diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index 4faf429b..a5310a2e 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -33,6 +33,6 @@ Aitchison and Shen (1980) define the logistic-normal family via the additive log - representable ALR coordinates `(710, 709)` and representable ILR coordinates round-trip through max-shifted inverses without exponential overflow; - extremes that would underflow a strictly positive `f64` simplex part fail closed; - equal shares map to the ALR and ILR origins; -- two-part ILR preserves Aitchison distance `√(1/2) |ln(0.8/0.2) - ln(0.5/0.5)|` between `(0.8, 0.2)` and `(0.5, 0.5)` (Egozcue et al., 2003); +- two-part ILR preserves Aitchison distance `√(1/2) |ln(0.8/0.2) - ln(0.5/0.5)|` between `(0.8, 0.2)` and `(0.5, 0.5)`, and pairwise ILR Euclidean distance recovers CLR Aitchison distance between two non-origin compositions (Egozcue et al., 2003); - zero, negative, non-unit-sum, non-finite, empty, and one-part vectors fail closed; - `tfidf`, `bm25`, and `keyword` labels are refused. From 578961ac929c5cc02190e7babb714129ad0a0bfb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 21:51:33 +0900 Subject: [PATCH 29/38] style(topic): rustfmt ILR recovery contract Match the CI rustfmt import order and assertion wrapping so the format/lint job can pass on the pairwise Aitchison head. --- .../topic_measurement/tests/ilr_recovery_contract.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/topic_measurement/tests/ilr_recovery_contract.rs b/crates/topic_measurement/tests/ilr_recovery_contract.rs index 9e1b58ca..469ac77b 100644 --- a/crates/topic_measurement/tests/ilr_recovery_contract.rs +++ b/crates/topic_measurement/tests/ilr_recovery_contract.rs @@ -2,8 +2,8 @@ #![allow(clippy::cast_precision_loss)] use topic_measurement::{ - additive_log_ratio, aitchison_distance, from_isometric_log_ratio, isometric_log_ratio, - TopicMeasurementError, + TopicMeasurementError, additive_log_ratio, aitchison_distance, from_isometric_log_ratio, + isometric_log_ratio, }; fn euclidean(left: &[f64], right: &[f64]) -> f64 { @@ -128,9 +128,11 @@ fn pairwise_ilr_euclidean_recovers_aitchison_distance_away_from_the_origin() { #[test] fn large_finite_ilr_coordinates_round_trip_or_fail_closed() { let representable = from_isometric_log_ratio(&[40.0]).expect("representable"); - assert!(representable - .iter() - .all(|part| part.is_finite() && *part > 0.0)); + assert!( + representable + .iter() + .all(|part| part.is_finite() && *part > 0.0) + ); let recovered = isometric_log_ratio(&representable).expect("forward"); assert!(rmse(&[40.0], &recovered) < 1e-12); From 1dd3953aa828c75e05c7b0df85641f8698d819c0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 21:55:10 +0900 Subject: [PATCH 30/38] fix(topic): satisfy clippy similar-names on ILR contract Rename three-part ILR and ALR bindings so -D warnings clippy can compile the pairwise Aitchison recovery test. --- .../topic_measurement/tests/ilr_recovery_contract.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/topic_measurement/tests/ilr_recovery_contract.rs b/crates/topic_measurement/tests/ilr_recovery_contract.rs index 469ac77b..a97dd153 100644 --- a/crates/topic_measurement/tests/ilr_recovery_contract.rs +++ b/crates/topic_measurement/tests/ilr_recovery_contract.rs @@ -107,18 +107,18 @@ fn pairwise_ilr_euclidean_recovers_aitchison_distance_away_from_the_origin() { let three_left = [2.0 / 6.0, 3.0 / 6.0, 1.0 / 6.0]; let three_right = [1.0 / 6.0, 1.0 / 6.0, 4.0 / 6.0]; - let three_left_ilr = isometric_log_ratio(&three_left).expect("three left"); - let three_right_ilr = isometric_log_ratio(&three_right).expect("three right"); - let three_ilr = euclidean(&three_left_ilr, &three_right_ilr); + let isometric_left = isometric_log_ratio(&three_left).expect("three left"); + let isometric_right = isometric_log_ratio(&three_right).expect("three right"); + let three_ilr = euclidean(&isometric_left, &isometric_right); let three_distance = aitchison_distance(&three_left, &three_right).expect("three"); assert!( (three_ilr - three_distance).abs() < 1e-15, "three-part ILR Euclidean {three_ilr} must equal Aitchison {three_distance}" ); - let three_left_alr = additive_log_ratio(&three_left).expect("alr left"); - let three_right_alr = additive_log_ratio(&three_right).expect("alr right"); - let alr_euclidean = euclidean(&three_left_alr, &three_right_alr); + let additive_left = additive_log_ratio(&three_left).expect("alr left"); + let additive_right = additive_log_ratio(&three_right).expect("alr right"); + let alr_euclidean = euclidean(&additive_left, &additive_right); assert!( (alr_euclidean - three_distance).abs() > 1e-6, "ALR Euclidean must not be treated as Aitchison distance" From b7214a8e9dc140e622124fc046d9aa5147dd7b27 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 22:04:37 +0900 Subject: [PATCH 31/38] fix(coverage): track nested Rust block-comment depth A boolean comment flag closed on the first */ and let quotes inside the outer comment start a string, hiding later executable DA rows. The scanner now increments and decrements comment depth, and the quality test drives execute() after a nested inner comment. --- scripts/check_coverage.py | 13 ++++++++----- tests/quality/test_check_coverage.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index 6f30a8ab..86c2a695 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -179,15 +179,18 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo in_string = False raw_hashes: int | None = None - in_block_comment = False + block_comment_depth = 0 for index, raw in enumerate(lines, start=1): if (in_string or raw_hashes is not None) and index == line_number: return True cursor = 0 while cursor < len(raw): - if in_block_comment: - if raw.startswith("*/", cursor): - in_block_comment = False + if block_comment_depth > 0: + if raw.startswith("/*", cursor): + block_comment_depth += 1 + cursor += 2 + elif raw.startswith("*/", cursor): + block_comment_depth -= 1 cursor += 2 else: cursor += 1 @@ -214,7 +217,7 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo if raw.startswith("//", cursor): break if raw.startswith("/*", cursor): - in_block_comment = True + block_comment_depth += 1 cursor += 2 continue raw_start = _raw_string_start(raw, cursor) diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index 4a62b2c9..23a75f37 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -540,6 +540,21 @@ def test_string_scanner_handles_comments_backslash_parity_and_methods(self) -> N self.assertFalse( coverage_contract._line_in_multiline_string_literal(block_comment, 5) ) + nested_block = [ + "/* outer", + " /* inner */", + r' still outer with " quote', + "*/", + "execute();", + ] + nested_path = Path(temporary) / "nested.rs" + nested_path.write_text("\n".join(nested_block) + "\n", encoding="utf-8") + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(nested_block, 5) + ) + self.assertTrue( + coverage_contract.is_executable_source_line(str(nested_path), 5) + ) self.assertTrue( coverage_contract._is_standalone_string_literal(r'"escaped\\",') ) From 16899f3f3fcb4361059fc8c7eafe9c6ebe368ed4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 23:26:34 +0900 Subject: [PATCH 32/38] fix(coverage): count code after multiline strings --- docs/research/topic-logratio-coordinates.md | 2 +- scripts/check_coverage.py | 13 +++++++++++-- tests/quality/test_check_coverage.py | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index a5310a2e..17ee68a2 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -25,7 +25,7 @@ Egozcue, J. J., Pawlowsky-Glahn, V., Mateu-Figueras, G., & Barceló-Vidal, C. (2 ## Application -Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore uses `additive_log_ratio` for logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Egozcue et al. (2003) construct the sequential orthonormal ILR basis whose Euclidean norm equals Aitchison distance; `isometric_log_ratio` implements that basis and `from_isometric_log_ratio` inverts it through a max-shifted centered-log-ratio reconstruction. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR rather than ALR (Egozcue et al., 2003). `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation (Aitchison & Shen, 1980). The forward ALR map subtracts logarithms rather than forming a potentially overflowing ratio (Aitchison & Shen, 1980; Blei & Lafferty, 2007). Both inverses fail closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980). +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore provides `additive_log_ratio` for future logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Egozcue et al. (2003) construct the sequential orthonormal ILR basis whose Euclidean norm equals Aitchison distance; `isometric_log_ratio` implements that basis and `from_isometric_log_ratio` inverts it through a max-shifted centered-log-ratio reconstruction. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR rather than ALR (Egozcue et al., 2003). `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation (Aitchison & Shen, 1980). The forward ALR map subtracts logarithms rather than forming a potentially overflowing ratio (Aitchison & Shen, 1980; Blei & Lafferty, 2007). Both inverses fail closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980). ## Verification diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index 86c2a695..b79705ac 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -181,8 +181,8 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo raw_hashes: int | None = None block_comment_depth = 0 for index, raw in enumerate(lines, start=1): - if (in_string or raw_hashes is not None) and index == line_number: - return True + target_continuation = (in_string or raw_hashes is not None) and index == line_number + target_closing_cursor: int | None = None cursor = 0 while cursor < len(raw): if block_comment_depth > 0: @@ -203,6 +203,8 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo else: raw_hashes = None cursor = closing + len(delimiter) + if target_continuation and target_closing_cursor is None: + target_closing_cursor = cursor continue if in_string: character = raw[cursor] @@ -211,6 +213,8 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo elif character == '"': in_string = False cursor += 1 + if target_continuation and target_closing_cursor is None: + target_closing_cursor = cursor else: cursor += 1 continue @@ -234,6 +238,11 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo cursor = character_end continue cursor += 1 + if target_continuation: + if target_closing_cursor is None: + return True + suffix = raw[target_closing_cursor:].strip().lstrip(",;)]}").lstrip() + return not suffix or suffix.startswith(("//", "/*")) return False diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index 23a75f37..f0f11175 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -590,6 +590,21 @@ def test_string_scanner_handles_comments_backslash_parity_and_methods(self) -> N coverage_contract._line_in_multiline_string_literal(byte_raw_string, 2) ) + closing_with_code = [ + ' let text = "first', + ' second"; execute(text);', + ] + closing_with_comment = [ + ' let text = "first', + ' second"; // no executable suffix', + ] + self.assertFalse( + coverage_contract._line_in_multiline_string_literal(closing_with_code, 2) + ) + self.assertTrue( + coverage_contract._line_in_multiline_string_literal(closing_with_comment, 2) + ) + character_and_lifetime = [ "fn query<'a>() {", " let quote: char = '\"';", From e3373912891263d4f0c45625cc6312591120bf80 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 23:33:52 +0900 Subject: [PATCH 33/38] fix(coverage): scan code after block comments --- CHANGELOG.md | 2 +- docs/research/topic-logratio-coordinates.md | 2 +- scripts/check_coverage.py | 12 ++++++++++-- tests/quality/test_check_coverage.py | 9 +++++++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47459edc..cf8b5808 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added -- `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, pairwise CLR Aitchison distance recovered by ILR Euclidean isometry away from the equal-share origin, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). +- `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, pairwise CLR Aitchison distance recovered by ILR Euclidean isometry for valid composition pairs, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. - Coverage source classification now scans Rust normal/raw/byte strings, comments, and character literals with escape-aware state, preserving executable string method calls and ignoring quoted comments. - `tepp_api` naruon live loopback HTTP/1.1 listener: `serve_one` installs a read/write deadline, requires a loopback `Host`, refuses `Transfer-Encoding` and NIM/proxy credential headers, parses `knowledge_cutoff` as RFC 3339 and refuses a future cutoff, keys analysis-run idempotency by tenant plus key, and proves both analysis-run and export POSTs over a real `TcpStream`. Not a production TLS/`$PORT` service (ADR 0011). diff --git a/docs/research/topic-logratio-coordinates.md b/docs/research/topic-logratio-coordinates.md index 17ee68a2..b24e706d 100644 --- a/docs/research/topic-logratio-coordinates.md +++ b/docs/research/topic-logratio-coordinates.md @@ -25,7 +25,7 @@ Egozcue, J. J., Pawlowsky-Glahn, V., Mateu-Figueras, G., & Barceló-Vidal, C. (2 ## Application -Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore provides `additive_log_ratio` for future logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Egozcue et al. (2003) construct the sequential orthonormal ILR basis whose Euclidean norm equals Aitchison distance; `isometric_log_ratio` implements that basis and `from_isometric_log_ratio` inverts it through a max-shifted centered-log-ratio reconstruction. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR rather than ALR (Egozcue et al., 2003). `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation (Aitchison & Shen, 1980). The forward ALR map subtracts logarithms rather than forming a potentially overflowing ratio (Aitchison & Shen, 1980; Blei & Lafferty, 2007). Both inverses fail closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980). +Aitchison and Shen (1980) define the logistic-normal family via the additive log-ratio map; Aitchison (1982) is the compositional-data authority that forbids treating parts of a whole as unconstrained Euclidean coordinates. Blei and Lafferty (2007) use that same reference-dependent map for correlated topic models. TEPP therefore provides `additive_log_ratio` for future logistic-normal regression and psychometric interfaces, but does not claim that ALR preserves Aitchison distance. Egozcue et al. (2003) construct the sequential orthonormal ILR basis whose Euclidean distance between two coordinate vectors equals the corresponding Aitchison distance; a single vector's norm is its distance from the equal-share origin. `isometric_log_ratio` implements that basis and `from_isometric_log_ratio` inverts it through a max-shifted centered-log-ratio reconstruction. Analyses whose estimand is orthonormal Euclidean Aitchison geometry must use ILR rather than ALR (Egozcue et al., 2003). `from_additive_log_ratio` treats the omitted reference component as logit zero, max-shifts all `K` logits together, and normalizes only after exponentiation (Aitchison & Shen, 1980). The forward ALR map subtracts logarithms rather than forming a potentially overflowing ratio (Aitchison & Shen, 1980; Blei & Lafferty, 2007). Both inverses fail closed when an `f64` simplex part would underflow to zero (Aitchison & Shen, 1980). ## Verification diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index b79705ac..65e140a7 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -183,6 +183,7 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo for index, raw in enumerate(lines, start=1): target_continuation = (in_string or raw_hashes is not None) and index == line_number target_closing_cursor: int | None = None + target_has_executable_suffix = False cursor = 0 while cursor < len(raw): if block_comment_depth > 0: @@ -218,12 +219,20 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo else: cursor += 1 continue + if raw[cursor].isspace(): + cursor += 1 + continue if raw.startswith("//", cursor): break if raw.startswith("/*", cursor): block_comment_depth += 1 cursor += 2 continue + if target_continuation and target_closing_cursor is not None: + if raw[cursor] in ",;)]}": + cursor += 1 + continue + target_has_executable_suffix = True raw_start = _raw_string_start(raw, cursor) if raw_start is not None: raw_hashes, cursor = raw_start @@ -241,8 +250,7 @@ def _line_in_multiline_string_literal(lines: list[str], line_number: int) -> boo if target_continuation: if target_closing_cursor is None: return True - suffix = raw[target_closing_cursor:].strip().lstrip(",;)]}").lstrip() - return not suffix or suffix.startswith(("//", "/*")) + return not target_has_executable_suffix return False diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index f0f11175..45433b2b 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -598,12 +598,21 @@ def test_string_scanner_handles_comments_backslash_parity_and_methods(self) -> N ' let text = "first', ' second"; // no executable suffix', ] + closing_with_block_comment_and_code = [ + ' let text = "first', + ' second"; /* note */ execute(text);', + ] self.assertFalse( coverage_contract._line_in_multiline_string_literal(closing_with_code, 2) ) self.assertTrue( coverage_contract._line_in_multiline_string_literal(closing_with_comment, 2) ) + self.assertFalse( + coverage_contract._line_in_multiline_string_literal( + closing_with_block_comment_and_code, 2 + ) + ) character_and_lifetime = [ "fn query<'a>() {", From b3dc7dab36e94728af7bea45a7eb3a065b616db5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:04:42 -0700 Subject: [PATCH 34/38] feat(topic): publish bounded TRSL topic lineage artifacts (#191) * feat(model): statistical Pareto K gates refuse LLM numerical authority ADR 0012 requires held-out likelihood/complexity comparison before any blinded LLM review. This crate admits K>=2 statistical candidates, drops dominated alternatives, and recovers known truth K with computed RMSE. * feat(topic): keep one identity across dormancy and reactivation ADR 0012 selects one global topic identity for the modeled period. Activity may become dormant or reactivated without minting a new identity; recovery is the computed match rate against known truth. * feat(api): serve naruon POSTs on loopback with a live deadline PR #87 accepted analysis-run bodies with knowledge_cutoff "k" and hung when a client sent a partial request. The named live listener now installs a read/write deadline, requires a loopback Host, refuses Transfer-Encoding and NIM/proxy headers, parses RFC 3339 cutoffs, keys idempotency by tenant plus key, and proves export over TCP. Co-authored-by: Seongho Bae * test(api): reproduce missing LineageWeave consumer contract The live TEPP listener currently accepts only tepp-consumer: naruon and keys idempotency without the consumer identity. These regressions require a credential-free LineageWeave exchange, a published consumer code, accepted 202 handling, cross-consumer idempotency isolation, and fail-closed unknown consumers. * feat(api): admit LineageWeave on the modular run boundary Publish a credential-free LineageWeave analysis-run exchange and a consumer-neutral loopback ingress. Accepted-run idempotency is isolated by consumer, tenant, and caller key; unpublished consumers and hostile headers fail closed. The acknowledgement remains asynchronous and does not claim a completed psychometric result. * ci: stage LineageWeave contract formatting repair The one-shot workflow removes test-only imports from production code, runs the pinned Rust formatter, verifies formatting, commits the repair, and removes itself. * fix(api): format and compile the LineageWeave contract * feat(api): publish terminal analysis result contract * fix(api): preserve existing contract documentation * fix(api): restore strict terminal result implementation * test(api): cover terminal analysis result contract * test(api): require cutoff-safe LineageWeave project history * feat(api): expose cutoff-safe project history contracts * feat(api): add cutoff-safe project history projection * feat(api): publish the LineageWeave project history exchange * ci: materialize the PR 159 availability-clock repair * ci: verify and publish the project-history availability contract * ci: verify TEPP LineageWeave project-history contract * feat(api): add analysis run status contract * fix(api): declare temporal core workspace version * fix(api): align project-history clocks and non-causal evidence * fix(api): harden analysis result serialization bindings * docs(adr): record consumer-scoped analysis-run ingress * docs(adr): index modular consumer ingress * test(api): require live project-history service route * ci: prove and implement the live project-history route * fix(api): validate localhost ports in live host checks * fix(api): share strict loopback host validation * fix(api): satisfy strict contract lint * fix(ci): align the TEPP history repair with the live contract * fix(ci): make the TEPP live-route repair exact-head compatible * fix(ci): install pinned Rust components correctly * test(api): complete naruon live branch coverage * fix(ci): remove the superseded analysis-run body-limit import * ci: remove superseded PR 159 verification workflow * test(api): close project-history line and branch coverage gaps * ci: verify the TEPP history coverage contract before publish * chore: close accidental placeholder issue * chore: remove accidental placeholder cleanup workflow * test(api): close analysis-run live coverage gaps * chore(ci): remove completed project-history repair workflow * test(coverage): merge branch outcomes by source coordinate * ci: finalize TEPP project-history contract * test(api): close analysis-run live coverage gaps * ci: pin project history verification actions * ci: pin finalization workflow actions * ci: pin and rerun TEPP project-history finalization * ci: dispatch pinned PR 159 finalizer * test(api): close project-history coverage edges * ci: remove completed project-history finalizers * docs: doctor the LineageWeave project-history contract * test(api): reproduce idempotency delimiter collision * fix(api): reject control characters in wire identities * test(api): preserve multiline wire text * fix(api): bound accepted analysis run payloads * test(api): align control character contract * test: close project history coverage gaps * fix: validate terminal result bindings * test(topic): use independent identity recovery oracle * fix(api): revalidate project history projections * test(api): cover empty https origin * test(api): close unreachable HTTP branch * test(api): cover localhost live host acceptance * test(api): close naruon HTTP branch coverage gap * test(api): close project history coverage gaps * test(api): cover project history invariants * test(api): cover project history response invariants * test(api): remove timing-sensitive timeout assertion * test(topic-lineage): complete identity branch contracts * test(model-selection): complete pareto gate coverage * ci: restack LineageWeave consumer contract on merged ingress * fix(docs): align naruon maturity with protected main * ci: trigger LineageWeave consumer restack from PR * fix(api): complete lineageweave restack safely * docs: bind consumer ingress to merged main lineage * ci: verify and repair PR 155 review findings * test: stage PR 155 review-finding repair * ci: execute the PR 155 repair through a recognized workflow * test: stage PR 159 timeout contract repair * ci: verify PR 159 loopback timeout contract * fix: close PR 155 review findings * fix: complete PR 155 coverage gates * test: strengthen coverage report regressions * fix(api): harden accepted receipts and provider headers * fix(ci): keep timeout verification in committed tests * Remove unreachable project history host branch * fix: enforce project history response size symmetry * docs: record project history service boundary * docs: remove ADR trailing whitespace * fix: harden analysis result contract boundaries * fix: enforce strict project history timestamps * docs: keep ADR index wording current * test: close coverage and match guarded arms * test: cover provider credential header branches * fix(api): reject delimiter-free credential headers * test: configure repository root for pytest * fix(coverage): preserve multiline match guards * fix(coverage): respect match arm boundaries * fix(coverage): reject block-boundary false guards * fix coverage guard after destructuring match arm * cover nested and long match guards * cover split nested match guard * retain guards after sibling match arms * style(api): apply rustfmt to project history tests * fix(api): close project history live ingress gaps * test(model-selection): validate repeated truth recovery * fix(model-selection): validate llm candidate K * feat(api): expose temporal evidence context for LineageWeave Ask (#158) * test(api): define LineageWeave temporal context contract * feat(api): add temporal context contract * fix(api): declare temporal core workspace version * fix(api): share strict loopback host validation * test(api): close live contract branch coverage * test(api): adapt temporal parser coverage to shared iterator * fix(api): enforce temporal context trust boundaries * test(api): cover temporal tie ordering branches * fix: remove fabricated temporal context idempotency * fix(api): allow temporal context reads without idempotency * feat(engine): execute cutoff-safe analysis runs (#178) * feat(engine): execute cutoff-safe analysis runs * docs(api): record analysis execution boundary * fix(engine): propagate artifact serialization errors * docs(engine): separate local and hosted verification * docs(engine): avoid duplicate gap-register landing file * docs(engine): remove absent register reference * docs(engine): preserve canonical documentation map * fix(engine): bound opaque analysis identifiers * Align analysis execution with accepted receipt contract * Guard analysis execution receipt identity * docs: register analysis gap doctoring * docs: align LineageWeave wire evidence * fix(api): bound temporal context serialization * fix(api): bound serialization before allocation * feat(api): package loopback temporal context service (#186) * feat(api): package loopback temporal context service * chore(api): healthcheck temporal context sidecar * test(api): execute packaged loopback ingress * fix(api): keep loopback service alive after request errors * feat(topic): add bounded TRSL reference estimator * feat(analysis): publish topic lineage artifacts * fix(deps): version topic workspace paths --------- Co-authored-by: Cursor Agent Co-authored-by: Seongho Bae Co-authored-by: CWL TEPP Contract Repair --- .dockerignore | 4 + .github/workflows/ci.yml | 6 +- .../hourly-nim-product-development.yml | 6 +- ARCHITECTURE.md | 10 +- CHANGELOG.d/lineageweave-project-history.md | 7 + .../lineageweave-temporal-context-service.md | 3 + CHANGELOG.md | 37 +- Cargo.lock | 35 + Cargo.toml | 6 + DOCUMENTATION.md | 2 + Dockerfile | 20 + README.md | 13 +- crates/analysis_engine/Cargo.toml | 31 + crates/analysis_engine/src/lib.rs | 726 +++++++++++++ .../src/topic_lineage_artifact.rs | 471 +++++++++ .../analysis_engine/tests/crate_contract.rs | 6 + .../tests/end_to_end_contract.rs | 107 ++ .../tests/topic_lineage_execution_contract.rs | 238 +++++ crates/model_selection/Cargo.toml | 17 + crates/model_selection/src/candidate.rs | 156 +++ crates/model_selection/src/error.rs | 60 ++ crates/model_selection/src/gate.rs | 123 +++ crates/model_selection/src/lib.rs | 22 + .../model_selection/tests/crate_contract.rs | 7 + .../tests/pareto_k_gate_contract.rs | 138 +++ crates/tepp_api/Cargo.toml | 6 + crates/tepp_api/src/analysis_result.rs | 362 +++++++ crates/tepp_api/src/analysis_run.rs | 213 +++- crates/tepp_api/src/analysis_run_live.rs | 970 ++++++++++++++++++ crates/tepp_api/src/bin/tepp_loopback.rs | 24 + crates/tepp_api/src/lib.rs | 109 +- crates/tepp_api/src/lineageweave_http.rs | 135 +++ crates/tepp_api/src/live_http.rs | 238 +++++ crates/tepp_api/src/naruon_http.rs | 82 +- crates/tepp_api/src/naruon_live.rs | 218 +--- crates/tepp_api/src/project_history.rs | 850 +++++++++++++++ crates/tepp_api/src/temporal_context.rs | 414 ++++++++ crates/tepp_api/src/wire.rs | 56 +- .../tests/analysis_result_contract.rs | 574 +++++++++++ crates/tepp_api/tests/example_contracts.rs | 83 +- .../tests/lineageweave_http_contract.rs | 155 +++ .../lineageweave_project_history_contract.rs | 569 ++++++++++ .../lineageweave_temporal_context_contract.rs | 444 ++++++++ .../tests/loopback_binary_contract.rs | 31 + crates/tepp_api/tests/naruon_http_contract.rs | 44 + crates/topic_lineage/Cargo.toml | 20 + crates/topic_lineage/src/activity.rs | 116 +++ crates/topic_lineage/src/error.rs | 53 + crates/topic_lineage/src/identity.rs | 94 ++ crates/topic_lineage/src/lib.rs | 24 + .../tests/activity_identity_contract.rs | 48 + crates/topic_lineage/tests/crate_contract.rs | 7 + crates/topic_measurement/Cargo.toml | 10 + crates/topic_measurement/src/error.rs | 28 + crates/topic_measurement/src/lib.rs | 18 + crates/topic_measurement/src/reference.rs | 877 ++++++++++++++++ crates/topic_measurement/src/sparse.rs | 222 ++++ .../tests/reference_estimator_contract.rs | 516 ++++++++++ docs/API_CONTRACT.md | 33 +- docs/DOCUMENTATION_ASSESSMENT.md | 2 +- docs/TRACEABILITY.md | 9 +- .../0011-standalone-modular-msa-boundary.md | 2 +- ...ational-shared-latent-topic-measurement.md | 65 +- ...17-consumer-scoped-analysis-run-ingress.md | 104 ++ ...0018-project-history-wire-size-symmetry.md | 70 ++ ...9-lineageweave-project-history-boundary.md | 91 ++ ...20-deterministic-analysis-run-execution.md | 99 ++ docs/adr/README.md | 12 +- docs/connectors/naruon-artifact-consumer.md | 2 +- ...LINEAGEWEAVE_PROJECT_HISTORY_REFERENCES.md | 46 + docs/doctoring/analysis-engine-gap-closure.md | 46 + docs/doctoring/analysis-engine-v1.md | 55 + docs/research/model-selection-pareto-gates.md | 43 + docs/research/rust-quality-tooling.md | 4 +- docs/research/standards-and-literature.md | 19 +- docs/research/topic-activity-identity.md | 34 + docs/validation/temporal-event-foundation.md | 4 +- .../consumer-ingress-main-base.md | 30 + .../project-history-parent-restack.md | 22 + pytest.ini | 2 + scripts/check_coverage.py | 109 +- scripts/check_workspace_contract.py | 3 + scripts/validate_documentation.py | 1 + tests/quality/test_check_coverage.py | 253 ++++- tests/quality/test_check_docstrings.py | 2 +- tests/quality/test_ci_coverage_diagnostics.py | 2 +- .../test_hourly_nim_product_development.py | 2 + 87 files changed, 10769 insertions(+), 258 deletions(-) create mode 100644 .dockerignore create mode 100644 CHANGELOG.d/lineageweave-project-history.md create mode 100644 CHANGELOG.d/lineageweave-temporal-context-service.md create mode 100644 Dockerfile create mode 100644 crates/analysis_engine/Cargo.toml create mode 100644 crates/analysis_engine/src/lib.rs create mode 100644 crates/analysis_engine/src/topic_lineage_artifact.rs create mode 100644 crates/analysis_engine/tests/crate_contract.rs create mode 100644 crates/analysis_engine/tests/end_to_end_contract.rs create mode 100644 crates/analysis_engine/tests/topic_lineage_execution_contract.rs create mode 100644 crates/model_selection/Cargo.toml create mode 100644 crates/model_selection/src/candidate.rs create mode 100644 crates/model_selection/src/error.rs create mode 100644 crates/model_selection/src/gate.rs create mode 100644 crates/model_selection/src/lib.rs create mode 100644 crates/model_selection/tests/crate_contract.rs create mode 100644 crates/model_selection/tests/pareto_k_gate_contract.rs create mode 100644 crates/tepp_api/src/analysis_result.rs create mode 100644 crates/tepp_api/src/analysis_run_live.rs create mode 100644 crates/tepp_api/src/bin/tepp_loopback.rs create mode 100644 crates/tepp_api/src/lineageweave_http.rs create mode 100644 crates/tepp_api/src/live_http.rs create mode 100644 crates/tepp_api/src/project_history.rs create mode 100644 crates/tepp_api/src/temporal_context.rs create mode 100644 crates/tepp_api/tests/analysis_result_contract.rs create mode 100644 crates/tepp_api/tests/lineageweave_http_contract.rs create mode 100644 crates/tepp_api/tests/lineageweave_project_history_contract.rs create mode 100644 crates/tepp_api/tests/lineageweave_temporal_context_contract.rs create mode 100644 crates/tepp_api/tests/loopback_binary_contract.rs create mode 100644 crates/topic_lineage/Cargo.toml create mode 100644 crates/topic_lineage/src/activity.rs create mode 100644 crates/topic_lineage/src/error.rs create mode 100644 crates/topic_lineage/src/identity.rs create mode 100644 crates/topic_lineage/src/lib.rs create mode 100644 crates/topic_lineage/tests/activity_identity_contract.rs create mode 100644 crates/topic_lineage/tests/crate_contract.rs create mode 100644 crates/topic_measurement/src/reference.rs create mode 100644 crates/topic_measurement/src/sparse.rs create mode 100644 crates/topic_measurement/tests/reference_estimator_contract.rs create mode 100644 docs/adr/0017-consumer-scoped-analysis-run-ingress.md create mode 100644 docs/adr/0018-project-history-wire-size-symmetry.md create mode 100644 docs/adr/0019-lineageweave-project-history-boundary.md create mode 100644 docs/adr/0020-deterministic-analysis-run-execution.md create mode 100644 docs/doctoring/LINEAGEWEAVE_PROJECT_HISTORY_REFERENCES.md create mode 100644 docs/doctoring/analysis-engine-gap-closure.md create mode 100644 docs/doctoring/analysis-engine-v1.md create mode 100644 docs/research/model-selection-pareto-gates.md create mode 100644 docs/research/topic-activity-identity.md create mode 100644 docs/verification/consumer-ingress-main-base.md create mode 100644 docs/verification/project-history-parent-restack.md create mode 100644 pytest.ini diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..fbf0a7ef --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.codegraph +.git +node_modules +target diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0d2d080..3f27d3a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -223,7 +223,7 @@ jobs: with: persist-credentials: false - name: Install pinned nightly with LLVM tools - run: rustup toolchain install nightly-2026-08-01 --profile minimal --component llvm-tools-preview + run: rustup toolchain install nightly-2026-08-21 --profile minimal --component llvm-tools-preview - name: Restore pinned cargo-llvm-cov id: llvm-cov-cache uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 @@ -237,12 +237,12 @@ jobs: run: cargo llvm-cov --version | grep -F "$CARGO_LLVM_COV_VERSION" - name: Generate exact branch coverage id: branch-report - run: cargo +nightly-2026-08-01 llvm-cov --branch --workspace --all-features --json --summary-only --output-path coverage-branches.json --ignore-filename-regex 'sqlx_live\.rs' + run: cargo +nightly-2026-08-21 llvm-cov --branch --workspace --all-features --json --output-path coverage-branches.json --ignore-filename-regex 'sqlx_live\.rs' - name: Enforce complete branch coverage run: python3 scripts/check_coverage.py coverage-branches.json --kind branches - name: Show exact missing branch diagnostics if: ${{ failure() && steps.branch-report.outcome == 'success' }} - run: cargo +nightly-2026-08-01 llvm-cov report --branch --text --show-missing-lines + run: cargo +nightly-2026-08-21 llvm-cov report --branch --text --show-missing-lines - name: Upload exact branch coverage diagnostics if: ${{ failure() && steps.branch-report.outcome == 'success' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/hourly-nim-product-development.yml b/.github/workflows/hourly-nim-product-development.yml index 93ec6061..7602ae0e 100644 --- a/.github/workflows/hourly-nim-product-development.yml +++ b/.github/workflows/hourly-nim-product-development.yml @@ -408,7 +408,7 @@ jobs: if [ "${{ steps.llvm-cov-cache.outputs.cache-hit }}" != true ]; then cargo install cargo-llvm-cov --locked --version 0.8.6 fi - rustup toolchain install nightly-2026-08-01 --profile minimal --component llvm-tools-preview + rustup toolchain install nightly-2026-08-21 --profile minimal --component llvm-tools-preview - name: Run every release-quality gate env: @@ -437,9 +437,9 @@ jobs: cargo deny check line_coverage="$RUNNER_TEMP/coverage.lcov" branch_coverage="$RUNNER_TEMP/coverage-branches.json" - cargo llvm-cov --workspace --all-features --lcov --output-path "$line_coverage" + cargo llvm-cov --workspace --all-features --lcov --output-path "$line_coverage" --ignore-filename-regex 'sqlx_live\.rs' python3 scripts/check_coverage.py "$line_coverage" --kind lines --format lcov - cargo +nightly-2026-08-01 llvm-cov --branch --workspace --all-features --json --summary-only --output-path "$branch_coverage" + cargo +nightly-2026-08-21 llvm-cov --branch --workspace --all-features --json --output-path "$branch_coverage" --ignore-filename-regex 'sqlx_live\.rs' python3 scripts/check_coverage.py "$branch_coverage" --kind branches [ -z "$(git diff --name-only)" ] [ -z "$(git ls-files --others --exclude-standard)" ] diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f10bf73a..8c049120 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -43,6 +43,11 @@ flowchart LR Every boundary must be independently usable and expose versioned contracts for integration with organization repositories, `naruon`, and `contextual-orchestrator`. +The `analysis_engine` vertical slice is intentionally separate from `tepp_api`: +the API owns wire contracts while the engine owns deterministic execution. It +does not replace the future topic or psychometric estimators and does not read +another service's application tables. + ## Implemented foundation topology Task 1 materializes the first storage-independent workspace boundaries. The @@ -60,8 +65,11 @@ boundaries above remain the target modular MSA architecture. | `corpus_split` | cutoff-safe, relation-aware partitioning | | `tepp_simulation` | known-truth temporal/event data generation | | `validation_core` | RMSE, bias, coverage, graph, and Monte Carlo metrics | -| `tepp_api` | versioned DTO, schema, and export contracts | +| `tepp_api` | versioned DTO, schema, terminal-result, and export contracts | | `topic_measurement` | logistic-normal ALR and sequential Egozcue ILR topic coordinates | +| `model_selection` | statistical/Pareto candidate-`K` gates; LLM votes are not numerical authority | +| `topic_lineage` | global topic identity across active/dormant/reactivated states | +| `analysis_engine` | bounded cutoff-safe temporal evidence readiness execution and digest-bound terminal artifacts | No crate exposes placeholder production behavior in Task 1. This prevents an empty façade from becoming a de facto public API before its invariants and tests diff --git a/CHANGELOG.d/lineageweave-project-history.md b/CHANGELOG.d/lineageweave-project-history.md new file mode 100644 index 00000000..6ef64bb3 --- /dev/null +++ b/CHANGELOG.d/lineageweave-project-history.md @@ -0,0 +1,7 @@ +# LineageWeave project-history projection + +- `tepp_api` projects already-authorized LineageWeave evidence into a strict, cutoff-safe project history, preserves explicit source-event identities, validates deterministic chronological ordering, recomputes non-causal findings, and rejects fabricated, credential-bearing, or oversized payloads. +- Request and generated-projection serialization now share the 256 KiB wire limit, preventing a successful projection that cannot pass TEPP's own response parser. +- ADR 0019 records the credential-free bounded service boundary and its split of authorization (LineageWeave) from temporal projection (TEPP). +- This fragment preserves the child release note while the stacked branch retains the parent consumer-ingress changelog during the ordinary parent merge. +- The loopback timeout regression is now asserted in the committed Rust test; documentation CI is read-only and no longer mutates contributor branches. diff --git a/CHANGELOG.d/lineageweave-temporal-context-service.md b/CHANGELOG.d/lineageweave-temporal-context-service.md new file mode 100644 index 00000000..e4a1fea8 --- /dev/null +++ b/CHANGELOG.d/lineageweave-temporal-context-service.md @@ -0,0 +1,3 @@ +### Added + +- Package the existing cutoff-safe `POST /v1/temporal-context` contract as the loopback-only `tepp-loopback` binary and container for trusted same-host consumers such as LineageWeave. diff --git a/CHANGELOG.md b/CHANGELOG.md index cf8b5808..f85fb5d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,15 +6,31 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added +- `topic_measurement` bounded deterministic CPU `f64` TRSL-TM reference estimator: canonical CSR/CSC inputs, cutoff-safe documents, standardized event time, weighted multiple memberships, prevalence covariates, explicit predecessor/successor regularization, multi-seed generalized EM, diagonal Laplace uncertainty, and fitted topic-lineage counts with known-truth RMSE plus exact line/branch coverage (ADR 0012; no persistence or accelerated-backend claim). - `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, pairwise CLR Aitchison distance recovered by ILR Euclidean isometry for valid composition pairs, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. - Coverage source classification now scans Rust normal/raw/byte strings, comments, and character literals with escape-aware state, preserving executable string method calls and ignoring quoted comments. +- `model_selection` candidate-`K` gates: statistical candidates require `K >= 2` and finite held-out log-likelihood/complexity, a Pareto front excludes dominated alternatives, LLM votes cannot define the numerical optimum, and selected `K` recovers known truth with computed RMSE. +- `topic_lineage` global P0 topic identity: activity may become dormant or reactivated without minting a new identity, and recovered identities match known truth at a higher computed rate than mint-on-reactivate replacements. +- `tepp_api` LineageWeave temporal-context contract (v1): cutoff-safe event eligibility, deterministic event-time ordering, explicit non-causal association/gap boundaries, HTTPS interchange construction, and loopback listener handling at `POST /v1/temporal-context`; read-only context requests no longer require the write-only idempotency header, and no causal inference or completed-result service is included. +- `tepp_api` LineageWeave consumer-scoped analysis-run ingress: versioned, credential-free requests use a published consumer identity and isolate idempotency by consumer, tenant workspace, and opaque caller key; the one-shot restack workflow is removed after the protected-main merge is verified. +- ADR 0017 records the consumer-scoped analysis-run ingress, its in-memory loopback maturity, and the persistence boundary required before production use. +- ADR 0019 records the credential-free bounded LineageWeave project-history service boundary and keeps source authorization with LineageWeave while TEPP owns temporal validation and deterministic projection. +- `tepp_api` project-history wire-size symmetry (ADR 0018): request and projection serialization enforce the shared 256 KiB limit, and generated projections fail closed before returning when their deterministic response would exceed it. +- Registered the analysis-engine gap-closure doctoring in the canonical documentation map so its product and scientific traceability record is discoverable. +- Authored Rust coverage classification now ignores standalone structural closing parentheses, preventing formatting-only LCOV rows from appearing as uncovered production behavior. +- `analysis_engine` vertical slice (ADR 0020): bounded Rust execution from an accepted analysis run to either a cutoff-safe readiness result or a validated `tepp.trsl_topic_lineage.v1` artifact from the ADR-0012 estimator. Topic artifacts preserve fitted predecessor/successor edges, connectable-post and lineage counts, request/snapshot/cutoff bindings, SHA-256 identity, and fail-closed non-convergence/tamper behavior with exact line/branch coverage. This remains active-PR evidence and does not claim causal or psychometric authority. +- Coverage classification preserves the final expression line of multiline Rust `match` guards while respecting preceding-arm boundaries, keeping the 100% authored-line gate conservative. +- `tepp_api` fail-closed analysis-result boundaries: status constructors reject + terminal envelopes that cannot fit the default 64 KiB status limit, and + standalone terminal results reject knowledge cutoffs in the future. +- `tepp_api` request-bound terminal analysis results and typed analysis-run status/read responses: accepted/running states cannot carry measurement evidence, terminal results bind exact request and receipt identities, and succeeded/failed payloads remain digest-bound or content-redacted. - `tepp_api` naruon live loopback HTTP/1.1 listener: `serve_one` installs a read/write deadline, requires a loopback `Host`, refuses `Transfer-Encoding` and NIM/proxy credential headers, parses `knowledge_cutoff` as RFC 3339 and refuses a future cutoff, keys analysis-run idempotency by tenant plus key, and proves both analysis-run and export POSTs over a real `TcpStream`. Not a production TLS/`$PORT` service (ADR 0011). - `tepp_api` adaptive orchestration router (ADR 0010): versioned `direct`/`verify`/`committee`/`conductor`/`abstain` selection from CPU `f64` risk, ambiguity, evidence, and token-budget inputs; recorded stages, recursion, decomposition, access lists, and role-specific reasoning effort; fail-closed document-controlled policy/access/credentials; LLM plans remain proposals under deterministic statistical authority; comparable-budget ablation requires a direct baseline; credential-free contextual-orchestrator binding. Live NIM HTTP remains accepted-target. - `tepp_api` purpose-bound provider-payload minimization: time-bounded `PurposeGrant` evaluation, fail-closed expired/not-yet-valid/inverted/cross-tenant/impossible-calendar denial, semantic UTC calendar validation, refusal to copy identity mappings into model-provider payloads or ordinary logs, preservation of opaque analytical identifiers and membership roles (no blanket PII mask), a separately authorized scientific re-identification path, and an internally bound FIPS 180-4 SHA-256 audit digest appended through `ReidentificationAuditSink` before disclosure. - `persistence_postgres` backup/restore integrity: restored snapshots stay unusable until tenant, canonical `SHA-256`, knowledge-cutoff eligibility, temporal window order, and append-only triggers revalidate; SQL probes raise `restore integrity failed` (ADR 0013). - `persistence_postgres` concurrent document-write stress: atomic revise `DO` block that requires exactly one open `system_to` close, SQLSTATE mapping onto `ConcurrentWriteConflict` / `DuplicateDocumentRecord`, and live multi-session insert/revise/append-only proofs. No new migration number. -- `tepp_api` naruon HTTP interchange: versioned `https` POST contracts for analysis-run create and modular export authorization that refuse table-access URLs, review/Copilot credential headers, reserved standard-header redefinition, principal-only export idempotency keys, and lexical inference claims (ADR 0011). +- `tepp_api` naruon HTTP interchange: versioned `https` POST contracts for analysis-run create and modular export authorization that refuse table-access URLs, provider-specific API-key/secret and review/Copilot credential headers, malformed extra HTTP fields, reserved standard-header redefinition, principal-only export idempotency keys, and lexical inference claims (ADR 0011). - `persistence_postgres` audit-event SQL contracts: append-only insert that refuses empty, oversized, or hostile `action_code` values before SQL is rendered. - `persistence_postgres` event-instance SQL contracts: bitemporal insert and as-known-at lookup that refuse inverted valid/system windows and hostile type/lifecycle labels before SQL is rendered. - `persistence_postgres` event-mention SQL contracts: mention identity cannot equal the instance it supports; confidence must be finite and in `(0, 1]`. @@ -78,6 +94,23 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Changed +- The LineageWeave temporal-context read exchange no longer emits a fabricated + `idempotency-key`; that header remains reserved for retryable write/export + operations with a caller-owned operation key. +- `tepp_api` project-history requests and projections now share the strict + `temporal_core` RFC 3339 parser and nominal `KnowledgeCutoff` boundary, + rejecting unknown offsets and other timestamp forms that the transport + parser could otherwise accept. +- Coverage validation now ignores LLVM rows for multiline call and iterator + syntax that have no independently executable source coordinate, while + retaining the authored-line 100% gate. +- Removed the temporary PR-155 review-repair workflows and source-fix helper after the bounded repair; subsequent changes use the normal reviewed branch path. +- Pinned Rust branch-coverage workflows to `nightly-2026-08-21`, which is newer than the workspace Rust 1.97.1 MSRV and avoids the previous nightly/MSRV mismatch. +- Applied the documented `sqlx_live.rs` authored-coverage exclusion to the hourly release gate so live-PostgreSQL success-path coverage is not reported as a false source failure. +- Removed unreachable duplicate Naruon host-control validation because the shared `require_nonempty` boundary already rejects C0/C1 controls; retained a C1 regression case alongside the existing C0 case. +- Rust LCOV quality gating now ignores visibility-qualified function signatures + and structural match-arm labels that LLVM reports as zero-hit non-executable + lines. - Clarified ADR 0001 so it owns Rust-first numerical/reference-backend authority while ADR 0011 owns cross-service MSA/service authority. - Clarified ADR 0006 so it owns GPU/VRAM and model-credential boundaries; ADR 0010 now owns LLM orchestration policy and ADR 0015 owns autonomous repository-write/review/merge authority. - Expanded ADR 0002–0005 and 0009–0011 with explicit implementation maturity, alternatives, failure/recovery, compatibility/migration, verification, and rollback/supersession boundaries where they were previously implicit. @@ -101,7 +134,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang - Required 100% production line and branch coverage and complete public API docstrings. - Required true-parameter recovery, RMSE, bias, interval coverage, temporal leakage, graph recovery, invariance, and CPU/GPU parity evidence. -- Expanded documentation contracts to require the canonical threat/privacy/assurance/API/orchestration/fitness documents, ADR policy, and every numbered ADR 0001–0016 to remain indexed and structurally complete. +- Expanded documentation contracts to require the canonical threat/privacy/assurance/API/orchestration/fitness documents, ADR policy, and every numbered ADR present in the canonical index to remain indexed and structurally complete. - Added deterministic validation that ADR files and the index have identical decision numbers and that every ADR declares valid decision status, implementation maturity, supersession scope, core decision sections, verification, and rollback behavior. - Added 100% statement and branch coverage for the repository quality-gate scripts. - Made a zero executable-code coverage denominator explicit for the skeleton-only slice rather than treating it as evidence of implemented behavior. diff --git a/Cargo.lock b/Cargo.lock index e3018765..8eadc1a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,22 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "analysis_engine" +version = "0.1.0" +dependencies = [ + "corpus_split", + "membership_core", + "relation_graph", + "serde", + "serde_json", + "sha2", + "temporal_core", + "tepp_api", + "topic_measurement", + "uuid", +] + [[package]] name = "atoi" version = "2.0.0" @@ -733,6 +749,10 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "model_selection" +version = "0.1.0" + [[package]] name = "num-traits" version = "0.2.19" @@ -1373,9 +1393,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "topic_lineage" +version = "0.1.0" +dependencies = [ + "uuid", +] + [[package]] name = "topic_measurement" version = "0.1.0" +dependencies = [ + "corpus_split", + "membership_core", + "relation_graph", + "temporal_core", + "uuid", + "validation_core", +] [[package]] name = "tracing" diff --git a/Cargo.toml b/Cargo.toml index 35ba7a1e..948e805c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,9 @@ members = [ "crates/validation_core", "crates/tepp_api", "crates/topic_measurement", + "crates/model_selection", + "crates/topic_lineage", + "crates/analysis_engine", ] default-members = [ "crates/evidence_core", @@ -25,6 +28,9 @@ default-members = [ "crates/validation_core", "crates/tepp_api", "crates/topic_measurement", + "crates/model_selection", + "crates/topic_lineage", + "crates/analysis_engine", ] [workspace.package] diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 9e93a508..6dfeadd5 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -38,6 +38,8 @@ TEPP's approved PRD v0.4 and implementation plan are the primary product baselin | Adaptive orchestration router doctoring | [`docs/research/adaptive-orchestration-router.md`](docs/research/adaptive-orchestration-router.md) | | Topic log-ratio coordinate doctoring | [`docs/research/topic-logratio-coordinates.md`](docs/research/topic-logratio-coordinates.md) | | Hourly NIM OpenCode doctoring | [`docs/doctoring/hourly-nim-opencode-development.md`](docs/doctoring/hourly-nim-opencode-development.md) | +| Analysis engine v1 doctoring | [`docs/doctoring/analysis-engine-v1.md`](docs/doctoring/analysis-engine-v1.md) | +| Analysis engine gap-closure doctoring | [`docs/doctoring/analysis-engine-gap-closure.md`](docs/doctoring/analysis-engine-gap-closure.md) | | Change history | [`CHANGELOG.md`](CHANGELOG.md) | ## Maturity vocabulary diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..ce294a14 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM rust:1.97.1-bookworm AS build +WORKDIR /src +COPY . . +RUN cargo build --locked --release -p tepp_api --bin tepp-loopback + +FROM debian:bookworm-slim +RUN apt-get update \ + && apt-get install --yes --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /src/target/release/tepp-loopback /usr/local/bin/tepp-loopback +USER 65532:65532 +HEALTHCHECK --interval=10s --timeout=3s --start-period=2s --retries=5 \ + CMD curl --fail --silent --show-error \ + --header "content-type: application/json" \ + --header "tepp-consumer: lineageweave" \ + --header "tepp-contract-version: 1" \ + --data '{"contract_version":1,"consumer_code":"lineageweave","knowledge_cutoff":"2026-08-20T00:00:00Z","subject_post_id":"health-post","events":[{"event_id":"health-event","source_post_id":"health-post","event_type_code":"health_probe","event_label":"Health probe","event_time":"2026-08-20T00:00:00Z","available_time":"2026-08-20T00:00:00Z","project_reference":null,"actor_references":["health-actor"]}]}' \ + http://127.0.0.1:18081/v1/temporal-context >/dev/null \ + || exit 1 +ENTRYPOINT ["/usr/local/bin/tepp-loopback"] diff --git a/README.md b/README.md index ed7a2d74..996f2a46 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ implemented in Rust. ## Current implementation state -This branch establishes the Task 1 Rust workspace and quality-gate foundation. -The eleven bounded crates compile independently. Domain crates expose only -validated production APIs; placeholder surfaces are prohibited. +This branch establishes the Rust workspace and quality-gate foundation. The +fourteen bounded crates compile independently; domain behavior includes immutable +evidence records, topic measurement, and cutoff-safe analysis execution. ```text crates/evidence_core @@ -22,6 +22,9 @@ crates/tepp_simulation crates/validation_core crates/tepp_api crates/topic_measurement +crates/model_selection +crates/topic_lineage +crates/analysis_engine ``` ## Local verification @@ -56,3 +59,7 @@ this skeleton-only slice; it must never conceal uncovered production behavior. No release, production-readiness, GPU, database, or statistical-recovery claim is made by this foundation slice. + +The active stacked analysis-engine slice adds a bounded executable readiness path +from an accepted run to a digest-bound terminal artifact. It is not yet +implemented-main and does not replace scientific estimator contracts. diff --git a/crates/analysis_engine/Cargo.toml b/crates/analysis_engine/Cargo.toml new file mode 100644 index 00000000..995ad914 --- /dev/null +++ b/crates/analysis_engine/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "analysis_engine" +description = "Deterministic cutoff-safe temporal evidence readiness execution." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +tepp_api = { path = "../tepp_api", version = "0.1.0" } +temporal_core = { path = "../temporal_core", version = "0.1.0" } +topic_measurement = { path = "../topic_measurement", version = "0.1.0" } +uuid.workspace = true + +[dev-dependencies] +corpus_split = { path = "../corpus_split", version = "0.1.0" } +membership_core = { path = "../membership_core", version = "0.1.0" } +relation_graph = { path = "../relation_graph", version = "0.1.0" } + +[lints] +workspace = true diff --git a/crates/analysis_engine/src/lib.rs b/crates/analysis_engine/src/lib.rs new file mode 100644 index 00000000..b807e607 --- /dev/null +++ b/crates/analysis_engine/src/lib.rs @@ -0,0 +1,726 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +//! Deterministic, cutoff-safe execution for the first TEPP analysis vertical slice. +//! +//! The engine consumes identity-free evidence metadata, excludes evidence that +//! was unavailable at the requested knowledge cutoff, counts multiple-membership +//! assignments without collapsing them, and emits a digest-bound terminal result +//! through [`tepp_api`]. It deliberately does not claim latent-variable or topic +//! estimation authority; it invokes estimators through their scientific crate +//! contracts and preserves their artifact meaning. + +mod topic_lineage_artifact; + +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use std::fmt; +use std::fmt::Write as _; +use temporal_core::{AvailableTime, EventTime, KnowledgeCutoff}; +use tepp_api::{ + AnalysisResultSummary, AnalysisRunAccepted, AnalysisRunRequest, AnalysisRunTerminalResult, + ApiError, +}; +use topic_measurement::TopicMeasurementError; + +/// Topic-lineage artifact and execution contracts from this engine. +pub use topic_lineage_artifact::{ + TOPIC_LINEAGE_ARTIFACT_BYTE_LIMIT, TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION, + TOPIC_LINEAGE_MODEL_CONTRACT_VERSION, TOPIC_LINEAGE_OUTPUT_PROFILE, TopicLineageArtifact, + TopicLineageArtifactEdge, TopicLineageExecution, execute_topic_lineage_run, +}; + +/// Versioned artifact schema emitted by this engine. +pub const ANALYSIS_ARTIFACT_SCHEMA_VERSION: &str = "tepp.temporal_evidence_readiness.v1"; +/// Number of deterministic statistics represented in the artifact summary. +pub const ANALYSIS_STATISTIC_COUNT: u64 = 4; +/// Maximum number of evidence units accepted by one in-memory execution. +pub const MAX_EVIDENCE_UNITS: usize = 100_000; +/// Maximum UTF-8 byte length of one snapshot or opaque evidence identifier. +pub const MAX_ANALYSIS_IDENTIFIER_BYTES: usize = 256; + +/// A bounded identity-free evidence unit offered to one analysis run. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AnalysisEvidenceUnit { + evidence_id: String, + event_time: EventTime, + available_time: AvailableTime, + membership_count: u32, +} + +impl AnalysisEvidenceUnit { + /// Construct an evidence unit with explicit event, availability, and + /// multiple-membership metadata. + /// + /// # Errors + /// + /// Returns [`AnalysisEngineError::InvalidEvidence`] when the identity is + /// empty or no membership assignment is supplied. + pub fn new( + evidence_id: impl Into, + event_time: EventTime, + available_time: AvailableTime, + membership_count: u32, + ) -> Result { + let evidence_id = evidence_id.into(); + if !valid_identifier(&evidence_id) || membership_count == 0 { + return Err(AnalysisEngineError::InvalidEvidence); + } + Ok(Self { + evidence_id, + event_time, + available_time, + membership_count, + }) + } + + /// Return the opaque evidence identity. + #[must_use] + pub fn evidence_id(&self) -> &str { + &self.evidence_id + } + + /// Return the event-valid time. + #[must_use] + pub const fn event_time(&self) -> EventTime { + self.event_time + } + + /// Return the evidence availability time. + #[must_use] + pub const fn available_time(&self) -> AvailableTime { + self.available_time + } + + /// Return the number of simultaneous membership assignments. + #[must_use] + pub const fn membership_count(&self) -> u32 { + self.membership_count + } +} + +/// A bounded snapshot of evidence metadata for one analysis run. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AnalysisCorpus { + snapshot_id: String, + evidence_units: Vec, +} + +impl AnalysisCorpus { + /// Construct a snapshot-owned corpus. + /// + /// # Errors + /// + /// Returns [`AnalysisEngineError::InvalidEvidence`] for an empty snapshot + /// identity or [`AnalysisEngineError::LimitExceeded`] for an oversized + /// in-memory corpus. + pub fn new( + snapshot_id: impl Into, + evidence_units: Vec, + ) -> Result { + let snapshot_id = snapshot_id.into(); + if !valid_identifier(&snapshot_id) { + return Err(AnalysisEngineError::InvalidEvidence); + } + if evidence_units.len() > MAX_EVIDENCE_UNITS { + return Err(AnalysisEngineError::LimitExceeded); + } + Ok(Self { + snapshot_id, + evidence_units, + }) + } + + /// Return the immutable snapshot identity. + #[must_use] + pub fn snapshot_id(&self) -> &str { + &self.snapshot_id + } + + /// Return the evidence units in source order. + #[must_use] + pub fn evidence_units(&self) -> &[AnalysisEvidenceUnit] { + &self.evidence_units + } +} + +/// Digest-bound, identity-free output artifact for one successful execution. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct AnalysisArtifact { + /// Versioned artifact schema. + pub schema_version: String, + /// Opaque accepted-run identity. + pub run_id: String, + /// Immutable source snapshot identity. + pub snapshot_id: String, + /// Historical cutoff applied to availability. + pub knowledge_cutoff: String, + /// Number of evidence units available by the cutoff. + pub eligible_evidence_count: u64, + /// Sum of preserved multiple-membership assignments. + pub eligible_membership_count: u64, + /// Earliest event-valid time among eligible evidence. + pub earliest_event_time: String, + /// Latest event-valid time among eligible evidence. + pub latest_event_time: String, +} + +impl AnalysisArtifact { + /// Serialize the canonical artifact bytes used for digesting. + /// + /// # Errors + /// + /// Returns [`AnalysisEngineError::SerializationFailure`] if serialization + /// unexpectedly fails. + pub fn to_json(&self) -> Result { + serde_json::to_string(self).map_err(|_| AnalysisEngineError::SerializationFailure) + } + + /// Return the lowercase SHA-256 digest of the canonical artifact JSON. + /// + /// # Errors + /// + /// Returns [`AnalysisEngineError::SerializationFailure`] if serialization + /// unexpectedly fails. + pub fn sha256(&self) -> Result { + self.to_json() + .map(|json| format_digest(Sha256::digest(json.into_bytes()))) + } +} + +/// One complete execution response, including the internal artifact and the +/// request-bound terminal wire result. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AnalysisExecution { + /// Digest-bound artifact, present only when the terminal result succeeded. + pub artifact: Option, + /// Request-bound terminal result returned to the service boundary. + pub terminal_result: AnalysisRunTerminalResult, +} + +/// Fail-closed errors from the deterministic analysis vertical slice. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AnalysisEngineError { + /// Request or accepted receipt failed its API contract. + Api(ApiError), + /// Evidence metadata was empty or structurally invalid. + InvalidEvidence, + /// Two evidence units reused one opaque identity. + DuplicateEvidence, + /// Corpus snapshot identity differed from the request snapshot. + SnapshotMismatch, + /// A bounded integer aggregation overflowed. + ArithmeticOverflow, + /// A serialized artifact could not be produced. + SerializationFailure, + /// The in-memory corpus exceeded the execution bound. + LimitExceeded, + /// A topic-measurement estimator rejected or could not complete the fit. + TopicMeasurement(TopicMeasurementError), + /// A topic-lineage artifact violated its bounded schema or count invariants. + InvalidTopicLineageArtifact, +} + +impl fmt::Display for AnalysisEngineError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::Api(error) => return error.fmt(formatter), + Self::InvalidEvidence => "invalid analysis evidence", + Self::DuplicateEvidence => "duplicate analysis evidence identity", + Self::SnapshotMismatch => "analysis snapshot identity mismatch", + Self::ArithmeticOverflow => "analysis evidence count overflow", + Self::SerializationFailure => "analysis artifact serialization failed", + Self::LimitExceeded => "analysis corpus exceeded its execution bound", + Self::TopicMeasurement(error) => return error.fmt(formatter), + Self::InvalidTopicLineageArtifact => "invalid topic lineage artifact", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for AnalysisEngineError {} + +impl From for AnalysisEngineError { + fn from(error: ApiError) -> Self { + Self::Api(error) + } +} + +impl From for AnalysisEngineError { + fn from(error: TopicMeasurementError) -> Self { + Self::TopicMeasurement(error) + } +} + +/// Execute the cutoff-safe temporal evidence readiness analysis. +/// +/// Evidence whose `available_time` is later than the request cutoff is excluded +/// before aggregation. Event time remains a separate clock, and all membership +/// assignments are summed rather than collapsed to one group. Successful output +/// contains only bounded counts and temporal extrema; source text, credentials, +/// and direct identities never enter the artifact. +/// +/// # Errors +/// +/// Returns a fail-closed error for invalid contracts, snapshot mismatch, +/// duplicate evidence identities, or invalid arithmetic/serialization state. +pub fn execute_analysis_run( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + corpus: &AnalysisCorpus, + completed_at: impl Into, +) -> Result { + request.to_json()?; + accepted.to_json()?; + require_receipt_identity(request, accepted)?; + if request.snapshot_id != corpus.snapshot_id { + return Err(AnalysisEngineError::SnapshotMismatch); + } + let cutoff = KnowledgeCutoff::parse_rfc3339(&request.knowledge_cutoff) + .map_err(|_| AnalysisEngineError::Api(ApiError::InvalidWirePayload))?; + let mut identities = BTreeSet::new(); + let mut eligible = Vec::new(); + for unit in &corpus.evidence_units { + if !identities.insert(unit.evidence_id.clone()) { + return Err(AnalysisEngineError::DuplicateEvidence); + } + if unit.available_time.instant() <= cutoff.instant() { + eligible.push(unit); + } + } + + let completed_at = completed_at.into(); + if eligible.is_empty() { + let terminal_result = AnalysisRunTerminalResult::failed( + request, + accepted, + completed_at, + "no_eligible_evidence", + )?; + return Ok(AnalysisExecution { + artifact: None, + terminal_result, + }); + } + + // The corpus bound makes this conversion and sum strictly smaller than + // `u64::MAX`: 100,000 * u32::MAX is below the 64-bit range. + let eligible_evidence_count = eligible.len() as u64; + let eligible_membership_count = eligible + .iter() + .fold(0_u64, |sum, unit| sum + u64::from(unit.membership_count)); + let (earliest, latest) = eligible.iter().fold( + (eligible[0].event_time, eligible[0].event_time), + |(earliest, latest), unit| (earliest.min(unit.event_time), latest.max(unit.event_time)), + ); + let artifact = AnalysisArtifact { + schema_version: ANALYSIS_ARTIFACT_SCHEMA_VERSION.to_owned(), + run_id: accepted.run_id.clone(), + snapshot_id: request.snapshot_id.clone(), + knowledge_cutoff: cutoff.to_rfc3339(), + eligible_evidence_count, + eligible_membership_count, + earliest_event_time: earliest.to_rfc3339(), + latest_event_time: latest.to_rfc3339(), + }; + artifact.sha256().and_then(move |digest| { + let artifact_id = format!("analysis_artifact_{}", &digest[..16]); + let summary = AnalysisResultSummary { + analysis_family: "temporal_evidence_readiness".to_owned(), + evidence_count: eligible_evidence_count, + statistic_count: ANALYSIS_STATISTIC_COUNT, + validation_status: "validated".to_owned(), + }; + let terminal_result = AnalysisRunTerminalResult::succeeded( + request, + accepted, + artifact_id, + digest, + ANALYSIS_ARTIFACT_SCHEMA_VERSION, + completed_at, + summary, + ) + .map_err(AnalysisEngineError::from)?; + Ok(AnalysisExecution { + artifact: Some(artifact), + terminal_result, + }) + }) +} + +/// Require the accepted receipt to carry the request's idempotency identity. +fn require_receipt_identity( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, +) -> Result<(), AnalysisEngineError> { + if request.idempotency_key != accepted.idempotency_key { + return Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)); + } + Ok(()) +} + +fn format_digest(digest: impl AsRef<[u8]>) -> String { + let mut output = String::with_capacity(digest.as_ref().len() * 2); + for byte in digest.as_ref() { + let _ = write!(output, "{byte:02x}"); + } + output +} + +fn valid_identifier(value: &str) -> bool { + !value.trim().is_empty() + && value.len() <= MAX_ANALYSIS_IDENTIFIER_BYTES + && !value.chars().any(char::is_control) +} + +#[cfg(test)] +mod tests { + use super::{ + ANALYSIS_ARTIFACT_SCHEMA_VERSION, ANALYSIS_STATISTIC_COUNT, AnalysisCorpus, + AnalysisEngineError, AnalysisEvidenceUnit, MAX_ANALYSIS_IDENTIFIER_BYTES, + MAX_EVIDENCE_UNITS, TopicMeasurementError, execute_analysis_run, + }; + use temporal_core::{AvailableTime, EventTime}; + use tepp_api::{AnalysisRunAccepted, AnalysisRunRequest, AnalysisRunTerminalState, ApiError}; + + fn request() -> AnalysisRunRequest { + AnalysisRunRequest { + contract_version: 1, + idempotency_key: "idem-analysis-1".into(), + tenant_workspace_id: "tenant-workspace-1".into(), + snapshot_id: "snapshot-1".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "temporal-evidence-v1".into(), + output_profile: "validation-report".into(), + } + } + + fn accepted() -> AnalysisRunAccepted { + AnalysisRunAccepted::new("run-1", "accepted", "idem-analysis-1").expect("accepted") + } + + fn unit(id: &str, event: &str, available: &str, memberships: u32) -> AnalysisEvidenceUnit { + AnalysisEvidenceUnit::new( + id, + EventTime::parse_rfc3339(event).expect("event"), + AvailableTime::parse_rfc3339(available).expect("available"), + memberships, + ) + .expect("unit") + } + + #[test] + fn successful_run_is_cutoff_safe_and_preserves_multiple_memberships() { + let corpus = AnalysisCorpus::new( + "snapshot-1", + vec![ + unit( + "evidence-1", + "2026-07-01T00:00:00Z", + "2026-07-15T00:00:00Z", + 2, + ), + unit( + "evidence-2", + "2026-07-20T00:00:00Z", + "2026-08-01T00:00:00Z", + 3, + ), + unit( + "late-evidence", + "2026-07-25T00:00:00Z", + "2026-08-02T00:00:00Z", + 9, + ), + ], + ) + .expect("corpus"); + let execution = + execute_analysis_run(&request(), &accepted(), &corpus, "2026-08-03T00:00:00Z") + .expect("execution"); + let artifact = execution.artifact.expect("artifact"); + assert_eq!(artifact.schema_version, ANALYSIS_ARTIFACT_SCHEMA_VERSION); + assert_eq!(artifact.eligible_evidence_count, 2); + assert_eq!(artifact.eligible_membership_count, 5); + assert_eq!(artifact.earliest_event_time, "2026-07-01T00:00:00Z"); + assert_eq!(artifact.latest_event_time, "2026-07-20T00:00:00Z"); + assert_eq!( + execution.terminal_result.run_state, + AnalysisRunTerminalState::Succeeded + ); + let summary = execution.terminal_result.summary.as_ref().expect("summary"); + assert_eq!(summary.evidence_count, 2); + assert_eq!(summary.statistic_count, ANALYSIS_STATISTIC_COUNT); + assert_eq!(summary.validation_status, "validated"); + assert!(execution.terminal_result.result_sha256.is_some()); + assert!(execution.terminal_result.to_json().is_ok()); + } + + #[test] + fn no_eligible_evidence_returns_a_redacted_failure_result() { + let corpus = AnalysisCorpus::new( + "snapshot-1", + vec![unit( + "late", + "2026-07-25T00:00:00Z", + "2026-08-02T00:00:00Z", + 1, + )], + ) + .expect("corpus"); + let execution = + execute_analysis_run(&request(), &accepted(), &corpus, "2026-08-03T00:00:00Z") + .expect("failure result"); + assert!(execution.artifact.is_none()); + assert_eq!( + execution.terminal_result.run_state, + AnalysisRunTerminalState::Failed + ); + assert_eq!( + execution.terminal_result.failure_code.as_deref(), + Some("no_eligible_evidence") + ); + assert!(execution.terminal_result.summary.is_none()); + } + + #[test] + fn trust_boundary_and_shape_errors_fail_closed() { + assert_eq!( + AnalysisEvidenceUnit::new( + "", + EventTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("event"), + AvailableTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("available"), + 1, + ), + Err(AnalysisEngineError::InvalidEvidence) + ); + assert_eq!( + AnalysisCorpus::new("", Vec::new()), + Err(AnalysisEngineError::InvalidEvidence) + ); + assert_eq!( + AnalysisCorpus::new("\n", Vec::new()), + Err(AnalysisEngineError::InvalidEvidence) + ); + assert_eq!( + AnalysisCorpus::new("s".repeat(MAX_ANALYSIS_IDENTIFIER_BYTES + 1), Vec::new()), + Err(AnalysisEngineError::InvalidEvidence) + ); + assert_eq!( + AnalysisEvidenceUnit::new( + "e", + EventTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("event"), + AvailableTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("available"), + 0, + ), + Err(AnalysisEngineError::InvalidEvidence) + ); + assert_eq!( + AnalysisEvidenceUnit::new( + "e".repeat(MAX_ANALYSIS_IDENTIFIER_BYTES + 1), + EventTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("event"), + AvailableTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("available"), + 1, + ), + Err(AnalysisEngineError::InvalidEvidence) + ); + let corpus = AnalysisCorpus::new( + "snapshot-2", + vec![unit( + "evidence-1", + "2026-07-01T00:00:00Z", + "2026-07-01T00:00:00Z", + 1, + )], + ) + .expect("corpus"); + assert_eq!( + execute_analysis_run(&request(), &accepted(), &corpus, "2026-08-03T00:00:00Z"), + Err(AnalysisEngineError::SnapshotMismatch) + ); + let mismatched_receipt = + AnalysisRunAccepted::new("run-1", "accepted", "other-idempotency").expect("receipt"); + let matching_corpus = AnalysisCorpus::new( + "snapshot-1", + vec![unit( + "evidence-1", + "2026-07-01T00:00:00Z", + "2026-07-01T00:00:00Z", + 1, + )], + ) + .expect("corpus"); + assert_eq!( + execute_analysis_run( + &request(), + &mismatched_receipt, + &matching_corpus, + "2026-08-03T00:00:00Z" + ), + Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)) + ); + let duplicate = AnalysisCorpus::new( + "snapshot-1", + vec![ + unit("same", "2026-07-01T00:00:00Z", "2026-07-01T00:00:00Z", 1), + unit("same", "2026-07-02T00:00:00Z", "2026-07-02T00:00:00Z", 1), + ], + ) + .expect("corpus"); + assert_eq!( + execute_analysis_run(&request(), &accepted(), &duplicate, "2026-08-03T00:00:00Z"), + Err(AnalysisEngineError::DuplicateEvidence) + ); + assert_eq!( + AnalysisEngineError::Api(ApiError::LimitExceeded).to_string(), + "API request exceeded configured limits" + ); + assert_eq!( + AnalysisEngineError::SerializationFailure.to_string(), + "analysis artifact serialization failed" + ); + } + + #[test] + fn public_accessors_limits_and_error_messages_are_executable() { + let evidence = unit( + "evidence-accessor", + "2026-07-01T00:00:00Z", + "2026-07-01T00:00:00Z", + 4, + ); + assert_eq!(evidence.evidence_id(), "evidence-accessor"); + assert_eq!( + evidence.event_time(), + EventTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("event") + ); + assert_eq!( + evidence.available_time(), + AvailableTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("available") + ); + assert_eq!(evidence.membership_count(), 4); + let corpus = + AnalysisCorpus::new("snapshot-accessor", vec![evidence.clone()]).expect("corpus"); + assert_eq!(corpus.snapshot_id(), "snapshot-accessor"); + assert_eq!(corpus.evidence_units(), &[evidence]); + + let oversized = AnalysisCorpus::new( + "snapshot-limit", + vec![ + unit("bounded", "2026-07-01T00:00:00Z", "2026-07-01T00:00:00Z", 1,); + MAX_EVIDENCE_UNITS + 1 + ], + ); + assert_eq!(oversized, Err(AnalysisEngineError::LimitExceeded)); + + let messages = [ + ( + AnalysisEngineError::InvalidEvidence, + "invalid analysis evidence", + ), + ( + AnalysisEngineError::DuplicateEvidence, + "duplicate analysis evidence identity", + ), + ( + AnalysisEngineError::SnapshotMismatch, + "analysis snapshot identity mismatch", + ), + ( + AnalysisEngineError::ArithmeticOverflow, + "analysis evidence count overflow", + ), + ( + AnalysisEngineError::SerializationFailure, + "analysis artifact serialization failed", + ), + ( + AnalysisEngineError::LimitExceeded, + "analysis corpus exceeded its execution bound", + ), + ( + AnalysisEngineError::TopicMeasurement(TopicMeasurementError::DidNotConverge), + "topic estimator did not converge", + ), + ( + AnalysisEngineError::InvalidTopicLineageArtifact, + "invalid topic lineage artifact", + ), + ]; + for (error, message) in messages { + assert_eq!(error.to_string(), message); + } + let converted: AnalysisEngineError = ApiError::InvalidWirePayload.into(); + assert_eq!(converted.to_string(), "invalid API wire payload"); + } + + #[test] + fn malformed_request_receipt_cutoff_and_completion_fail_closed() { + let corpus = AnalysisCorpus::new( + "snapshot-1", + vec![unit( + "evidence-1", + "2026-07-01T00:00:00Z", + "2026-07-01T00:00:00Z", + 1, + )], + ) + .expect("corpus"); + + let mut invalid_request = request(); + invalid_request.idempotency_key.clear(); + assert_eq!( + execute_analysis_run( + &invalid_request, + &accepted(), + &corpus, + "2026-08-03T00:00:00Z" + ), + Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)) + ); + + let mut invalid_accepted = accepted(); + invalid_accepted.run_id.clear(); + assert_eq!( + execute_analysis_run( + &request(), + &invalid_accepted, + &corpus, + "2026-08-03T00:00:00Z" + ), + Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)) + ); + + let mut invalid_cutoff = request(); + invalid_cutoff.knowledge_cutoff = "not-a-time".into(); + assert_eq!( + execute_analysis_run( + &invalid_cutoff, + &accepted(), + &corpus, + "2026-08-03T00:00:00Z" + ), + Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)) + ); + + assert_eq!( + execute_analysis_run(&request(), &accepted(), &corpus, "not-a-time"), + Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)) + ); + + let no_evidence = AnalysisCorpus::new( + "snapshot-1", + vec![unit( + "late", + "2026-07-01T00:00:00Z", + "2026-08-02T00:00:00Z", + 1, + )], + ) + .expect("corpus"); + assert_eq!( + execute_analysis_run(&request(), &accepted(), &no_evidence, "not-a-time"), + Err(AnalysisEngineError::Api(ApiError::InvalidWirePayload)) + ); + } +} diff --git a/crates/analysis_engine/src/topic_lineage_artifact.rs b/crates/analysis_engine/src/topic_lineage_artifact.rs new file mode 100644 index 00000000..9b33ce17 --- /dev/null +++ b/crates/analysis_engine/src/topic_lineage_artifact.rs @@ -0,0 +1,471 @@ +//! Digest-bound completed artifacts from the ADR-0012 topic estimator. + +use std::collections::BTreeSet; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use temporal_core::KnowledgeCutoff; +use tepp_api::{ + AnalysisResultSummary, AnalysisRunAccepted, AnalysisRunRequest, AnalysisRunTerminalResult, +}; +use topic_measurement::{ + ReferenceTopicInput, ReferenceTopicModelConfig, fit_reference_topic_model, +}; +use uuid::Uuid; + +use crate::{AnalysisEngineError, format_digest, require_receipt_identity, valid_identifier}; + +/// Versioned schema for a completed TRSL topic-lineage artifact. +pub const TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION: &str = "tepp.trsl_topic_lineage.v1"; +/// Model contract required by the CPU `f64` reference execution path. +pub const TOPIC_LINEAGE_MODEL_CONTRACT_VERSION: &str = "trsl_tm_cpu_f64_v1"; +/// Analysis-run output profile required for a topic-lineage artifact. +pub const TOPIC_LINEAGE_OUTPUT_PROFILE: &str = "trsl_topic_lineage_v1"; +/// Maximum canonical artifact JSON size. +pub const TOPIC_LINEAGE_ARTIFACT_BYTE_LIMIT: usize = 256 * 1024; +const TOPIC_LINEAGE_EDGE_LIMIT: usize = 100_000; + +/// One fitted same-topic predecessor/successor association. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TopicLineageArtifactEdge { + /// Opaque predecessor document identity. + pub predecessor_document_id: String, + /// Opaque successor document identity. + pub successor_document_id: String, + /// Artifact-local global topic index. + pub topic_index: u64, + /// Minimum dominant-topic posterior mean across the linked documents. + pub association_strength: f64, +} + +/// Completed, bounded topic-lineage result consumed by product-history clients. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TopicLineageArtifact { + /// Exact versioned schema identity. + pub schema_version: String, + /// Opaque accepted-run identity. + pub run_id: String, + /// Immutable source snapshot identity. + pub snapshot_id: String, + /// Historical evidence cutoff used by the estimator. + pub knowledge_cutoff: String, + /// Selected deterministic initialization seed. + pub selected_seed: u64, + /// Iterations used by the selected converged fit. + pub iterations: u64, + /// Final finite penalized objective. + pub objective: f64, + /// Number of global topics in the fitted model. + pub topic_count: u64, + /// Number of modeled evidence documents. + pub evidence_count: u64, + /// Documents incident to at least one fitted same-topic sequence edge. + pub connected_post_count: u64, + /// Topics represented by at least one fitted sequence edge. + pub lineage_count: u64, + /// Fitted edges restricted to explicit forward predecessor/successor input. + pub sequence_edges: Vec, + /// Fixed claim boundary for consumer copy. + pub inference_status: String, +} + +impl TopicLineageArtifact { + /// Parse and fully validate a bounded artifact JSON payload. + /// + /// # Errors + /// + /// Returns [`AnalysisEngineError::InvalidTopicLineageArtifact`] when the + /// schema, dimensions, identifiers, counts, edges, or claim boundary fail. + pub fn from_json(payload: &str) -> Result { + if payload.len() > TOPIC_LINEAGE_ARTIFACT_BYTE_LIMIT { + return Err(AnalysisEngineError::LimitExceeded); + } + let artifact: Self = serde_json::from_str(payload) + .map_err(|_| AnalysisEngineError::InvalidTopicLineageArtifact)?; + artifact.validate()?; + Ok(artifact) + } + + /// Serialize canonical validated artifact JSON. + /// + /// # Errors + /// + /// Returns a typed validation, serialization, or size failure. + pub fn to_json(&self) -> Result { + self.validate()?; + let payload = + serde_json::to_string(self).map_err(|_| AnalysisEngineError::SerializationFailure)?; + if payload.len() > TOPIC_LINEAGE_ARTIFACT_BYTE_LIMIT { + return Err(AnalysisEngineError::LimitExceeded); + } + Ok(payload) + } + + /// Return the lowercase SHA-256 digest of canonical artifact JSON. + /// + /// # Errors + /// + /// Returns a typed validation or serialization failure. + pub fn sha256(&self) -> Result { + self.to_json() + .map(|json| format_digest(Sha256::digest(json.into_bytes()))) + } + + fn validate(&self) -> Result<(), AnalysisEngineError> { + if self.schema_version != TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION + || !valid_identifier(&self.run_id) + || !valid_identifier(&self.snapshot_id) + || KnowledgeCutoff::parse_rfc3339(&self.knowledge_cutoff).is_err() + || self.iterations == 0 + || !self.objective.is_finite() + || self.topic_count < 2 + || self.evidence_count < 2 + || self.connected_post_count > self.evidence_count + || self.lineage_count > self.topic_count + || self.sequence_edges.len() > TOPIC_LINEAGE_EDGE_LIMIT + || self.inference_status != "fitted_topic_association_not_causation" + { + return Err(AnalysisEngineError::InvalidTopicLineageArtifact); + } + let mut pairs = BTreeSet::new(); + let mut connected = BTreeSet::new(); + let mut lineages = BTreeSet::new(); + for edge in &self.sequence_edges { + let predecessor = Uuid::parse_str(&edge.predecessor_document_id) + .map_err(|_| AnalysisEngineError::InvalidTopicLineageArtifact)?; + let successor = Uuid::parse_str(&edge.successor_document_id) + .map_err(|_| AnalysisEngineError::InvalidTopicLineageArtifact)?; + if predecessor == successor + || edge.topic_index >= self.topic_count + || !edge.association_strength.is_finite() + || edge.association_strength <= 0.0 + || edge.association_strength > 1.0 + || !pairs.insert((predecessor, successor)) + { + return Err(AnalysisEngineError::InvalidTopicLineageArtifact); + } + connected.insert(predecessor); + connected.insert(successor); + lineages.insert(edge.topic_index); + } + if self.connected_post_count != connected.len() as u64 + || self.lineage_count != lineages.len() as u64 + { + return Err(AnalysisEngineError::InvalidTopicLineageArtifact); + } + Ok(()) + } +} + +/// One completed topic-lineage artifact and its request-bound terminal result. +#[derive(Clone, Debug, PartialEq)] +pub struct TopicLineageExecution { + /// Digest-bound completed model artifact. + pub artifact: TopicLineageArtifact, + /// Terminal result carrying the artifact identity, digest, and schema. + pub terminal_result: AnalysisRunTerminalResult, +} + +/// Execute the validated ADR-0012 CPU `f64` reference estimator. +/// +/// The caller supplies the exact snapshot identity and cutoff used to construct +/// `input`; both must exactly match the already-validated analysis request. +/// This executor preserves the estimator result and does not select `K`, infer +/// causal edges, or emit a partial artifact. +/// +/// # Errors +/// +/// Returns a request/receipt/snapshot/cutoff/profile error, estimator failure, +/// arithmetic error, or invalid/oversized artifact error. +pub fn execute_topic_lineage_run( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + snapshot_id: &str, + knowledge_cutoff: KnowledgeCutoff, + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + completed_at: impl Into, +) -> Result { + request.to_json()?; + accepted.to_json()?; + require_receipt_identity(request, accepted)?; + if request.snapshot_id != snapshot_id { + return Err(AnalysisEngineError::SnapshotMismatch); + } + if request.knowledge_cutoff != knowledge_cutoff.to_rfc3339() + || request.model_contract_version != TOPIC_LINEAGE_MODEL_CONTRACT_VERSION + || request.output_profile != TOPIC_LINEAGE_OUTPUT_PROFILE + { + return Err(AnalysisEngineError::InvalidEvidence); + } + + let model = fit_reference_topic_model(input, config)?; + let topic_count = u64::try_from(model.topic_term_probabilities.len()) + .map_err(|_| AnalysisEngineError::ArithmeticOverflow)?; + let evidence_count = u64::try_from(input.document_count()) + .map_err(|_| AnalysisEngineError::ArithmeticOverflow)?; + let connected_post_count = u64::try_from(model.connected_post_count) + .map_err(|_| AnalysisEngineError::ArithmeticOverflow)?; + let lineage_count = + u64::try_from(model.lineage_count).map_err(|_| AnalysisEngineError::ArithmeticOverflow)?; + let sequence_edges: Vec<_> = model + .sequence_edges + .iter() + .map(|edge| { + Ok(TopicLineageArtifactEdge { + predecessor_document_id: edge.predecessor_document_id.to_string(), + successor_document_id: edge.successor_document_id.to_string(), + topic_index: u64::try_from(edge.topic_index) + .map_err(|_| AnalysisEngineError::ArithmeticOverflow)?, + association_strength: edge.association_strength, + }) + }) + .collect::>()?; + let artifact = TopicLineageArtifact { + schema_version: TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION.into(), + run_id: accepted.run_id.clone(), + snapshot_id: snapshot_id.to_owned(), + knowledge_cutoff: knowledge_cutoff.to_rfc3339(), + selected_seed: model.seed, + iterations: u64::try_from(model.iterations) + .map_err(|_| AnalysisEngineError::ArithmeticOverflow)?, + objective: model.objective, + topic_count, + evidence_count, + connected_post_count, + lineage_count, + sequence_edges, + inference_status: "fitted_topic_association_not_causation".into(), + }; + let digest = artifact.sha256()?; + let statistic_count = u64::try_from(artifact.sequence_edges.len()) + .map_err(|_| AnalysisEngineError::ArithmeticOverflow)? + .checked_add(2) + .ok_or(AnalysisEngineError::ArithmeticOverflow)?; + let summary = AnalysisResultSummary::new( + "trsl_topic_lineage", + evidence_count, + statistic_count, + "reference_estimator_converged", + ); + let summary = summary?; + let terminal_result = AnalysisRunTerminalResult::succeeded( + request, + accepted, + format!("topic_lineage_artifact_{}", &digest[..16]), + digest, + TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION, + completed_at, + summary, + ); + let terminal_result = terminal_result?; + Ok(TopicLineageExecution { + artifact, + terminal_result, + }) +} + +#[cfg(test)] +mod tests { + use super::{ + TOPIC_LINEAGE_ARTIFACT_BYTE_LIMIT, TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION, + TOPIC_LINEAGE_EDGE_LIMIT, TopicLineageArtifact, TopicLineageArtifactEdge, + }; + use crate::AnalysisEngineError; + + fn artifact() -> TopicLineageArtifact { + TopicLineageArtifact { + schema_version: TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION.into(), + run_id: "run-1".into(), + snapshot_id: "snapshot-1".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + selected_seed: 7, + iterations: 4, + objective: -1.0, + topic_count: 2, + evidence_count: 2, + connected_post_count: 2, + lineage_count: 1, + sequence_edges: vec![TopicLineageArtifactEdge { + predecessor_document_id: "00000000-0000-0000-0000-000000000001".into(), + successor_document_id: "00000000-0000-0000-0000-000000000002".into(), + topic_index: 0, + association_strength: 0.8, + }], + inference_status: "fitted_topic_association_not_causation".into(), + } + } + + fn assert_invalid(artifact: &TopicLineageArtifact) { + assert_eq!( + artifact.to_json(), + Err(AnalysisEngineError::InvalidTopicLineageArtifact) + ); + } + + #[test] + fn artifact_round_trip_and_size_bounds_fail_closed() { + let artifact = artifact(); + let payload = artifact.to_json().expect("json"); + assert_eq!( + TopicLineageArtifact::from_json(&payload), + Ok(artifact.clone()) + ); + assert_eq!(artifact.sha256().expect("digest").len(), 64); + assert_eq!( + TopicLineageArtifact::from_json("{}"), + Err(AnalysisEngineError::InvalidTopicLineageArtifact) + ); + assert_eq!( + TopicLineageArtifact::from_json(&"x".repeat(TOPIC_LINEAGE_ARTIFACT_BYTE_LIMIT + 1)), + Err(AnalysisEngineError::LimitExceeded) + ); + + let mut oversized = artifact; + oversized.sequence_edges = (1_u128..=2_000) + .map(|index| TopicLineageArtifactEdge { + predecessor_document_id: uuid::Uuid::from_u128(index).to_string(), + successor_document_id: uuid::Uuid::from_u128(index + 1).to_string(), + topic_index: 0, + association_strength: 0.8, + }) + .collect(); + oversized.evidence_count = 2_001; + oversized.connected_post_count = 2_001; + assert_eq!(oversized.to_json(), Err(AnalysisEngineError::LimitExceeded)); + } + + #[test] + fn artifact_metadata_tampering_fails_closed() { + let artifact = artifact(); + let invalid_artifacts = [ + { + let mut value = artifact.clone(); + value.schema_version.clear(); + value + }, + { + let mut value = artifact.clone(); + value.run_id.clear(); + value + }, + { + let mut value = artifact.clone(); + value.snapshot_id.clear(); + value + }, + { + let mut value = artifact.clone(); + value.knowledge_cutoff = "invalid".into(); + value + }, + { + let mut value = artifact.clone(); + value.iterations = 0; + value + }, + { + let mut value = artifact.clone(); + value.objective = f64::NAN; + value + }, + { + let mut value = artifact.clone(); + value.topic_count = 1; + value + }, + { + let mut value = artifact.clone(); + value.evidence_count = 1; + value + }, + { + let mut value = artifact.clone(); + value.connected_post_count = 3; + value + }, + { + let mut value = artifact.clone(); + value.lineage_count = 3; + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges = + vec![value.sequence_edges[0].clone(); TOPIC_LINEAGE_EDGE_LIMIT + 1]; + value + }, + { + let mut value = artifact.clone(); + value.inference_status.clear(); + value + }, + ]; + for invalid in invalid_artifacts { + assert_invalid(&invalid); + } + } + + #[test] + fn artifact_edge_tampering_fails_closed() { + let artifact = artifact(); + let invalid_artifacts = [ + { + let mut value = artifact.clone(); + value.sequence_edges[0].successor_document_id = + value.sequence_edges[0].predecessor_document_id.clone(); + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges[0].topic_index = 2; + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges[0].association_strength = f64::NAN; + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges[0].association_strength = 0.0; + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges[0].association_strength = 1.1; + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges.push(value.sequence_edges[0].clone()); + value + }, + { + let mut value = artifact.clone(); + value.connected_post_count = 1; + value + }, + { + let mut value = artifact.clone(); + value.lineage_count = 0; + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges[0].predecessor_document_id = "invalid".into(); + value + }, + { + let mut value = artifact.clone(); + value.sequence_edges[0].successor_document_id = "invalid".into(); + value + }, + ]; + for invalid in invalid_artifacts { + assert_invalid(&invalid); + } + } +} diff --git a/crates/analysis_engine/tests/crate_contract.rs b/crates/analysis_engine/tests/crate_contract.rs new file mode 100644 index 00000000..c401c287 --- /dev/null +++ b/crates/analysis_engine/tests/crate_contract.rs @@ -0,0 +1,6 @@ +//! Package identity contract for the analysis engine. + +#[test] +fn package_identity_is_stable() { + assert_eq!(env!("CARGO_PKG_NAME"), "analysis_engine"); +} diff --git a/crates/analysis_engine/tests/end_to_end_contract.rs b/crates/analysis_engine/tests/end_to_end_contract.rs new file mode 100644 index 00000000..829e56f5 --- /dev/null +++ b/crates/analysis_engine/tests/end_to_end_contract.rs @@ -0,0 +1,107 @@ +//! Realistic cutoff-safe end-to-end analysis execution. + +use analysis_engine::{ + AnalysisCorpus, AnalysisEngineError, AnalysisEvidenceUnit, execute_analysis_run, +}; +use temporal_core::{AvailableTime, EventTime}; +use tepp_api::{AnalysisRunAccepted, AnalysisRunRequest, AnalysisRunTerminalState}; + +fn evidence(id: &str, available: &str, memberships: u32) -> AnalysisEvidenceUnit { + AnalysisEvidenceUnit::new( + id, + EventTime::parse_rfc3339("2026-07-10T12:00:00Z").expect("event time"), + AvailableTime::parse_rfc3339(available).expect("available time"), + memberships, + ) + .expect("evidence") +} + +#[test] +fn production_shape_run_excludes_future_available_evidence() { + let request = AnalysisRunRequest { + contract_version: 1, + idempotency_key: "customer-run-2026-08-01".into(), + tenant_workspace_id: "workspace-opaque-1".into(), + snapshot_id: "snapshot-customer-2026-08-01".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "temporal-evidence-v1".into(), + output_profile: "validation-report".into(), + }; + let accepted = + AnalysisRunAccepted::new("run-customer-1", "accepted", "customer-run-2026-08-01") + .expect("accepted"); + let corpus = AnalysisCorpus::new( + "snapshot-customer-2026-08-01", + vec![ + evidence("invoice-renewal", "2026-07-31T23:59:59Z", 2), + evidence("later-correction", "2026-08-01T00:00:01Z", 4), + ], + ) + .expect("snapshot"); + let execution = execute_analysis_run(&request, &accepted, &corpus, "2026-08-01T00:01:00Z") + .expect("execute"); + assert_eq!( + execution.terminal_result.run_state, + AnalysisRunTerminalState::Succeeded + ); + assert_eq!( + execution + .artifact + .expect("artifact") + .eligible_evidence_count, + 1 + ); +} + +#[test] +fn snapshot_identity_is_not_inferred_from_customer_payload() { + let request = AnalysisRunRequest { + contract_version: 1, + idempotency_key: "run".into(), + tenant_workspace_id: "workspace".into(), + snapshot_id: "request-snapshot".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "model-v1".into(), + output_profile: "report".into(), + }; + let accepted = AnalysisRunAccepted::new("run", "accepted", "run").expect("accepted"); + let corpus = AnalysisCorpus::new( + "other-snapshot", + vec![evidence("evidence", "2026-07-01T00:00:00Z", 1)], + ) + .expect("snapshot"); + assert_eq!( + execute_analysis_run(&request, &accepted, &corpus, "2026-08-01T00:01:00Z"), + Err(AnalysisEngineError::SnapshotMismatch) + ); +} + +#[test] +fn mismatched_receipt_identity_is_rejected_before_corpus_scan() { + let request = AnalysisRunRequest { + contract_version: 1, + idempotency_key: "request-idempotency".into(), + tenant_workspace_id: "workspace".into(), + snapshot_id: "snapshot".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "model-v1".into(), + output_profile: "report".into(), + }; + let accepted = + AnalysisRunAccepted::new("run", "accepted", "receipt-idempotency").expect("accepted"); + let corpus = AnalysisCorpus::new( + "snapshot", + vec![ + evidence("duplicate-evidence", "2026-07-01T00:00:00Z", 1), + evidence("duplicate-evidence", "2026-07-02T00:00:00Z", 1), + ], + ) + .expect("snapshot"); + + assert_eq!( + execute_analysis_run(&request, &accepted, &corpus, "2026-08-01T00:01:00Z"), + Err(AnalysisEngineError::Api( + tepp_api::ApiError::InvalidWirePayload + )) + ); +} diff --git a/crates/analysis_engine/tests/topic_lineage_execution_contract.rs b/crates/analysis_engine/tests/topic_lineage_execution_contract.rs new file mode 100644 index 00000000..045a8946 --- /dev/null +++ b/crates/analysis_engine/tests/topic_lineage_execution_contract.rs @@ -0,0 +1,238 @@ +//! End-to-end contract for the completed TRSL topic-lineage artifact. + +use analysis_engine::{ + AnalysisEngineError, TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION, + TOPIC_LINEAGE_MODEL_CONTRACT_VERSION, TOPIC_LINEAGE_OUTPUT_PROFILE, execute_topic_lineage_run, +}; +use corpus_split::{CorpusDocument, CorpusSnapshot}; +use membership_core::{ + GroupId, MemberId, MembershipAssignment, MembershipNetwork, MembershipRole, MembershipWeight, +}; +use relation_graph::{ + RelationEdge, RelationEndpointId, RelationEvidenceStatus, RelationGraph, RelationKind, +}; +use temporal_core::{ + AvailableTime, EventTime, KnowledgeCutoff, TemporalBoundary, TemporalInterval, + TemporalPrecision, +}; +use tepp_api::{AnalysisRunAccepted, AnalysisRunRequest, AnalysisRunTerminalState}; +use topic_measurement::{ReferenceTopicInput, ReferenceTopicModelConfig, SparseMatrix}; +use uuid::Uuid; + +fn event_time(day: u8) -> EventTime { + EventTime::parse_rfc3339(&format!("2026-07-{day:02}T00:00:00Z")).expect("event time") +} + +fn fixture() -> ( + CorpusSnapshot, + Vec, + Vec, + MembershipNetwork, + RelationGraph, +) { + let ids: Vec<_> = (1_u128..=4).map(Uuid::from_u128).collect(); + let times: Vec<_> = (1_u8..=4).map(event_time).collect(); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-08-01T00:00:00Z").expect("cutoff"); + let available = AvailableTime::parse_rfc3339("2026-07-01T00:00:00Z").expect("available"); + let mut snapshot = CorpusSnapshot::new(); + let mut memberships = MembershipNetwork::new(); + for id in &ids { + snapshot + .insert_if_eligible(CorpusDocument::new(*id, available), &cutoff) + .expect("eligible"); + memberships + .insert( + MembershipAssignment::new( + MemberId::from_uuid(*id), + GroupId::from_uuid(Uuid::from_u128(100)), + MembershipRole::Project, + MembershipWeight::full().expect("weight"), + event_time(1), + event_time(9), + ) + .expect("membership"), + ) + .expect("insert"); + } + let mut relations = RelationGraph::new(); + for (source, target, source_day, target_day) in [(0, 1, 1, 2), (1, 2, 2, 3), (2, 3, 3, 4)] { + let interval = |day| { + TemporalInterval::bounded( + TemporalBoundary::Included(event_time(day)), + TemporalBoundary::Included( + EventTime::parse_rfc3339(&format!("2026-07-{day:02}T12:00:00Z")).expect("end"), + ), + TemporalPrecision::Second, + ) + .expect("interval") + }; + relations + .insert( + RelationEdge::new( + RelationKind::TransitionsTo, + RelationEndpointId::from_uuid(ids[source]), + RelationEndpointId::from_uuid(ids[target]), + RelationEvidenceStatus::Observed, + interval(source_day), + interval(target_day), + ) + .expect("relation"), + ) + .expect("insert relation"); + } + (snapshot, ids, times, memberships, relations) +} + +fn request() -> AnalysisRunRequest { + AnalysisRunRequest { + contract_version: 1, + idempotency_key: "topic-lineage-idem".into(), + tenant_workspace_id: "tenant-workspace".into(), + snapshot_id: "snapshot-topic-lineage".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: TOPIC_LINEAGE_MODEL_CONTRACT_VERSION.into(), + output_profile: TOPIC_LINEAGE_OUTPUT_PROFILE.into(), + } +} + +#[test] +fn fitted_topics_emit_digest_bound_predecessor_successor_counts() { + let (snapshot, ids, times, memberships, relations) = fixture(); + let counts = SparseMatrix::from_csr( + 4, + 4, + vec![0, 2, 4, 6, 8], + vec![0, 1, 0, 1, 2, 3, 2, 3], + vec![90.0, 10.0, 85.0, 15.0, 10.0, 90.0, 15.0, 85.0], + ) + .expect("counts"); + let input = ReferenceTopicInput::new( + &snapshot, + ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("input"); + let config = ReferenceTopicModelConfig::new(2, vec![7, 11], 2_000, 1e-5) + .expect("config") + .with_hyperparameters(1.0, 0.5, 0.01, 0.05, 0.2) + .expect("hyperparameters"); + let request = request(); + let accepted = + AnalysisRunAccepted::new("run-topic-lineage", "accepted", &request.idempotency_key) + .expect("accepted"); + let execution = execute_topic_lineage_run( + &request, + &accepted, + "snapshot-topic-lineage", + KnowledgeCutoff::parse_rfc3339("2026-08-01T00:00:00Z").expect("cutoff"), + &input, + &config, + "2026-08-02T00:00:00Z", + ) + .expect("execution"); + + assert_eq!( + execution.artifact.schema_version, + TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION + ); + assert_eq!(execution.artifact.connected_post_count, 4); + assert_eq!(execution.artifact.lineage_count, 2); + assert_eq!(execution.artifact.sequence_edges.len(), 2); + assert_eq!( + execution.terminal_result.run_state, + AnalysisRunTerminalState::Succeeded + ); + assert_eq!( + execution.terminal_result.result_sha256.as_deref(), + Some(execution.artifact.sha256().expect("digest").as_str()) + ); + assert_eq!( + execution.terminal_result.result_schema_version.as_deref(), + Some(TOPIC_LINEAGE_ARTIFACT_SCHEMA_VERSION) + ); + assert!(execution.artifact.to_json().is_ok()); +} + +#[test] +fn execution_refuses_binding_and_nonconvergence_without_an_artifact() { + let (snapshot, ids, times, memberships, relations) = fixture(); + let counts = SparseMatrix::from_csr(4, 2, vec![0, 1, 2, 3, 4], vec![0, 0, 1, 1], vec![1.0; 4]) + .expect("counts"); + let input = ReferenceTopicInput::new( + &snapshot, + ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("input"); + let request = request(); + let accepted = + AnalysisRunAccepted::new("run-topic-lineage", "accepted", &request.idempotency_key) + .expect("accepted"); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-08-01T00:00:00Z").expect("cutoff"); + let config = ReferenceTopicModelConfig::new(2, vec![1], 2, 1e-12).expect("config"); + + assert_eq!( + execute_topic_lineage_run( + &request, + &accepted, + "other-snapshot", + cutoff, + &input, + &config, + "2026-08-02T00:00:00Z", + ), + Err(AnalysisEngineError::SnapshotMismatch) + ); + for invalid_request in [ + { + let mut value = request.clone(); + value.knowledge_cutoff = "2026-08-02T00:00:00Z".into(); + value + }, + { + let mut value = request.clone(); + value.model_contract_version = "other-model".into(); + value + }, + { + let mut value = request.clone(); + value.output_profile = "other-profile".into(); + value + }, + ] { + assert_eq!( + execute_topic_lineage_run( + &invalid_request, + &accepted, + "snapshot-topic-lineage", + cutoff, + &input, + &config, + "2026-08-02T00:00:00Z", + ), + Err(AnalysisEngineError::InvalidEvidence) + ); + } + assert_eq!( + execute_topic_lineage_run( + &request, + &accepted, + "snapshot-topic-lineage", + cutoff, + &input, + &config, + "2026-08-02T00:00:00Z", + ), + Err(AnalysisEngineError::TopicMeasurement( + topic_measurement::TopicMeasurementError::DidNotConverge + )) + ); +} diff --git a/crates/model_selection/Cargo.toml b/crates/model_selection/Cargo.toml new file mode 100644 index 00000000..ae636952 --- /dev/null +++ b/crates/model_selection/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "model_selection" +description = "Statistical and Pareto candidate-K gates that refuse LLM numerical authority." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[lints] +workspace = true diff --git a/crates/model_selection/src/candidate.rs b/crates/model_selection/src/candidate.rs new file mode 100644 index 00000000..b692ef13 --- /dev/null +++ b/crates/model_selection/src/candidate.rs @@ -0,0 +1,156 @@ +//! Candidate topic counts with statistical diagnostics. + +use crate::ModelSelectionError; + +/// One candidate `K` together with the diagnostics that may admit it. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ModelCandidate { + candidate_k: u32, + held_out_log_likelihood: Option, + complexity: Option, + llm_vote_only: bool, +} + +impl ModelCandidate { + /// Construct a statistically supported candidate. + /// + /// # Errors + /// + /// Returns [`ModelSelectionError::NonPositiveCandidateK`] when `candidate_k` + /// is less than two, or [`ModelSelectionError::InvalidDiagnostic`] when a + /// diagnostic is non-finite. + pub fn statistical( + candidate_k: u32, + held_out_log_likelihood: f64, + complexity: f64, + ) -> Result { + if candidate_k < 2 { + return Err(ModelSelectionError::NonPositiveCandidateK); + } + if !held_out_log_likelihood.is_finite() || !complexity.is_finite() || complexity < 0.0 { + return Err(ModelSelectionError::InvalidDiagnostic); + } + Ok(Self { + candidate_k, + held_out_log_likelihood: Some(held_out_log_likelihood), + complexity: Some(complexity), + llm_vote_only: false, + }) + } + + /// Construct a candidate whose only support is an LLM vote. + /// + /// The vote may later recommend among statistically admissible candidates. + /// It cannot itself define the numerical optimum. + /// + /// # Errors + /// + /// Returns [`ModelSelectionError::NonPositiveCandidateK`] when `candidate_k` + /// is less than two. + pub fn llm_vote_only(candidate_k: u32) -> Result { + if candidate_k < 2 { + return Err(ModelSelectionError::NonPositiveCandidateK); + } + Ok(Self { + candidate_k, + held_out_log_likelihood: None, + complexity: None, + llm_vote_only: true, + }) + } + + /// Return the candidate topic count. + #[must_use] + pub const fn candidate_k(self) -> u32 { + self.candidate_k + } + + /// Return whether this candidate carries finite statistical diagnostics. + #[must_use] + pub const fn is_statistically_supported(self) -> bool { + match (self.held_out_log_likelihood, self.complexity) { + (Some(_), Some(_)) => !self.llm_vote_only, + _ => false, + } + } + + /// Held-out log-likelihood when the candidate is statistically supported. + #[must_use] + pub const fn held_out_log_likelihood(self) -> Option { + self.held_out_log_likelihood + } + + /// Complexity penalty (larger is worse) when statistically supported. + #[must_use] + pub const fn complexity(self) -> Option { + self.complexity + } + + /// Return whether the candidate is an LLM vote without statistical support. + #[must_use] + pub const fn is_llm_vote_only(self) -> bool { + self.llm_vote_only + } + + /// Return whether `self` Pareto-dominates `other` on likelihood and complexity. + #[must_use] + pub fn dominates(self, other: Self) -> bool { + let (Some(self_ll), Some(self_complexity), Some(other_ll), Some(other_complexity)) = ( + self.held_out_log_likelihood, + self.complexity, + other.held_out_log_likelihood, + other.complexity, + ) else { + return false; + }; + let no_worse = self_ll >= other_ll && self_complexity <= other_complexity; + let strictly_better = self_ll > other_ll || self_complexity < other_complexity; + no_worse && strictly_better + } +} + +#[cfg(test)] +mod tests { + use super::ModelCandidate; + use crate::ModelSelectionError; + + #[test] + fn statistical_candidate_accessors_and_dominance_cover_branches() { + let better = ModelCandidate::statistical(4, -10.0, 5.0).expect("better"); + let worse = ModelCandidate::statistical(8, -20.0, 9.0).expect("worse"); + assert_eq!(better.candidate_k(), 4); + assert_eq!(better.held_out_log_likelihood(), Some(-10.0)); + assert_eq!(better.complexity(), Some(5.0)); + assert!(better.is_statistically_supported()); + assert!(!better.is_llm_vote_only()); + assert!(better.dominates(worse)); + assert!(!worse.dominates(better)); + assert!(!better.dominates(better)); + + let llm = ModelCandidate::llm_vote_only(3).expect("valid llm candidate"); + assert!(llm.is_llm_vote_only()); + assert!(!llm.is_statistically_supported()); + assert!(!llm.dominates(better)); + assert!(!better.dominates(llm)); + assert_eq!( + ModelCandidate::statistical(0, -1.0, 1.0), + Err(ModelSelectionError::NonPositiveCandidateK) + ); + assert_eq!( + ModelCandidate::statistical(2, -1.0, -0.1), + Err(ModelSelectionError::InvalidDiagnostic) + ); + assert_eq!( + ModelCandidate::statistical(2, f64::NAN, 1.0), + Err(ModelSelectionError::InvalidDiagnostic) + ); + assert_eq!( + ModelCandidate::statistical(2, -1.0, f64::INFINITY), + Err(ModelSelectionError::InvalidDiagnostic) + ); + assert_eq!( + ModelCandidate::llm_vote_only(1), + Err(ModelSelectionError::NonPositiveCandidateK) + ); + } +} diff --git a/crates/model_selection/src/error.rs b/crates/model_selection/src/error.rs new file mode 100644 index 00000000..9c34391e --- /dev/null +++ b/crates/model_selection/src/error.rs @@ -0,0 +1,60 @@ +//! Fail-closed model-selection errors. + +use std::fmt; + +/// A fail-closed model-selection error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum ModelSelectionError { + /// Candidate `K` was less than two. + NonPositiveCandidateK, + /// A diagnostic was non-finite or otherwise unusable. + InvalidDiagnostic, + /// No candidates were supplied. + EmptyCandidateSet, + /// An LLM vote was asked to define the numerical optimum. + LlmVoteIsNotStatisticalAuthority, +} + +impl fmt::Display for ModelSelectionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::NonPositiveCandidateK => "candidate k must be at least two", + Self::InvalidDiagnostic => "invalid model-selection diagnostic", + Self::EmptyCandidateSet => "empty model-selection candidate set", + Self::LlmVoteIsNotStatisticalAuthority => "llm vote is not statistical authority", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for ModelSelectionError {} + +#[cfg(test)] +mod tests { + use super::ModelSelectionError; + + #[test] + fn error_messages_are_stable() { + for (error, message) in [ + ( + ModelSelectionError::NonPositiveCandidateK, + "candidate k must be at least two", + ), + ( + ModelSelectionError::InvalidDiagnostic, + "invalid model-selection diagnostic", + ), + ( + ModelSelectionError::EmptyCandidateSet, + "empty model-selection candidate set", + ), + ( + ModelSelectionError::LlmVoteIsNotStatisticalAuthority, + "llm vote is not statistical authority", + ), + ] { + assert_eq!(error.to_string(), message); + } + } +} diff --git a/crates/model_selection/src/gate.rs b/crates/model_selection/src/gate.rs new file mode 100644 index 00000000..72fc9aac --- /dev/null +++ b/crates/model_selection/src/gate.rs @@ -0,0 +1,123 @@ +//! Pareto admission and selection among statistically supported candidates. + +use crate::{ModelCandidate, ModelSelectionError}; + +/// Select the unique admissible `K` from a Pareto-filtered statistical front. +/// +/// LLM-only candidates are ignored as recommenders and never become the +/// numerical optimum. Among non-dominated statistical candidates the gate +/// prefers higher held-out log-likelihood, then smaller `K`; complexity is +/// applied while constructing the Pareto front. +/// +/// # Errors +/// +/// Returns [`ModelSelectionError::EmptyCandidateSet`] when no candidates are +/// supplied or [`ModelSelectionError::LlmVoteIsNotStatisticalAuthority`] when +/// every candidate is an LLM vote. +pub fn select_candidate_k(candidates: &[ModelCandidate]) -> Result { + if candidates.is_empty() { + return Err(ModelSelectionError::EmptyCandidateSet); + } + if candidates + .iter() + .all(|candidate| candidate.is_llm_vote_only()) + { + return Err(ModelSelectionError::LlmVoteIsNotStatisticalAuthority); + } + + let statistical: Vec = candidates + .iter() + .copied() + .filter(|candidate| candidate.is_statistically_supported()) + .collect(); + let mut front: Vec = statistical + .iter() + .copied() + .filter(|candidate| !statistical.iter().any(|other| other.dominates(*candidate))) + .collect(); + front.sort_by(|left, right| { + let ll_ord = right + .held_out_log_likelihood() + .partial_cmp(&left.held_out_log_likelihood()) + .unwrap_or(std::cmp::Ordering::Equal); + if ll_ord != std::cmp::Ordering::Equal { + return ll_ord; + } + left.candidate_k().cmp(&right.candidate_k()) + }); + Ok(front[0].candidate_k()) +} + +/// RMSE of selected `K` replications against a known-truth topic count. +/// +/// # Errors +/// +/// Returns [`ModelSelectionError::EmptyCandidateSet`] when `selected` is +/// empty, [`ModelSelectionError::NonPositiveCandidateK`] when `truth_k` is +/// less than two, or [`ModelSelectionError::InvalidDiagnostic`] when a +/// selected replication is less than two. +pub fn selected_k_root_mean_square_error( + selected: &[u32], + truth_k: u32, +) -> Result { + if selected.is_empty() { + return Err(ModelSelectionError::EmptyCandidateSet); + } + if truth_k < 2 { + return Err(ModelSelectionError::NonPositiveCandidateK); + } + let mut sum_squares = 0.0_f64; + for selected_k in selected { + if *selected_k < 2 { + return Err(ModelSelectionError::InvalidDiagnostic); + } + let residual = f64::from(*selected_k) - f64::from(truth_k); + sum_squares += residual * residual; + } + Ok((sum_squares / selected.len() as f64).sqrt()) +} + +#[cfg(test)] +mod tests { + use super::{select_candidate_k, selected_k_root_mean_square_error}; + use crate::{ModelCandidate, ModelSelectionError}; + + #[test] + fn gate_helpers_cover_local_branches() { + let a = ModelCandidate::statistical(2, -30.0, 8.0).expect("a"); + let b = ModelCandidate::statistical(4, -30.0, 8.0).expect("b"); + assert_eq!( + select_candidate_k(&[]), + Err(ModelSelectionError::EmptyCandidateSet) + ); + assert_eq!(select_candidate_k(&[a, b]).expect("tie"), 2); + let higher_likelihood = ModelCandidate::statistical(8, -20.0, 9.0).expect("likelihood"); + assert_eq!( + select_candidate_k(&[a, higher_likelihood]).expect("likelihood tie-break"), + 8 + ); + + assert_eq!( + selected_k_root_mean_square_error(&[], 4), + Err(ModelSelectionError::EmptyCandidateSet) + ); + assert_eq!( + selected_k_root_mean_square_error(&[4], 1), + Err(ModelSelectionError::NonPositiveCandidateK) + ); + assert_eq!( + selected_k_root_mean_square_error(&[1], 4), + Err(ModelSelectionError::InvalidDiagnostic) + ); + assert!( + selected_k_root_mean_square_error(&[4], 4) + .expect("valid rmse") + .abs() + < f64::EPSILON + ); + assert_eq!( + select_candidate_k(&[ModelCandidate::llm_vote_only(3).expect("valid llm candidate")]), + Err(ModelSelectionError::LlmVoteIsNotStatisticalAuthority) + ); + } +} diff --git a/crates/model_selection/src/lib.rs b/crates/model_selection/src/lib.rs new file mode 100644 index 00000000..599829af --- /dev/null +++ b/crates/model_selection/src/lib.rs @@ -0,0 +1,22 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +// Selected-K RMSE casts small finite topic counts to `f64`. +#![allow(clippy::cast_precision_loss)] +//! Statistical and Pareto candidate-`K` gates for TRSL-TM model selection. +//! +//! Model selection uses held-out log-likelihood and complexity before any +//! blinded LLM review. An LLM vote may recommend among statistically +//! admissible candidates but never defines the numerical optimum (ADR 0012). + +mod candidate; +mod error; +mod gate; + +/// One candidate `K` with statistical or LLM-only support. +pub use candidate::ModelCandidate; +/// Fail-closed model-selection errors. +pub use error::ModelSelectionError; +/// Select the admissible candidate `K` from a Pareto-filtered statistical front. +pub use gate::select_candidate_k; +/// RMSE of selected `K` replications against known truth. +pub use gate::selected_k_root_mean_square_error; diff --git a/crates/model_selection/tests/crate_contract.rs b/crates/model_selection/tests/crate_contract.rs new file mode 100644 index 00000000..cd4ec7ba --- /dev/null +++ b/crates/model_selection/tests/crate_contract.rs @@ -0,0 +1,7 @@ +//! Integration contract for the `model_selection` package identity. + +#[test] +fn package_identity_is_stable() { + let observed = std::hint::black_box(env!("CARGO_PKG_NAME")); + assert_eq!(observed, "model_selection"); +} diff --git a/crates/model_selection/tests/pareto_k_gate_contract.rs b/crates/model_selection/tests/pareto_k_gate_contract.rs new file mode 100644 index 00000000..52e43c7c --- /dev/null +++ b/crates/model_selection/tests/pareto_k_gate_contract.rs @@ -0,0 +1,138 @@ +//! Statistical/Pareto K gates are deterministic and recover known K without LLM authority. + +use model_selection::{ + ModelCandidate, ModelSelectionError, select_candidate_k, selected_k_root_mean_square_error, +}; + +fn candidate(k: u32, log_likelihood: f64, complexity: f64) -> ModelCandidate { + ModelCandidate::statistical(k, log_likelihood, complexity).expect("statistical candidate") +} + +fn synthetic_candidates(truth_k: u32, noisy_replication: bool) -> [ModelCandidate; 3] { + let true_log_likelihood = if noisy_replication { -20.0 } else { -10.0 }; + [ + candidate(truth_k, true_log_likelihood, f64::from(truth_k)), + candidate(truth_k - 1, -30.0, f64::from(truth_k - 1)), + candidate( + truth_k + 1, + if noisy_replication { -19.0 } else { -25.0 }, + f64::from(truth_k + 1), + ), + ] +} + +#[test] +fn non_positive_k_and_non_finite_diagnostics_fail_closed() { + assert_eq!( + ModelCandidate::statistical(1, -10.0, 4.0), + Err(ModelSelectionError::NonPositiveCandidateK) + ); + assert_eq!( + ModelCandidate::statistical(3, f64::NAN, 4.0), + Err(ModelSelectionError::InvalidDiagnostic) + ); + assert_eq!( + ModelCandidate::statistical(3, -10.0, f64::INFINITY), + Err(ModelSelectionError::InvalidDiagnostic) + ); + assert_eq!( + ModelCandidate::statistical(3, -10.0, -0.1), + Err(ModelSelectionError::InvalidDiagnostic) + ); +} + +#[test] +fn llm_vote_cannot_define_the_numerical_optimum() { + let only_llm = ModelCandidate::llm_vote_only(5).expect("valid llm candidate"); + assert_eq!( + select_candidate_k(&[only_llm]), + Err(ModelSelectionError::LlmVoteIsNotStatisticalAuthority) + ); +} + +#[test] +fn pareto_front_selects_known_truth_k_with_computed_rmse() { + let truth_k = 4_u32; + let candidates = [ + candidate(2, -100.0, 10.0), + candidate(truth_k, -40.0, 20.0), + candidate(8, -45.0, 40.0), + ModelCandidate::llm_vote_only(6).expect("valid llm candidate"), + ]; + + let selected = select_candidate_k(&candidates).expect("admissible statistical front"); + assert_eq!(selected, truth_k); + + let rmse = selected_k_root_mean_square_error(&[selected], truth_k).expect("rmse"); + let expected = { + let residual = f64::from(selected) - f64::from(truth_k); + (residual * residual).sqrt() + }; + assert!((rmse - expected).abs() < f64::EPSILON); + assert!(rmse < 0.5); + assert_eq!( + select_candidate_k(&[candidate(2, -30.0, 8.0), candidate(4, -30.0, 8.0)]), + Ok(2) + ); + assert_eq!( + selected_k_root_mean_square_error(&[], truth_k), + Err(ModelSelectionError::EmptyCandidateSet) + ); + assert_eq!( + selected_k_root_mean_square_error(&[selected], 1), + Err(ModelSelectionError::NonPositiveCandidateK) + ); +} + +#[test] +fn repeated_synthetic_truth_recovers_k_with_bounded_error_and_bias() { + let truth = [3_u32, 4, 5, 6, 7, 8]; + let selected: Vec = truth + .iter() + .enumerate() + .map(|(replication, truth_k)| { + select_candidate_k(&synthetic_candidates(*truth_k, replication == 4)) + .expect("synthetic statistical front") + }) + .collect(); + + let matching = truth + .iter() + .zip(&selected) + .filter(|(truth_k, selected_k)| truth_k == selected_k) + .count(); + let sum_squared_error: f64 = truth + .iter() + .zip(&selected) + .map(|(truth_k, selected_k)| { + let residual = f64::from(*selected_k) - f64::from(*truth_k); + residual * residual + }) + .sum(); + let bias: f64 = truth + .iter() + .zip(&selected) + .map(|(truth_k, selected_k)| f64::from(*selected_k) - f64::from(*truth_k)) + .sum::() + / f64::from(u32::try_from(truth.len()).expect("small fixture")); + + assert_eq!(selected, vec![3, 4, 5, 6, 8, 8]); + assert_eq!(matching, 5); + let replication_count = f64::from(u32::try_from(truth.len()).expect("small fixture")); + assert!((sum_squared_error / replication_count).sqrt() < 0.5); + assert!((bias - (1.0 / 6.0)).abs() < f64::EPSILON); + for (truth_k, selected_k) in truth.iter().zip(&selected) { + assert!( + selected_k_root_mean_square_error(&[*selected_k], *truth_k).expect("replication RMSE") + <= 1.0 + ); + } +} + +#[test] +fn empty_candidate_sets_abstain() { + assert_eq!( + select_candidate_k(&[]), + Err(ModelSelectionError::EmptyCandidateSet) + ); +} diff --git a/crates/tepp_api/Cargo.toml b/crates/tepp_api/Cargo.toml index b7d27cc7..1af51a29 100644 --- a/crates/tepp_api/Cargo.toml +++ b/crates/tepp_api/Cargo.toml @@ -20,5 +20,11 @@ serde_json = { workspace = true } sha2 = { workspace = true } temporal_core = { path = "../temporal_core", version = "0.1.0" } +[[bin]] +name = "tepp-loopback" +path = "src/bin/tepp_loopback.rs" +test = false +bench = false + [lints] workspace = true diff --git a/crates/tepp_api/src/analysis_result.rs b/crates/tepp_api/src/analysis_result.rs new file mode 100644 index 00000000..61bd13db --- /dev/null +++ b/crates/tepp_api/src/analysis_result.rs @@ -0,0 +1,362 @@ +//! Versioned terminal analysis-run result contracts. +//! +//! Submission acceptance and scientific completion are separate facts. +//! [`AnalysisRunAccepted`] is only a durable receipt. This module defines a +//! distinct, request-bound terminal result with a digest-bound artifact or a +//! redacted failure code. + +use crate::analysis_run::require_rfc3339_knowledge_cutoff; +use crate::wire::{ + from_json, require_byte_limit, require_contract_version, require_nonempty, to_json, +}; +use crate::{AnalysisRunAccepted, AnalysisRunRequest, ApiError}; +use serde::{Deserialize, Serialize}; +use temporal_core::SystemTime; + +/// Supported terminal analysis-result contract version. +pub const ANALYSIS_RESULT_CONTRACT_VERSION: u16 = 1; + +/// Default maximum terminal analysis-result JSON payload size in bytes. +pub const DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT: usize = 64 * 1024; + +const MAXIMUM_SUMMARY_COUNT: u64 = 1_000_000_000; +const MAXIMUM_FAILURE_CODE_BYTES: usize = 64; + +/// Canonical terminal lifecycle state for an analysis run. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AnalysisRunTerminalState { + /// Computation completed with a digest-bound result artifact. + Succeeded, + /// Computation ended without a result artifact. + Failed, +} + +/// Bounded, identity-free summary of one completed measurement artifact. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AnalysisResultSummary { + /// Versioned analysis family. + pub analysis_family: String, + /// Number of evidence units represented by the result. + pub evidence_count: u64, + /// Number of reported statistics or parameters. + pub statistic_count: u64, + /// Provider-authored validation status. + pub validation_status: String, +} + +impl AnalysisResultSummary { + /// Construct and validate a bounded, identity-free summary. + /// + /// # Errors + /// + /// Returns a fail-closed contract error for empty labels or unbounded + /// counts. + pub fn new( + analysis_family: impl Into, + evidence_count: u64, + statistic_count: u64, + validation_status: impl Into, + ) -> Result { + let value = Self { + analysis_family: analysis_family.into(), + evidence_count, + statistic_count, + validation_status: validation_status.into(), + }; + value.validate()?; + Ok(value) + } + + fn validate(&self) -> Result<(), ApiError> { + require_nonempty(&self.analysis_family)?; + require_nonempty(&self.validation_status)?; + if self.evidence_count > MAXIMUM_SUMMARY_COUNT + || self.statistic_count > MAXIMUM_SUMMARY_COUNT + { + return Err(ApiError::LimitExceeded); + } + Ok(()) + } +} + +/// Request-bound terminal outcome for one accepted analysis run. +/// +/// The succeeded shape excludes source text, credentials, direct identity, +/// respondent/item records, and unrestricted model output. The failed shape +/// contains no measurement artifact. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AnalysisRunTerminalResult { + /// Semantic contract version. + pub contract_version: u16, + /// Opaque remote run identity from [`AnalysisRunAccepted`]. + pub run_id: String, + /// Terminal lifecycle state. + pub run_state: AnalysisRunTerminalState, + /// Exact request idempotency key. + pub idempotency_key: String, + /// Authorized tenant/workspace opaque identity. + pub tenant_workspace_id: String, + /// Immutable corpus/evidence snapshot identity. + pub snapshot_id: String, + /// Exact request knowledge cutoff. + pub knowledge_cutoff: String, + /// Exact model/backend contract identity. + pub model_contract_version: String, + /// Exact requested output profile. + pub output_profile: String, + /// Opaque result artifact identity for a succeeded run. + pub result_artifact_id: Option, + /// Canonical lowercase SHA-256 result digest. + pub result_sha256: Option, + /// Versioned result-schema identity. + pub result_schema_version: Option, + /// Strict RFC 3339 system time at terminal completion. + pub completed_at: String, + /// Bounded summary for a succeeded run. + pub summary: Option, + /// Stable snake-case code for a failed run. + pub failure_code: Option, +} + +impl AnalysisRunTerminalResult { + /// Construct a succeeded terminal result bound to request and receipt. + /// + /// # Errors + /// + /// Returns a fail-closed error for invalid shape, digest, time, summary, or + /// request/receipt binding. + pub fn succeeded( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + result_artifact_id: impl Into, + result_sha256: impl Into, + result_schema_version: impl Into, + completed_at: impl Into, + summary: AnalysisResultSummary, + ) -> Result { + let value = Self { + contract_version: ANALYSIS_RESULT_CONTRACT_VERSION, + run_id: accepted.run_id.clone(), + run_state: AnalysisRunTerminalState::Succeeded, + idempotency_key: request.idempotency_key.clone(), + tenant_workspace_id: request.tenant_workspace_id.clone(), + snapshot_id: request.snapshot_id.clone(), + knowledge_cutoff: request.knowledge_cutoff.clone(), + model_contract_version: request.model_contract_version.clone(), + output_profile: request.output_profile.clone(), + result_artifact_id: Some(result_artifact_id.into()), + result_sha256: Some(result_sha256.into()), + result_schema_version: Some(result_schema_version.into()), + completed_at: completed_at.into(), + summary: Some(summary), + failure_code: None, + }; + value.validate()?; + require_terminal_binding(request, accepted, &value)?; + Ok(value) + } + + /// Construct a failed terminal result bound to request and receipt. + /// + /// # Errors + /// + /// Returns a fail-closed error for invalid time, failure code, or + /// request/receipt binding. + pub fn failed( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + completed_at: impl Into, + failure_code: impl Into, + ) -> Result { + let value = Self { + contract_version: ANALYSIS_RESULT_CONTRACT_VERSION, + run_id: accepted.run_id.clone(), + run_state: AnalysisRunTerminalState::Failed, + idempotency_key: request.idempotency_key.clone(), + tenant_workspace_id: request.tenant_workspace_id.clone(), + snapshot_id: request.snapshot_id.clone(), + knowledge_cutoff: request.knowledge_cutoff.clone(), + model_contract_version: request.model_contract_version.clone(), + output_profile: request.output_profile.clone(), + result_artifact_id: None, + result_sha256: None, + result_schema_version: None, + completed_at: completed_at.into(), + summary: None, + failure_code: Some(failure_code.into()), + }; + value.validate()?; + require_terminal_binding(request, accepted, &value)?; + Ok(value) + } + + /// Parse and validate a terminal result with the default byte limit. + /// + /// # Errors + /// + /// Returns wire, version, limit, time, digest, shape, or field errors. + pub fn from_json(payload: &str) -> Result { + Self::from_json_with_limit(payload, DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT) + } + + /// Parse and validate a terminal result with a caller-supplied byte limit. + /// + /// # Errors + /// + /// Returns wire, version, limit, time, digest, shape, or field errors. + pub fn from_json_with_limit(payload: &str, maximum_bytes: usize) -> Result { + require_byte_limit(payload, maximum_bytes)?; + let value: Self = from_json(payload)?; + value.validate()?; + Ok(value) + } + + /// Serialize this terminal result after complete validation. + /// + /// # Errors + /// + /// Returns validation or serialization errors. + pub fn to_json(&self) -> Result { + self.validate()?; + let payload = to_json(self)?; + require_byte_limit(&payload, DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT)?; + Ok(payload) + } + + pub(crate) fn validate(&self) -> Result<(), ApiError> { + require_contract_version(self.contract_version, ANALYSIS_RESULT_CONTRACT_VERSION)?; + for value in [ + &self.run_id, + &self.idempotency_key, + &self.tenant_workspace_id, + &self.snapshot_id, + &self.knowledge_cutoff, + &self.model_contract_version, + &self.output_profile, + &self.completed_at, + ] { + require_nonempty(value)?; + } + require_rfc3339_knowledge_cutoff(&self.knowledge_cutoff)?; + SystemTime::parse_rfc3339(&self.completed_at).map_err(|_| ApiError::InvalidWirePayload)?; + + match self.run_state { + AnalysisRunTerminalState::Succeeded => self.validate_succeeded(), + AnalysisRunTerminalState::Failed => self.validate_failed(), + } + } + + fn validate_succeeded(&self) -> Result<(), ApiError> { + let artifact_id = self + .result_artifact_id + .as_deref() + .ok_or(ApiError::InvalidWirePayload)?; + let digest = self + .result_sha256 + .as_deref() + .ok_or(ApiError::InvalidWirePayload)?; + let schema = self + .result_schema_version + .as_deref() + .ok_or(ApiError::InvalidWirePayload)?; + let summary = self.summary.as_ref().ok_or(ApiError::InvalidWirePayload)?; + require_nonempty(artifact_id)?; + require_nonempty(schema)?; + require_canonical_sha256(digest)?; + summary.validate()?; + if self.failure_code.is_some() { + return Err(ApiError::InvalidWirePayload); + } + Ok(()) + } + + fn validate_failed(&self) -> Result<(), ApiError> { + if self.result_artifact_id.is_some() + || self.result_sha256.is_some() + || self.result_schema_version.is_some() + || self.summary.is_some() + { + return Err(ApiError::InvalidWirePayload); + } + require_failure_code( + self.failure_code + .as_deref() + .ok_or(ApiError::InvalidWirePayload)?, + ) + } +} + +/// Return whether a terminal result exactly binds to its submitted request. +#[must_use] +pub fn terminal_result_matches_request( + request: &AnalysisRunRequest, + result: &AnalysisRunTerminalResult, +) -> bool { + result.idempotency_key == request.idempotency_key + && result.tenant_workspace_id == request.tenant_workspace_id + && result.snapshot_id == request.snapshot_id + && result.knowledge_cutoff == request.knowledge_cutoff + && result.model_contract_version == request.model_contract_version + && result.output_profile == request.output_profile +} + +/// Return whether a terminal result exactly binds to an accepted receipt. +#[must_use] +pub fn terminal_result_matches_accepted( + accepted: &AnalysisRunAccepted, + result: &AnalysisRunTerminalResult, +) -> bool { + result.run_id == accepted.run_id && result.idempotency_key == accepted.idempotency_key +} + +/// Require exact request and accepted-receipt binding. +/// +/// # Errors +/// +/// Returns [`ApiError::InvalidWirePayload`] when either binding differs. +pub fn require_terminal_binding( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + result: &AnalysisRunTerminalResult, +) -> Result<(), ApiError> { + request.validate()?; + accepted.validate()?; + result.validate()?; + if terminal_result_matches_request(request, result) + && terminal_result_matches_accepted(accepted, result) + { + Ok(()) + } else { + Err(ApiError::InvalidWirePayload) + } +} + +fn require_canonical_sha256(value: &str) -> Result<(), ApiError> { + let valid = value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)); + if valid { + Ok(()) + } else { + Err(ApiError::InvalidWirePayload) + } +} + +fn require_failure_code(value: &str) -> Result<(), ApiError> { + let bytes = value.as_bytes(); + let valid = !bytes.is_empty() + && bytes.len() <= MAXIMUM_FAILURE_CODE_BYTES + && bytes[0].is_ascii_lowercase() + && bytes + .iter() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'_'); + if valid { + Ok(()) + } else { + Err(ApiError::InvalidWirePayload) + } +} diff --git a/crates/tepp_api/src/analysis_run.rs b/crates/tepp_api/src/analysis_run.rs index b263a1a3..2dd80374 100644 --- a/crates/tepp_api/src/analysis_run.rs +++ b/crates/tepp_api/src/analysis_run.rs @@ -4,6 +4,7 @@ use crate::ApiError; use crate::wire::{ from_json, require_byte_limit, require_contract_version, require_nonempty, to_json, }; +use crate::{AnalysisRunTerminalResult, AnalysisRunTerminalState, require_terminal_binding}; use jiff::Timestamp; use serde::{Deserialize, Serialize}; use temporal_core::KnowledgeCutoff; @@ -14,6 +15,9 @@ pub const ANALYSIS_RUN_CONTRACT_VERSION: u16 = 1; /// Default maximum analysis-run JSON payload size in bytes. pub const DEFAULT_ANALYSIS_RUN_BYTE_LIMIT: usize = 64 * 1024; +/// Supported analysis-run status/read contract version. +pub const ANALYSIS_RUN_STATUS_CONTRACT_VERSION: u16 = 1; + /// Request to create a durable analysis run. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(deny_unknown_fields)] @@ -48,6 +52,36 @@ pub struct AnalysisRunAccepted { pub idempotency_key: String, } +/// Lifecycle state returned by the typed analysis-run status contract. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AnalysisRunStatusState { + /// The server durably accepted the run. + Accepted, + /// The server is processing the accepted run. + Running, + /// The run completed with a measurement artifact. + Succeeded, + /// The run completed without a measurement artifact. + Failed, +} + +/// Typed status/read response for an accepted analysis run. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AnalysisRunStatus { + /// Semantic contract version for this status payload family. + pub contract_version: u16, + /// Opaque server-assigned run identity. + pub run_id: String, + /// Current lifecycle state. + pub run_state: AnalysisRunStatusState, + /// Exact request idempotency key. + pub idempotency_key: String, + /// Validated terminal result, present only for terminal states. + pub terminal_result: Option, +} + impl AnalysisRunRequest { /// Parse and validate a JSON analysis-run request with default size limit. /// @@ -77,10 +111,12 @@ impl AnalysisRunRequest { /// Returns field-validation or serialization errors. pub fn to_json(&self) -> Result { self.validate()?; - to_json(self) + let payload = to_json(self)?; + require_byte_limit(&payload, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT)?; + Ok(payload) } - fn validate(&self) -> Result<(), ApiError> { + pub(crate) fn validate(&self) -> Result<(), ApiError> { require_contract_version(self.contract_version, ANALYSIS_RUN_CONTRACT_VERSION)?; require_nonempty(&self.idempotency_key)?; require_nonempty(&self.tenant_workspace_id)?; @@ -96,7 +132,7 @@ impl AnalysisRunRequest { /// /// A buyer cannot claim analysis of evidence that is not yet available. The /// request receipt instant is treated as availability of the command itself. -fn require_rfc3339_knowledge_cutoff(knowledge_cutoff: &str) -> Result<(), ApiError> { +pub(crate) fn require_rfc3339_knowledge_cutoff(knowledge_cutoff: &str) -> Result<(), ApiError> { require_nonempty(knowledge_cutoff)?; let cutoff = KnowledgeCutoff::parse_rfc3339(knowledge_cutoff) .map_err(|_| ApiError::InvalidWirePayload)?; @@ -135,6 +171,16 @@ impl AnalysisRunAccepted { /// /// Returns wire, version, or field-validation errors. pub fn from_json(payload: &str) -> Result { + Self::from_json_with_limit(payload, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT) + } + + /// Parse an accepted-run payload with a caller-supplied byte limit. + /// + /// # Errors + /// + /// Returns wire, version, limit, or field-validation errors. + pub fn from_json_with_limit(payload: &str, maximum_bytes: usize) -> Result { + require_byte_limit(payload, maximum_bytes)?; let accepted: Self = from_json(payload)?; accepted.validate()?; Ok(accepted) @@ -147,14 +193,143 @@ impl AnalysisRunAccepted { /// Returns validation or serialization errors. pub fn to_json(&self) -> Result { self.validate()?; - to_json(self) + let payload = to_json(self)?; + require_byte_limit(&payload, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT)?; + Ok(payload) } - fn validate(&self) -> Result<(), ApiError> { + pub(crate) fn validate(&self) -> Result<(), ApiError> { require_contract_version(self.contract_version, ANALYSIS_RUN_CONTRACT_VERSION)?; require_nonempty(&self.run_id)?; - require_nonempty(&self.run_state)?; + if self.run_state != "accepted" { + return Err(ApiError::InvalidWirePayload); + } + require_nonempty(&self.idempotency_key)?; + Ok(()) + } +} + +impl AnalysisRunStatus { + /// Construct an accepted status from a durable receipt. + /// + /// # Errors + /// + /// Returns a fail-closed error when the receipt is invalid. + pub fn accepted(accepted: &AnalysisRunAccepted) -> Result { + Self::new(accepted, AnalysisRunStatusState::Accepted, None) + } + + /// Construct a running status from a durable receipt. + /// + /// # Errors + /// + /// Returns a fail-closed error when the receipt is invalid. + pub fn running(accepted: &AnalysisRunAccepted) -> Result { + Self::new(accepted, AnalysisRunStatusState::Running, None) + } + + /// Construct a terminal status bound to the submitted request and receipt. + /// + /// # Errors + /// + /// Returns a fail-closed error when the result or its binding is invalid. + pub fn terminal( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + result: AnalysisRunTerminalResult, + ) -> Result { + require_terminal_binding(request, accepted, &result)?; + let state = match result.run_state { + AnalysisRunTerminalState::Succeeded => AnalysisRunStatusState::Succeeded, + AnalysisRunTerminalState::Failed => AnalysisRunStatusState::Failed, + }; + Self::new(accepted, state, Some(result)) + } + + /// Parse and validate a status/read payload with the default byte limit. + /// + /// # Errors + /// + /// Returns wire, version, limit, shape, or field-validation errors. + pub fn from_json(payload: &str) -> Result { + Self::from_json_with_limit(payload, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT) + } + + /// Parse and validate a status/read payload with a caller-supplied limit. + /// + /// # Errors + /// + /// Returns wire, version, limit, shape, or field-validation errors. + pub fn from_json_with_limit(payload: &str, maximum_bytes: usize) -> Result { + require_byte_limit(payload, maximum_bytes)?; + let status: Self = from_json(payload)?; + status.validate()?; + Ok(status) + } + + /// Serialize a status/read payload after complete validation. + /// + /// # Errors + /// + /// Returns validation or serialization errors. + pub fn to_json(&self) -> Result { + self.validate()?; + let payload = to_json(self)?; + require_byte_limit(&payload, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT)?; + Ok(payload) + } + + fn new( + accepted: &AnalysisRunAccepted, + run_state: AnalysisRunStatusState, + terminal_result: Option, + ) -> Result { + accepted.validate()?; + let status = Self { + contract_version: ANALYSIS_RUN_STATUS_CONTRACT_VERSION, + run_id: accepted.run_id.clone(), + run_state, + idempotency_key: accepted.idempotency_key.clone(), + terminal_result, + }; + status.validate()?; + status.require_serialized_size()?; + Ok(status) + } + + fn require_serialized_size(&self) -> Result<(), ApiError> { + let payload = to_json(self)?; + require_byte_limit(&payload, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT) + } + + fn validate(&self) -> Result<(), ApiError> { + require_contract_version(self.contract_version, ANALYSIS_RUN_STATUS_CONTRACT_VERSION)?; + require_nonempty(&self.run_id)?; require_nonempty(&self.idempotency_key)?; + match self.run_state { + AnalysisRunStatusState::Accepted | AnalysisRunStatusState::Running => { + if self.terminal_result.is_some() { + return Err(ApiError::InvalidWirePayload); + } + } + AnalysisRunStatusState::Succeeded | AnalysisRunStatusState::Failed => { + let result = self + .terminal_result + .as_ref() + .ok_or(ApiError::InvalidWirePayload)?; + result.validate()?; + let expected_state = match result.run_state { + AnalysisRunTerminalState::Succeeded => AnalysisRunStatusState::Succeeded, + AnalysisRunTerminalState::Failed => AnalysisRunStatusState::Failed, + }; + if expected_state != self.run_state + || result.run_id != self.run_id + || result.idempotency_key != self.idempotency_key + { + return Err(ApiError::InvalidWirePayload); + } + } + } Ok(()) } } @@ -168,6 +343,32 @@ pub fn requests_are_idempotent_matches( left == right } +/// Require exact status identity and, for terminal states, request binding. +/// +/// # Errors +/// +/// Returns [`ApiError::InvalidWirePayload`] when the status does not match the +/// receipt or its terminal result does not match the request. +pub fn require_status_binding( + request: &AnalysisRunRequest, + accepted: &AnalysisRunAccepted, + status: &AnalysisRunStatus, +) -> Result<(), ApiError> { + request.validate()?; + accepted.validate()?; + status.validate()?; + if request.idempotency_key != accepted.idempotency_key + || status.run_id != accepted.run_id + || status.idempotency_key != accepted.idempotency_key + { + return Err(ApiError::InvalidWirePayload); + } + if let Some(result) = status.terminal_result.as_ref() { + require_terminal_binding(request, accepted, result)?; + } + Ok(()) +} + #[cfg(test)] mod tests { use super::{ diff --git a/crates/tepp_api/src/analysis_run_live.rs b/crates/tepp_api/src/analysis_run_live.rs new file mode 100644 index 00000000..6768c6ef --- /dev/null +++ b/crates/tepp_api/src/analysis_run_live.rs @@ -0,0 +1,970 @@ +//! Consumer-neutral live analysis-run ingress for modular CWL services. +//! +//! This module keeps the Naruon compatibility listener intact while providing +//! the shared `/v1/analysis-runs` and cutoff-safe `/v1/temporal-context` +//! boundaries needed by Naruon and `LineageWeave`. It accepts transport +//! acknowledgements and temporal evidence context only; completed psychometric +//! results remain outside this crate. + +use std::collections::HashMap; +use std::io::Write; +use std::net::{SocketAddr, TcpListener}; + +use crate::lineageweave_http::{LINEAGEWEAVE_CONSUMER_CODE, consumer_is_supported}; +use crate::live_http::{ + header_value, map_io_error, parse_headers, parse_request_line, read_http_request_with_limit, + split_request_with_limit, validate_common_headers, +}; +use crate::naruon_http::NARUON_ANALYSIS_RUN_PATH; +use crate::{ + AnalysisRunAccepted, AnalysisRunRequest, ApiError, DEFAULT_PROJECT_HISTORY_BYTE_LIMIT, + ErrorEnvelope, NARUON_LIVE_IO_TIMEOUT, NaruonLiveResponse, PROJECT_HISTORY_PATH, + ProjectHistoryProjection, ProjectHistoryRequest, TEMPORAL_CONTEXT_PATH, TemporalContextRequest, + build_temporal_context, project_history_projection, requests_are_idempotent_matches, +}; + +const MAX_LIVE_REQUEST_BODY_BYTES: usize = DEFAULT_PROJECT_HISTORY_BYTE_LIMIT; + +#[cfg(test)] +use crate::live_http::{declared_content_length, host_implies_table_access, split_header_line}; + +/// Loopback HTTP/1.1 analysis-run service shared by published CWL consumers. +/// +/// The service accepts only Naruon and `LineageWeave` consumer identities. Its +/// idempotency namespace includes consumer, tenant, and caller key so one +/// product cannot replay or conflict with another product's accepted run. +#[derive(Debug)] +pub struct AnalysisRunLiveService { + listener: Option, + bound_addr: Option, + next_run_serial: u64, + next_request_serial: u64, + accepted_runs: HashMap, + accepted_project_histories: HashMap, +} + +impl Default for AnalysisRunLiveService { + fn default() -> Self { + Self::new() + } +} + +impl AnalysisRunLiveService { + /// Construct an in-memory handler with no bound socket. + #[must_use] + pub fn new() -> Self { + Self { + listener: None, + bound_addr: None, + next_run_serial: 1, + next_request_serial: 1, + accepted_runs: HashMap::new(), + accepted_project_histories: HashMap::new(), + } + } + + /// Bind an ephemeral IPv4 loopback port. + /// + /// # Errors + /// + /// Returns [`ApiError::InvalidWirePayload`] when the operating system + /// refuses the loopback bind. + pub fn bind_loopback() -> Result { + Self::bind(SocketAddr::from(([127, 0, 0, 1], 0))) + } + + /// Bind a caller-supplied loopback address. + /// + /// # Errors + /// + /// Returns [`ApiError::AuthorizationDenied`] for a non-loopback address + /// and [`ApiError::InvalidWirePayload`] when the socket cannot be opened. + pub fn bind(addr: SocketAddr) -> Result { + if !addr.ip().is_loopback() { + return Err(ApiError::AuthorizationDenied); + } + let listener = TcpListener::bind(addr).map_err(|error| map_io_error(&error))?; + let bound_addr = listener + .local_addr() + .map_err(|error| map_io_error(&error))?; + Ok(Self { + listener: Some(listener), + bound_addr: Some(bound_addr), + ..Self::new() + }) + } + + /// Return the bound loopback address. + /// + /// # Errors + /// + /// Returns [`ApiError::InvalidWirePayload`] when no socket is bound. + pub fn local_addr(&self) -> Result { + self.bound_addr.ok_or(ApiError::InvalidWirePayload) + } + + /// Accept and serve one HTTP/1.1 request. + /// + /// # Errors + /// + /// Returns a fail-closed API error when no socket is bound or socket I/O + /// fails. Protocol errors are returned as redacted HTTP responses. + pub fn serve_one(&mut self) -> Result { + let listener = self.listener.as_ref().ok_or(ApiError::InvalidWirePayload)?; + let (mut stream, _) = listener.accept().map_err(|error| map_io_error(&error))?; + stream + .set_read_timeout(Some(NARUON_LIVE_IO_TIMEOUT)) + .map_err(|error| map_io_error(&error))?; + stream + .set_write_timeout(Some(NARUON_LIVE_IO_TIMEOUT)) + .map_err(|error| map_io_error(&error))?; + let response = match read_http_request_with_limit(&mut stream, MAX_LIVE_REQUEST_BODY_BYTES) + { + Ok(request) => self.handle_http_request(&request), + Err(error) => self.response_from_error(error), + }; + stream + .write_all(&response.to_http_bytes()) + .map_err(|error| map_io_error(&error))?; + stream.flush().map_err(|error| map_io_error(&error))?; + Ok(response) + } + + /// Parse and handle one complete HTTP/1.1 request already in memory. + #[must_use] + pub fn handle_http_request(&mut self, request: &str) -> NaruonLiveResponse { + match self.dispatch_http_request(request) { + Ok(response) => response, + Err(error) => self.response_from_error(error), + } + } + + fn dispatch_http_request(&mut self, request: &str) -> Result { + let (header_block, body) = split_request_with_limit(request, MAX_LIVE_REQUEST_BODY_BYTES)?; + let mut lines = header_block.split("\r\n"); + let (method, path) = parse_request_line(lines.next().unwrap_or(""))?; + if method != "POST" + || (path != NARUON_ANALYSIS_RUN_PATH + && path != TEMPORAL_CONTEXT_PATH + && path != PROJECT_HISTORY_PATH) + { + return Err(ApiError::InvalidWirePayload); + } + let headers = parse_headers(&mut lines)?; + let consumer = require_headers( + &headers, + self.bound_addr, + path == NARUON_ANALYSIS_RUN_PATH || path == PROJECT_HISTORY_PATH, + )?; + if path == TEMPORAL_CONTEXT_PATH { + if consumer != LINEAGEWEAVE_CONSUMER_CODE { + return Err(ApiError::InvalidWirePayload); + } + let context_request = TemporalContextRequest::from_json(body)?; + let response = build_temporal_context(&context_request)?; + return Ok(json_response(200, "OK", response.to_json()?)); + } + if path == PROJECT_HISTORY_PATH { + return self.accept_project_history(consumer, &headers, body); + } + self.accept_analysis_run(consumer, &headers, body) + } + + fn accept_analysis_run( + &mut self, + consumer: &str, + headers: &HashMap, + body: &str, + ) -> Result { + let request = AnalysisRunRequest::from_json(body)?; + let idempotency_key = header_value(headers, "idempotency-key")?; + if idempotency_key != request.idempotency_key { + return Err(ApiError::InvalidWirePayload); + } + let replay_key = consumer_tenant_idempotency_key( + consumer, + &request.tenant_workspace_id, + idempotency_key, + ); + if let Some((stored_request, stored_accepted)) = self.accepted_runs.get(&replay_key) { + if requests_are_idempotent_matches(stored_request, &request) { + return Ok(json_response(202, "Accepted", stored_accepted.to_json()?)); + } + return Err(ApiError::InvalidWirePayload); + } + let run_id = format!("tepp-run-{}", self.next_run_serial); + self.next_run_serial += 1; + let accepted = + AnalysisRunAccepted::new(run_id, "accepted", request.idempotency_key.clone())?; + let response_body = accepted.to_json()?; + self.accepted_runs.insert(replay_key, (request, accepted)); + Ok(json_response(202, "Accepted", response_body)) + } + + fn accept_project_history( + &mut self, + consumer: &str, + headers: &HashMap, + body: &str, + ) -> Result { + if consumer != LINEAGEWEAVE_CONSUMER_CODE { + return Err(ApiError::InvalidWirePayload); + } + let request = ProjectHistoryRequest::from_json(body)?; + let idempotency_key = header_value(headers, "idempotency-key")?; + if idempotency_key != request.idempotency_key { + return Err(ApiError::InvalidWirePayload); + } + let replay_key = consumer_tenant_idempotency_key( + consumer, + &request.tenant_workspace_id, + idempotency_key, + ); + if let Some((stored_request, stored_projection)) = + self.accepted_project_histories.get(&replay_key) + { + if stored_request == &request { + return Ok(json_response(200, "OK", stored_projection.to_json()?)); + } + return Err(ApiError::InvalidWirePayload); + } + let projection = project_history_projection(&request)?; + let response_body = projection.to_json()?; + self.accepted_project_histories + .insert(replay_key, (request, projection)); + Ok(json_response(200, "OK", response_body)) + } + + fn response_from_error(&mut self, error: ApiError) -> NaruonLiveResponse { + let request_id = format!("analysis-run-live-{}", self.next_request_serial); + self.next_request_serial += 1; + let (status_code, reason_phrase) = status_for(error); + let body = error_envelope_json(error, request_id); + json_response(status_code, reason_phrase, body) + } +} + +fn require_headers( + headers: &HashMap, + bound_addr: Option, + require_idempotency_key: bool, +) -> Result<&str, ApiError> { + validate_common_headers(headers, bound_addr)?; + if header_value(headers, "tepp-contract-version")? != "1" { + return Err(ApiError::InvalidWirePayload); + } + let consumer = header_value(headers, "tepp-consumer")?; + if !consumer_is_supported(consumer) { + return Err(ApiError::InvalidWirePayload); + } + if require_idempotency_key { + let _idempotency_key = header_value(headers, "idempotency-key")?; + } + Ok(consumer) +} + +fn consumer_tenant_idempotency_key( + consumer: &str, + tenant_workspace_id: &str, + idempotency_key: &str, +) -> String { + format!("{consumer}\u{1f}{tenant_workspace_id}\u{1f}{idempotency_key}") +} + +fn status_for(error: ApiError) -> (u16, &'static str) { + match error { + ApiError::InvalidWirePayload => (400, "Bad Request"), + ApiError::AuthorizationDenied => (403, "Forbidden"), + ApiError::LimitExceeded => (413, "Payload Too Large"), + ApiError::UnsupportedContractVersion => (422, "Unprocessable Entity"), + } +} + +fn error_envelope_json(error: ApiError, request_id: String) -> String { + ErrorEnvelope::from_api_error(error, request_id) + .and_then(|envelope| envelope.to_json()) + .unwrap_or_else(|_| { + "{\"error_code\":\"invalid_wire_payload\",\"message\":\"invalid API wire payload\",\"request_id\":\"analysis-run-live-fallback\",\"retryable\":false}".to_owned() + }) +} + +fn json_response( + status_code: u16, + reason_phrase: &'static str, + body: String, +) -> NaruonLiveResponse { + NaruonLiveResponse { + status_code, + reason_phrase, + body, + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::fmt::Write as _; + use std::io::{Cursor, Read, Write}; + use std::net::TcpStream; + use std::thread; + use std::time::Duration; + + use super::{ + AnalysisRunLiveService, consumer_tenant_idempotency_key, declared_content_length, + error_envelope_json, host_implies_table_access, map_io_error, parse_headers, + require_headers, split_header_line, status_for, + }; + use crate::live_http::{host_is_loopback, read_http_request, split_request}; + use crate::{ + ANALYSIS_RUN_CONTRACT_VERSION, AnalysisRunRequest, ApiError, + DEFAULT_ANALYSIS_RUN_BYTE_LIMIT, ErrorEnvelope, LINEAGEWEAVE_CONSUMER_CODE, + NARUON_ANALYSIS_RUN_PATH, NARUON_CONSUMER_CODE, NARUON_LIVE_HEADER_BYTE_LIMIT, + NARUON_LIVE_HEADER_COUNT_LIMIT, NARUON_LIVE_IO_TIMEOUT, TEMPORAL_CONTEXT_PATH, + }; + + fn sample_run() -> AnalysisRunRequest { + AnalysisRunRequest { + contract_version: ANALYSIS_RUN_CONTRACT_VERSION, + idempotency_key: "analysis-live-idem-001".into(), + tenant_workspace_id: "analysis-live-tenant".into(), + snapshot_id: "analysis-live-snapshot".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "tepp-analysis-run-v1".into(), + output_profile: "calibrated_event_measurement".into(), + } + } + + fn http_request(body: &str, headers: &[(&str, &str)]) -> String { + let mut request = format!("POST {NARUON_ANALYSIS_RUN_PATH} HTTP/1.1\r\n"); + for (name, value) in headers { + write!(request, "{name}: {value}\r\n").expect("header"); + } + write!(request, "content-length: {}\r\n\r\n{body}", body.len()).expect("body"); + request + } + + fn valid_request(run: &AnalysisRunRequest, consumer: &str, host: &str) -> String { + let body = run.to_json().expect("run json"); + http_request( + &body, + &[ + ("Host", host), + ("content-type", "application/json"), + ("tepp-consumer", consumer), + ("tepp-contract-version", "1"), + ("idempotency-key", run.idempotency_key.as_str()), + ], + ) + } + + fn envelope(body: &str) -> ErrorEnvelope { + serde_json::from_str(body).expect("error envelope") + } + + #[test] + fn helper_contracts_cover_consumer_identity_and_loopback_ports() { + assert_eq!( + consumer_tenant_idempotency_key(LINEAGEWEAVE_CONSUMER_CODE, "tenant", "key"), + "lineageweave\u{1f}tenant\u{1f}key" + ); + assert_ne!( + consumer_tenant_idempotency_key(LINEAGEWEAVE_CONSUMER_CODE, "tenant", "key"), + consumer_tenant_idempotency_key(NARUON_CONSUMER_CODE, "tenant", "key") + ); + assert!(host_is_loopback("localhost:8080", None)); + assert!(!host_is_loopback("localhost:not-a-port", None)); + assert_eq!( + AnalysisRunLiveService::bind("0.0.0.0:0".parse().expect("addr")).expect_err("denied"), + ApiError::AuthorizationDenied + ); + assert_eq!( + AnalysisRunLiveService::new() + .local_addr() + .expect_err("unbound"), + ApiError::InvalidWirePayload + ); + } + + #[test] + fn bind_and_error_helpers_cover_loopback_and_fail_closed_edges() { + let default_service = AnalysisRunLiveService::default(); + assert_eq!( + default_service + .local_addr() + .expect_err("default is unbound"), + ApiError::InvalidWirePayload + ); + let service = AnalysisRunLiveService::bind_loopback().expect("loopback bind"); + let addr = service.local_addr().expect("bound address"); + assert!(addr.ip().is_loopback()); + assert_eq!( + AnalysisRunLiveService::bind(addr).expect_err("in-use address"), + ApiError::InvalidWirePayload + ); + assert_eq!( + AnalysisRunLiveService::new() + .serve_one() + .expect_err("unbound serve"), + ApiError::InvalidWirePayload + ); + + assert_eq!( + status_for(ApiError::InvalidWirePayload), + (400, "Bad Request") + ); + assert_eq!( + status_for(ApiError::AuthorizationDenied), + (403, "Forbidden") + ); + assert_eq!( + status_for(ApiError::LimitExceeded), + (413, "Payload Too Large") + ); + assert_eq!( + status_for(ApiError::UnsupportedContractVersion), + (422, "Unprocessable Entity") + ); + assert_eq!( + map_io_error(&std::io::Error::new( + std::io::ErrorKind::TimedOut, + "timeout" + )), + ApiError::LimitExceeded + ); + assert_eq!( + map_io_error(&std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "would block" + )), + ApiError::LimitExceeded + ); + assert_eq!( + map_io_error(&std::io::Error::other("broken")), + ApiError::InvalidWirePayload + ); + assert!(host_implies_table_access("db.postgres.local")); + assert!(host_implies_table_access("jdbc.local")); + assert!(host_implies_table_access("127.0.0.1/sql")); + assert!(host_implies_table_access("127.0.0.1/tables/x")); + assert!(host_implies_table_access("bad host")); + assert!(host_implies_table_access("bad;host")); + assert!(host_implies_table_access("bad'host")); + assert!(host_implies_table_access("bad\\host")); + assert!(host_implies_table_access("bad\u{0001}host")); + assert!(!host_implies_table_access("127.0.0.1:43789")); + assert!( + error_envelope_json(ApiError::InvalidWirePayload, String::new()) + .contains("analysis-run-live-fallback") + ); + } + + #[test] + #[allow(clippy::too_many_lines)] + fn handler_covers_acceptance_replay_and_header_security() { + let run = sample_run(); + let body = run.to_json().expect("body"); + let mut service = AnalysisRunLiveService::new(); + assert_eq!( + service + .handle_http_request(&format!( + "GET {NARUON_ANALYSIS_RUN_PATH} HTTP/1.1\r\ncontent-length: 0\r\n\r\n" + )) + .status_code, + 400 + ); + for request_line in [ + "POST /wrong HTTP/1.1", + "POST /v1/analysis-runs HTTP/1.0", + "POST /v1/analysis-runs HTTP/1.1 extra", + ] { + assert_eq!( + service + .handle_http_request(&format!("{request_line}\r\ncontent-length: 0\r\n\r\n")) + .status_code, + 400 + ); + } + + let naruon = + service.handle_http_request(&valid_request(&run, NARUON_CONSUMER_CODE, "127.0.0.1")); + let lineageweave = service.handle_http_request(&valid_request( + &run, + LINEAGEWEAVE_CONSUMER_CODE, + "127.0.0.1", + )); + assert_eq!(naruon.status_code, 202); + assert_eq!(lineageweave.status_code, 202); + let replay = service.handle_http_request(&valid_request( + &run, + LINEAGEWEAVE_CONSUMER_CODE, + "127.0.0.1", + )); + assert_eq!(replay.status_code, 202); + assert_eq!(replay.body, lineageweave.body); + + let mut conflict = run.clone(); + conflict.snapshot_id = "different-snapshot".into(); + assert_eq!( + service + .handle_http_request(&valid_request( + &conflict, + LINEAGEWEAVE_CONSUMER_CODE, + "127.0.0.1", + )) + .status_code, + 400 + ); + + let mismatch = http_request( + &body, + &[ + ("Host", "127.0.0.1"), + ("content-type", "application/json"), + ("tepp-consumer", NARUON_CONSUMER_CODE), + ("tepp-contract-version", "1"), + ("idempotency-key", "different-key"), + ], + ); + assert_eq!(service.handle_http_request(&mismatch).status_code, 400); + + let unsupported = body.replace("\"contract_version\":1", "\"contract_version\":9"); + let unsupported_response = service.handle_http_request(&http_request( + &unsupported, + &[ + ("Host", "127.0.0.1"), + ("content-type", "application/json"), + ("tepp-consumer", NARUON_CONSUMER_CODE), + ("tepp-contract-version", "1"), + ("idempotency-key", run.idempotency_key.as_str()), + ], + )); + assert_eq!(unsupported_response.status_code, 422); + assert_eq!( + envelope(&unsupported_response.body).error_code(), + "unsupported_contract_version" + ); + + for (name, value) in [ + ("authorization", "Bearer secret"), + ("proxy-authorization", "Basic secret"), + ("cookie", "session=secret"), + ("x-api-key", "secret"), + ] { + let response = service.handle_http_request(&http_request( + &body, + &[ + ("Host", "127.0.0.1"), + ("content-type", "application/json"), + ("tepp-consumer", NARUON_CONSUMER_CODE), + ("tepp-contract-version", "1"), + ("idempotency-key", run.idempotency_key.as_str()), + (name, value), + ], + )); + assert_eq!(response.status_code, 403, "header={name}"); + assert!(!response.body.contains(value)); + } + + for (headers, status) in [ + ( + vec![ + ("content-type", "application/json"), + ("tepp-consumer", NARUON_CONSUMER_CODE), + ("tepp-contract-version", "1"), + ("idempotency-key", run.idempotency_key.as_str()), + ], + 400, + ), + ( + vec![ + ("Host", ""), + ("content-type", "application/json"), + ("tepp-consumer", NARUON_CONSUMER_CODE), + ("tepp-contract-version", "1"), + ("idempotency-key", run.idempotency_key.as_str()), + ], + 400, + ), + ( + vec![ + ("Host", "127.0.0.1/sql"), + ("content-type", "application/json"), + ("tepp-consumer", NARUON_CONSUMER_CODE), + ("tepp-contract-version", "1"), + ("idempotency-key", run.idempotency_key.as_str()), + ], + 400, + ), + ( + vec![ + ("Host", "8.8.8.8"), + ("content-type", "application/json"), + ("tepp-consumer", NARUON_CONSUMER_CODE), + ("tepp-contract-version", "1"), + ("idempotency-key", run.idempotency_key.as_str()), + ], + 403, + ), + ( + vec![ + ("Host", "127.0.0.1"), + ("content-type", "text/plain"), + ("tepp-consumer", NARUON_CONSUMER_CODE), + ("tepp-contract-version", "1"), + ("idempotency-key", run.idempotency_key.as_str()), + ], + 400, + ), + ( + vec![ + ("Host", "127.0.0.1"), + ("content-type", "application/json"), + ("tepp-consumer", NARUON_CONSUMER_CODE), + ("tepp-contract-version", "2"), + ("idempotency-key", run.idempotency_key.as_str()), + ], + 400, + ), + ( + vec![ + ("Host", "127.0.0.1"), + ("content-type", "application/json"), + ("tepp-consumer", "unpublished"), + ("tepp-contract-version", "1"), + ("idempotency-key", run.idempotency_key.as_str()), + ], + 400, + ), + ( + vec![ + ("Host", "127.0.0.1"), + ("content-type", "application/json"), + ("tepp-consumer", NARUON_CONSUMER_CODE), + ("tepp-contract-version", "1"), + ("idempotency-key", ""), + ], + 400, + ), + ] { + assert_eq!( + service + .handle_http_request(&http_request(&body, &headers)) + .status_code, + status + ); + } + + let transfer = format!( + "POST {NARUON_ANALYSIS_RUN_PATH} HTTP/1.1\r\nHost: 127.0.0.1\r\ncontent-type: application/json\r\ntepp-consumer: naruon\r\ntepp-contract-version: 1\r\nidempotency-key: key\r\ntransfer-encoding: chunked\r\ncontent-length: {}\r\n\r\n{body}", + body.len() + ); + assert_eq!(service.handle_http_request(&transfer).status_code, 400); + } + + #[test] + fn temporal_read_headers_and_defensive_write_edges_are_covered() { + let run = sample_run(); + let body = run.to_json().expect("body"); + let mut service = AnalysisRunLiveService::new(); + + for missing_header in ["tepp-contract-version", "tepp-consumer"] { + let headers = [ + ("Host", "127.0.0.1"), + ("content-type", "application/json"), + ("tepp-consumer", NARUON_CONSUMER_CODE), + ("tepp-contract-version", "1"), + ("idempotency-key", run.idempotency_key.as_str()), + ] + .into_iter() + .filter(|(name, _)| *name != missing_header) + .collect::>(); + assert_eq!( + service + .handle_http_request(&http_request(&body, &headers)) + .status_code, + 400, + "missing={missing_header}" + ); + } + + let temporal_body = r#"{"contract_version":1,"consumer_code":"lineageweave","knowledge_cutoff":"2026-08-20T00:00:00Z","subject_post_id":null,"events":[{"event_id":"event-1","source_post_id":"post-1","event_type_code":"order_awarded","event_label":"Order awarded","event_time":"2026-08-01T09:00:00Z","available_time":"2026-08-01T10:00:00Z","project_reference":null,"actor_references":["actor-1"]}]}"#; + let temporal_request = format!( + "POST {TEMPORAL_CONTEXT_PATH} HTTP/1.1\r\nHost: 127.0.0.1\r\ncontent-type: application/json\r\ntepp-consumer: lineageweave\r\ntepp-contract-version: 1\r\ncontent-length: {}\r\n\r\n{temporal_body}", + temporal_body.len() + ); + assert_eq!( + service.handle_http_request(&temporal_request).status_code, + 200 + ); + + let mut headers = HashMap::from([ + ("host".to_owned(), "127.0.0.1".to_owned()), + ("content-type".to_owned(), "application/json".to_owned()), + ("tepp-consumer".to_owned(), NARUON_CONSUMER_CODE.to_owned()), + ("tepp-contract-version".to_owned(), "1".to_owned()), + ]); + assert_eq!( + service.accept_analysis_run(NARUON_CONSUMER_CODE, &headers, &body), + Err(ApiError::InvalidWirePayload) + ); + headers.insert("idempotency-key".to_owned(), run.idempotency_key.clone()); + assert_eq!( + require_headers(&headers, None, true), + Ok(NARUON_CONSUMER_CODE) + ); + headers.insert("tepp-contract-version".to_owned(), "2".to_owned()); + assert_eq!( + require_headers(&headers, None, true), + Err(ApiError::InvalidWirePayload) + ); + headers.insert("tepp-contract-version".to_owned(), "1".to_owned()); + headers.insert("tepp-consumer".to_owned(), "unpublished".to_owned()); + assert_eq!( + require_headers(&headers, None, true), + Err(ApiError::InvalidWirePayload) + ); + headers.insert("tepp-consumer".to_owned(), NARUON_CONSUMER_CODE.to_owned()); + let accepted = service + .accept_analysis_run(NARUON_CONSUMER_CODE, &headers, &body) + .expect("accepted"); + assert_eq!(accepted.status_code, 202); + let replay = service + .accept_analysis_run(NARUON_CONSUMER_CODE, &headers, &body) + .expect("replay"); + assert_eq!(replay.body, accepted.body); + } + + #[test] + fn parser_helpers_cover_framing_header_and_limit_edges() { + assert_eq!( + split_request("").expect_err("empty"), + ApiError::InvalidWirePayload + ); + assert_eq!( + split_request(&"x".repeat(NARUON_LIVE_HEADER_BYTE_LIMIT)), + Err(ApiError::LimitExceeded) + ); + assert_eq!( + split_request(&format!( + "{}\r\n\r\n", + "x".repeat(NARUON_LIVE_HEADER_BYTE_LIMIT + 1) + )), + Err(ApiError::LimitExceeded) + ); + let oversized_body = "x".repeat(DEFAULT_ANALYSIS_RUN_BYTE_LIMIT + 1); + assert_eq!( + split_request(&format!( + "POST {NARUON_ANALYSIS_RUN_PATH} HTTP/1.1\r\ncontent-length: {}\r\n\r\n{oversized_body}", + oversized_body.len() + )), + Err(ApiError::LimitExceeded) + ); + assert_eq!( + split_header_line("NoColon"), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + split_header_line(": empty-name"), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + split_header_line("Bad Name: value"), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + split_header_line("Bad\u{0001}Name: value"), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + split_header_line("Host: 127.0.0.1").expect("header"), + ("Host", "127.0.0.1") + ); + assert_eq!( + declared_content_length( + "POST /x HTTP/1.1\r\ncontent-length: 1\r\ncontent-length: 1\r\n\r\n" + ), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + declared_content_length("POST /x HTTP/1.1\r\ncontent-length: \r\n\r\n"), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + declared_content_length("POST /x HTTP/1.1\r\ncontent-length: +1\r\n\r\n"), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + declared_content_length("POST /x HTTP/1.1\r\nHost: 127.0.0.1\r\n"), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + declared_content_length( + "POST /x HTTP/1.1\r\ncontent-length: 999999999999999999999\r\n\r\n" + ), + Err(ApiError::InvalidWirePayload) + ); + let mut crowded = (0..=NARUON_LIVE_HEADER_COUNT_LIMIT) + .map(|index| Box::leak(format!("x-{index}: value").into_boxed_str()) as &str); + assert_eq!(parse_headers(&mut crowded), Err(ApiError::LimitExceeded)); + let mut duplicate = ["x-header: one", "X-HEADER: two"].into_iter(); + assert_eq!( + parse_headers(&mut duplicate), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + split_request("POST /v1/analysis-runs HTTP/1.1\r\ncontent-length: 2\r\n\r\na"), + Err(ApiError::InvalidWirePayload) + ); + } + + #[test] + fn read_http_request_covers_transport_utf8_and_body_limits() { + assert_eq!( + read_http_request(&mut Cursor::new(Vec::::new())).expect_err("eof"), + ApiError::InvalidWirePayload + ); + assert_eq!( + read_http_request(&mut ScriptedRead::error(std::io::ErrorKind::TimedOut)) + .expect_err("timeout"), + ApiError::LimitExceeded + ); + assert_eq!( + read_http_request(&mut ScriptedRead::error(std::io::ErrorKind::Other)) + .expect_err("other"), + ApiError::InvalidWirePayload + ); + assert_eq!( + read_http_request(&mut Cursor::new(vec![ + b'x'; + NARUON_LIVE_HEADER_BYTE_LIMIT + 1 + ])) + .expect_err("header limit"), + ApiError::LimitExceeded + ); + + let run = sample_run(); + let request = valid_request(&run, NARUON_CONSUMER_CODE, "127.0.0.1"); + assert_eq!( + read_http_request(&mut ScriptedRead::bytes(request.as_bytes())).expect("request"), + request + ); + let zero = request.replace(&run.to_json().expect("body"), ""); + let zero = zero.replace( + &format!("content-length: {}", run.to_json().expect("body").len()), + "content-length: 0", + ); + assert!(read_http_request(&mut Cursor::new(zero.into_bytes())).is_ok()); + + let mut invalid_header = b"POST /v1/analysis-runs HTTP/1.1\r\n".to_vec(); + invalid_header.push(0xff); + invalid_header.extend_from_slice(b"\r\ncontent-length: 0\r\n\r\n"); + assert_eq!( + read_http_request(&mut Cursor::new(invalid_header)).expect_err("header utf8"), + ApiError::InvalidWirePayload + ); + let header = format!( + "POST {NARUON_ANALYSIS_RUN_PATH} HTTP/1.1\r\nHost: 127.0.0.1\r\ncontent-length: 1\r\n\r\n" + ); + let invalid_body = [header.as_bytes(), &[0xff]].concat(); + assert_eq!( + read_http_request(&mut Cursor::new(invalid_body)).expect_err("body utf8"), + ApiError::InvalidWirePayload + ); + let truncated = + format!("POST {NARUON_ANALYSIS_RUN_PATH} HTTP/1.1\r\ncontent-length: 4\r\n\r\nab"); + assert_eq!( + read_http_request(&mut Cursor::new(truncated.into_bytes())).expect_err("short body"), + ApiError::InvalidWirePayload + ); + let huge = format!( + "POST {NARUON_ANALYSIS_RUN_PATH} HTTP/1.1\r\ncontent-length: {}\r\n\r\n", + DEFAULT_ANALYSIS_RUN_BYTE_LIMIT + 1 + ); + assert_eq!( + read_http_request(&mut Cursor::new(huge.into_bytes())).expect_err("body limit"), + ApiError::LimitExceeded + ); + } + + #[test] + fn serve_one_covers_loopback_success_disconnect_and_timeout() { + let run = sample_run(); + let mut service = AnalysisRunLiveService::bind_loopback().expect("bind"); + let addr = service.local_addr().expect("address"); + let worker = thread::spawn(move || service.serve_one()); + let mut stream = TcpStream::connect(addr).expect("connect"); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .expect("read timeout"); + stream + .write_all(valid_request(&run, NARUON_CONSUMER_CODE, &addr.to_string()).as_bytes()) + .expect("request"); + let mut response = String::new(); + stream.read_to_string(&mut response).expect("response"); + assert!(response.starts_with("HTTP/1.1 202 Accepted")); + assert_eq!( + worker.join().expect("join").expect("served").status_code, + 202 + ); + + let mut idle = AnalysisRunLiveService::bind_loopback().expect("idle bind"); + let idle_addr = idle.local_addr().expect("idle address"); + let idle_worker = thread::spawn(move || idle.serve_one()); + drop(TcpStream::connect(idle_addr).expect("idle connect")); + assert_eq!( + idle_worker + .join() + .expect("idle join") + .expect("idle served") + .status_code, + 400 + ); + + let mut timeout = AnalysisRunLiveService::bind_loopback().expect("timeout bind"); + let timeout_addr = timeout.local_addr().expect("timeout address"); + let timeout_worker = thread::spawn(move || timeout.serve_one()); + let stream = TcpStream::connect(timeout_addr).expect("timeout connect"); + let started = std::time::Instant::now(); + let timeout_response = timeout_worker + .join() + .expect("timeout join") + .expect("timeout served"); + drop(stream); + assert!(started.elapsed() >= NARUON_LIVE_IO_TIMEOUT); + assert_eq!(timeout_response.status_code, 413); + assert_eq!( + envelope(&timeout_response.body).error_code(), + "limit_exceeded" + ); + } + + struct ScriptedRead { + reader: Cursor>, + first_error: Option, + } + + impl ScriptedRead { + fn bytes(bytes: &[u8]) -> Self { + Self { + reader: Cursor::new(bytes.to_vec()), + first_error: None, + } + } + + fn error(kind: std::io::ErrorKind) -> Self { + Self { + reader: Cursor::new(Vec::new()), + first_error: Some(kind), + } + } + } + + impl Read for ScriptedRead { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if let Some(kind) = self.first_error.take() { + return Err(std::io::Error::new(kind, "scripted error")); + } + self.reader.read(buffer) + } + } +} diff --git a/crates/tepp_api/src/bin/tepp_loopback.rs b/crates/tepp_api/src/bin/tepp_loopback.rs new file mode 100644 index 00000000..90800a8c --- /dev/null +++ b/crates/tepp_api/src/bin/tepp_loopback.rs @@ -0,0 +1,24 @@ +//! Runnable loopback ingress for trusted same-host TEPP consumers. + +use std::net::SocketAddr; + +use tepp_api::AnalysisRunLiveService; + +const DEFAULT_BIND_ADDR: &str = "127.0.0.1:18081"; + +fn main() -> Result<(), Box> { + let mut arguments = std::env::args().skip(1); + let bind_addr = arguments + .next() + .unwrap_or(DEFAULT_BIND_ADDR.to_owned()) + .parse::()?; + let request_limit = arguments + .next() + .map(|value| value.parse::()) + .transpose()? + .unwrap_or(usize::MAX); + let mut service = AnalysisRunLiveService::bind(bind_addr)?; + println!("{}", service.local_addr()?); + (0..request_limit).for_each(|_| drop(service.serve_one())); + Ok(()) +} diff --git a/crates/tepp_api/src/lib.rs b/crates/tepp_api/src/lib.rs index 1c6fdf0f..884a7634 100644 --- a/crates/tepp_api/src/lib.rs +++ b/crates/tepp_api/src/lib.rs @@ -5,32 +5,65 @@ //! These pure wire contracts let TEPP operate standalone and as a modular CWL //! component without sharing application tables. Domain estimation remains in //! scientific crates; this crate only defines fail-closed interchange shapes. -//! naruon HTTP interchange is a versioned `https` POST to analysis-run and -//! export paths; table-access URLs, review/Copilot/NIM/proxy headers, and -//! lexical inference claims fail closed. A loopback live listener proves -//! those POSTs over TCP without claiming production TLS (ADR 0011). +//! Naruon and `LineageWeave` use the versioned analysis-run contract; `LineageWeave` +//! may also request a cutoff-safe project-history projection from explicit +//! source evidence. Naruon owns the current purpose-bound export adapter. +//! Loopback listeners prove the HTTP boundary without claiming production TLS, +//! causality, or completed psychometric model results. +mod analysis_result; mod analysis_run; +mod analysis_run_live; mod authorization; mod envelope; mod error; mod export; +mod lineageweave_http; +mod live_http; mod naruon_http; mod naruon_live; mod orchestration; +mod project_history; mod provider_payload; +mod temporal_context; mod wire; +/// Terminal analysis-result contract version constant. +pub use analysis_result::ANALYSIS_RESULT_CONTRACT_VERSION; +/// Bounded identity-free terminal result summary. +pub use analysis_result::AnalysisResultSummary; +/// Request-bound terminal analysis outcome. +pub use analysis_result::AnalysisRunTerminalResult; +/// Canonical terminal analysis-run state. +pub use analysis_result::AnalysisRunTerminalState; +/// Default terminal analysis-result payload byte limit. +pub use analysis_result::DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT; +/// Require exact terminal result binding to request and accepted receipt. +pub use analysis_result::require_terminal_binding; +/// Compare a terminal result with an accepted receipt. +pub use analysis_result::terminal_result_matches_accepted; +/// Compare a terminal result with its submitted request. +pub use analysis_result::terminal_result_matches_request; /// Analysis-run contract version constant. pub use analysis_run::ANALYSIS_RUN_CONTRACT_VERSION; +/// Analysis-run status/read contract version constant. +pub use analysis_run::ANALYSIS_RUN_STATUS_CONTRACT_VERSION; /// Accepted analysis-run response. pub use analysis_run::AnalysisRunAccepted; /// Analysis-run create request. pub use analysis_run::AnalysisRunRequest; +/// Typed analysis-run status/read response. +pub use analysis_run::AnalysisRunStatus; +/// Analysis-run status/read lifecycle state. +pub use analysis_run::AnalysisRunStatusState; /// Default analysis-run payload byte limit. pub use analysis_run::DEFAULT_ANALYSIS_RUN_BYTE_LIMIT; /// Idempotent request equality helper. pub use analysis_run::requests_are_idempotent_matches; +/// Require exact status binding to a request and accepted receipt. +pub use analysis_run::require_status_binding; +/// Consumer-neutral loopback analysis-run service. +pub use analysis_run_live::AnalysisRunLiveService; /// Content-redacting error envelope. pub use envelope::ErrorEnvelope; /// Fail-closed API errors. @@ -54,19 +87,29 @@ pub use authorization::ExportAuthorizationRequest; pub use authorization::authorize_export; /// Fail closed when an export decision is denied. pub use authorization::require_export_allowed; -/// Versioned analysis-run path naruon may call. +/// Published `LineageWeave` modular-consumer identity. +pub use lineageweave_http::LINEAGEWEAVE_CONSUMER_CODE; +/// Published Naruon modular-consumer identity. +pub use lineageweave_http::NARUON_CONSUMER_CODE; +/// Build a `LineageWeave` analysis-run exchange without provider credentials. +pub use lineageweave_http::lineageweave_analysis_run_exchange; +/// Build a `LineageWeave` project-history exchange without provider credentials. +pub use lineageweave_http::lineageweave_project_history_exchange; +/// Build a credential-free `LineageWeave` temporal-context exchange. +pub use lineageweave_http::lineageweave_temporal_context_exchange; +/// Versioned analysis-run path modular consumers may call. pub use naruon_http::NARUON_ANALYSIS_RUN_PATH; -/// Versioned export path naruon may call. +/// Versioned export path Naruon may call. pub use naruon_http::NARUON_EXPORT_PATH; -/// Allowed TEPP inference method code naruon may claim. +/// Allowed TEPP inference method code Naruon may claim. pub use naruon_http::NARUON_TEPP_INFERENCE_METHOD; -/// Fail-closed HTTP exchange naruon may send to TEPP. +/// Fail-closed HTTP exchange a modular consumer may send to TEPP. pub use naruon_http::NaruonHttpExchange; -/// Build a naruon analysis-run create exchange. +/// Build a Naruon analysis-run create exchange. pub use naruon_http::naruon_analysis_run_exchange; -/// Build an analysis-run exchange and refuse credential headers. +/// Build a Naruon analysis-run exchange and refuse credential headers. pub use naruon_http::naruon_analysis_run_exchange_with_headers; -/// Build a naruon export-authorization exchange. +/// Build a Naruon export-authorization exchange. pub use naruon_http::naruon_export_exchange; /// Refuse lexical heuristics as TEPP inference claims. pub use naruon_http::naruon_may_claim_tepp_inference; @@ -76,9 +119,9 @@ pub use naruon_live::NARUON_LIVE_HEADER_BYTE_LIMIT; pub use naruon_live::NARUON_LIVE_HEADER_COUNT_LIMIT; /// Accepted-stream read/write deadline. pub use naruon_live::NARUON_LIVE_IO_TIMEOUT; -/// HTTP/1.1 response from the naruon live listener. +/// HTTP/1.1 response from the loopback listener. pub use naruon_live::NaruonLiveResponse; -/// Loopback live HTTP/1.1 service for naruon POSTs. +/// Backward-compatible Naruon loopback HTTP/1.1 service. pub use naruon_live::NaruonLiveService; /// Comparable-budget ablation record. pub use orchestration::BudgetAblationRecord; @@ -116,6 +159,26 @@ pub use orchestration::bind_contextual_orchestrator; pub use orchestration::record_budget_ablation; /// Route a task onto a versioned orchestration plan. pub use orchestration::route_orchestration; +/// Default maximum serialized project-history request bytes. +pub use project_history::DEFAULT_PROJECT_HISTORY_BYTE_LIMIT; +/// Default maximum project-history event count. +pub use project_history::DEFAULT_PROJECT_HISTORY_EVENT_LIMIT; +/// Supported project-history contract version. +pub use project_history::PROJECT_HISTORY_CONTRACT_VERSION; +/// Versioned project-history path. +pub use project_history::PROJECT_HISTORY_PATH; +/// Explicit source-grounded project event. +pub use project_history::ProjectHistoryEvent; +/// One non-causal temporal finding. +pub use project_history::ProjectHistoryFinding; +/// Project-history HTTP exchange. +pub use project_history::ProjectHistoryHttpExchange; +/// Deterministic TEPP project-history projection. +pub use project_history::ProjectHistoryProjection; +/// Versioned project-history request. +pub use project_history::ProjectHistoryRequest; +/// Build a cutoff-safe project-history projection. +pub use project_history::project_history_projection; /// Elevated re-identification result. pub use provider_payload::DisclosedIdentityMapping; /// Separately protected identity mapping. @@ -138,3 +201,23 @@ pub use provider_payload::ReidentificationAuditSink; pub use provider_payload::disclose_identity_mapping; /// Minimize evidence for a model provider. pub use provider_payload::minimize_provider_payload; +/// Temporal association claim boundary. +pub use temporal_context::TEMPORAL_ASSOCIATION_CLAIM_BOUNDARY; +/// Temporal-context contract version constant. +pub use temporal_context::TEMPORAL_CONTEXT_CONTRACT_VERSION; +/// Versioned temporal-context HTTP path. +pub use temporal_context::TEMPORAL_CONTEXT_PATH; +/// One opaque event in a temporal-context request. +pub use temporal_context::TemporalContextEvent; +/// One adjacent temporal relation. +pub use temporal_context::TemporalContextRelation; +/// Temporal-context request. +pub use temporal_context::TemporalContextRequest; +/// Temporal-context response. +pub use temporal_context::TemporalContextResponse; +/// One ordered event in a temporal-context response. +pub use temporal_context::TemporalContextTimelineEvent; +/// One non-causal transition-gap candidate. +pub use temporal_context::TemporalTransitionGapCandidate; +/// Build a cutoff-safe, non-causal temporal context. +pub use temporal_context::build_temporal_context; diff --git a/crates/tepp_api/src/lineageweave_http.rs b/crates/tepp_api/src/lineageweave_http.rs new file mode 100644 index 00000000..6094760e --- /dev/null +++ b/crates/tepp_api/src/lineageweave_http.rs @@ -0,0 +1,135 @@ +//! Published modular-consumer identity and `LineageWeave` TEPP exchanges. + +use crate::naruon_http::compose_https_target; +use crate::project_history::build_project_history_exchange; +use crate::{ + AnalysisRunRequest, ApiError, NaruonHttpExchange, ProjectHistoryHttpExchange, + ProjectHistoryRequest, TEMPORAL_CONTEXT_CONTRACT_VERSION, TEMPORAL_CONTEXT_PATH, + TemporalContextRequest, naruon_analysis_run_exchange, +}; + +/// Stable consumer identity used by the Naruon adapter. +pub const NARUON_CONSUMER_CODE: &str = "naruon"; + +/// Stable consumer identity used by the `LineageWeave` adapter. +pub const LINEAGEWEAVE_CONSUMER_CODE: &str = "lineageweave"; + +/// Build a `LineageWeave` → TEPP analysis-run exchange without provider credentials. +/// +/// The function reuses TEPP's existing origin, body, and header validation, +/// then replaces only the published modular-consumer identity. The accepted +/// response remains an asynchronous transport acknowledgement, not a completed +/// psychometric result. +/// +/// # Errors +/// +/// Returns the same fail-closed errors as [`naruon_analysis_run_exchange`]. +pub fn lineageweave_analysis_run_exchange( + origin: &str, + request: &AnalysisRunRequest, +) -> Result { + let mut exchange = naruon_analysis_run_exchange(origin, request)?; + let consumer_header = exchange + .headers + .iter_mut() + .find(|(name, _)| name.eq_ignore_ascii_case("tepp-consumer")) + .ok_or(ApiError::InvalidWirePayload)?; + LINEAGEWEAVE_CONSUMER_CODE.clone_into(&mut consumer_header.1); + Ok(exchange) +} + +/// Build a credential-free `LineageWeave` temporal-context exchange. +/// +/// # Errors +/// +/// Returns a fail-closed error for a hostile origin or invalid temporal-context +/// request. +pub fn lineageweave_temporal_context_exchange( + origin: &str, + request: &TemporalContextRequest, +) -> Result { + let target_url = compose_https_target(origin, TEMPORAL_CONTEXT_PATH)?; + let body = request.to_json()?; + let headers = vec![ + ("content-type".into(), "application/json".into()), + ("tepp-consumer".into(), LINEAGEWEAVE_CONSUMER_CODE.into()), + ( + "tepp-contract-version".into(), + TEMPORAL_CONTEXT_CONTRACT_VERSION.to_string(), + ), + ]; + Ok(NaruonHttpExchange { + method: "POST", + target_url, + headers, + body, + }) +} + +/// Build a `LineageWeave` → TEPP project-history exchange without credentials. +/// +/// The request contains only bounded source evidence selected after +/// `LineageWeave` authorization. TEPP validates the cutoff and returns a +/// deterministic temporal-association projection, never a causal score. +/// +/// # Errors +/// +/// Returns a fail-closed origin, request, version, size, or timestamp error. +pub fn lineageweave_project_history_exchange( + origin: &str, + request: &ProjectHistoryRequest, +) -> Result { + build_project_history_exchange(origin, LINEAGEWEAVE_CONSUMER_CODE, request) +} + +/// Return whether a modular analysis-run consumer is published by TEPP. +pub(crate) fn consumer_is_supported(consumer_code: &str) -> bool { + matches!( + consumer_code, + NARUON_CONSUMER_CODE | LINEAGEWEAVE_CONSUMER_CODE + ) +} + +#[cfg(test)] +mod tests { + use super::{ + LINEAGEWEAVE_CONSUMER_CODE, NARUON_CONSUMER_CODE, consumer_is_supported, + lineageweave_analysis_run_exchange, + }; + use crate::{ANALYSIS_RUN_CONTRACT_VERSION, AnalysisRunRequest, ApiError}; + + fn sample_run() -> AnalysisRunRequest { + AnalysisRunRequest { + contract_version: ANALYSIS_RUN_CONTRACT_VERSION, + idempotency_key: "idem-1".into(), + tenant_workspace_id: "tenant-1".into(), + snapshot_id: "snapshot-1".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "tepp-analysis-run-v1".into(), + output_profile: "calibrated_event_measurement".into(), + } + } + + #[test] + fn supported_consumer_set_is_closed() { + assert!(consumer_is_supported(NARUON_CONSUMER_CODE)); + assert!(consumer_is_supported(LINEAGEWEAVE_CONSUMER_CODE)); + assert!(!consumer_is_supported("unknown")); + } + + #[test] + fn lineageweave_exchange_preserves_existing_fail_closed_validation() { + let run = sample_run(); + let exchange = lineageweave_analysis_run_exchange("https://tepp.example.test", &run) + .expect("exchange"); + assert!( + exchange + .headers + .contains(&("tepp-consumer".into(), LINEAGEWEAVE_CONSUMER_CODE.into())) + ); + assert_eq!( + lineageweave_analysis_run_exchange("http://tepp.example.test", &run), + Err(ApiError::InvalidWirePayload) + ); + } +} diff --git a/crates/tepp_api/src/live_http.rs b/crates/tepp_api/src/live_http.rs new file mode 100644 index 00000000..ed8d7e56 --- /dev/null +++ b/crates/tepp_api/src/live_http.rs @@ -0,0 +1,238 @@ +//! Shared fail-closed framing and host validation for loopback HTTP listeners. + +use std::collections::HashMap; +use std::io::Read; +use std::net::{IpAddr, SocketAddr}; + +use crate::naruon_http::header_is_credential; +use crate::{ApiError, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT}; + +/// Maximum request-line plus header bytes accepted before the body. +pub const NARUON_LIVE_HEADER_BYTE_LIMIT: usize = 8 * 1024; + +/// Maximum number of HTTP header lines on one live request. +pub const NARUON_LIVE_HEADER_COUNT_LIMIT: usize = 32; + +/// Read one HTTP/1.1 request, including its declared UTF-8 body. +pub(crate) fn read_http_request(reader: &mut R) -> Result { + read_http_request_with_limit(reader, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT) +} + +/// Read one HTTP/1.1 request with a caller-selected body limit. +pub(crate) fn read_http_request_with_limit( + reader: &mut R, + maximum_body_bytes: usize, +) -> Result { + let mut header_bytes = Vec::new(); + let mut byte = [0_u8; 1]; + while !header_bytes.ends_with(b"\r\n\r\n") { + if header_bytes.len() >= NARUON_LIVE_HEADER_BYTE_LIMIT { + return Err(ApiError::LimitExceeded); + } + let read = reader + .read(&mut byte) + .map_err(|error| map_io_error(&error))?; + if read == 0 { + return Err(ApiError::InvalidWirePayload); + } + header_bytes.push(byte[0]); + } + let header_text = + std::str::from_utf8(&header_bytes).map_err(|_| ApiError::InvalidWirePayload)?; + let content_length = declared_content_length(header_text)?; + if content_length > maximum_body_bytes { + return Err(ApiError::LimitExceeded); + } + let mut body = vec![0_u8; content_length]; + if content_length > 0 { + reader + .read_exact(&mut body) + .map_err(|error| map_io_error(&error))?; + } + let body_text = std::str::from_utf8(&body).map_err(|_| ApiError::InvalidWirePayload)?; + Ok(format!("{header_text}{body_text}")) +} + +/// Split one complete request into its header block and UTF-8 body. +pub(crate) fn split_request(request: &str) -> Result<(&str, &str), ApiError> { + split_request_with_limit(request, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT) +} + +/// Split one complete request with a caller-selected body limit. +pub(crate) fn split_request_with_limit( + request: &str, + maximum_body_bytes: usize, +) -> Result<(&str, &str), ApiError> { + let Some(index) = request.find("\r\n\r\n") else { + if request.len() >= NARUON_LIVE_HEADER_BYTE_LIMIT { + return Err(ApiError::LimitExceeded); + } + return Err(ApiError::InvalidWirePayload); + }; + if index > NARUON_LIVE_HEADER_BYTE_LIMIT { + return Err(ApiError::LimitExceeded); + } + let header_block = &request[..index]; + let body = &request[index + 4..]; + let declared = declared_content_length(&format!("{header_block}\r\n\r\n"))?; + if declared != body.len() { + return Err(ApiError::InvalidWirePayload); + } + if declared > maximum_body_bytes { + return Err(ApiError::LimitExceeded); + } + Ok((header_block, body)) +} + +/// Parse the single decimal content length from a complete header block. +pub(crate) fn declared_content_length(header_text: &str) -> Result { + let header_block = header_text + .strip_suffix("\r\n\r\n") + .ok_or(ApiError::InvalidWirePayload)?; + let mut found = None; + for line in header_block.split("\r\n").skip(1) { + let (name, value) = split_header_line(line)?; + if name.eq_ignore_ascii_case("content-length") { + if found.is_some() + || value.is_empty() + || !value.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(ApiError::InvalidWirePayload); + } + found = Some(value.parse().map_err(|_| ApiError::InvalidWirePayload)?); + } + } + found.ok_or(ApiError::InvalidWirePayload) +} + +/// Parse a strict HTTP/1.1 request line into method and path. +pub(crate) fn parse_request_line(line: &str) -> Result<(&str, &str), ApiError> { + let mut parts = line.split(' '); + let method = parts.next().ok_or(ApiError::InvalidWirePayload)?; + let path = parts.next().ok_or(ApiError::InvalidWirePayload)?; + let version = parts.next().ok_or(ApiError::InvalidWirePayload)?; + if parts.next().is_some() || version != "HTTP/1.1" { + return Err(ApiError::InvalidWirePayload); + } + if !path.starts_with('/') || path.contains('?') || path.contains('#') || path.contains("://") { + return Err(ApiError::InvalidWirePayload); + } + Ok((method, path)) +} + +/// Parse and normalize bounded, unique HTTP headers. +pub(crate) fn parse_headers<'a, I>(lines: I) -> Result, ApiError> +where + I: Iterator, +{ + let mut headers = HashMap::new(); + let mut count = 0_usize; + for line in lines { + count += 1; + if count > NARUON_LIVE_HEADER_COUNT_LIMIT { + return Err(ApiError::LimitExceeded); + } + let (name, value) = split_header_line(line)?; + let key = name.to_ascii_lowercase(); + if headers.contains_key(&key) { + return Err(ApiError::InvalidWirePayload); + } + headers.insert(key, value.to_owned()); + } + Ok(headers) +} + +/// Split one header line while rejecting malformed names. +pub(crate) fn split_header_line(line: &str) -> Result<(&str, &str), ApiError> { + let Some((name, value)) = line.split_once(':') else { + return Err(ApiError::InvalidWirePayload); + }; + if name.is_empty() || name.chars().any(|ch| ch.is_whitespace() || ch.is_control()) { + return Err(ApiError::InvalidWirePayload); + } + Ok((name, value.trim())) +} + +/// Return one required non-empty normalized header value. +pub(crate) fn header_value<'a>( + headers: &'a HashMap, + name: &str, +) -> Result<&'a str, ApiError> { + let value = headers.get(name).ok_or(ApiError::InvalidWirePayload)?; + if value.is_empty() { + return Err(ApiError::InvalidWirePayload); + } + Ok(value.as_str()) +} + +/// Validate common credential, framing, content-type, and loopback boundaries. +pub(crate) fn validate_common_headers( + headers: &HashMap, + bound_addr: Option, +) -> Result<(), ApiError> { + for name in headers.keys() { + if header_is_credential(name) { + return Err(ApiError::AuthorizationDenied); + } + } + if headers.contains_key("transfer-encoding") { + return Err(ApiError::InvalidWirePayload); + } + let host = header_value(headers, "host")?; + if host_implies_table_access(host) { + return Err(ApiError::InvalidWirePayload); + } + if !host_is_loopback(host, bound_addr) { + return Err(ApiError::AuthorizationDenied); + } + if header_value(headers, "content-type")? != "application/json" { + return Err(ApiError::InvalidWirePayload); + } + Ok(()) +} + +/// Return whether a host value implies direct database or table access. +pub(crate) fn host_implies_table_access(host: &str) -> bool { + let lowered = host.to_ascii_lowercase(); + lowered.contains("postgres") + || lowered.contains("jdbc") + || lowered.contains("/sql") + || lowered.contains("/tables/") + || lowered.contains('\'') + || lowered.contains(';') + || lowered.contains('\\') + || lowered.contains(' ') + || lowered.chars().any(char::is_control) +} + +/// Return whether a host resolves to loopback or the bound loopback socket. +pub(crate) fn host_is_loopback(host: &str, bound_addr: Option) -> bool { + if let Some(bound) = bound_addr + && (host == bound.to_string() || host == bound.ip().to_string()) + { + return true; + } + let lowered = host.to_ascii_lowercase(); + if lowered == "localhost" + || lowered + .strip_prefix("localhost:") + .is_some_and(|port| !port.is_empty() && port.parse::().is_ok()) + { + return true; + } + if let Ok(addr) = host.parse::() { + return addr.ip().is_loopback(); + } + if let Ok(ip) = host.parse::() { + return ip.is_loopback(); + } + false +} + +/// Map socket timeout and transport failures to redacted API errors. +pub(crate) fn map_io_error(error: &std::io::Error) -> ApiError { + match error.kind() { + std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock => ApiError::LimitExceeded, + _ => ApiError::InvalidWirePayload, + } +} diff --git a/crates/tepp_api/src/naruon_http.rs b/crates/tepp_api/src/naruon_http.rs index 2d1a6c2f..e9afde2d 100644 --- a/crates/tepp_api/src/naruon_http.rs +++ b/crates/tepp_api/src/naruon_http.rs @@ -116,7 +116,7 @@ pub fn naruon_may_claim_tepp_inference(method_code: &str) -> Result<(), ApiError } } -fn compose_https_target(origin: &str, path: &str) -> Result { +pub(crate) fn compose_https_target(origin: &str, path: &str) -> Result { require_nonempty(origin)?; if !origin.starts_with("https://") { return Err(ApiError::InvalidWirePayload); @@ -155,6 +155,15 @@ pub(crate) fn header_is_credential(name: &str) -> bool { || lowered == "proxy-authorization" || lowered == "cookie" || lowered == "x-api-key" + || lowered.contains("api-key") + || lowered.contains("api_key") + || lowered.contains("apikey") + || lowered.contains("secret") + || lowered.contains("credential") + || lowered.contains("openai") + || lowered.contains("anthropic") + || lowered.contains("bytez") + || lowered.contains("openrouter") || lowered.contains("token") || lowered.contains("copilot") || lowered.contains("github") @@ -163,7 +172,10 @@ pub(crate) fn header_is_credential(name: &str) -> bool { } fn refuse_credential_headers(extra_headers: &[(&str, &str)]) -> Result<(), ApiError> { - for (name, _) in extra_headers { + for (name, value) in extra_headers { + if !is_http_field_name(name) || value.chars().any(char::is_control) { + return Err(ApiError::InvalidWirePayload); + } if header_is_reserved_standard(name) { return Err(ApiError::InvalidWirePayload); } @@ -174,7 +186,31 @@ fn refuse_credential_headers(extra_headers: &[(&str, &str)]) -> Result<(), ApiEr Ok(()) } -fn standard_headers(idempotency_key: &str) -> Vec<(String, String)> { +fn is_http_field_name(name: &str) -> bool { + !name.is_empty() + && name.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'!' | b'#' + | b'$' + | b'%' + | b'&' + | b'\'' + | b'*' + | b'+' + | b'-' + | b'.' + | b'^' + | b'_' + | b'`' + | b'|' + | b'~' + ) + }) +} + +pub(crate) fn standard_headers(idempotency_key: &str) -> Vec<(String, String)> { vec![ ("content-type".into(), "application/json".into()), ("tepp-consumer".into(), "naruon".into()), @@ -246,6 +282,11 @@ mod tests { compose_https_target("https://ho\u{0001}st", "/v1/x"), Err(ApiError::InvalidWirePayload) ); + let c1_control_origin = format!("https://host{}example", char::from_u32(0x80).unwrap()); + assert_eq!( + compose_https_target(&c1_control_origin, "/v1/x"), + Err(ApiError::InvalidWirePayload) + ); assert_eq!( compose_https_target("https://db.postgres.example", "/v1/x"), Err(ApiError::InvalidWirePayload) @@ -316,6 +357,41 @@ mod tests { refuse_credential_headers(&[("x-nvidia-nim-key", "nvapi-x")]), Err(ApiError::AuthorizationDenied) ); + for name in [ + "x-openai-api-key", + "x-anthropic-key", + "x-bytez-api-key", + "x-openrouter-api-key", + "x-api_key", + "x-secret", + "x-credential", + "x-openai", + "x-bytez", + "x-openrouter", + "x-provider-api_key", + "x-provider-secret", + "x-provider-credential", + "x-provider-openai", + "x-provider-bytez", + "x-provider-openrouter", + ] { + assert_eq!( + refuse_credential_headers(&[(name, "provider-secret")]), + Err(ApiError::AuthorizationDenied), + "header={name}" + ); + } + for (name, value) in [ + ("", "value"), + ("bad name", "value"), + ("x-trace", "ok\r\nx-injected: 1"), + ] { + assert_eq!( + refuse_credential_headers(&[(name, value)]), + Err(ApiError::InvalidWirePayload), + "header={name:?}" + ); + } } #[test] diff --git a/crates/tepp_api/src/naruon_live.rs b/crates/tepp_api/src/naruon_live.rs index b068fedd..f9b4ca32 100644 --- a/crates/tepp_api/src/naruon_live.rs +++ b/crates/tepp_api/src/naruon_live.rs @@ -2,24 +2,35 @@ use std::collections::HashMap; use std::io::{Read, Write}; -use std::net::{IpAddr, SocketAddr, TcpListener, TcpStream}; +use std::net::{SocketAddr, TcpListener, TcpStream}; use std::time::Duration; use crate::authorization::{ AnalyticalPurpose, ExportAuthorizationRequest, authorize_export, require_export_allowed, }; -use crate::naruon_http::{NARUON_ANALYSIS_RUN_PATH, NARUON_EXPORT_PATH, header_is_credential}; +use crate::lineageweave_http::NARUON_CONSUMER_CODE; +use crate::live_http::{ + header_value, map_io_error, parse_headers, parse_request_line, read_http_request, + split_request, validate_common_headers, +}; +use crate::naruon_http::{NARUON_ANALYSIS_RUN_PATH, NARUON_EXPORT_PATH}; use crate::wire::{from_json, to_json}; use crate::{ - AnalysisRunAccepted, AnalysisRunRequest, ApiError, DEFAULT_ANALYSIS_RUN_BYTE_LIMIT, - ErrorEnvelope, requests_are_idempotent_matches, + AnalysisRunAccepted, AnalysisRunRequest, ApiError, ErrorEnvelope, + requests_are_idempotent_matches, }; -/// Maximum request-line plus header bytes accepted before the body. -pub const NARUON_LIVE_HEADER_BYTE_LIMIT: usize = 8 * 1024; +#[cfg(test)] +use crate::DEFAULT_ANALYSIS_RUN_BYTE_LIMIT; +#[cfg(test)] +use crate::live_http::{ + declared_content_length, host_implies_table_access, host_is_loopback, split_header_line, +}; -/// Maximum number of HTTP header lines on one live request. -pub const NARUON_LIVE_HEADER_COUNT_LIMIT: usize = 32; +/// Maximum live HTTP header-block bytes. +pub use crate::live_http::NARUON_LIVE_HEADER_BYTE_LIMIT; +/// Maximum live HTTP header count. +pub use crate::live_http::NARUON_LIVE_HEADER_COUNT_LIMIT; /// Read and write deadline installed on every accepted stream. pub const NARUON_LIVE_IO_TIMEOUT: Duration = Duration::from_secs(1); @@ -164,37 +175,7 @@ impl NaruonLiveService { /// [`NARUON_LIVE_HEADER_BYTE_LIMIT`]. Other read/framing failures are /// [`ApiError::InvalidWirePayload`]. pub fn read_http_request(reader: &mut R) -> Result { - let mut header_bytes = Vec::new(); - let mut byte = [0_u8; 1]; - loop { - if header_bytes.len() >= NARUON_LIVE_HEADER_BYTE_LIMIT { - return Err(ApiError::LimitExceeded); - } - let read = reader - .read(&mut byte) - .map_err(|error| map_io_error(&error))?; - if read == 0 { - return Err(ApiError::InvalidWirePayload); - } - header_bytes.push(byte[0]); - if header_bytes.ends_with(b"\r\n\r\n") { - break; - } - } - let header_text = - std::str::from_utf8(&header_bytes).map_err(|_| ApiError::InvalidWirePayload)?; - let content_length = declared_content_length(header_text)?; - if content_length > DEFAULT_ANALYSIS_RUN_BYTE_LIMIT { - return Err(ApiError::LimitExceeded); - } - let mut body = vec![0_u8; content_length]; - if content_length > 0 { - reader - .read_exact(&mut body) - .map_err(|error| map_io_error(&error))?; - } - let body_text = std::str::from_utf8(&body).map_err(|_| ApiError::InvalidWirePayload)?; - Ok(format!("{header_text}{body_text}")) + read_http_request(reader) } /// Write one HTTP/1.1 response to `writer`. @@ -342,123 +323,12 @@ fn status_for(error: ApiError) -> (u16, &'static str) { } } -fn map_io_error(error: &std::io::Error) -> ApiError { - match error.kind() { - std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock => ApiError::LimitExceeded, - _ => ApiError::InvalidWirePayload, - } -} - -fn split_request(request: &str) -> Result<(&str, &str), ApiError> { - let Some(index) = request.find("\r\n\r\n") else { - if request.len() >= NARUON_LIVE_HEADER_BYTE_LIMIT { - return Err(ApiError::LimitExceeded); - } - return Err(ApiError::InvalidWirePayload); - }; - if index > NARUON_LIVE_HEADER_BYTE_LIMIT { - return Err(ApiError::LimitExceeded); - } - let header_block = &request[..index]; - let body = &request[index + 4..]; - let declared = declared_content_length(&format!("{header_block}\r\n\r\n"))?; - if declared != body.len() { - return Err(ApiError::InvalidWirePayload); - } - if declared > DEFAULT_ANALYSIS_RUN_BYTE_LIMIT { - return Err(ApiError::LimitExceeded); - } - Ok((header_block, body)) -} - -fn declared_content_length(header_text: &str) -> Result { - let header_block = header_text - .strip_suffix("\r\n\r\n") - .ok_or(ApiError::InvalidWirePayload)?; - let mut found = None; - for line in header_block.split("\r\n").skip(1) { - let (name, value) = split_header_line(line)?; - if name.eq_ignore_ascii_case("content-length") { - if found.is_some() { - return Err(ApiError::InvalidWirePayload); - } - if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { - return Err(ApiError::InvalidWirePayload); - } - found = Some(value.parse().map_err(|_| ApiError::InvalidWirePayload)?); - } - } - found.ok_or(ApiError::InvalidWirePayload) -} - -fn parse_request_line(line: &str) -> Result<(&str, &str), ApiError> { - let mut parts = line.split(' '); - let method = parts.next().ok_or(ApiError::InvalidWirePayload)?; - let path = parts.next().ok_or(ApiError::InvalidWirePayload)?; - let version = parts.next().ok_or(ApiError::InvalidWirePayload)?; - if parts.next().is_some() || version != "HTTP/1.1" { - return Err(ApiError::InvalidWirePayload); - } - if !path.starts_with('/') || path.contains('?') || path.contains('#') || path.contains("://") { - return Err(ApiError::InvalidWirePayload); - } - Ok((method, path)) -} - -fn parse_headers<'a, I>(lines: I) -> Result, ApiError> -where - I: Iterator, -{ - let mut headers = HashMap::new(); - let mut count = 0_usize; - for line in lines { - count += 1; - if count > NARUON_LIVE_HEADER_COUNT_LIMIT { - return Err(ApiError::LimitExceeded); - } - let (name, value) = split_header_line(line)?; - let key = name.to_ascii_lowercase(); - if headers.contains_key(&key) { - return Err(ApiError::InvalidWirePayload); - } - headers.insert(key, value.to_owned()); - } - Ok(headers) -} - -fn split_header_line(line: &str) -> Result<(&str, &str), ApiError> { - let Some((name, value)) = line.split_once(':') else { - return Err(ApiError::InvalidWirePayload); - }; - if name.is_empty() || name.chars().any(|ch| ch.is_whitespace() || ch.is_control()) { - return Err(ApiError::InvalidWirePayload); - } - Ok((name, value.trim())) -} - fn refuse_live_headers( headers: &HashMap, bound_addr: Option, ) -> Result<(), ApiError> { - for name in headers.keys() { - if header_is_credential(name) { - return Err(ApiError::AuthorizationDenied); - } - } - if headers.contains_key("transfer-encoding") { - return Err(ApiError::InvalidWirePayload); - } - let host = header_value(headers, "host")?; - if host_implies_table_access(host) { - return Err(ApiError::InvalidWirePayload); - } - if !host_is_loopback(host, bound_addr) { - return Err(ApiError::AuthorizationDenied); - } - if header_value(headers, "content-type")? != "application/json" { - return Err(ApiError::InvalidWirePayload); - } - if header_value(headers, "tepp-consumer")? != "naruon" { + validate_common_headers(headers, bound_addr)?; + if header_value(headers, "tepp-consumer")? != NARUON_CONSUMER_CODE { return Err(ApiError::InvalidWirePayload); } if header_value(headers, "tepp-contract-version")? != "1" { @@ -468,50 +338,6 @@ fn refuse_live_headers( Ok(()) } -fn header_value<'a>(headers: &'a HashMap, name: &str) -> Result<&'a str, ApiError> { - let value = headers.get(name).ok_or(ApiError::InvalidWirePayload)?; - if value.is_empty() { - return Err(ApiError::InvalidWirePayload); - } - Ok(value.as_str()) -} - -fn host_implies_table_access(host: &str) -> bool { - let lowered = host.to_ascii_lowercase(); - lowered.contains("postgres") - || lowered.contains("jdbc") - || lowered.contains("/sql") - || lowered.contains("/tables/") - || lowered.contains('\'') - || lowered.contains(';') - || lowered.contains('\\') - || lowered.contains(' ') - || lowered.chars().any(char::is_control) -} - -fn host_is_loopback(host: &str, bound_addr: Option) -> bool { - if let Some(bound) = bound_addr - && (host == bound.to_string() || host == bound.ip().to_string()) - { - return true; - } - let lowered = host.to_ascii_lowercase(); - if lowered == "localhost" - || lowered - .strip_prefix("localhost:") - .is_some_and(|port| !port.is_empty() && port.parse::().is_ok()) - { - return true; - } - if let Ok(addr) = host.parse::() { - return addr.ip().is_loopback(); - } - if let Ok(ip) = host.parse::() { - return ip.is_loopback(); - } - false -} - #[cfg(test)] mod tests { use super::{ diff --git a/crates/tepp_api/src/project_history.rs b/crates/tepp_api/src/project_history.rs new file mode 100644 index 00000000..433fab45 --- /dev/null +++ b/crates/tepp_api/src/project_history.rs @@ -0,0 +1,850 @@ +//! Cutoff-safe project-history projection for `LineageWeave` buyer surfaces. +//! +//! TEPP owns temporal validation and deterministic ordering. `LineageWeave` owns +//! authorization and selects the bounded source evidence supplied here. The +//! projection reports explicit temporal associations only; it never upgrades +//! sequence into causality or emits a psychometric score. + +use std::collections::{BTreeSet, HashSet}; +use std::net::{IpAddr, Ipv6Addr}; + +use jiff::Timestamp; +use serde::{Deserialize, Serialize}; +use temporal_core::{KnowledgeCutoff, TemporalInstant}; + +use crate::ApiError; +use crate::wire::{ + from_json, require_byte_limit, require_contract_version, require_nonempty, to_json_with_limit, +}; + +/// Supported project-history request and response contract version. +pub const PROJECT_HISTORY_CONTRACT_VERSION: u16 = 1; + +/// Versioned project-history path exposed by a TEPP service adapter. +pub const PROJECT_HISTORY_PATH: &str = "/v1/project-histories"; + +/// Maximum serialized request size accepted by the project-history contract. +pub const DEFAULT_PROJECT_HISTORY_BYTE_LIMIT: usize = 256 * 1024; + +/// Maximum event count accepted in one project-history request. +pub const DEFAULT_PROJECT_HISTORY_EVENT_LIMIT: usize = 128; + +/// Explicit event evidence supplied by an authorized modular consumer. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ProjectHistoryEvent { + /// Consumer-owned opaque event identity. + pub event_id: String, + /// Bounded machine event type, such as `voc_received`. + pub event_type_code: String, + /// Buyer-readable event title grounded in the source evidence. + pub event_title: String, + /// Event occurrence instant as RFC 3339. + pub occurred_at: String, + /// Instant at which this evidence was available to the analysis. + pub available_at: String, + /// Authorized `LineageWeave` source-post identity. + pub source_post_id: String, + /// Bounded evidence excerpt; never an instruction or causal conclusion. + pub evidence_text: String, + /// Opaque actor identities explicitly attached to this event. + pub actor_ids: Vec, +} + +/// Versioned request for a deterministic TEPP project-history projection. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ProjectHistoryRequest { + /// Semantic contract version. + pub contract_version: u16, + /// Caller-supplied opaque idempotency key. + pub idempotency_key: String, + /// Authorized tenant or workspace identity. + pub tenant_workspace_id: String, + /// Consumer-owned stable project key. + pub project_key: String, + /// Buyer-readable project label. + pub project_name: String, + /// Maximum evidence-availability instant as RFC 3339. + pub knowledge_cutoff: String, + /// Event around which before/after findings are evaluated. + pub focus_event_id: String, + /// Explicit source-grounded events. + pub events: Vec, +} + +/// One evidence-grounded temporal association in a project history. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ProjectHistoryFinding { + /// Stable finding code interpreted by consumer UI copy. + pub finding_code: String, + /// Non-causal explanation of the explicit event ordering. + pub summary: String, + /// Event identities supporting this finding. + pub related_event_ids: Vec, + /// Source-post identities supporting this finding. + pub evidence_post_ids: Vec, +} + +/// Deterministically ordered project-history response. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ProjectHistoryProjection { + /// Semantic contract version. + pub contract_version: u16, + /// Consumer-owned stable project key. + pub project_key: String, + /// Buyer-readable project label. + pub project_name: String, + /// Focus event echoed after validation. + pub focus_event_id: String, + /// Knowledge cutoff applied to every event in the response. + pub knowledge_cutoff: String, + /// Earliest event instant in the response. + pub history_span_start: String, + /// Latest event instant in the response. + pub history_span_end: String, + /// Distinct explicit actor count across the supplied events. + pub participant_count: usize, + /// Fixed claim boundary: sequence is association, not causation. + pub inference_status: String, + /// Events ordered by occurrence instant and stable identity. + pub events: Vec, + /// Findings derived only from explicit known event types. + pub findings: Vec, +} + +/// HTTP exchange for a project-history request. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProjectHistoryHttpExchange { + /// HTTP method, always `POST`. + pub method: &'static str, + /// Absolute HTTPS target ending in [`PROJECT_HISTORY_PATH`]. + pub target_url: String, + /// Exact version, consumer, content, and idempotency headers. + pub headers: Vec<(String, String)>, + /// Validated JSON request body. + pub body: String, +} + +impl ProjectHistoryRequest { + /// Parse and validate a project-history request using the default limit. + /// + /// # Errors + /// + /// Returns a version, size, JSON, timestamp, leakage, or field error. + pub fn from_json(payload: &str) -> Result { + Self::from_json_with_limit(payload, DEFAULT_PROJECT_HISTORY_BYTE_LIMIT) + } + + /// Parse and validate a project-history request using a caller limit. + /// + /// # Errors + /// + /// Returns a version, size, JSON, timestamp, leakage, or field error. + pub fn from_json_with_limit(payload: &str, maximum_bytes: usize) -> Result { + require_byte_limit(payload, maximum_bytes)?; + let request: Self = from_json(payload)?; + request.validate()?; + Ok(request) + } + + /// Serialize a validated project-history request. + /// + /// # Errors + /// + /// Returns a field-validation or serialization error. + pub fn to_json(&self) -> Result { + self.validate()?; + to_json_with_limit(self, DEFAULT_PROJECT_HISTORY_BYTE_LIMIT) + } + + fn validate(&self) -> Result<(), ApiError> { + require_contract_version(self.contract_version, PROJECT_HISTORY_CONTRACT_VERSION)?; + validate_bounded_text(&self.idempotency_key, 256)?; + validate_bounded_text(&self.tenant_workspace_id, 256)?; + validate_bounded_text(&self.project_key, 256)?; + validate_bounded_text(&self.project_name, 512)?; + validate_bounded_text(&self.focus_event_id, 256)?; + if self.events.is_empty() || self.events.len() > DEFAULT_PROJECT_HISTORY_EVENT_LIMIT { + return Err(ApiError::LimitExceeded); + } + let cutoff = parse_knowledge_cutoff(&self.knowledge_cutoff)?; + if cutoff_is_in_future(cutoff)? { + return Err(ApiError::InvalidWirePayload); + } + let mut event_ids = HashSet::with_capacity(self.events.len()); + let mut focus_found = false; + for event in &self.events { + validate_event(event, cutoff.instant())?; + if !event_ids.insert(event.event_id.as_str()) { + return Err(ApiError::InvalidWirePayload); + } + focus_found |= event.event_id == self.focus_event_id; + } + if !focus_found { + return Err(ApiError::InvalidWirePayload); + } + Ok(()) + } +} + +impl ProjectHistoryProjection { + /// Parse and validate a serialized TEPP projection. + /// + /// # Errors + /// + /// Returns a JSON, version, field, or claim-boundary error. + pub fn from_json(payload: &str) -> Result { + Self::from_json_with_limit(payload, DEFAULT_PROJECT_HISTORY_BYTE_LIMIT) + } + + /// Parse and validate a serialized TEPP projection with a caller limit. + /// + /// # Errors + /// + /// Returns a size, JSON, version, field, or claim-boundary error. + pub fn from_json_with_limit(payload: &str, maximum_bytes: usize) -> Result { + require_byte_limit(payload, maximum_bytes)?; + let projection: Self = from_json(payload)?; + projection.validate()?; + Ok(projection) + } + + /// Serialize a validated TEPP projection. + /// + /// # Errors + /// + /// Returns a validation or serialization error. + pub fn to_json(&self) -> Result { + self.validate()?; + to_json_with_limit(self, DEFAULT_PROJECT_HISTORY_BYTE_LIMIT) + } + + fn validate(&self) -> Result<(), ApiError> { + require_contract_version(self.contract_version, PROJECT_HISTORY_CONTRACT_VERSION)?; + validate_bounded_text(&self.project_key, 256)?; + validate_bounded_text(&self.project_name, 512)?; + validate_bounded_text(&self.focus_event_id, 256)?; + if self.inference_status != "temporal_association_only" || self.events.is_empty() { + return Err(ApiError::InvalidWirePayload); + } + if self.events.len() > DEFAULT_PROJECT_HISTORY_EVENT_LIMIT { + return Err(ApiError::LimitExceeded); + } + let cutoff = parse_knowledge_cutoff(&self.knowledge_cutoff)?; + if cutoff_is_in_future(cutoff)? { + return Err(ApiError::InvalidWirePayload); + } + let mut event_ids = HashSet::with_capacity(self.events.len()); + let mut focus_index = None; + for (index, event) in self.events.iter().enumerate() { + validate_event(event, cutoff.instant())?; + if !event_ids.insert(event.event_id.as_str()) { + return Err(ApiError::InvalidWirePayload); + } + if event.event_id == self.focus_event_id { + focus_index = Some(index); + } + } + let focus_index = focus_index.ok_or(ApiError::InvalidWirePayload)?; + let mut previous = None; + for event in &self.events { + let occurred_at = parse_timestamp(&event.occurred_at)?; + if let Some((previous_time, previous_id)) = previous + && (occurred_at < previous_time + || (occurred_at == previous_time && event.event_id.as_str() <= previous_id)) + { + return Err(ApiError::InvalidWirePayload); + } + previous = Some((occurred_at, event.event_id.as_str())); + } + let start = parse_timestamp(&self.history_span_start)?; + let end = parse_timestamp(&self.history_span_end)?; + let first_event_time = parse_timestamp(&self.events[0].occurred_at)?; + // The non-empty guard above makes this index safe and removes an + // unreachable second empty-events error path from the response contract. + let last_event_time = parse_timestamp(&self.events[self.events.len() - 1].occurred_at)?; + if start > end || start != first_event_time || end != last_event_time { + return Err(ApiError::InvalidWirePayload); + } + let participant_count = self + .events + .iter() + .flat_map(|event| event.actor_ids.iter().map(String::as_str)) + .collect::>() + .len(); + if self.participant_count != participant_count + || self.findings != build_findings(&self.events, focus_index) + { + return Err(ApiError::InvalidWirePayload); + } + Ok(()) + } +} + +/// Build a deterministic, cutoff-safe project-history projection. +/// +/// Findings are created only from explicit event type codes around the focus +/// event. The function does not infer causality, missing events, or latent +/// scores. +/// +/// # Errors +/// +/// Returns a fail-closed request validation error. +pub fn project_history_projection( + request: &ProjectHistoryRequest, +) -> Result { + request.validate()?; + let mut ordered = request.events.clone(); + ordered.sort_by_key(|event| { + ( + parse_timestamp(&event.occurred_at).ok(), + event.event_id.clone(), + ) + }); + let focus_index = ordered + .iter() + .position(|event| event.event_id == request.focus_event_id) + .ok_or(ApiError::InvalidWirePayload)?; + let findings = build_findings(&ordered, focus_index); + let participant_count = ordered + .iter() + .flat_map(|event| event.actor_ids.iter().map(String::as_str)) + .collect::>() + .len(); + let history_span_start = ordered + .first() + .map(|event| event.occurred_at.clone()) + .ok_or(ApiError::InvalidWirePayload)?; + let history_span_end = ordered + .last() + .map(|event| event.occurred_at.clone()) + .ok_or(ApiError::InvalidWirePayload)?; + let projection = ProjectHistoryProjection { + contract_version: PROJECT_HISTORY_CONTRACT_VERSION, + project_key: request.project_key.clone(), + project_name: request.project_name.clone(), + focus_event_id: request.focus_event_id.clone(), + knowledge_cutoff: request.knowledge_cutoff.clone(), + history_span_start, + history_span_end, + participant_count, + inference_status: "temporal_association_only".into(), + events: ordered, + findings, + }; + projection.to_json()?; + Ok(projection) +} + +pub(crate) fn build_project_history_exchange( + origin: &str, + consumer_code: &str, + request: &ProjectHistoryRequest, +) -> Result { + validate_bounded_text(consumer_code, 64)?; + let target_url = compose_https_target(origin)?; + let body = request.to_json()?; + Ok(ProjectHistoryHttpExchange { + method: "POST", + target_url, + headers: vec![ + ("content-type".into(), "application/json".into()), + ("tepp-consumer".into(), consumer_code.to_owned()), + ( + "tepp-contract-version".into(), + PROJECT_HISTORY_CONTRACT_VERSION.to_string(), + ), + ("idempotency-key".into(), request.idempotency_key.clone()), + ], + body, + }) +} + +fn validate_event(event: &ProjectHistoryEvent, cutoff: TemporalInstant) -> Result<(), ApiError> { + validate_bounded_text(&event.event_id, 256)?; + validate_code(&event.event_type_code)?; + validate_bounded_text(&event.event_title, 512)?; + validate_bounded_text(&event.source_post_id, 256)?; + validate_bounded_text(&event.evidence_text, 4096)?; + if event.actor_ids.len() > 64 { + return Err(ApiError::LimitExceeded); + } + for actor_id in &event.actor_ids { + validate_bounded_text(actor_id, 256)?; + } + parse_timestamp(&event.occurred_at)?; + let available_at = parse_timestamp(&event.available_at)?; + if available_at > cutoff { + return Err(ApiError::InvalidWirePayload); + } + Ok(()) +} + +fn validate_bounded_text(value: &str, maximum_bytes: usize) -> Result<(), ApiError> { + require_nonempty(value)?; + if value.len() > maximum_bytes { + return Err(ApiError::LimitExceeded); + } + Ok(()) +} + +fn validate_code(value: &str) -> Result<(), ApiError> { + validate_bounded_text(value, 64)?; + if !value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + { + return Err(ApiError::InvalidWirePayload); + } + Ok(()) +} + +fn parse_knowledge_cutoff(value: &str) -> Result { + KnowledgeCutoff::parse_rfc3339(value).map_err(|_| ApiError::InvalidWirePayload) +} + +fn parse_timestamp(value: &str) -> Result { + TemporalInstant::parse_rfc3339(value).map_err(|_| ApiError::InvalidWirePayload) +} + +fn cutoff_is_in_future(cutoff: KnowledgeCutoff) -> Result { + let now = KnowledgeCutoff::parse_rfc3339(&Timestamp::now().to_string()) + .map_err(|_| ApiError::InvalidWirePayload)?; + Ok(cutoff > now) +} + +fn build_findings( + ordered: &[ProjectHistoryEvent], + focus_index: usize, +) -> Vec { + let before = &ordered[..focus_index]; + let after = &ordered[focus_index + 1..]; + let specification = first_type(before, "specification_changed"); + let handoff = first_type(before, "handoff_recorded"); + let mut findings = Vec::new(); + append_single_finding( + &mut findings, + first_type(before, "contract_awarded"), + "contract_award_before_focus", + "An explicit contract-award event precedes the focus event.", + ); + append_single_finding( + &mut findings, + specification, + "specification_change_before_focus", + "An explicit specification-change event precedes the focus event.", + ); + append_single_finding( + &mut findings, + first_type(before, "delivered"), + "delivery_before_focus", + "An explicit delivery event precedes the focus event.", + ); + append_single_finding( + &mut findings, + handoff, + "handoff_before_focus", + "An explicit operational-handoff event precedes the focus event.", + ); + append_single_finding( + &mut findings, + first_type(after, "rebid_started"), + "rebid_after_focus", + "An explicit rebid event follows the focus event.", + ); + if let (Some(specification), Some(handoff)) = (specification, handoff) { + findings.push(combined_finding(specification, handoff)); + } + findings +} + +fn first_type<'a>( + events: &'a [ProjectHistoryEvent], + event_type_code: &str, +) -> Option<&'a ProjectHistoryEvent> { + events + .iter() + .find(|event| event.event_type_code == event_type_code) +} + +fn append_single_finding( + findings: &mut Vec, + event: Option<&ProjectHistoryEvent>, + finding_code: &str, + summary: &str, +) { + if let Some(event) = event { + findings.push(ProjectHistoryFinding { + finding_code: finding_code.to_owned(), + summary: summary.to_owned(), + related_event_ids: vec![event.event_id.clone()], + evidence_post_ids: vec![event.source_post_id.clone()], + }); + } +} + +fn combined_finding( + specification: &ProjectHistoryEvent, + handoff: &ProjectHistoryEvent, +) -> ProjectHistoryFinding { + let evidence_post_ids = [ + specification.source_post_id.clone(), + handoff.source_post_id.clone(), + ] + .into_iter() + .collect::>() + .into_iter() + .collect(); + ProjectHistoryFinding { + finding_code: "specification_change_and_handoff_before_focus".into(), + summary: "Explicit specification-change and handoff events precede the focus event; this is a temporal association, not a causal conclusion.".into(), + related_event_ids: vec![ + specification.event_id.clone(), + handoff.event_id.clone(), + ], + evidence_post_ids, + } +} + +fn compose_https_target(origin: &str) -> Result { + validate_bounded_text(origin, 2048)?; + let authority = origin + .strip_prefix("https://") + .ok_or(ApiError::InvalidWirePayload)?; + validate_https_authority(authority)?; + let lowered = authority.to_ascii_lowercase(); + if lowered.contains("postgres") || lowered.contains("jdbc") { + return Err(ApiError::InvalidWirePayload); + } + Ok(format!("{origin}{PROJECT_HISTORY_PATH}")) +} + +fn validate_https_authority(authority: &str) -> Result<(), ApiError> { + if authority.is_empty() + || authority.contains('@') + || authority.contains('/') + || authority.contains('?') + || authority.contains('#') + || authority + .chars() + .any(|character| matches!(character, '\'' | ';' | '\\' | ' ') || character.is_control()) + { + return Err(ApiError::InvalidWirePayload); + } + + if let Some(bracketed) = authority.strip_prefix('[') { + let close = bracketed.find(']').ok_or(ApiError::InvalidWirePayload)?; + let host = &bracketed[..close]; + host.parse::() + .map_err(|_| ApiError::InvalidWirePayload)?; + let suffix = &bracketed[close + 1..]; + if suffix.is_empty() { + return Ok(()); + } + let port = suffix + .strip_prefix(':') + .ok_or(ApiError::InvalidWirePayload)?; + return validate_https_port(port); + } + + if authority.contains(']') || authority.matches(':').count() > 1 { + return Err(ApiError::InvalidWirePayload); + } + let (host, port) = authority + .rsplit_once(':') + .map_or((authority, None), |(host, port)| (host, Some(port))); + validate_https_host(host)?; + if let Some(port) = port { + validate_https_port(port)?; + } + Ok(()) +} + +fn validate_https_host(host: &str) -> Result<(), ApiError> { + if host.is_empty() || host.len() > 253 { + return Err(ApiError::InvalidWirePayload); + } + if host.parse::().is_ok() { + return Ok(()); + } + for label in host.split('.') { + if label.is_empty() + || label.len() > 63 + || label.starts_with('-') + || label.ends_with('-') + || !label + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + { + return Err(ApiError::InvalidWirePayload); + } + } + Ok(()) +} + +fn validate_https_port(port: &str) -> Result<(), ApiError> { + if port.is_empty() || port.parse::().map_or(true, |value| value == 0) { + return Err(ApiError::InvalidWirePayload); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ + PROJECT_HISTORY_CONTRACT_VERSION, ProjectHistoryEvent, ProjectHistoryProjection, + ProjectHistoryRequest, build_project_history_exchange, compose_https_target, + project_history_projection, validate_code, + }; + use crate::ApiError; + + fn request_with_single_event() -> ProjectHistoryRequest { + ProjectHistoryRequest { + contract_version: PROJECT_HISTORY_CONTRACT_VERSION, + idempotency_key: "idem".into(), + tenant_workspace_id: "tenant".into(), + project_key: "project".into(), + project_name: "Project".into(), + knowledge_cutoff: "2026-08-19T23:59:59Z".into(), + focus_event_id: "focus".into(), + events: vec![ProjectHistoryEvent { + event_id: "focus".into(), + event_type_code: "voc_received".into(), + event_title: "VOC".into(), + occurred_at: "2026-08-19T09:00:00Z".into(), + available_at: "2026-08-19T10:00:00Z".into(), + source_post_id: "post".into(), + evidence_text: "explicit evidence".into(), + actor_ids: Vec::new(), + }], + } + } + + #[test] + fn projection_round_trip_preserves_the_non_causal_claim_boundary() { + let request = request_with_single_event(); + let projection = project_history_projection(&request).expect("projection"); + let json = projection.to_json().expect("json"); + assert_eq!( + ProjectHistoryProjection::from_json(&json).expect("decode"), + projection + ); + assert!(projection.findings.is_empty()); + assert_eq!(projection.participant_count, 0); + } + + #[test] + fn projection_counts_memberships_and_emits_explicit_findings() { + let mut request = request_with_single_event(); + let event = |event_id: &str, event_type_code: &str, occurred_at: &str, actor_id: &str| { + ProjectHistoryEvent { + event_id: event_id.into(), + event_type_code: event_type_code.into(), + event_title: event_type_code.into(), + occurred_at: occurred_at.into(), + available_at: occurred_at.into(), + source_post_id: format!("post-{event_id}"), + evidence_text: "explicit evidence".into(), + actor_ids: vec![actor_id.into()], + } + }; + request.events = vec![ + event( + "award", + "contract_awarded", + "2026-08-19T08:00:00Z", + "actor-1", + ), + event( + "specification", + "specification_changed", + "2026-08-19T09:00:00Z", + "actor-1", + ), + event("delivery", "delivered", "2026-08-19T10:00:00Z", "actor-2"), + event( + "handoff", + "handoff_recorded", + "2026-08-19T11:00:00Z", + "actor-2", + ), + event("focus", "voc_received", "2026-08-19T12:00:00Z", "actor-3"), + event("rebid", "rebid_started", "2026-08-19T13:00:00Z", "actor-3"), + ]; + let projection = project_history_projection(&request).expect("projection"); + assert_eq!(projection.participant_count, 3); + assert_eq!(projection.findings.len(), 6); + let payload = projection.to_json().expect("projection json"); + assert_eq!( + ProjectHistoryProjection::from_json(&payload), + Ok(projection) + ); + } + + #[test] + fn request_refuses_missing_focus_bad_codes_and_excess_events() { + let mut missing_focus = request_with_single_event(); + missing_focus.focus_event_id = "missing".into(); + assert_eq!( + project_history_projection(&missing_focus), + Err(ApiError::InvalidWirePayload) + ); + + let mut bad_code = request_with_single_event(); + bad_code.events[0].event_type_code = "VOC Received".into(); + assert_eq!( + project_history_projection(&bad_code), + Err(ApiError::InvalidWirePayload) + ); + + let mut excess = request_with_single_event(); + excess.events = + vec![excess.events[0].clone(); super::DEFAULT_PROJECT_HISTORY_EVENT_LIMIT + 1]; + assert_eq!( + project_history_projection(&excess), + Err(ApiError::LimitExceeded) + ); + } + + #[test] + fn validation_edges_cover_cutoffs_bounds_projection_and_origins() { + let mut empty = request_with_single_event(); + empty.events.clear(); + assert_eq!( + project_history_projection(&empty), + Err(ApiError::LimitExceeded) + ); + + let mut future_cutoff = request_with_single_event(); + future_cutoff.knowledge_cutoff = "2999-01-01T00:00:00Z".into(); + assert_eq!( + project_history_projection(&future_cutoff), + Err(ApiError::InvalidWirePayload) + ); + + let mut occurred_after_cutoff = request_with_single_event(); + occurred_after_cutoff.events[0].occurred_at = "2026-08-20T00:00:00Z".into(); + assert!(project_history_projection(&occurred_after_cutoff).is_ok()); + + let mut available_after_cutoff = request_with_single_event(); + available_after_cutoff.events[0].available_at = "2026-08-20T00:00:00Z".into(); + assert_eq!( + project_history_projection(&available_after_cutoff), + Err(ApiError::InvalidWirePayload) + ); + + let mut too_many_actors = request_with_single_event(); + too_many_actors.events[0].actor_ids = vec!["actor".into(); 65]; + assert_eq!( + project_history_projection(&too_many_actors), + Err(ApiError::LimitExceeded) + ); + + let mut oversized_title = request_with_single_event(); + oversized_title.events[0].event_title = "x".repeat(513); + assert_eq!( + project_history_projection(&oversized_title), + Err(ApiError::LimitExceeded) + ); + assert_eq!(validate_code("_"), Ok(())); + assert_eq!(validate_code("1"), Ok(())); + + let projection = + project_history_projection(&request_with_single_event()).expect("projection"); + let mut wrong_status = projection.clone(); + wrong_status.inference_status = "causal_score".into(); + assert_eq!(wrong_status.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut empty_projection = projection.clone(); + empty_projection.events.clear(); + assert_eq!( + empty_projection.to_json(), + Err(ApiError::InvalidWirePayload) + ); + + let mut inverted_span = projection; + inverted_span.history_span_start = "2026-08-20T00:00:00Z".into(); + inverted_span.history_span_end = "2026-08-19T00:00:00Z".into(); + assert_eq!(inverted_span.to_json(), Err(ApiError::InvalidWirePayload)); + + let request = request_with_single_event(); + assert_eq!( + build_project_history_exchange("", "lineageweave", &request), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + build_project_history_exchange("https://example.test", &"x".repeat(65), &request), + Err(ApiError::LimitExceeded) + ); + assert!( + build_project_history_exchange("https://example.test", "lineageweave", &request) + .is_ok() + ); + } + + #[test] + fn malformed_https_authorities_are_rejected() { + for origin in [ + "http://example.test", + "https://", + "https://:", + "https:///path", + "https://user@example.test", + "https://example.test/path", + "https://example.test?query", + "https://example.test#fragment", + "https://example.test:", + "https://example.test:not-a-port", + "https://example.test:65536", + "https://[::1", + "https://[not-ipv6]", + "https://::1", + "https://example]test", + "https://example test", + "https://example'test", + "https://example;test", + "https://example\\test", + "https://example\ntest", + "https://postgres.example.test", + "https://jdbc.example.test", + ] { + assert_eq!( + compose_https_target(origin), + Err(ApiError::InvalidWirePayload), + "origin must be rejected: {origin:?}" + ); + } + assert_eq!( + compose_https_target("https://example.test").expect("origin"), + "https://example.test/v1/project-histories" + ); + assert!(compose_https_target("https://example.test:443").is_ok()); + assert!(compose_https_target("https://127.0.0.1").is_ok()); + assert!(compose_https_target("https://[::1]").is_ok()); + assert!(compose_https_target("https://[::1]:443").is_ok()); + for origin in [ + "https://-example.test", + "https://example-.test", + "https://example..test", + "https://example_.test", + ] { + assert_eq!( + compose_https_target(origin), + Err(ApiError::InvalidWirePayload) + ); + } + let long_host = format!("https://{}", "a.".repeat(127) + "a"); + assert_eq!( + compose_https_target(&long_host), + Err(ApiError::InvalidWirePayload) + ); + let long_label = format!("https://{}.test", "a".repeat(64)); + assert_eq!( + compose_https_target(&long_label), + Err(ApiError::InvalidWirePayload) + ); + } +} diff --git a/crates/tepp_api/src/temporal_context.rs b/crates/tepp_api/src/temporal_context.rs new file mode 100644 index 00000000..f1c6792a --- /dev/null +++ b/crates/tepp_api/src/temporal_context.rs @@ -0,0 +1,414 @@ +//! Versioned, cutoff-safe temporal evidence context for modular consumers. + +use crate::ApiError; +use crate::lineageweave_http::LINEAGEWEAVE_CONSUMER_CODE; +use crate::wire::{ + from_json, require_byte_limit, require_contract_version, require_nonempty, to_json_with_limit, +}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use temporal_core::{AvailableTime, EventTime, KnowledgeCutoff, TemporalInstant}; + +/// Supported temporal-context contract version. +pub const TEMPORAL_CONTEXT_CONTRACT_VERSION: u16 = 1; + +/// Versioned temporal-context HTTP path. +pub const TEMPORAL_CONTEXT_PATH: &str = "/v1/temporal-context"; + +/// Claim boundary for temporal association output. +pub const TEMPORAL_ASSOCIATION_CLAIM_BOUNDARY: &str = "association_not_causal"; + +const DEFAULT_TEMPORAL_CONTEXT_BYTE_LIMIT: usize = 64 * 1024; +const MAXIMUM_TEMPORAL_CONTEXT_EVENTS: usize = 1024; + +/// One opaque event offered for cutoff-safe temporal ordering. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalContextEvent { + /// Opaque event identity. + pub event_id: String, + /// Opaque source-post identity. + pub source_post_id: String, + /// Stable event-type code. + pub event_type_code: String, + /// Bounded display label for the event. + pub event_label: String, + /// Event or valid time. + pub event_time: String, + /// Availability time for historical eligibility. + pub available_time: String, + /// Opaque project identity, when known. + pub project_reference: Option, + /// Opaque actor identities participating in the event. + pub actor_references: Vec, +} + +/// Request for one bounded temporal evidence context. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalContextRequest { + /// Semantic contract version. + pub contract_version: u16, + /// Published modular-consumer identity. + pub consumer_code: String, + /// Latest availability time permitted in the context. + pub knowledge_cutoff: String, + /// Optional opaque post identity whose event is the subject. + pub subject_post_id: Option, + /// Bounded source events. + pub events: Vec, +} + +/// One ordered event in a temporal context response. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalContextTimelineEvent { + /// Opaque event identity. + pub event_id: String, + /// Opaque source-post identity. + pub source_post_id: String, + /// Stable event-type code. + pub event_type_code: String, + /// Bounded display label for the event. + pub event_label: String, + /// Original event-time representation. + pub event_time: String, + /// Optional opaque project identity. + pub project_reference: Option, + /// Opaque actor identities. + pub actor_references: Vec, + /// Zero-based deterministic temporal sequence position. + pub sequence_ordinal: usize, + /// Whether this event is associated with the requested subject post. + pub is_subject: bool, +} + +/// One non-causal forward temporal relation in a context response. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalContextRelation { + /// Earlier event identity. + pub from_event_id: String, + /// Later event identity. + pub to_event_id: String, + /// Stable relation code. + pub relation_code: String, +} + +/// One candidate transition gap that is not a causal claim. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalTransitionGapCandidate { + /// Earlier event identity. + pub from_event_id: String, + /// Later event identity. + pub to_event_id: String, + /// Explicit non-causal evidence-status code. + pub evidence_status_code: String, +} + +/// Ordered temporal evidence context returned to a modular consumer. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalContextResponse { + /// Semantic contract version. + pub contract_version: u16, + /// Explicit claim boundary for every relation and gap candidate. + pub claim_boundary: String, + /// Events ordered by absolute event time. + pub timeline_events: Vec, + /// Adjacent forward-only temporal relations. + pub temporal_relations: Vec, + /// Adjacent candidate gaps that are not causal claims. + pub transition_gap_candidates: Vec, + /// Source-post identities in response order. + pub source_post_ids: Vec, +} + +impl TemporalContextRequest { + /// Parse and validate a temporal-context request with the default limit. + /// + /// # Errors + /// + /// Returns a fail-closed wire, version, limit, consumer, time, or shape + /// error. + pub fn from_json(payload: &str) -> Result { + Self::from_json_with_limit(payload, DEFAULT_TEMPORAL_CONTEXT_BYTE_LIMIT) + } + + /// Parse and validate a temporal-context request with a caller limit. + /// + /// # Errors + /// + /// Returns a fail-closed wire, version, limit, consumer, time, or shape + /// error. + pub fn from_json_with_limit(payload: &str, maximum_bytes: usize) -> Result { + require_byte_limit(payload, maximum_bytes)?; + let request: Self = from_json(payload)?; + request.validate()?; + Ok(request) + } + + /// Serialize a temporal-context request after validation. + /// + /// # Errors + /// + /// Returns a fail-closed validation or serialization error. + pub fn to_json(&self) -> Result { + self.validate()?; + to_json_with_limit(self, DEFAULT_TEMPORAL_CONTEXT_BYTE_LIMIT) + } + + fn validate(&self) -> Result<(), ApiError> { + self.validated_ordered_events().map(|_| ()) + } + + fn validated_ordered_events( + &self, + ) -> Result, ApiError> { + require_contract_version(self.contract_version, TEMPORAL_CONTEXT_CONTRACT_VERSION)?; + if self.consumer_code != LINEAGEWEAVE_CONSUMER_CODE { + return Err(ApiError::InvalidWirePayload); + } + let cutoff = KnowledgeCutoff::parse_rfc3339(&self.knowledge_cutoff) + .map_err(|_| ApiError::InvalidWirePayload)?; + if self.events.is_empty() { + return Err(ApiError::InvalidWirePayload); + } + if self.events.len() > MAXIMUM_TEMPORAL_CONTEXT_EVENTS { + return Err(ApiError::LimitExceeded); + } + if let Some(subject_post_id) = &self.subject_post_id { + require_nonempty(subject_post_id)?; + } + let mut event_ids = HashSet::with_capacity(self.events.len()); + let mut ordered = Vec::with_capacity(self.events.len()); + for event in &self.events { + let event_time = validate_event(event, cutoff.instant(), &mut event_ids)?; + ordered.push((event.clone(), event_time)); + } + if let Some(subject_post_id) = &self.subject_post_id + && !self + .events + .iter() + .any(|event| event.source_post_id == *subject_post_id) + { + return Err(ApiError::InvalidWirePayload); + } + ordered.sort_by(|left, right| { + left.1 + .cmp(&right.1) + .then_with(|| left.0.event_id.cmp(&right.0.event_id)) + }); + Ok(ordered) + } +} + +impl TemporalContextResponse { + /// Parse and validate a temporal-context response. + /// + /// # Errors + /// + /// Returns a fail-closed wire, version, limit, or response-shape error. + pub fn from_json(payload: &str) -> Result { + Self::from_json_with_limit(payload, DEFAULT_TEMPORAL_CONTEXT_BYTE_LIMIT) + } + + /// Parse and validate a temporal-context response with a caller limit. + /// + /// # Errors + /// + /// Returns a fail-closed wire, version, limit, or response-shape error. + pub fn from_json_with_limit(payload: &str, maximum_bytes: usize) -> Result { + require_byte_limit(payload, maximum_bytes)?; + let response: Self = from_json(payload)?; + response.validate()?; + Ok(response) + } + + /// Serialize a temporal-context response after validation. + /// + /// # Errors + /// + /// Returns a fail-closed validation or serialization error. + pub fn to_json(&self) -> Result { + self.validate()?; + to_json_with_limit(self, DEFAULT_TEMPORAL_CONTEXT_BYTE_LIMIT) + } + + fn validate(&self) -> Result<(), ApiError> { + require_contract_version(self.contract_version, TEMPORAL_CONTEXT_CONTRACT_VERSION)?; + if self.claim_boundary != TEMPORAL_ASSOCIATION_CLAIM_BOUNDARY { + return Err(ApiError::InvalidWirePayload); + } + if self.timeline_events.is_empty() + || self.timeline_events.len() != self.source_post_ids.len() + || self.temporal_relations.len().checked_add(1) != Some(self.timeline_events.len()) + || self.transition_gap_candidates.len().checked_add(1) + != Some(self.timeline_events.len()) + { + return Err(ApiError::InvalidWirePayload); + } + let mut event_ids = HashSet::with_capacity(self.timeline_events.len()); + let mut previous_key: Option<(TemporalInstant, &str)> = None; + for (ordinal, event) in self.timeline_events.iter().enumerate() { + if event.sequence_ordinal != ordinal + || event.event_id.is_empty() + || event.source_post_id.is_empty() + || event.event_type_code.is_empty() + || event.event_label.is_empty() + || event.event_time.is_empty() + || event.actor_references.is_empty() + || self.source_post_ids[ordinal] != event.source_post_id + { + return Err(ApiError::InvalidWirePayload); + } + if !event_ids.insert(event.event_id.clone()) { + return Err(ApiError::InvalidWirePayload); + } + let event_time = EventTime::parse_rfc3339(&event.event_time) + .map_err(|_| ApiError::InvalidWirePayload)? + .instant(); + if let Some((previous_time, previous_id)) = previous_key + && (event_time < previous_time + || (event_time == previous_time && event.event_id.as_str() <= previous_id)) + { + return Err(ApiError::InvalidWirePayload); + } + previous_key = Some((event_time, event.event_id.as_str())); + if let Some(project_reference) = &event.project_reference { + require_nonempty(project_reference)?; + } + for actor_reference in &event.actor_references { + require_nonempty(actor_reference)?; + } + } + for (index, relation) in self.temporal_relations.iter().enumerate() { + if relation.from_event_id != self.timeline_events[index].event_id + || relation.to_event_id != self.timeline_events[index + 1].event_id + || relation.relation_code != "before" + { + return Err(ApiError::InvalidWirePayload); + } + } + for (index, candidate) in self.transition_gap_candidates.iter().enumerate() { + if candidate.from_event_id != self.timeline_events[index].event_id + || candidate.to_event_id != self.timeline_events[index + 1].event_id + || candidate.evidence_status_code != "candidate_not_causal" + { + return Err(ApiError::InvalidWirePayload); + } + } + Ok(()) + } +} + +/// Build an ordered, cutoff-safe temporal context without causal inference. +/// +/// # Errors +/// +/// Returns a fail-closed error when the request contains future-available, +/// duplicate, malformed, or otherwise invalid evidence. +pub fn build_temporal_context( + request: &TemporalContextRequest, +) -> Result { + let ordered = request.validated_ordered_events()?; + let timeline_events = ordered + .iter() + .enumerate() + .map( + |(sequence_ordinal, (event, _))| TemporalContextTimelineEvent { + event_id: event.event_id.clone(), + source_post_id: event.source_post_id.clone(), + event_type_code: event.event_type_code.clone(), + event_label: event.event_label.clone(), + event_time: event.event_time.clone(), + project_reference: event.project_reference.clone(), + actor_references: event.actor_references.clone(), + sequence_ordinal, + is_subject: request + .subject_post_id + .as_ref() + .is_some_and(|subject| subject == &event.source_post_id), + }, + ) + .collect::>(); + let temporal_relations = adjacent_relations(&ordered); + let transition_gap_candidates = adjacent_gaps(&ordered); + let response = TemporalContextResponse { + contract_version: TEMPORAL_CONTEXT_CONTRACT_VERSION, + claim_boundary: TEMPORAL_ASSOCIATION_CLAIM_BOUNDARY.into(), + source_post_ids: timeline_events + .iter() + .map(|event| event.source_post_id.clone()) + .collect(), + timeline_events, + temporal_relations, + transition_gap_candidates, + }; + response.validate()?; + Ok(response) +} + +fn validate_event( + event: &TemporalContextEvent, + cutoff: TemporalInstant, + event_ids: &mut HashSet, +) -> Result { + for value in [ + &event.event_id, + &event.source_post_id, + &event.event_type_code, + &event.event_label, + &event.event_time, + &event.available_time, + ] { + require_nonempty(value)?; + } + if !event_ids.insert(event.event_id.clone()) { + return Err(ApiError::InvalidWirePayload); + } + if let Some(project_reference) = &event.project_reference { + require_nonempty(project_reference)?; + } + if event.actor_references.is_empty() { + return Err(ApiError::InvalidWirePayload); + } + for actor_reference in &event.actor_references { + require_nonempty(actor_reference)?; + } + let event_time = + EventTime::parse_rfc3339(&event.event_time).map_err(|_| ApiError::InvalidWirePayload)?; + let available_time = AvailableTime::parse_rfc3339(&event.available_time) + .map_err(|_| ApiError::InvalidWirePayload)?; + if available_time.instant() > cutoff { + return Err(ApiError::InvalidWirePayload); + } + Ok(event_time.instant()) +} + +fn adjacent_relations( + ordered: &[(TemporalContextEvent, TemporalInstant)], +) -> Vec { + ordered + .windows(2) + .map(|events| TemporalContextRelation { + from_event_id: events[0].0.event_id.clone(), + to_event_id: events[1].0.event_id.clone(), + relation_code: "before".into(), + }) + .collect() +} + +fn adjacent_gaps( + ordered: &[(TemporalContextEvent, TemporalInstant)], +) -> Vec { + ordered + .windows(2) + .map(|events| TemporalTransitionGapCandidate { + from_event_id: events[0].0.event_id.clone(), + to_event_id: events[1].0.event_id.clone(), + evidence_status_code: "candidate_not_causal".into(), + }) + .collect() +} diff --git a/crates/tepp_api/src/wire.rs b/crates/tepp_api/src/wire.rs index 13b2a3fb..3586f10f 100644 --- a/crates/tepp_api/src/wire.rs +++ b/crates/tepp_api/src/wire.rs @@ -2,6 +2,28 @@ use crate::ApiError; use serde::{Deserialize, Serialize}; +use std::io::{self, Write}; + +struct LimitedWriter { + bytes: Vec, + maximum_bytes: usize, + limit_exceeded: bool, +} + +impl Write for LimitedWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + if self.bytes.len().saturating_add(buffer.len()) > self.maximum_bytes { + self.limit_exceeded = true; + return Err(io::Error::other("JSON byte limit exceeded")); + } + self.bytes.extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} /// Serialize a wire DTO to canonical JSON. /// @@ -12,6 +34,31 @@ pub fn to_json(value: &T) -> Result { serde_json::to_string(value).map_err(|_| ApiError::InvalidWirePayload) } +/// Serialize a wire DTO without buffering more than `maximum_bytes`. +/// +/// # Errors +/// +/// Returns [`ApiError::LimitExceeded`] when serialization crosses the limit, +/// or [`ApiError::InvalidWirePayload`] for another serialization failure. +pub fn to_json_with_limit( + value: &T, + maximum_bytes: usize, +) -> Result { + let mut writer = LimitedWriter { + bytes: Vec::with_capacity(maximum_bytes.min(4096)), + maximum_bytes, + limit_exceeded: false, + }; + if serde_json::to_writer(&mut writer, value).is_err() { + return Err(if writer.limit_exceeded { + ApiError::LimitExceeded + } else { + ApiError::InvalidWirePayload + }); + } + String::from_utf8(writer.bytes).map_err(|_| ApiError::InvalidWirePayload) +} + /// Deserialize a strict wire DTO from JSON text. /// /// # Errors @@ -63,6 +110,7 @@ pub fn require_contract_version(version: u16, expected: u16) -> Result<(), ApiEr mod tests { use super::{ from_json, require_byte_limit, require_contract_version, require_nonempty, to_json, + to_json_with_limit, }; use crate::ApiError; use serde::Serialize; @@ -85,6 +133,12 @@ mod tests { to_json(&SerializationFailure), Err(ApiError::InvalidWirePayload) ); + assert_eq!( + to_json_with_limit(&SerializationFailure, 8), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!(to_json_with_limit(&"abc", 5), Ok("\"abc\"".into())); + assert_eq!(to_json_with_limit(&"abc", 4), Err(ApiError::LimitExceeded)); assert_eq!( from_json::("not-json"), Err(ApiError::InvalidWirePayload) @@ -94,7 +148,7 @@ mod tests { assert_eq!(require_nonempty(" "), Err(ApiError::InvalidWirePayload)); assert_eq!(require_nonempty(""), Err(ApiError::InvalidWirePayload)); assert_eq!( - require_nonempty("topic\u{1f}unit"), + require_nonempty("tenant\u{1f}workspace"), Err(ApiError::InvalidWirePayload) ); require_byte_limit("abc", 3).expect("ok"); diff --git a/crates/tepp_api/tests/analysis_result_contract.rs b/crates/tepp_api/tests/analysis_result_contract.rs new file mode 100644 index 00000000..c18e536e --- /dev/null +++ b/crates/tepp_api/tests/analysis_result_contract.rs @@ -0,0 +1,574 @@ +//! Contract tests for request-bound terminal analysis results. + +use tepp_api::{ + ANALYSIS_RESULT_CONTRACT_VERSION, ANALYSIS_RUN_CONTRACT_VERSION, + ANALYSIS_RUN_STATUS_CONTRACT_VERSION, AnalysisResultSummary, AnalysisRunAccepted, + AnalysisRunRequest, AnalysisRunStatus, AnalysisRunStatusState, AnalysisRunTerminalResult, + AnalysisRunTerminalState, ApiError, DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT, + DEFAULT_ANALYSIS_RUN_BYTE_LIMIT, require_status_binding, require_terminal_binding, + terminal_result_matches_accepted, terminal_result_matches_request, +}; + +const DIGEST: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +fn request() -> AnalysisRunRequest { + AnalysisRunRequest { + contract_version: ANALYSIS_RUN_CONTRACT_VERSION, + idempotency_key: "idem-1".into(), + tenant_workspace_id: "tenant-ws-1".into(), + snapshot_id: "snapshot-1".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "temporal-model-v1".into(), + output_profile: "validation-report".into(), + } +} + +fn accepted() -> AnalysisRunAccepted { + AnalysisRunAccepted::new("run-1", "accepted", "idem-1").expect("accepted") +} + +#[test] +fn accepted_receipt_rejects_non_accepted_lifecycle_states() { + assert_eq!( + AnalysisRunAccepted::new("run-1", "running", "idem-1"), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + AnalysisRunAccepted::new("run-1", "failed", "idem-1"), + Err(ApiError::InvalidWirePayload) + ); +} + +fn summary() -> AnalysisResultSummary { + AnalysisResultSummary::new("temporal_topic_measurement", 120, 42, "validated").expect("summary") +} + +fn succeeded() -> AnalysisRunTerminalResult { + AnalysisRunTerminalResult::succeeded( + &request(), + &accepted(), + "artifact-1", + DIGEST, + "tepp-result-v1", + "2026-08-02T03:04:05Z", + summary(), + ) + .expect("succeeded") +} + +fn failed() -> AnalysisRunTerminalResult { + AnalysisRunTerminalResult::failed( + &request(), + &accepted(), + "2026-08-02T03:04:05Z", + "estimation_failed", + ) + .expect("failed") +} + +#[test] +fn terminal_success_and_failure_round_trip_without_receipt_confusion() { + let success = succeeded(); + assert_eq!(success.run_state, AnalysisRunTerminalState::Succeeded); + assert!(terminal_result_matches_request(&request(), &success)); + assert!(terminal_result_matches_accepted(&accepted(), &success)); + assert_eq!( + require_terminal_binding(&request(), &accepted(), &success), + Ok(()) + ); + let json = success.to_json().expect("json"); + assert!(json.len() <= DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT); + assert_eq!( + AnalysisRunTerminalResult::from_json(&json).expect("decoded"), + success + ); + + let failure = failed(); + assert_eq!(failure.run_state, AnalysisRunTerminalState::Failed); + assert_eq!(failure.result_artifact_id, None); + assert_eq!(failure.summary, None); + let json = failure.to_json().expect("json"); + assert_eq!( + AnalysisRunTerminalResult::from_json(&json).expect("decoded"), + failure + ); + + let accepted_json = accepted().to_json().expect("accepted json"); + assert_eq!( + AnalysisRunTerminalResult::from_json(&accepted_json), + Err(ApiError::InvalidWirePayload) + ); +} + +#[test] +fn wire_version_limit_extension_and_time_validation_fail_closed() { + let mut value: serde_json::Value = + serde_json::from_str(&succeeded().to_json().expect("json")).expect("value"); + value["extra"] = serde_json::json!(true); + assert_eq!( + AnalysisRunTerminalResult::from_json(&value.to_string()), + Err(ApiError::InvalidWirePayload) + ); + + let json = succeeded().to_json().expect("json"); + assert_eq!( + AnalysisRunTerminalResult::from_json_with_limit(&json, 8), + Err(ApiError::LimitExceeded) + ); + + let mut value = succeeded(); + value.contract_version = ANALYSIS_RESULT_CONTRACT_VERSION + 1; + assert_eq!(value.to_json(), Err(ApiError::UnsupportedContractVersion)); + + let mut value = succeeded(); + value.knowledge_cutoff = "yesterday".into(); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.knowledge_cutoff = "2099-01-01T00:00:00Z".into(); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.completed_at = "2026-99-99T25:00:00Z".into(); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + // System time is distinct from the knowledge cutoff; a pre-cutoff run may + // legitimately publish a result for a historical snapshot. + let mut value = succeeded(); + value.completed_at = "2026-07-31T23:59:59Z".into(); + assert!(value.to_json().is_ok()); +} + +#[test] +fn serialization_enforces_default_result_and_status_limits() { + let mut result = succeeded(); + result.summary.as_mut().expect("summary").analysis_family = + "x".repeat(DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT); + assert_eq!(result.to_json(), Err(ApiError::LimitExceeded)); + + let oversized_accepted = AnalysisRunAccepted::new( + "x".repeat(DEFAULT_ANALYSIS_RUN_BYTE_LIMIT), + "accepted", + "idem-1", + ) + .expect("accepted"); + assert_eq!(oversized_accepted.to_json(), Err(ApiError::LimitExceeded)); + assert_eq!( + AnalysisRunAccepted::from_json(&format!( + "{{\"contract_version\":1,\"run_id\":\"{}\",\"run_state\":\"accepted\",\"idempotency_key\":\"idem-1\"}}", + "x".repeat(DEFAULT_ANALYSIS_RUN_BYTE_LIMIT) + )), + Err(ApiError::LimitExceeded) + ); + assert_eq!( + AnalysisRunStatus::accepted(&oversized_accepted), + Err(ApiError::LimitExceeded) + ); + + let mut near_limit_result = succeeded(); + let initial_size = near_limit_result.to_json().expect("initial result").len(); + near_limit_result + .summary + .as_mut() + .expect("summary") + .analysis_family + .push_str(&"x".repeat(DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT - 1 - initial_size)); + let near_limit_json = near_limit_result.to_json().expect("near-limit result"); + assert!(near_limit_json.len() < DEFAULT_ANALYSIS_RESULT_BYTE_LIMIT); + assert_eq!( + AnalysisRunStatus::terminal(&request(), &accepted(), near_limit_result), + Err(ApiError::LimitExceeded) + ); +} + +#[test] +fn every_required_binding_field_is_nonempty() { + for index in 0..8 { + let mut value = succeeded(); + match index { + 0 => value.run_id.clear(), + 1 => value.idempotency_key.clear(), + 2 => value.tenant_workspace_id.clear(), + 3 => value.snapshot_id.clear(), + 4 => value.knowledge_cutoff.clear(), + 5 => value.model_contract_version.clear(), + 6 => value.output_profile.clear(), + 7 => value.completed_at.clear(), + _ => unreachable!(), + } + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + } +} + +#[test] +fn succeeded_shape_requires_complete_digest_bound_result() { + let mut value = succeeded(); + value.result_artifact_id = None; + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.result_artifact_id = Some(String::new()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.result_sha256 = None; + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + for digest in [ + String::new(), + "abcd".into(), + DIGEST.to_uppercase(), + format!("{DIGEST}0"), + format!("g{}", &DIGEST[1..]), + ] { + let mut value = succeeded(); + value.result_sha256 = Some(digest); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + } + + let mut value = succeeded(); + value.result_schema_version = None; + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.result_schema_version = Some(String::new()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.summary = None; + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = succeeded(); + value.failure_code = Some("unexpected_failure".into()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); +} + +#[test] +fn summary_is_nonempty_and_bounded_in_constructor_and_wire_shape() { + assert_eq!( + AnalysisResultSummary::new("", 0, 0, "validated"), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + AnalysisResultSummary::new("family", 0, 0, ""), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + AnalysisResultSummary::new("family", 1_000_000_001, 0, "validated"), + Err(ApiError::LimitExceeded) + ); + assert_eq!( + AnalysisResultSummary::new("family", 0, 1_000_000_001, "validated"), + Err(ApiError::LimitExceeded) + ); + + let invalid_summaries = [ + AnalysisResultSummary { + analysis_family: String::new(), + evidence_count: 0, + statistic_count: 0, + validation_status: "validated".into(), + }, + AnalysisResultSummary { + analysis_family: "family".into(), + evidence_count: 0, + statistic_count: 0, + validation_status: String::new(), + }, + AnalysisResultSummary { + analysis_family: "family".into(), + evidence_count: 1_000_000_001, + statistic_count: 0, + validation_status: "validated".into(), + }, + AnalysisResultSummary { + analysis_family: "family".into(), + evidence_count: 0, + statistic_count: 1_000_000_001, + validation_status: "validated".into(), + }, + ]; + for invalid_summary in invalid_summaries { + let mut value = succeeded(); + value.summary = Some(invalid_summary); + assert!(matches!( + value.to_json(), + Err(ApiError::InvalidWirePayload | ApiError::LimitExceeded) + )); + } +} + +#[test] +fn failed_shape_refuses_measurement_fields_and_invalid_failure_codes() { + let base = failed(); + let digit_code = AnalysisRunTerminalResult::failed( + &request(), + &accepted(), + "2026-08-02T03:04:05Z", + "estimation_failed_2", + ) + .expect("digits are valid failure-code characters"); + assert_eq!( + digit_code.failure_code.as_deref(), + Some("estimation_failed_2") + ); + + let mut value = base.clone(); + value.result_artifact_id = Some("artifact".into()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = base.clone(); + value.result_sha256 = Some(DIGEST.into()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = base.clone(); + value.result_schema_version = Some("schema".into()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut value = base.clone(); + value.summary = Some(summary()); + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + + for code in [ + None, + Some(String::new()), + Some("UPPER_CASE".into()), + Some("_leading".into()), + Some("contains-hyphen".into()), + Some("x".repeat(65)), + ] { + let mut value = base.clone(); + value.failure_code = code; + assert_eq!(value.to_json(), Err(ApiError::InvalidWirePayload)); + } +} + +#[test] +fn every_request_binding_dimension_and_receipt_identity_is_checked() { + let result = succeeded(); + + let mut tampered = result.clone(); + tampered.failure_code = Some("late_failure".into()); + assert_eq!( + require_terminal_binding(&request(), &accepted(), &tampered), + Err(ApiError::InvalidWirePayload) + ); + + for index in 0..6 { + let mut mismatched = request(); + match index { + 0 => mismatched.idempotency_key = "other".into(), + 1 => mismatched.tenant_workspace_id = "other".into(), + 2 => mismatched.snapshot_id = "other".into(), + 3 => mismatched.knowledge_cutoff = "2026-07-31T00:00:00Z".into(), + 4 => mismatched.model_contract_version = "other".into(), + 5 => mismatched.output_profile = "other".into(), + _ => unreachable!(), + } + assert!(!terminal_result_matches_request(&mismatched, &result)); + assert_eq!( + require_terminal_binding(&mismatched, &accepted(), &result), + Err(ApiError::InvalidWirePayload) + ); + } + + let other_run = AnalysisRunAccepted::new("other-run", "accepted", "idem-1").expect("accepted"); + assert!(!terminal_result_matches_accepted(&other_run, &result)); + assert_eq!( + require_terminal_binding(&request(), &other_run, &result), + Err(ApiError::InvalidWirePayload) + ); + + let other_key = AnalysisRunAccepted::new("run-1", "accepted", "other-key").expect("accepted"); + assert!(!terminal_result_matches_accepted(&other_key, &result)); + assert_eq!( + require_terminal_binding(&request(), &other_key, &result), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + AnalysisRunTerminalResult::succeeded( + &request(), + &other_key, + "artifact", + DIGEST, + "schema", + "2026-08-02T03:04:05Z", + summary(), + ), + Err(ApiError::InvalidWirePayload) + ); + assert_eq!( + AnalysisRunTerminalResult::failed( + &request(), + &other_key, + "2026-08-02T03:04:05Z", + "provider_timeout", + ), + Err(ApiError::InvalidWirePayload) + ); + + let mut invalid_request = request(); + invalid_request.contract_version += 1; + assert_eq!( + require_terminal_binding(&invalid_request, &accepted(), &result), + Err(ApiError::UnsupportedContractVersion) + ); + let mut invalid_accepted = accepted(); + invalid_accepted.contract_version += 1; + assert_eq!( + require_terminal_binding(&request(), &invalid_accepted, &result), + Err(ApiError::UnsupportedContractVersion) + ); +} + +#[test] +fn status_read_contract_round_trips_lifecycle_and_terminal_results() { + let accepted_status = AnalysisRunStatus::accepted(&accepted()).expect("accepted status"); + assert_eq!(accepted_status.run_state, AnalysisRunStatusState::Accepted); + assert_eq!(accepted_status.terminal_result, None); + assert_eq!( + require_status_binding(&request(), &accepted(), &accepted_status), + Ok(()) + ); + let accepted_json = accepted_status.to_json().expect("accepted json"); + assert_eq!( + AnalysisRunStatus::from_json(&accepted_json).expect("accepted decode"), + accepted_status + ); + + let running_status = AnalysisRunStatus::running(&accepted()).expect("running status"); + assert_eq!(running_status.run_state, AnalysisRunStatusState::Running); + assert_eq!( + require_status_binding(&request(), &accepted(), &running_status), + Ok(()) + ); + + for (result, expected_state) in [ + (succeeded(), AnalysisRunStatusState::Succeeded), + (failed(), AnalysisRunStatusState::Failed), + ] { + let status = + AnalysisRunStatus::terminal(&request(), &accepted(), result).expect("terminal status"); + assert_eq!(status.run_state, expected_state); + assert!(status.terminal_result.is_some()); + assert_eq!( + require_status_binding(&request(), &accepted(), &status), + Ok(()) + ); + let json = status.to_json().expect("terminal json"); + assert_eq!( + AnalysisRunStatus::from_json(&json).expect("terminal decode"), + status + ); + } + + assert_eq!( + ANALYSIS_RUN_STATUS_CONTRACT_VERSION, + ANALYSIS_RUN_CONTRACT_VERSION + ); +} + +#[test] +fn status_read_contract_rejects_unknown_oversized_and_invalid_shapes() { + let accepted_status = AnalysisRunStatus::accepted(&accepted()).expect("status"); + let mut value: serde_json::Value = + serde_json::from_str(&accepted_status.to_json().expect("json")).expect("value"); + value["extra"] = serde_json::json!(true); + assert_eq!( + AnalysisRunStatus::from_json(&value.to_string()), + Err(ApiError::InvalidWirePayload) + ); + let json = accepted_status.to_json().expect("json"); + assert_eq!( + AnalysisRunStatus::from_json_with_limit(&json, 1), + Err(ApiError::LimitExceeded) + ); + + let mut invalid = accepted_status.clone(); + invalid.contract_version = ANALYSIS_RUN_STATUS_CONTRACT_VERSION + 1; + assert_eq!(invalid.to_json(), Err(ApiError::UnsupportedContractVersion)); + invalid = accepted_status.clone(); + invalid.run_id.clear(); + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + invalid = accepted_status.clone(); + invalid.idempotency_key.clear(); + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut invalid = accepted_status.clone(); + invalid.terminal_result = Some(succeeded()); + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + + let terminal = + AnalysisRunStatus::terminal(&request(), &accepted(), succeeded()).expect("terminal status"); + let mut invalid = terminal.clone(); + invalid.terminal_result = None; + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + invalid = terminal.clone(); + invalid.run_state = AnalysisRunStatusState::Failed; + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut invalid = terminal.clone(); + invalid.terminal_result.as_mut().expect("result").run_id = "other".into(); + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + let mut invalid = terminal; + invalid + .terminal_result + .as_mut() + .expect("result") + .idempotency_key = "other".into(); + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); +} + +#[test] +fn status_read_binding_rejects_receipt_and_request_mismatches() { + let accepted_status = AnalysisRunStatus::accepted(&accepted()).expect("status"); + let other_run = AnalysisRunAccepted::new("other-run", "accepted", "idem-1").expect("run"); + assert_eq!( + require_status_binding(&request(), &other_run, &accepted_status), + Err(ApiError::InvalidWirePayload) + ); + let other_key = AnalysisRunAccepted::new("run-1", "accepted", "other-key").expect("key"); + assert_eq!( + require_status_binding(&request(), &other_key, &accepted_status), + Err(ApiError::InvalidWirePayload) + ); + + let terminal = + AnalysisRunStatus::terminal(&request(), &accepted(), succeeded()).expect("terminal status"); + let mut other_request = request(); + other_request.snapshot_id = "other-snapshot".into(); + assert_eq!( + require_status_binding(&other_request, &accepted(), &terminal), + Err(ApiError::InvalidWirePayload) + ); + + let accepted_status = AnalysisRunStatus::accepted(&accepted()).expect("status"); + let mut other_idempotency = request(); + other_idempotency.idempotency_key = "other-key".into(); + assert_eq!( + require_status_binding(&other_idempotency, &accepted(), &accepted_status), + Err(ApiError::InvalidWirePayload) + ); + + let mut invalid_status = accepted_status.clone(); + invalid_status.idempotency_key = "other-key".into(); + assert_eq!( + require_status_binding(&request(), &accepted(), &invalid_status), + Err(ApiError::InvalidWirePayload) + ); + + let mut invalid_request = request(); + invalid_request.contract_version += 1; + assert_eq!( + require_status_binding(&invalid_request, &accepted(), &accepted_status), + Err(ApiError::UnsupportedContractVersion) + ); + assert_eq!( + AnalysisRunStatus::terminal( + &other_request, + &accepted(), + terminal.terminal_result.unwrap() + ), + Err(ApiError::InvalidWirePayload) + ); +} diff --git a/crates/tepp_api/tests/example_contracts.rs b/crates/tepp_api/tests/example_contracts.rs index a286dd98..592d896d 100644 --- a/crates/tepp_api/tests/example_contracts.rs +++ b/crates/tepp_api/tests/example_contracts.rs @@ -1,7 +1,10 @@ //! Example payloads under `/examples` must parse through the live contracts. use std::path::PathBuf; -use tepp_api::{AnalysisRunRequest, ReproducibilityManifest}; +use tepp_api::{ + AnalysisRunLiveService, AnalysisRunRequest, NARUON_ANALYSIS_RUN_PATH, NaruonLiveService, + ReproducibilityManifest, +}; fn repo_example(name: &str) -> String { let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -30,3 +33,81 @@ fn committed_examples_parse_through_live_contracts() { .expect("manifest example"); assert_eq!(manifest.engine_version, "0.1.0"); } + +#[test] +fn example_contracts_prove_live_idempotency_and_bound_loopback_identity() { + let mut analysis_service = AnalysisRunLiveService::new(); + let run = AnalysisRunRequest::from_json(&repo_example("analysis_run_request_v1.json")) + .expect("analysis example"); + let body = run.to_json().expect("run json"); + let request = format!( + "POST {NARUON_ANALYSIS_RUN_PATH} HTTP/1.1\r\nHost: 127.0.0.1\r\ncontent-type: application/json\r\ntepp-consumer: naruon\r\ntepp-contract-version: 1\r\nidempotency-key: {}\r\ncontent-length: {}\r\n\r\n{body}", + run.idempotency_key, + body.len() + ); + let accepted = analysis_service.handle_http_request(&request); + assert_eq!(accepted.status_code, 202); + assert_eq!( + analysis_service.handle_http_request(&request).body, + accepted.body + ); + let mut conflict = run.clone(); + conflict.snapshot_id.push_str("-changed"); + let conflict_body = conflict.to_json().expect("conflict json"); + let conflict_request = request + .replace( + &format!("content-length: {}", body.len()), + &format!("content-length: {}", conflict_body.len()), + ) + .replace(&body, &conflict_body); + assert_eq!( + analysis_service + .handle_http_request(&conflict_request) + .status_code, + 400 + ); + + let service = NaruonLiveService::bind_loopback().expect("loopback bind"); + let bound = service.local_addr().expect("bound address"); + let bound_request = |host: &str| { + format!( + "POST {NARUON_ANALYSIS_RUN_PATH} HTTP/1.1\r\nHost: {host}\r\ncontent-type: application/json\r\ntepp-consumer: naruon\r\ntepp-contract-version: 1\r\nidempotency-key: {}\r\ncontent-length: {}\r\n\r\n{body}", + run.idempotency_key, + body.len() + ) + }; + let mut bound_service = service; + assert_eq!( + bound_service + .handle_http_request(&bound_request(&bound.to_string())) + .status_code, + 202 + ); + assert_eq!( + bound_service + .handle_http_request(&bound_request(&bound.ip().to_string())) + .status_code, + 202 + ); + let mut conflicting = run.clone(); + conflicting.snapshot_id.push_str("-changed"); + let conflicting_body = conflicting.to_json().expect("bound conflict json"); + let conflicting_request = bound_request(&bound.to_string()) + .replace( + &format!("content-length: {}", body.len()), + &format!("content-length: {}", conflicting_body.len()), + ) + .replace(&body, &conflicting_body); + assert_eq!( + bound_service + .handle_http_request(&conflicting_request) + .status_code, + 400 + ); + assert_eq!( + bound_service + .handle_http_request(&bound_request("8.8.8.8")) + .status_code, + 403 + ); +} diff --git a/crates/tepp_api/tests/lineageweave_http_contract.rs b/crates/tepp_api/tests/lineageweave_http_contract.rs new file mode 100644 index 00000000..19b3e352 --- /dev/null +++ b/crates/tepp_api/tests/lineageweave_http_contract.rs @@ -0,0 +1,155 @@ +//! `LineageWeave` uses the published asynchronous TEPP analysis-run boundary. + +use std::fmt::Write as _; +use std::io::{Read, Write}; +use std::net::TcpStream; +use std::thread; +use std::time::Duration; + +use tepp_api::{ + ANALYSIS_RUN_CONTRACT_VERSION, AnalysisRunAccepted, AnalysisRunLiveService, AnalysisRunRequest, + ApiError, LINEAGEWEAVE_CONSUMER_CODE, NARUON_ANALYSIS_RUN_PATH, NARUON_CONSUMER_CODE, + NARUON_LIVE_HEADER_BYTE_LIMIT, lineageweave_analysis_run_exchange, +}; + +fn sample_run() -> AnalysisRunRequest { + AnalysisRunRequest { + contract_version: ANALYSIS_RUN_CONTRACT_VERSION, + idempotency_key: "shared-idempotency-key".into(), + tenant_workspace_id: "shared-tenant-workspace".into(), + snapshot_id: "lineageweave-snapshot-001".into(), + knowledge_cutoff: "2026-08-01T00:00:00Z".into(), + model_contract_version: "tepp-analysis-run-v1".into(), + output_profile: "calibrated_event_measurement".into(), + } +} + +fn http_request(consumer: &str, run: &AnalysisRunRequest) -> String { + let body = run.to_json().expect("run json"); + let mut request = format!("POST {NARUON_ANALYSIS_RUN_PATH} HTTP/1.1\r\n"); + for (name, value) in [ + ("Host", "127.0.0.1"), + ("content-type", "application/json"), + ("tepp-consumer", consumer), + ("tepp-contract-version", "1"), + ("idempotency-key", run.idempotency_key.as_str()), + ] { + write!(request, "{name}: {value}\r\n").expect("header"); + } + write!(request, "content-length: {}\r\n\r\n{body}", body.len()).expect("body"); + request +} + +#[test] +fn lineageweave_exchange_uses_the_published_consumer_header_without_credentials() { + let run = sample_run(); + let exchange = lineageweave_analysis_run_exchange("https://tepp.example.test", &run) + .expect("lineageweave exchange"); + assert_eq!(exchange.method, "POST"); + assert_eq!( + exchange.target_url, + "https://tepp.example.test/v1/analysis-runs" + ); + assert!( + exchange + .headers + .contains(&("tepp-consumer".into(), LINEAGEWEAVE_CONSUMER_CODE.into())) + ); + assert!( + exchange + .headers + .contains(&("idempotency-key".into(), run.idempotency_key.clone())) + ); + assert!(exchange.headers.iter().all(|(name, _)| { + !matches!( + name.to_ascii_lowercase().as_str(), + "authorization" | "proxy-authorization" | "cookie" | "x-api-key" + ) + })); +} + +#[test] +fn live_listener_accepts_lineageweave_and_isolates_consumer_idempotency() { + let loopback = AnalysisRunLiveService::bind_loopback().expect("loopback bind"); + assert!( + loopback + .local_addr() + .expect("loopback address") + .ip() + .is_loopback() + ); + assert_eq!( + AnalysisRunLiveService::bind("0.0.0.0:0".parse().expect("non-loopback address")) + .expect_err("non-loopback bind must fail"), + ApiError::AuthorizationDenied + ); + + let run = sample_run(); + let mut service = AnalysisRunLiveService::new(); + + let naruon = service.handle_http_request(&http_request(NARUON_CONSUMER_CODE, &run)); + let lineageweave = service.handle_http_request(&http_request(LINEAGEWEAVE_CONSUMER_CODE, &run)); + + assert_eq!(naruon.status_code, 202); + assert_eq!(lineageweave.status_code, 202); + let naruon_accepted = AnalysisRunAccepted::from_json(&naruon.body).expect("naruon ack"); + let lineageweave_accepted = + AnalysisRunAccepted::from_json(&lineageweave.body).expect("lineageweave ack"); + assert_ne!(naruon_accepted.run_id, lineageweave_accepted.run_id); + assert_eq!(lineageweave_accepted.run_state, "accepted"); + assert_eq!(lineageweave_accepted.idempotency_key, run.idempotency_key); + + let replay = service.handle_http_request(&http_request(LINEAGEWEAVE_CONSUMER_CODE, &run)); + assert_eq!(replay.status_code, 202); + assert_eq!(replay.body, lineageweave.body); + + let mut conflict = run.clone(); + conflict.snapshot_id = "lineageweave-snapshot-conflict".into(); + let conflict_response = + service.handle_http_request(&http_request(LINEAGEWEAVE_CONSUMER_CODE, &conflict)); + assert_eq!(conflict_response.status_code, 400); +} + +#[test] +fn live_listener_refuses_an_unpublished_consumer() { + let mut service = AnalysisRunLiveService::new(); + assert_eq!(service.handle_http_request("").status_code, 400); + assert_eq!( + service + .handle_http_request(&"x".repeat(NARUON_LIVE_HEADER_BYTE_LIMIT)) + .status_code, + 413 + ); + let duplicate_headers = format!( + "POST {NARUON_ANALYSIS_RUN_PATH} HTTP/1.1\r\nHost: 127.0.0.1\r\nhost: 127.0.0.1\r\ncontent-length: 0\r\n\r\n" + ); + assert_eq!( + service.handle_http_request(&duplicate_headers).status_code, + 400 + ); + let response = + service.handle_http_request(&http_request("unpublished-consumer", &sample_run())); + assert_eq!(response.status_code, 400); +} + +#[test] +fn live_listener_serves_lineageweave_over_loopback() { + let run = sample_run(); + let mut service = AnalysisRunLiveService::bind_loopback().expect("loopback bind"); + let address = service.local_addr().expect("loopback address"); + let worker = thread::spawn(move || service.serve_one()); + let mut stream = TcpStream::connect(address).expect("connect"); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .expect("read timeout"); + stream + .write_all(http_request(LINEAGEWEAVE_CONSUMER_CODE, &run).as_bytes()) + .expect("request"); + let mut response = String::new(); + stream.read_to_string(&mut response).expect("response"); + assert!(response.starts_with("HTTP/1.1 202 Accepted")); + assert_eq!( + worker.join().expect("join").expect("served").status_code, + 202 + ); +} diff --git a/crates/tepp_api/tests/lineageweave_project_history_contract.rs b/crates/tepp_api/tests/lineageweave_project_history_contract.rs new file mode 100644 index 00000000..9bfaf7ff --- /dev/null +++ b/crates/tepp_api/tests/lineageweave_project_history_contract.rs @@ -0,0 +1,569 @@ +//! `LineageWeave` project-history requests remain cutoff-safe and non-causal. + +use std::io::{Read, Write}; +use std::net::TcpStream; +use std::thread; + +use tepp_api::{ + AnalysisRunLiveService, ApiError, LINEAGEWEAVE_CONSUMER_CODE, NARUON_CONSUMER_CODE, + PROJECT_HISTORY_CONTRACT_VERSION, PROJECT_HISTORY_PATH, ProjectHistoryEvent, + ProjectHistoryProjection, ProjectHistoryRequest, lineageweave_project_history_exchange, + project_history_projection, +}; + +fn event( + event_id: &str, + event_type_code: &str, + event_title: &str, + occurred_at: &str, + source_post_id: &str, + actor_ids: &[&str], +) -> ProjectHistoryEvent { + ProjectHistoryEvent { + event_id: event_id.into(), + event_type_code: event_type_code.into(), + event_title: event_title.into(), + occurred_at: occurred_at.into(), + available_at: occurred_at.into(), + source_post_id: source_post_id.into(), + evidence_text: format!("evidence for {event_title}"), + actor_ids: actor_ids.iter().map(|value| (*value).to_owned()).collect(), + } +} + +fn sample_request() -> ProjectHistoryRequest { + ProjectHistoryRequest { + contract_version: PROJECT_HISTORY_CONTRACT_VERSION, + idempotency_key: "lineageweave-project-acme-voc-1".into(), + tenant_workspace_id: "tenant-demo".into(), + project_key: "project-acme".into(), + project_name: "Acme renewal".into(), + knowledge_cutoff: "2026-08-19T23:59:59Z".into(), + focus_event_id: "event-voc".into(), + events: vec![ + event( + "event-rebid", + "rebid_started", + "Rebid", + "2026-08-10T09:00:00Z", + "post-rebid", + &["person-3"], + ), + event( + "event-award", + "contract_awarded", + "Contract award", + "2022-03-11T09:00:00Z", + "post-award", + &["person-1"], + ), + event( + "event-spec", + "specification_changed", + "Specification change", + "2023-06-15T09:00:00Z", + "post-spec", + &["person-1", "person-2"], + ), + event( + "event-delivery", + "delivered", + "Delivery", + "2024-02-20T09:00:00Z", + "post-delivery", + &["person-2"], + ), + event( + "event-handoff", + "handoff_recorded", + "Operational handoff", + "2024-03-01T09:00:00Z", + "post-handoff", + &["person-2", "person-3"], + ), + event( + "event-voc", + "voc_received", + "VOC received", + "2026-07-30T09:00:00Z", + "post-voc", + &["person-3"], + ), + ], + } +} + +fn live_request(consumer: &str, body: &str, idempotency_key: &str) -> String { + format!( + "POST {PROJECT_HISTORY_PATH} HTTP/1.1\r\nHost: 127.0.0.1\r\ncontent-type: application/json\r\ntepp-consumer: {consumer}\r\ntepp-contract-version: {PROJECT_HISTORY_CONTRACT_VERSION}\r\nidempotency-key: {idempotency_key}\r\ncontent-length: {}\r\n\r\n{body}", + body.len() + ) +} + +fn request_near_the_serialized_byte_limit() -> ProjectHistoryRequest { + fn build(evidence_bytes: usize) -> ProjectHistoryRequest { + let events = (0..64) + .map(|index| { + let month = index / 28 + 1; + let day = index % 28 + 1; + let occurred_at = format!("2026-{month:02}-{day:02}T09:00:00Z"); + ProjectHistoryEvent { + event_id: format!("event-{index:03}"), + event_type_code: if index == 63 { + "voc_received".into() + } else { + "note_recorded".into() + }, + event_title: format!("Event {index}"), + occurred_at: occurred_at.clone(), + available_at: occurred_at, + source_post_id: format!("post-{index:03}"), + evidence_text: "e".repeat(evidence_bytes), + actor_ids: Vec::new(), + } + }) + .collect(); + ProjectHistoryRequest { + contract_version: PROJECT_HISTORY_CONTRACT_VERSION, + idempotency_key: "near-limit-request".into(), + tenant_workspace_id: "tenant-demo".into(), + project_key: "project-demo".into(), + project_name: "Project demo".into(), + knowledge_cutoff: "2026-08-19T23:59:59Z".into(), + focus_event_id: "event-063".into(), + events, + } + } + + let mut low: usize = 0; + let mut high: usize = 4096; + while low < high { + let evidence_bytes = (low + high).div_ceil(2); + let request = build(evidence_bytes); + let payload = serde_json::to_string(&request).expect("request json"); + if payload.len() <= tepp_api::DEFAULT_PROJECT_HISTORY_BYTE_LIMIT { + low = evidence_bytes; + } else { + high = evidence_bytes - 1; + } + } + build(low) +} + +#[test] +fn projection_orders_the_cycle_and_explains_only_explicit_temporal_evidence() { + let projection = project_history_projection(&sample_request()).expect("projection"); + + assert_eq!( + projection.contract_version, + PROJECT_HISTORY_CONTRACT_VERSION + ); + assert_eq!(projection.focus_event_id, "event-voc"); + assert_eq!( + projection.knowledge_cutoff, + sample_request().knowledge_cutoff + ); + assert_eq!(projection.inference_status, "temporal_association_only"); + assert_eq!(projection.participant_count, 3); + assert_eq!( + projection + .events + .iter() + .map(|item| item.event_type_code.as_str()) + .collect::>(), + vec![ + "contract_awarded", + "specification_changed", + "delivered", + "handoff_recorded", + "voc_received", + "rebid_started", + ] + ); + let finding_codes = projection + .findings + .iter() + .map(|finding| finding.finding_code.as_str()) + .collect::>(); + assert!(finding_codes.contains(&"specification_change_before_focus")); + assert!(finding_codes.contains(&"handoff_before_focus")); + assert!(finding_codes.contains(&"rebid_after_focus")); + assert!(finding_codes.contains(&"specification_change_and_handoff_before_focus")); + assert!( + projection + .findings + .iter() + .all(|finding| !finding.evidence_post_ids.is_empty()) + ); +} + +#[test] +fn projection_rejects_future_evidence_duplicates_and_unknown_json_fields() { + let mut future = sample_request(); + future.events[0].available_at = "2026-08-20T00:00:00Z".into(); + assert_eq!( + project_history_projection(&future), + Err(ApiError::InvalidWirePayload) + ); + + let mut known_future_occurrence = sample_request(); + known_future_occurrence.events[0].occurred_at = "2026-08-20T00:00:00Z".into(); + assert!(project_history_projection(&known_future_occurrence).is_ok()); + + let mut duplicate = sample_request(); + duplicate.events[1].event_id = duplicate.events[0].event_id.clone(); + assert_eq!( + project_history_projection(&duplicate), + Err(ApiError::InvalidWirePayload) + ); + + let json = sample_request().to_json().expect("json"); + let hostile = json.replacen('{', "{\"unpublished_causal_score\":1,", 1); + assert_eq!( + ProjectHistoryRequest::from_json(&hostile), + Err(ApiError::InvalidWirePayload) + ); +} + +#[test] +fn generated_projection_rejects_output_that_exceeds_the_wire_limit() { + let request = request_near_the_serialized_byte_limit(); + let request_payload = request.to_json().expect("request remains within the limit"); + assert!(request_payload.len() <= tepp_api::DEFAULT_PROJECT_HISTORY_BYTE_LIMIT); + + assert_eq!( + project_history_projection(&request), + Err(ApiError::LimitExceeded) + ); +} + +#[test] +fn project_history_rejects_unknown_offset_temporal_values() { + let mut request = sample_request(); + request.knowledge_cutoff = "2026-08-19T23:59:59-00:00".into(); + assert_eq!( + project_history_projection(&request), + Err(ApiError::InvalidWirePayload) + ); +} + +#[test] +fn request_json_with_explicit_limit_round_trips_a_valid_contract() { + let request = sample_request(); + let payload = request.to_json().expect("request json"); + let parsed = ProjectHistoryRequest::from_json_with_limit(&payload, payload.len() + 1) + .expect("request with explicit limit"); + assert_eq!(parsed, request); +} + +#[test] +fn live_project_history_route_is_cutoff_safe_and_idempotent() { + let request = sample_request(); + let body = request.to_json().expect("request json"); + let mut service = AnalysisRunLiveService::new(); + + let first = service.handle_http_request(&live_request( + LINEAGEWEAVE_CONSUMER_CODE, + &body, + &request.idempotency_key, + )); + assert_eq!(first.status_code, 200); + let projection = ProjectHistoryProjection::from_json(&first.body).expect("projection"); + assert_eq!(projection.project_key, request.project_key); + assert_eq!(projection.inference_status, "temporal_association_only"); + + let replay = service.handle_http_request(&live_request( + LINEAGEWEAVE_CONSUMER_CODE, + &body, + &request.idempotency_key, + )); + assert_eq!(replay.status_code, 200); + assert_eq!(replay.body, first.body); + + let mut conflict = request.clone(); + conflict.project_name = "Conflicting project".into(); + let conflict_body = conflict.to_json().expect("conflicting request json"); + assert_eq!( + service + .handle_http_request(&live_request( + LINEAGEWEAVE_CONSUMER_CODE, + &conflict_body, + &conflict.idempotency_key, + )) + .status_code, + 400 + ); + + let unknown = body.replacen('{', "{\"unpublished_causal_score\":1,", 1); + assert_eq!( + service + .handle_http_request(&live_request( + LINEAGEWEAVE_CONSUMER_CODE, + &unknown, + &request.idempotency_key, + )) + .status_code, + 400 + ); + + let mut unavailable = request.clone(); + unavailable.events[0].available_at = "2026-08-20T00:00:00Z".into(); + let unavailable_body = serde_json::to_string(&unavailable).expect("unavailable json"); + assert_eq!( + service + .handle_http_request(&live_request( + LINEAGEWEAVE_CONSUMER_CODE, + &unavailable_body, + &unavailable.idempotency_key, + )) + .status_code, + 400 + ); + + let mut oversized = request.clone(); + oversized.project_name = "x".repeat(513); + let oversized_body = serde_json::to_string(&oversized).expect("oversized json"); + assert_eq!( + service + .handle_http_request(&live_request( + LINEAGEWEAVE_CONSUMER_CODE, + &oversized_body, + &oversized.idempotency_key, + )) + .status_code, + 413 + ); + + let credentialed = live_request(LINEAGEWEAVE_CONSUMER_CODE, &body, &request.idempotency_key) + .replace( + "content-length:", + "authorization: Bearer secret\r\ncontent-length:", + ); + let credentialed_response = service.handle_http_request(&credentialed); + assert_eq!(credentialed_response.status_code, 403); + assert!(!credentialed_response.body.contains("Bearer secret")); + + assert_eq!( + service + .handle_http_request(&live_request( + NARUON_CONSUMER_CODE, + &body, + &request.idempotency_key, + )) + .status_code, + 400 + ); + + let mismatched_header = live_request( + LINEAGEWEAVE_CONSUMER_CODE, + &body, + "different-header-idempotency-key", + ); + assert_eq!( + service.handle_http_request(&mismatched_header).status_code, + 400 + ); +} + +#[test] +fn live_project_history_route_serves_over_loopback() { + let request = sample_request(); + let body = request.to_json().expect("request json"); + let mut service = AnalysisRunLiveService::bind_loopback().expect("loopback bind"); + let address = service.local_addr().expect("loopback address"); + let worker = thread::spawn(move || service.serve_one()); + let mut stream = TcpStream::connect(address).expect("connect"); + stream + .write_all( + live_request(LINEAGEWEAVE_CONSUMER_CODE, &body, &request.idempotency_key).as_bytes(), + ) + .expect("request"); + let mut response = String::new(); + stream.read_to_string(&mut response).expect("response"); + assert!(response.starts_with("HTTP/1.1 200 OK")); + assert_eq!( + worker.join().expect("join").expect("served").status_code, + 200 + ); +} + +#[test] +fn lineageweave_exchange_uses_the_versioned_credential_free_tepp_path() { + let exchange = + lineageweave_project_history_exchange("https://tepp.example.test", &sample_request()) + .expect("exchange"); + + assert_eq!( + exchange.target_url, + format!("https://tepp.example.test{PROJECT_HISTORY_PATH}") + ); + assert_eq!(exchange.method, "POST"); + assert!( + exchange + .headers + .contains(&("tepp-consumer".into(), LINEAGEWEAVE_CONSUMER_CODE.into())) + ); + assert!( + exchange + .headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case("authorization")) + ); + assert_eq!( + lineageweave_project_history_exchange("http://tepp.example.test", &sample_request()), + Err(ApiError::InvalidWirePayload) + ); +} + +#[test] +fn projection_response_revalidates_cutoff_order_findings_and_payload_size() { + let projection = project_history_projection(&sample_request()).expect("projection"); + let payload = projection.to_json().expect("projection json"); + assert_eq!( + ProjectHistoryProjection::from_json_with_limit(&payload, payload.len() - 1), + Err(ApiError::LimitExceeded) + ); + + let mut too_many: serde_json::Value = serde_json::from_str(&payload).expect("value"); + let events = too_many["events"].as_array_mut().expect("events"); + let template = events[0].clone(); + while events.len() <= 128 { + events.push(template.clone()); + } + let too_many_json = serde_json::to_string(&too_many).expect("too many json"); + assert_eq!( + ProjectHistoryProjection::from_json(&too_many_json), + Err(ApiError::LimitExceeded) + ); + + let mut future_cutoff: serde_json::Value = serde_json::from_str(&payload).expect("value"); + future_cutoff["knowledge_cutoff"] = serde_json::Value::String("2999-01-01T00:00:00Z".into()); + let future_cutoff_json = serde_json::to_string(&future_cutoff).expect("future cutoff json"); + assert_eq!( + ProjectHistoryProjection::from_json(&future_cutoff_json), + Err(ApiError::InvalidWirePayload) + ); + + let mut duplicate: serde_json::Value = serde_json::from_str(&payload).expect("value"); + duplicate["events"][1]["event_id"] = duplicate["events"][0]["event_id"].clone(); + let duplicate_json = serde_json::to_string(&duplicate).expect("duplicate json"); + assert_eq!( + ProjectHistoryProjection::from_json(&duplicate_json), + Err(ApiError::InvalidWirePayload) + ); + + let mut reversed: serde_json::Value = serde_json::from_str(&payload).expect("value"); + reversed["events"][1]["occurred_at"] = serde_json::Value::String("2020-01-01T00:00:00Z".into()); + let reversed_json = serde_json::to_string(&reversed).expect("reversed json"); + assert_eq!( + ProjectHistoryProjection::from_json(&reversed_json), + Err(ApiError::InvalidWirePayload) + ); + + let mut equal_time: serde_json::Value = serde_json::from_str(&payload).expect("value"); + equal_time["events"][1]["occurred_at"] = equal_time["events"][0]["occurred_at"].clone(); + equal_time["events"][1]["available_at"] = equal_time["events"][0]["available_at"].clone(); + let equal_time_json = serde_json::to_string(&equal_time).expect("equal time json"); + assert!(ProjectHistoryProjection::from_json(&equal_time_json).is_ok()); + + equal_time["events"][1]["event_id"] = serde_json::Value::String("event-aaa".into()); + let equal_id_regression = serde_json::to_string(&equal_time).expect("equal id json"); + assert_eq!( + ProjectHistoryProjection::from_json(&equal_id_regression), + Err(ApiError::InvalidWirePayload) + ); + + let mut value: serde_json::Value = serde_json::from_str(&payload).expect("value"); + value["events"][0]["available_at"] = serde_json::Value::String("2026-08-20T00:00:00Z".into()); + let future = serde_json::to_string(&value).expect("future json"); + assert_eq!( + ProjectHistoryProjection::from_json(&future), + Err(ApiError::InvalidWirePayload) + ); + + let mut mismatched_span_start: serde_json::Value = + serde_json::from_str(&payload).expect("value"); + mismatched_span_start["history_span_start"] = + serde_json::Value::String("2022-03-10T09:00:00Z".into()); + let mismatched_span_start_json = + serde_json::to_string(&mismatched_span_start).expect("span start json"); + assert_eq!( + ProjectHistoryProjection::from_json(&mismatched_span_start_json), + Err(ApiError::InvalidWirePayload) + ); + + let mut mismatched_span_end: serde_json::Value = serde_json::from_str(&payload).expect("value"); + mismatched_span_end["history_span_end"] = + serde_json::Value::String("2026-08-11T09:00:00Z".into()); + let mismatched_span_end_json = + serde_json::to_string(&mismatched_span_end).expect("span end json"); + assert_eq!( + ProjectHistoryProjection::from_json(&mismatched_span_end_json), + Err(ApiError::InvalidWirePayload) + ); + + let mut mismatched_participant_count: serde_json::Value = + serde_json::from_str(&payload).expect("value"); + mismatched_participant_count["participant_count"] = serde_json::Value::from(99); + let mismatched_participant_count_json = + serde_json::to_string(&mismatched_participant_count).expect("participant count json"); + assert_eq!( + ProjectHistoryProjection::from_json(&mismatched_participant_count_json), + Err(ApiError::InvalidWirePayload) + ); + + let mut value: serde_json::Value = serde_json::from_str(&payload).expect("value"); + value["findings"] = serde_json::json!([{ + "finding_code": "causal_score", + "summary": "causal", + "related_event_ids": ["event-award"], + "evidence_post_ids": ["post-award"] + }]); + let fabricated = serde_json::to_string(&value).expect("fabricated json"); + assert_eq!( + ProjectHistoryProjection::from_json(&fabricated), + Err(ApiError::InvalidWirePayload) + ); +} + +#[test] +fn projection_response_rejects_inconsistent_span_and_participant_count() { + let projection = project_history_projection(&sample_request()).expect("projection"); + let payload = projection.to_json().expect("projection json"); + + let mut reversed_span: serde_json::Value = serde_json::from_str(&payload).expect("value"); + reversed_span["history_span_start"] = serde_json::Value::String("2999-01-01T00:00:00Z".into()); + let reversed_span_json = serde_json::to_string(&reversed_span).expect("reversed span json"); + assert_eq!( + ProjectHistoryProjection::from_json(&reversed_span_json), + Err(ApiError::InvalidWirePayload) + ); + + let mut mismatched_start: serde_json::Value = serde_json::from_str(&payload).expect("value"); + mismatched_start["history_span_start"] = + serde_json::Value::String("2022-03-10T00:00:00Z".into()); + let mismatched_start_json = + serde_json::to_string(&mismatched_start).expect("mismatched start json"); + assert_eq!( + ProjectHistoryProjection::from_json(&mismatched_start_json), + Err(ApiError::InvalidWirePayload) + ); + + let mut mismatched_end: serde_json::Value = serde_json::from_str(&payload).expect("value"); + mismatched_end["history_span_end"] = serde_json::Value::String("2026-08-09T00:00:00Z".into()); + let mismatched_end_json = serde_json::to_string(&mismatched_end).expect("mismatched end json"); + assert_eq!( + ProjectHistoryProjection::from_json(&mismatched_end_json), + Err(ApiError::InvalidWirePayload) + ); + + let mut mismatched_participants: serde_json::Value = + serde_json::from_str(&payload).expect("value"); + mismatched_participants["participant_count"] = serde_json::Value::Number(0.into()); + let mismatched_participants_json = + serde_json::to_string(&mismatched_participants).expect("participants json"); + assert_eq!( + ProjectHistoryProjection::from_json(&mismatched_participants_json), + Err(ApiError::InvalidWirePayload) + ); +} diff --git a/crates/tepp_api/tests/lineageweave_temporal_context_contract.rs b/crates/tepp_api/tests/lineageweave_temporal_context_contract.rs new file mode 100644 index 00000000..66502d35 --- /dev/null +++ b/crates/tepp_api/tests/lineageweave_temporal_context_contract.rs @@ -0,0 +1,444 @@ +//! Contract tests for TEPP temporal evidence used by `LineageWeave` Ask surfaces. + +use std::fmt::Write as _; + +use tepp_api::{ + AnalysisRunLiveService, ApiError, LINEAGEWEAVE_CONSUMER_CODE, NARUON_CONSUMER_CODE, + NaruonHttpExchange, TEMPORAL_ASSOCIATION_CLAIM_BOUNDARY, TEMPORAL_CONTEXT_CONTRACT_VERSION, + TEMPORAL_CONTEXT_PATH, TemporalContextEvent, TemporalContextRequest, TemporalContextResponse, + build_temporal_context, lineageweave_temporal_context_exchange, +}; + +fn event( + event_id: &str, + post_id: &str, + event_type: &str, + label: &str, + event_time: &str, + available_time: &str, + actors: &[&str], +) -> TemporalContextEvent { + TemporalContextEvent { + event_id: event_id.into(), + source_post_id: post_id.into(), + event_type_code: event_type.into(), + event_label: label.into(), + event_time: event_time.into(), + available_time: available_time.into(), + project_reference: Some("project-alpha".into()), + actor_references: actors.iter().map(|value| (*value).to_owned()).collect(), + } +} + +fn request() -> TemporalContextRequest { + TemporalContextRequest { + contract_version: TEMPORAL_CONTEXT_CONTRACT_VERSION, + consumer_code: LINEAGEWEAVE_CONSUMER_CODE.into(), + knowledge_cutoff: "2026-08-20T00:00:00Z".into(), + subject_post_id: Some("post-voc".into()), + events: vec![ + event( + "event-voc", + "post-voc", + "voc_received", + "VOC 접수", + "2026-08-01T09:00:00Z", + "2026-08-01T10:00:00Z", + &["actor-support"], + ), + event( + "event-order", + "post-order", + "order_awarded", + "수주", + "2022-03-01T09:00:00Z", + "2022-03-01T10:00:00Z", + &["actor-sales"], + ), + event( + "event-delivery", + "post-delivery", + "delivered", + "납품", + "2024-02-01T09:00:00Z", + "2024-02-01T10:00:00Z", + &["actor-operations"], + ), + event( + "event-spec", + "post-spec", + "specification_changed", + "사양 변경", + "2023-06-01T09:00:00Z", + "2023-06-01T10:00:00Z", + &["actor-engineering"], + ), + ], + } +} + +fn http_request_for_consumer(body: &str, consumer: &str) -> String { + let mut value = format!("POST {TEMPORAL_CONTEXT_PATH} HTTP/1.1\r\n"); + for (name, header_value) in [ + ("Host", "127.0.0.1"), + ("content-type", "application/json"), + ("tepp-consumer", consumer), + ("tepp-contract-version", "1"), + ("idempotency-key", "timeline-001"), + ] { + write!(value, "{name}: {header_value}\r\n").expect("header"); + } + write!(value, "content-length: {}\r\n\r\n{body}", body.len()).expect("body"); + value +} + +fn http_request_from_exchange(exchange: &NaruonHttpExchange) -> String { + let mut value = format!("POST {TEMPORAL_CONTEXT_PATH} HTTP/1.1\r\nHost: 127.0.0.1\r\n"); + for (name, header_value) in &exchange.headers { + write!(value, "{name}: {header_value}\r\n").expect("header"); + } + write!( + value, + "content-length: {}\r\n\r\n{}", + exchange.body.len(), + exchange.body + ) + .expect("body"); + value +} + +#[test] +fn temporal_context_orders_events_and_marks_only_candidate_gaps() { + let response = build_temporal_context(&request()).expect("context"); + assert_eq!(response.contract_version, TEMPORAL_CONTEXT_CONTRACT_VERSION); + assert_eq!(response.claim_boundary, TEMPORAL_ASSOCIATION_CLAIM_BOUNDARY); + assert_eq!( + response + .timeline_events + .iter() + .map(|item| item.event_id.as_str()) + .collect::>(), + vec!["event-order", "event-spec", "event-delivery", "event-voc"] + ); + assert_eq!( + response + .timeline_events + .iter() + .map(|item| item.sequence_ordinal) + .collect::>(), + vec![0, 1, 2, 3] + ); + assert!(response.timeline_events[3].is_subject); + assert_eq!(response.temporal_relations.len(), 3); + assert!( + response + .temporal_relations + .iter() + .all(|item| item.relation_code == "before") + ); + assert!( + response + .transition_gap_candidates + .iter() + .all(|item| item.evidence_status_code == "candidate_not_causal") + ); + assert!( + response + .transition_gap_candidates + .iter() + .any(|item| item.from_event_id == "event-spec" && item.to_event_id == "event-delivery") + ); + assert_eq!( + response.source_post_ids, + vec!["post-order", "post-spec", "post-delivery", "post-voc"] + ); +} + +#[test] +fn temporal_context_rejects_leakage_duplicates_and_unpublished_consumers() { + let mut future = request(); + future.events[0].available_time = "2026-09-01T00:00:00Z".into(); + assert_eq!( + build_temporal_context(&future), + Err(ApiError::InvalidWirePayload) + ); + + let mut duplicate = request(); + duplicate.events[1].event_id = duplicate.events[0].event_id.clone(); + assert_eq!( + build_temporal_context(&duplicate), + Err(ApiError::InvalidWirePayload) + ); + + let mut hostile = request(); + hostile.consumer_code = "unpublished-consumer".into(); + assert_eq!( + build_temporal_context(&hostile), + Err(ApiError::InvalidWirePayload) + ); +} + +#[test] +fn lineageweave_exchange_and_live_listener_return_the_same_context() { + let request = request(); + let exchange = lineageweave_temporal_context_exchange("https://tepp.example.test", &request) + .expect("exchange"); + assert_eq!(exchange.method, "POST"); + assert_eq!( + exchange.target_url, + "https://tepp.example.test/v1/temporal-context" + ); + assert!(exchange.headers.iter().all(|(name, _)| { + !name.eq_ignore_ascii_case("authorization") && !name.to_ascii_lowercase().contains("token") + })); + assert!( + exchange + .headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case("idempotency-key")) + ); + + let mut service = AnalysisRunLiveService::new(); + let response = service.handle_http_request(&http_request_from_exchange(&exchange)); + assert_eq!(response.status_code, 200); + let live = TemporalContextResponse::from_json(&response.body).expect("response"); + assert_eq!(live, build_temporal_context(&request).expect("direct")); +} + +fn single_event_response() -> TemporalContextResponse { + let mut value = request(); + value.events.truncate(1); + value.subject_post_id = None; + build_temporal_context(&value).expect("single event") +} + +fn two_event_response() -> TemporalContextResponse { + let mut value = request(); + value.events.truncate(2); + value.subject_post_id = None; + build_temporal_context(&value).expect("two events") +} + +#[test] +fn temporal_context_rejects_invalid_requests() { + let mut empty_events = request(); + empty_events.events.clear(); + assert_eq!( + build_temporal_context(&empty_events), + Err(ApiError::InvalidWirePayload) + ); + + let mut too_many_events = request(); + too_many_events.events = (0..1025) + .map(|index| { + let mut value = too_many_events.events[0].clone(); + value.event_id = format!("event-{index}"); + value + }) + .collect(); + assert_eq!( + build_temporal_context(&too_many_events), + Err(ApiError::LimitExceeded) + ); + + let mut empty_subject = request(); + empty_subject.subject_post_id = Some(String::new()); + assert_eq!( + build_temporal_context(&empty_subject), + Err(ApiError::InvalidWirePayload) + ); + + let mut unknown_subject = request(); + unknown_subject.subject_post_id = Some("missing-post".into()); + assert_eq!( + build_temporal_context(&unknown_subject), + Err(ApiError::InvalidWirePayload) + ); + + let mut empty_project = request(); + empty_project.events[0].project_reference = Some(" ".into()); + assert_eq!( + build_temporal_context(&empty_project), + Err(ApiError::InvalidWirePayload) + ); + + let mut empty_actors = request(); + empty_actors.events[0].actor_references.clear(); + assert_eq!( + build_temporal_context(&empty_actors), + Err(ApiError::InvalidWirePayload) + ); + + let mut no_project = request(); + no_project.events[0].project_reference = None; + assert!(build_temporal_context(&no_project).is_ok()); +} + +#[test] +fn temporal_context_serialization_enforces_the_shared_wire_limit() { + let mut oversized_request = request(); + oversized_request.events[0].event_label = "x".repeat(64 * 1024); + assert_eq!(oversized_request.to_json(), Err(ApiError::LimitExceeded)); + + let mut oversized_response = single_event_response(); + oversized_response.timeline_events[0].event_label = "x".repeat(64 * 1024); + assert_eq!(oversized_response.to_json(), Err(ApiError::LimitExceeded)); +} + +#[test] +fn temporal_context_rejects_invalid_response_shapes() { + let single_response = single_event_response(); + + let mut invalid_claim = single_response.clone(); + invalid_claim.claim_boundary = "causal".into(); + assert_eq!(invalid_claim.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut empty_timeline = single_response.clone(); + empty_timeline.timeline_events.clear(); + assert_eq!(empty_timeline.to_json(), Err(ApiError::InvalidWirePayload)); + + let mut missing_source = single_response.clone(); + missing_source.source_post_ids.clear(); + assert_eq!(missing_source.to_json(), Err(ApiError::InvalidWirePayload)); + + let two_response = two_event_response(); + + let mut missing_relation = two_response.clone(); + missing_relation.temporal_relations.clear(); + assert_eq!( + missing_relation.to_json(), + Err(ApiError::InvalidWirePayload) + ); + + let mut missing_gap = two_response.clone(); + missing_gap.transition_gap_candidates.clear(); + assert_eq!(missing_gap.to_json(), Err(ApiError::InvalidWirePayload)); +} + +#[test] +fn temporal_context_rejects_invalid_response_fields_and_edges() { + let single_response = single_event_response(); + for invalid in [ + { + let mut value = single_response.clone(); + value.timeline_events[0].sequence_ordinal = 1; + value + }, + { + let mut value = single_response.clone(); + value.timeline_events[0].event_id.clear(); + value + }, + { + let mut value = single_response.clone(); + value.timeline_events[0].source_post_id.clear(); + value + }, + { + let mut value = single_response.clone(); + value.timeline_events[0].event_type_code.clear(); + value + }, + { + let mut value = single_response.clone(); + value.timeline_events[0].event_label.clear(); + value + }, + { + let mut value = single_response.clone(); + value.timeline_events[0].event_time.clear(); + value + }, + { + let mut value = single_response.clone(); + value.timeline_events[0].actor_references.clear(); + value + }, + { + let mut value = single_response.clone(); + value.source_post_ids[0] = "different-post".into(); + value + }, + ] { + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + } + + let two_response = two_event_response(); + for invalid in [ + { + let mut value = two_response.clone(); + value.timeline_events[0].event_time = "2026-08-02T09:00:00Z".into(); + value + }, + { + let mut value = two_response.clone(); + value.timeline_events[1].event_id = value.timeline_events[0].event_id.clone(); + value + }, + { + let mut value = two_response.clone(); + value.timeline_events[0].project_reference = Some(" ".into()); + value + }, + { + let mut value = two_response.clone(); + value.timeline_events[0].actor_references = vec![" ".into()]; + value + }, + { + let mut value = two_response.clone(); + value.temporal_relations[0].from_event_id = "wrong".into(); + value + }, + { + let mut value = two_response.clone(); + value.temporal_relations[0].to_event_id = "wrong".into(); + value + }, + { + let mut value = two_response.clone(); + value.temporal_relations[0].relation_code = "after".into(); + value + }, + { + let mut value = two_response.clone(); + value.transition_gap_candidates[0].from_event_id = "wrong".into(); + value + }, + { + let mut value = two_response.clone(); + value.transition_gap_candidates[0].to_event_id = "wrong".into(); + value + }, + { + let mut value = two_response.clone(); + value.transition_gap_candidates[0].evidence_status_code = "causal".into(); + value + }, + ] { + assert_eq!(invalid.to_json(), Err(ApiError::InvalidWirePayload)); + } +} + +#[test] +fn temporal_context_rejects_equal_timestamp_id_regressions() { + let two_response = two_event_response(); + let mut equal_time = two_response.clone(); + equal_time.timeline_events[1].event_time = equal_time.timeline_events[0].event_time.clone(); + assert!(equal_time.to_json().is_ok()); + + let mut equal_time_out_of_order = equal_time; + equal_time_out_of_order.timeline_events[1].event_id = "event-aaa".into(); + assert_eq!( + equal_time_out_of_order.to_json(), + Err(ApiError::InvalidWirePayload) + ); +} + +#[test] +fn temporal_context_requires_matching_lineageweave_header() { + let body = request().to_json().expect("request json"); + let response = AnalysisRunLiveService::new() + .handle_http_request(&http_request_for_consumer(&body, NARUON_CONSUMER_CODE)); + assert_eq!(response.status_code, 400); +} diff --git a/crates/tepp_api/tests/loopback_binary_contract.rs b/crates/tepp_api/tests/loopback_binary_contract.rs new file mode 100644 index 00000000..20e47564 --- /dev/null +++ b/crates/tepp_api/tests/loopback_binary_contract.rs @@ -0,0 +1,31 @@ +//! The packaged loopback binary serves the published temporal-context wire. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpStream; +use std::process::{Command, Stdio}; + +#[test] +fn binary_serves_one_bounded_temporal_context_request() { + let mut child = Command::new(env!("CARGO_BIN_EXE_tepp-loopback")) + .args(["127.0.0.1:0", "1"]) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn loopback service"); + let mut address = String::new(); + BufReader::new(child.stdout.take().expect("stdout")) + .read_line(&mut address) + .expect("bound address"); + let body = r#"{"contract_version":1,"consumer_code":"lineageweave","knowledge_cutoff":"2026-08-20T00:00:00Z","subject_post_id":"post-1","events":[{"event_id":"event-1","source_post_id":"post-1","event_type_code":"health_probe","event_label":"Health probe","event_time":"2026-08-20T00:00:00Z","available_time":"2026-08-20T00:00:00Z","project_reference":null,"actor_references":["actor-1"]}]}"#; + let request = format!( + "POST /v1/temporal-context HTTP/1.1\r\nHost: {}\r\ncontent-type: application/json\r\ntepp-consumer: lineageweave\r\ntepp-contract-version: 1\r\ncontent-length: {}\r\n\r\n{body}", + address.trim(), + body.len() + ); + let mut stream = TcpStream::connect(address.trim()).expect("connect"); + stream.write_all(request.as_bytes()).expect("request"); + let mut response = String::new(); + stream.read_to_string(&mut response).expect("response"); + assert!(response.starts_with("HTTP/1.1 200 OK")); + assert!(response.contains("association_not_causal")); + assert!(child.wait().expect("wait").success()); +} diff --git a/crates/tepp_api/tests/naruon_http_contract.rs b/crates/tepp_api/tests/naruon_http_contract.rs index cb096cb5..1623e5bd 100644 --- a/crates/tepp_api/tests/naruon_http_contract.rs +++ b/crates/tepp_api/tests/naruon_http_contract.rs @@ -57,6 +57,7 @@ fn table_access_and_non_https_origins_fail_closed() { let run = sample_run(); for origin in [ "", + "https://", "postgres://tepp.example.test/tepp", "postgresql://tepp.example.test/tepp", "jdbc:postgresql://tepp.example.test/tepp", @@ -119,6 +120,49 @@ fn review_and_copilot_headers_are_authorization_denied() { ), Err(ApiError::AuthorizationDenied) ); + for name in [ + "x-openai-api-key", + "x-anthropic-key", + "x-bytez-api-key", + "x-openrouter-api-key", + "x-apikey", + "x_api_key", + "x-secret", + "x-credential", + "x_openai", + "x_bytez", + "x_openrouter", + ] { + assert_eq!( + naruon_analysis_run_exchange_with_headers( + "https://tepp.example.test", + &run, + &[(name, "provider-secret")] + ), + Err(ApiError::AuthorizationDenied), + "header={name}" + ); + } +} + +#[test] +fn malformed_extra_headers_fail_closed_before_forwarding() { + let run = sample_run(); + for (name, value) in [ + ("", "value"), + ("bad name", "value"), + ("x-trace", "ok\r\nx-injected: 1"), + ] { + assert_eq!( + naruon_analysis_run_exchange_with_headers( + "https://tepp.example.test", + &run, + &[(name, value)] + ), + Err(ApiError::InvalidWirePayload), + "header={name:?}" + ); + } } #[test] diff --git a/crates/topic_lineage/Cargo.toml b/crates/topic_lineage/Cargo.toml new file mode 100644 index 00000000..8941b7d9 --- /dev/null +++ b/crates/topic_lineage/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "topic_lineage" +description = "Global topic identity that survives dormancy and reactivation." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[dependencies] +uuid = { workspace = true } + +[lints] +workspace = true diff --git a/crates/topic_lineage/src/activity.rs b/crates/topic_lineage/src/activity.rs new file mode 100644 index 00000000..4c3c3566 --- /dev/null +++ b/crates/topic_lineage/src/activity.rs @@ -0,0 +1,116 @@ +//! Activity states that cannot change topic identity. + +use crate::{TopicIdentity, TopicLineageError}; + +/// Activity of one global topic identity over time. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TopicActivity { + /// The topic is currently expressed. + Active, + /// The topic is temporarily unexpressed without losing identity. + Dormant, + /// The topic has returned after dormancy under the same identity. + Reactivated, +} + +impl TopicActivity { + /// Stable wire name. + #[must_use] + pub const fn wire_name(self) -> &'static str { + match self { + Self::Active => "active", + Self::Dormant => "dormant", + Self::Reactivated => "reactivated", + } + } +} + +/// One topic identity together with its current activity state. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct TopicLineageRecord { + identity: TopicIdentity, + activity: TopicActivity, +} + +impl TopicLineageRecord { + /// Open an active topic identity. + #[must_use] + pub const fn active(identity: TopicIdentity) -> Self { + Self { + identity, + activity: TopicActivity::Active, + } + } + + /// Return the durable topic identity. + #[must_use] + pub const fn identity(self) -> TopicIdentity { + self.identity + } + + /// Return the current activity state. + #[must_use] + pub const fn activity(self) -> TopicActivity { + self.activity + } + + /// Move an active or reactivated topic into dormancy. + /// + /// # Errors + /// + /// Returns [`TopicLineageError::InvalidActivityTransition`] when the topic + /// is already dormant. + pub fn make_dormant(self) -> Result { + match self.activity { + TopicActivity::Active | TopicActivity::Reactivated => Ok(Self { + identity: self.identity, + activity: TopicActivity::Dormant, + }), + TopicActivity::Dormant => Err(TopicLineageError::InvalidActivityTransition), + } + } + + /// Reactivate a dormant topic without changing its identity. + /// + /// # Errors + /// + /// Returns [`TopicLineageError::InvalidActivityTransition`] when the topic + /// is not dormant. + pub fn reactivate(self) -> Result { + match self.activity { + TopicActivity::Dormant => Ok(Self { + identity: self.identity, + activity: TopicActivity::Reactivated, + }), + TopicActivity::Active | TopicActivity::Reactivated => { + Err(TopicLineageError::InvalidActivityTransition) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::{TopicActivity, TopicLineageRecord}; + use crate::{TopicIdentity, TopicLineageError}; + use uuid::Uuid; + + #[test] + fn illegal_transitions_fail_closed() { + let identity = TopicIdentity::from_uuid(Uuid::from_u128(5)); + let active = TopicLineageRecord::active(identity); + assert_eq!(active.activity().wire_name(), "active"); + assert_eq!( + active.reactivate(), + Err(TopicLineageError::InvalidActivityTransition) + ); + let dormant = active.make_dormant().expect("dormant"); + assert_eq!( + dormant.make_dormant(), + Err(TopicLineageError::InvalidActivityTransition) + ); + let reactivated = dormant.reactivate().expect("reactivated"); + assert_eq!(reactivated.activity().wire_name(), "reactivated"); + assert_eq!(TopicActivity::Dormant.wire_name(), "dormant"); + } +} diff --git a/crates/topic_lineage/src/error.rs b/crates/topic_lineage/src/error.rs new file mode 100644 index 00000000..488678ac --- /dev/null +++ b/crates/topic_lineage/src/error.rs @@ -0,0 +1,53 @@ +//! Fail-closed topic-lineage errors. + +use std::fmt; + +/// A fail-closed topic-lineage error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum TopicLineageError { + /// A reactivation tried to mint a new topic identity. + ReactivationIsNotNewTopic, + /// An activity transition was not allowed from the current state. + InvalidActivityTransition, + /// Identity slices were empty or length-mismatched. + InvalidIdentityPayload, +} + +impl fmt::Display for TopicLineageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::ReactivationIsNotNewTopic => "reactivation is not a new topic", + Self::InvalidActivityTransition => "invalid topic activity transition", + Self::InvalidIdentityPayload => "invalid topic identity payload", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for TopicLineageError {} + +#[cfg(test)] +mod tests { + use super::TopicLineageError; + + #[test] + fn error_messages_are_stable() { + for (error, message) in [ + ( + TopicLineageError::ReactivationIsNotNewTopic, + "reactivation is not a new topic", + ), + ( + TopicLineageError::InvalidActivityTransition, + "invalid topic activity transition", + ), + ( + TopicLineageError::InvalidIdentityPayload, + "invalid topic identity payload", + ), + ] { + assert_eq!(error.to_string(), message); + } + } +} diff --git a/crates/topic_lineage/src/identity.rs b/crates/topic_lineage/src/identity.rs new file mode 100644 index 00000000..09c4d186 --- /dev/null +++ b/crates/topic_lineage/src/identity.rs @@ -0,0 +1,94 @@ +//! Stable topic identity independent of activity state. + +use crate::TopicLineageError; +use uuid::Uuid; + +/// Opaque global topic identity (P0). +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct TopicIdentity(Uuid); + +impl TopicIdentity { + /// Reconstruct from a UUID. + #[must_use] + pub const fn from_uuid(value: Uuid) -> Self { + Self(value) + } + + /// Borrow the UUID value. + #[must_use] + pub const fn as_uuid(self) -> Uuid { + self.0 + } +} + +/// Fraction of recovered identities that match known-truth identities. +/// +/// # Errors +/// +/// Returns [`TopicLineageError::InvalidIdentityPayload`] when either slice is +/// empty or the lengths differ. +#[allow(clippy::cast_precision_loss)] +pub fn identity_recovery_rate( + truth: &[TopicIdentity], + decided: &[TopicIdentity], +) -> Result { + if truth.is_empty() || truth.len() != decided.len() { + return Err(TopicLineageError::InvalidIdentityPayload); + } + let mut matches = 0_usize; + for (truth_id, decided_id) in truth.iter().zip(decided) { + if truth_id == decided_id { + matches += 1; + } + } + Ok(matches as f64 / truth.len() as f64) +} + +/// Explicit refusal to treat reactivation as a newly minted topic. +/// +/// # Errors +/// +/// Returns [`TopicLineageError::ReactivationIsNotNewTopic`] when the proposed +/// identity differs from the incumbent. +pub fn refuse_new_identity_on_reactivation( + incumbent: TopicIdentity, + proposed: TopicIdentity, +) -> Result<(), TopicLineageError> { + if incumbent == proposed { + Ok(()) + } else { + Err(TopicLineageError::ReactivationIsNotNewTopic) + } +} + +#[cfg(test)] +mod tests { + use super::{TopicIdentity, identity_recovery_rate, refuse_new_identity_on_reactivation}; + use crate::TopicLineageError; + use uuid::Uuid; + + #[test] + fn identity_helpers_cover_local_branches() { + let identity = TopicIdentity::from_uuid(Uuid::from_u128(1)); + assert_eq!(identity.as_uuid(), Uuid::from_u128(1)); + assert_eq!( + refuse_new_identity_on_reactivation(identity, identity), + Ok(()) + ); + let other = TopicIdentity::from_uuid(Uuid::from_u128(2)); + assert_eq!( + refuse_new_identity_on_reactivation(identity, other), + Err(TopicLineageError::ReactivationIsNotNewTopic) + ); + assert_eq!( + identity_recovery_rate(&[identity], &[]), + Err(TopicLineageError::InvalidIdentityPayload) + ); + assert_eq!( + identity_recovery_rate(&[], &[identity]), + Err(TopicLineageError::InvalidIdentityPayload) + ); + assert_eq!(identity_recovery_rate(&[identity], &[identity]), Ok(1.0)); + assert_eq!(identity_recovery_rate(&[identity], &[other]), Ok(0.0)); + } +} diff --git a/crates/topic_lineage/src/lib.rs b/crates/topic_lineage/src/lib.rs new file mode 100644 index 00000000..5b7b7cbe --- /dev/null +++ b/crates/topic_lineage/src/lib.rs @@ -0,0 +1,24 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +//! Global topic identity that survives dormancy and reactivation. +//! +//! A P0 topic identity is selected once for the modeled period. Activity may +//! change without minting a new identity. Reactivation is not a new topic +//! (ADR 0012). + +mod activity; +mod error; +mod identity; + +/// Activity of one global topic identity. +pub use activity::TopicActivity; +/// Topic identity together with its current activity. +pub use activity::TopicLineageRecord; +/// Fail-closed topic-lineage errors. +pub use error::TopicLineageError; +/// Opaque global topic identity. +pub use identity::TopicIdentity; +/// Fraction of recovered identities that match known truth. +pub use identity::identity_recovery_rate; +/// Refuse to treat reactivation as a newly minted topic. +pub use identity::refuse_new_identity_on_reactivation; diff --git a/crates/topic_lineage/tests/activity_identity_contract.rs b/crates/topic_lineage/tests/activity_identity_contract.rs new file mode 100644 index 00000000..d749f32f --- /dev/null +++ b/crates/topic_lineage/tests/activity_identity_contract.rs @@ -0,0 +1,48 @@ +//! Topic identity survives dormancy and reactivation. + +use topic_lineage::{ + TopicActivity, TopicIdentity, TopicLineageError, TopicLineageRecord, identity_recovery_rate, + refuse_new_identity_on_reactivation, +}; +use uuid::Uuid; + +#[test] +fn reactivation_cannot_mint_a_new_topic_identity() { + let identity = TopicIdentity::from_uuid(Uuid::from_u128(11)); + let active = TopicLineageRecord::active(identity); + let dormant = active.make_dormant().expect("dormant"); + assert_eq!(dormant.activity(), TopicActivity::Dormant); + assert_eq!(dormant.identity(), identity); + + let reactivated = dormant.reactivate().expect("reactivated"); + assert_eq!(reactivated.activity(), TopicActivity::Reactivated); + assert_eq!(reactivated.identity(), identity); + + let other = TopicIdentity::from_uuid(Uuid::from_u128(99)); + assert_eq!( + refuse_new_identity_on_reactivation(identity, other), + Err(TopicLineageError::ReactivationIsNotNewTopic) + ); +} + +#[test] +fn recovered_identities_match_known_truth_better_than_minted_replacements() { + let stable = TopicIdentity::from_uuid(Uuid::from_u128(3)); + let truth = [stable, stable, stable]; + let recovered = [stable, stable, stable]; + let minted = [stable, stable, TopicIdentity::from_uuid(Uuid::from_u128(4))]; + + let recovered_rate = identity_recovery_rate(&truth, &recovered).expect("recovered"); + let minted_rate = identity_recovery_rate(&truth, &minted).expect("minted"); + assert!((recovered_rate - 1.0).abs() < f64::EPSILON); + assert!((minted_rate - (2.0 / 3.0)).abs() < 1e-12); + assert!(recovered_rate > minted_rate); +} + +#[test] +fn empty_identity_payloads_fail_closed() { + assert_eq!( + identity_recovery_rate(&[], &[]), + Err(TopicLineageError::InvalidIdentityPayload) + ); +} diff --git a/crates/topic_lineage/tests/crate_contract.rs b/crates/topic_lineage/tests/crate_contract.rs new file mode 100644 index 00000000..774d69f6 --- /dev/null +++ b/crates/topic_lineage/tests/crate_contract.rs @@ -0,0 +1,7 @@ +//! Integration contract for the `topic_lineage` package identity. + +#[test] +fn package_identity_is_stable() { + let observed = std::hint::black_box(env!("CARGO_PKG_NAME")); + assert_eq!(observed, "topic_lineage"); +} diff --git a/crates/topic_measurement/Cargo.toml b/crates/topic_measurement/Cargo.toml index 299f03c2..8d990d48 100644 --- a/crates/topic_measurement/Cargo.toml +++ b/crates/topic_measurement/Cargo.toml @@ -13,5 +13,15 @@ keywords.workspace = true categories.workspace = true publish = false +[dependencies] +corpus_split = { path = "../corpus_split", version = "0.1.0" } +membership_core = { path = "../membership_core", version = "0.1.0" } +relation_graph = { path = "../relation_graph", version = "0.1.0" } +temporal_core = { path = "../temporal_core", version = "0.1.0" } +uuid.workspace = true + +[dev-dependencies] +validation_core = { path = "../validation_core", version = "0.1.0" } + [lints] workspace = true diff --git a/crates/topic_measurement/src/error.rs b/crates/topic_measurement/src/error.rs index ea934e52..abf0b225 100644 --- a/crates/topic_measurement/src/error.rs +++ b/crates/topic_measurement/src/error.rs @@ -13,6 +13,14 @@ pub enum TopicMeasurementError { InvalidLogRatioDimension, /// TF-IDF, BM25, or keyword scores were offered as inferential coordinates. LexicalWeightForbidden, + /// A sparse matrix violated its compressed-storage contract. + InvalidSparseMatrix, + /// A reference-estimator input or configuration violated its scientific contract. + InvalidModelInput, + /// The estimator produced a non-finite intermediate and failed closed. + NonFiniteEstimate, + /// No seeded initialization converged within the bounded iteration budget. + DidNotConverge, } impl fmt::Display for TopicMeasurementError { @@ -21,6 +29,10 @@ impl fmt::Display for TopicMeasurementError { Self::InvalidComposition => "invalid compositional topic vector", Self::InvalidLogRatioDimension => "invalid log-ratio dimension", Self::LexicalWeightForbidden => "lexical inferential weights are forbidden", + Self::InvalidSparseMatrix => "invalid sparse matrix", + Self::InvalidModelInput => "invalid topic model input", + Self::NonFiniteEstimate => "non-finite topic estimate", + Self::DidNotConverge => "topic estimator did not converge", }; formatter.write_str(message) } @@ -46,5 +58,21 @@ mod tests { TopicMeasurementError::LexicalWeightForbidden.to_string(), "lexical inferential weights are forbidden" ); + assert_eq!( + TopicMeasurementError::InvalidSparseMatrix.to_string(), + "invalid sparse matrix" + ); + assert_eq!( + TopicMeasurementError::InvalidModelInput.to_string(), + "invalid topic model input" + ); + assert_eq!( + TopicMeasurementError::NonFiniteEstimate.to_string(), + "non-finite topic estimate" + ); + assert_eq!( + TopicMeasurementError::DidNotConverge.to_string(), + "topic estimator did not converge" + ); } } diff --git a/crates/topic_measurement/src/lib.rs b/crates/topic_measurement/src/lib.rs index 26c8d82b..38cd38ae 100644 --- a/crates/topic_measurement/src/lib.rs +++ b/crates/topic_measurement/src/lib.rs @@ -13,6 +13,8 @@ mod coordinates; mod error; mod lexical; +mod reference; +mod sparse; /// Additive log-ratio map from a simplex vector. pub use coordinates::additive_log_ratio; @@ -28,3 +30,19 @@ pub use coordinates::isometric_log_ratio; pub use error::TopicMeasurementError; /// Refuse lexical retrieval weights as inferential coordinates. pub use lexical::refuse_lexical_inferential_weight; +/// One admitted structural prevalence feature. +pub use reference::PrevalenceFeature; +/// Validated input for the CPU `f64` reference estimator. +pub use reference::ReferenceTopicInput; +/// A converged topic-model result with uncertainty and lineage counts. +pub use reference::ReferenceTopicModel; +/// Bounded deterministic reference-estimator configuration. +pub use reference::ReferenceTopicModelConfig; +/// One inferred predecessor/successor association within a fitted topic. +pub use reference::TopicSequenceEdge; +/// Fit the bounded deterministic CPU `f64` TRSL-TM reference estimator. +pub use reference::fit_reference_topic_model; +/// Validated compressed sparse numeric matrix. +pub use sparse::SparseMatrix; +/// Whether compressed values are grouped by row or by column. +pub use sparse::SparseOrientation; diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs new file mode 100644 index 00000000..ec5f223c --- /dev/null +++ b/crates/topic_measurement/src/reference.rs @@ -0,0 +1,877 @@ +//! Bounded CPU `f64` reference estimator for TRSL-TM prevalence and lineage. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use corpus_split::CorpusSnapshot; +use membership_core::{GroupId, MemberId, MembershipNetwork, MembershipRole}; +use relation_graph::RelationGraph; +use temporal_core::EventTime; +use uuid::Uuid; + +use crate::{SparseMatrix, TopicMeasurementError, from_additive_log_ratio}; + +const DEFAULT_PRIOR_VARIANCE: f64 = 1.0; +const DEFAULT_RELATION_STRENGTH: f64 = 0.25; +const DEFAULT_RIDGE: f64 = 0.01; +const DEFAULT_TOPIC_SMOOTHING: f64 = 0.05; +const DEFAULT_STEP_SIZE: f64 = 0.2; + +/// One column in the structural prevalence mean. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PrevalenceFeature { + /// Constant intercept. + Intercept, + /// Standardized event-time offset. + EventTime, + /// Caller-supplied admitted prevalence covariate. + Covariate(usize), + /// One active weighted cross-classified membership context. + Membership { + /// Contextual membership role. + role: MembershipRole, + /// Opaque analytical group identity. + group_id: GroupId, + }, +} + +/// Validated input for the CPU `f64` reference estimator. +#[derive(Clone, Debug)] +pub struct ReferenceTopicInput { + document_ids: Vec, + term_rows: Vec>, + vocabulary_size: usize, + design: Vec>, + features: Vec, + transition_pairs: Vec<(usize, usize)>, +} + +impl ReferenceTopicInput { + /// Build a cutoff-, membership-, time-, and relation-validated model input. + /// + /// `document_term` may be CSR or CSC. `covariates`, when present, may also + /// use either orientation. Every document must occur in `snapshot`, have a + /// nonempty nonnegative term row, span at least two event times, and have at + /// least one active membership across the modeled corpus. Only validated + /// forward transition edges with both endpoints in the corpus affect the + /// relational objective; all other relation kinds remain provenance only. + /// + /// # Errors + /// + /// Returns [`TopicMeasurementError::InvalidModelInput`] when any dimension, + /// cutoff, count, time, membership, covariate, or transition invariant fails. + pub fn new( + snapshot: &CorpusSnapshot, + document_ids: Vec, + document_term: &SparseMatrix, + event_times: &[EventTime], + covariates: Option<&SparseMatrix>, + memberships: &MembershipNetwork, + relations: &RelationGraph, + ) -> Result { + let document_count = document_ids.len(); + if document_count < 2 + || document_term.rows() != document_count + || document_term.columns() < 2 + || event_times.len() != document_count + || document_ids.iter().any(|id| !snapshot.contains(*id)) + { + return Err(TopicMeasurementError::InvalidModelInput); + } + let index_by_id: HashMap = document_ids + .iter() + .copied() + .enumerate() + .map(|(index, id)| (id, index)) + .collect(); + if index_by_id.len() != document_count { + return Err(TopicMeasurementError::InvalidModelInput); + } + + let term_rows = document_term.row_entries(); + for row in &term_rows { + if row.is_empty() + || row.iter().any(|(_, value)| *value < 0.0) + || row.iter().map(|(_, value)| value).sum::() <= 0.0 + { + return Err(TopicMeasurementError::InvalidModelInput); + } + } + + let (design, features) = build_design(&document_ids, event_times, covariates, memberships)?; + let transition_pairs = collect_transition_pairs(&index_by_id, relations)?; + + Ok(Self { + document_ids, + term_rows, + vocabulary_size: document_term.columns(), + design, + features, + transition_pairs, + }) + } + + /// Return the number of modeled documents. + #[must_use] + pub fn document_count(&self) -> usize { + self.document_ids.len() + } + + /// Return the vocabulary size. + #[must_use] + pub const fn vocabulary_size(&self) -> usize { + self.vocabulary_size + } + + /// Return the ordered structural prevalence features. + #[must_use] + pub fn features(&self) -> &[PrevalenceFeature] { + &self.features + } +} + +fn build_design( + document_ids: &[Uuid], + event_times: &[EventTime], + covariates: Option<&SparseMatrix>, + memberships: &MembershipNetwork, +) -> Result<(Vec>, Vec), TopicMeasurementError> { + let document_count = document_ids.len(); + let standardized_time = standardize_event_time(event_times)?; + let covariate_rows = match covariates { + Some(matrix) if matrix.rows() == document_count => Some(matrix.row_entries()), + Some(_) => return Err(TopicMeasurementError::InvalidModelInput), + None => None, + }; + let covariate_count = covariate_rows.as_ref().map_or(0, |rows| { + rows.iter() + .flatten() + .map(|(column, _)| *column) + .max() + .map_or(0, |value| value + 1) + }); + + let active: Vec<_> = document_ids + .iter() + .zip(event_times) + .map(|(id, time)| memberships.active_memberships_for(MemberId::from_uuid(*id), *time)) + .collect(); + let membership_keys: BTreeSet<_> = active + .iter() + .flatten() + .map(|assignment| (assignment.role(), assignment.group_id())) + .collect(); + if membership_keys.is_empty() { + return Err(TopicMeasurementError::InvalidModelInput); + } + let membership_columns: BTreeMap<_, _> = membership_keys + .iter() + .copied() + .enumerate() + .map(|(index, key)| (key, index)) + .collect(); + + let mut features = vec![PrevalenceFeature::Intercept, PrevalenceFeature::EventTime]; + features.extend((0..covariate_count).map(PrevalenceFeature::Covariate)); + features.extend( + membership_keys + .iter() + .map(|(role, group_id)| PrevalenceFeature::Membership { + role: *role, + group_id: *group_id, + }), + ); + let mut design = vec![vec![0.0; features.len()]; document_count]; + for row in 0..document_count { + design[row][0] = 1.0; + design[row][1] = standardized_time[row]; + if let Some(covariates) = &covariate_rows { + for &(column, value) in &covariates[row] { + design[row][2 + column] = value; + } + } + for assignment in &active[row] { + let column = membership_columns[&(assignment.role(), assignment.group_id())]; + design[row][2 + covariate_count + column] = assignment.weight().value(); + } + } + Ok((design, features)) +} + +fn collect_transition_pairs( + index_by_id: &HashMap, + relations: &RelationGraph, +) -> Result, TopicMeasurementError> { + let mut transition_pairs = BTreeSet::new(); + for edge in relations.edges().filter(|edge| edge.is_transition_edge()) { + let Some(&source) = index_by_id.get(&edge.source().as_uuid()) else { + continue; + }; + let Some(&target) = index_by_id.get(&edge.target().as_uuid()) else { + continue; + }; + transition_pairs.insert((source, target)); + } + if transition_pairs.is_empty() { + Err(TopicMeasurementError::InvalidModelInput) + } else { + Ok(transition_pairs.into_iter().collect()) + } +} + +/// Bounded deterministic reference-estimator configuration. +#[derive(Clone, Debug, PartialEq)] +pub struct ReferenceTopicModelConfig { + topic_count: usize, + seeds: Vec, + maximum_iterations: usize, + tolerance: f64, + prior_variance: f64, + relation_strength: f64, + ridge: f64, + topic_smoothing: f64, + step_size: f64, +} + +impl ReferenceTopicModelConfig { + /// Construct a reference configuration with ADR-owned v1 hyperparameters. + /// + /// # Errors + /// + /// Returns [`TopicMeasurementError::InvalidModelInput`] unless `topic_count` + /// is at least two, seeds are nonempty, the iteration budget is at least + /// two, and tolerance is finite and positive. + pub fn new( + topic_count: usize, + seeds: Vec, + maximum_iterations: usize, + tolerance: f64, + ) -> Result { + let value = Self { + topic_count, + seeds, + maximum_iterations, + tolerance, + prior_variance: DEFAULT_PRIOR_VARIANCE, + relation_strength: DEFAULT_RELATION_STRENGTH, + ridge: DEFAULT_RIDGE, + topic_smoothing: DEFAULT_TOPIC_SMOOTHING, + step_size: DEFAULT_STEP_SIZE, + }; + value.validate()?; + Ok(value) + } + + /// Replace numerical hyperparameters while retaining dimensional controls. + /// + /// # Errors + /// + /// Returns [`TopicMeasurementError::InvalidModelInput`] for any non-finite + /// or non-positive value, except `relation_strength` and `ridge`, which may + /// be exactly zero for a declared ablation. + pub fn with_hyperparameters( + mut self, + prior_variance: f64, + relation_strength: f64, + ridge: f64, + topic_smoothing: f64, + step_size: f64, + ) -> Result { + self.prior_variance = prior_variance; + self.relation_strength = relation_strength; + self.ridge = ridge; + self.topic_smoothing = topic_smoothing; + self.step_size = step_size; + self.validate()?; + Ok(self) + } + + fn validate(&self) -> Result<(), TopicMeasurementError> { + if self.topic_count < 2 + || self.seeds.is_empty() + || self.maximum_iterations < 2 + || !self.tolerance.is_finite() + || self.tolerance <= 0.0 + || !self.prior_variance.is_finite() + || self.prior_variance <= 0.0 + || !self.relation_strength.is_finite() + || self.relation_strength < 0.0 + || !self.ridge.is_finite() + || self.ridge < 0.0 + || !self.topic_smoothing.is_finite() + || self.topic_smoothing <= 0.0 + || !self.step_size.is_finite() + || self.step_size <= 0.0 + { + return Err(TopicMeasurementError::InvalidModelInput); + } + Ok(()) + } +} + +/// One inferred predecessor/successor association within a dominant topic. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct TopicSequenceEdge { + /// Opaque predecessor document identity. + pub predecessor_document_id: Uuid, + /// Opaque successor document identity. + pub successor_document_id: Uuid, + /// Artifact-local global topic index. + pub topic_index: usize, + /// Minimum dominant-topic posterior mean across the two documents. + pub association_strength: f64, +} + +/// A converged topic-model result with uncertainty and lineage counts. +#[derive(Clone, Debug, PartialEq)] +pub struct ReferenceTopicModel { + /// Selected deterministic initialization seed. + pub seed: u64, + /// Iterations used by the selected converged fit. + pub iterations: usize, + /// Final finite penalized objective. + pub objective: f64, + /// Global topic-by-term probability matrix. + pub topic_term_probabilities: Vec>, + /// Document-by-topic posterior mean proportions. + pub document_topic_proportions: Vec>, + /// Diagonal Laplace variance for each document ALR coordinate. + pub document_coordinate_variances: Vec>, + /// Structural prevalence coefficient matrix, feature by ALR coordinate. + pub prevalence_coefficients: Vec>, + /// Ordered structural feature meanings for coefficient rows. + pub prevalence_features: Vec, + /// Inferred dominant-topic links restricted to explicit forward transitions. + pub sequence_edges: Vec, + /// Distinct documents incident to at least one inferred sequence edge. + pub connected_post_count: usize, + /// Distinct global topics represented by at least one sequence edge. + pub lineage_count: usize, +} + +#[derive(Clone)] +struct FitState { + seed: u64, + iterations: usize, + objective: f64, + beta: Vec>, + eta: Vec>, + coefficients: Vec>, +} + +/// Fit the bounded deterministic CPU `f64` TRSL-TM reference estimator. +/// +/// # Errors +/// +/// Returns a typed invalid-input, non-finite, or convergence failure. The +/// function never returns a partial fit. +pub fn fit_reference_topic_model( + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, +) -> Result { + config.validate()?; + if config.topic_count > input.vocabulary_size { + return Err(TopicMeasurementError::InvalidModelInput); + } + let mut best = None; + for &seed in &config.seeds { + match fit_seed(input, config, seed) { + Ok(candidate) + if best.as_ref().is_none_or(|incumbent: &FitState| { + candidate.objective > incumbent.objective + }) => + { + best = Some(candidate); + } + Ok(_) | Err(TopicMeasurementError::DidNotConverge) => {} + Err(error) => return Err(error), + } + } + let state = best.ok_or(TopicMeasurementError::DidNotConverge)?; + build_result(input, config, state) +} + +fn fit_seed( + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + seed: u64, +) -> Result { + let document_count = input.document_ids.len(); + let coordinate_count = config.topic_count - 1; + let mut rng = seed.max(1); + let mut beta = vec![vec![0.0; input.vocabulary_size]; config.topic_count]; + for topic in &mut beta { + for value in topic.iter_mut() { + *value = config.topic_smoothing + next_unit(&mut rng); + } + normalize(topic)?; + } + let mut eta = vec![vec![0.0; coordinate_count]; document_count]; + for row in &mut eta { + for value in row { + *value = (next_unit(&mut rng) - 0.5) * 0.1; + } + } + let mut coefficients = vec![vec![0.0; coordinate_count]; input.features.len()]; + let mut previous = None; + + for iteration in 1..=config.maximum_iterations { + let theta = topic_proportions(&eta)?; + let (document_topic_counts, beta_counts, ll) = + expectation(input, &theta, &beta, config.topic_count)?; + let means = prevalence_means(&input.design, &coefficients); + let objective = objective(input, config, &theta, &eta, &means, &coefficients, ll)?; + if previous.is_some_and(|value: f64| { + (objective - value).abs() / (1.0 + value.abs()) <= config.tolerance + }) && iteration > 3 + { + return Ok(FitState { + seed, + iterations: iteration, + objective, + beta, + eta, + coefficients, + }); + } + previous = Some(objective); + beta = update_beta(beta_counts, config.topic_smoothing)?; + update_coefficients(input, config, &eta, &means, &mut coefficients)?; + let counts = &document_topic_counts; + update_eta(input, config, counts, &theta, &means, &mut eta)?; + } + Err(TopicMeasurementError::DidNotConverge) +} + +fn topic_proportions(eta: &[Vec]) -> Result>, TopicMeasurementError> { + eta.iter().map(|row| from_additive_log_ratio(row)).collect() +} + +type ExpectationOutput = (Vec>, Vec>, f64); + +fn expectation( + input: &ReferenceTopicInput, + theta: &[Vec], + beta: &[Vec], + topic_count: usize, +) -> Result { + let mut document_topic_counts = vec![vec![0.0; topic_count]; input.document_ids.len()]; + let mut beta_counts = vec![vec![0.0; input.vocabulary_size]; topic_count]; + let mut log_likelihood = 0.0; + for (document, terms) in input.term_rows.iter().enumerate() { + for &(term, count) in terms { + let probability = (0..topic_count) + .map(|topic| theta[document][topic] * beta[topic][term]) + .sum::(); + let log_probability = probability.ln(); + require_finite(log_probability)?; + log_likelihood += count * log_probability; + for topic in 0..topic_count { + let expected = count * theta[document][topic] * beta[topic][term] / probability; + document_topic_counts[document][topic] += expected; + beta_counts[topic][term] += expected; + } + } + } + if !log_likelihood.is_finite() { + return Err(TopicMeasurementError::NonFiniteEstimate); + } + Ok((document_topic_counts, beta_counts, log_likelihood)) +} + +fn update_beta( + mut counts: Vec>, + smoothing: f64, +) -> Result>, TopicMeasurementError> { + for topic in &mut counts { + for value in topic.iter_mut() { + *value += smoothing; + } + normalize(topic)?; + } + Ok(counts) +} + +fn prevalence_means(design: &[Vec], coefficients: &[Vec]) -> Vec> { + let coordinate_count = coefficients[0].len(); + design + .iter() + .map(|row| { + let mut mean = vec![0.0; coordinate_count]; + for (feature, value) in row.iter().enumerate() { + for (coordinate, target) in mean.iter_mut().enumerate() { + *target += value * coefficients[feature][coordinate]; + } + } + mean + }) + .collect() +} + +fn objective( + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + theta: &[Vec], + eta: &[Vec], + means: &[Vec], + coefficients: &[Vec], + log_likelihood: f64, +) -> Result { + let prior = eta + .iter() + .zip(means) + .flat_map(|(row, mean)| row.iter().zip(mean)) + .map(|(value, mean)| (value - mean).powi(2)) + .sum::() + / (2.0 * config.prior_variance); + let relation = input + .transition_pairs + .iter() + .map(|&(source, target)| { + theta[source] + .iter() + .zip(&theta[target]) + .map(|(left, right)| (left - right).powi(2)) + .sum::() + }) + .sum::() + * config.relation_strength + / 2.0; + let ridge = coefficients + .iter() + .flatten() + .map(|value| value * value) + .sum::() + * config.ridge + / 2.0; + let value = log_likelihood - prior - relation - ridge; + if value.is_finite() { + Ok(value) + } else { + Err(TopicMeasurementError::NonFiniteEstimate) + } +} + +fn update_coefficients( + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + eta: &[Vec], + means: &[Vec], + coefficients: &mut [Vec], +) -> Result<(), TopicMeasurementError> { + let scale = config.step_size / bounded_count(input.document_ids.len())?; + for feature in 0..coefficients.len() { + for coordinate in 0..coefficients[feature].len() { + let gradient = input + .design + .iter() + .enumerate() + .map(|(document, row)| { + row[feature] * (eta[document][coordinate] - means[document][coordinate]) + / config.prior_variance + }) + .sum::() + - config.ridge * coefficients[feature][coordinate]; + coefficients[feature][coordinate] += scale * gradient; + require_finite(coefficients[feature][coordinate])?; + } + } + Ok(()) +} + +fn update_eta( + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + counts: &[Vec], + theta: &[Vec], + means: &[Vec], + eta: &mut [Vec], +) -> Result<(), TopicMeasurementError> { + let coordinate_count = config.topic_count - 1; + let mut relation_gradient = vec![vec![0.0; coordinate_count]; input.document_ids.len()]; + for &(source, target) in &input.transition_pairs { + let delta: Vec = theta[source] + .iter() + .zip(&theta[target]) + .map(|(left, right)| left - right) + .collect(); + let source_dot = dot(&delta, &theta[source]); + let target_dot = dot(&delta, &theta[target]); + for coordinate in 0..coordinate_count { + relation_gradient[source][coordinate] -= config.relation_strength + * theta[source][coordinate] + * (delta[coordinate] - source_dot); + relation_gradient[target][coordinate] += config.relation_strength + * theta[target][coordinate] + * (delta[coordinate] - target_dot); + } + } + for document in 0..eta.len() { + let token_count = counts[document].iter().sum::(); + let scale = config.step_size / (1.0 + token_count); + for coordinate in 0..coordinate_count { + let gradient = counts[document][coordinate] + - token_count * theta[document][coordinate] + - (eta[document][coordinate] - means[document][coordinate]) / config.prior_variance + + relation_gradient[document][coordinate]; + eta[document][coordinate] += scale * gradient; + require_finite(eta[document][coordinate])?; + } + } + Ok(()) +} + +fn build_result( + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + state: FitState, +) -> Result { + let theta = topic_proportions(&state.eta)?; + let mut degrees = vec![0_usize; input.document_ids.len()]; + for &(source, target) in &input.transition_pairs { + degrees[source] += 1; + degrees[target] += 1; + } + let mut variances = Vec::with_capacity(theta.len()); + for (document, proportions) in theta.iter().enumerate() { + let token_count = input.term_rows[document] + .iter() + .map(|(_, count)| count) + .sum::(); + let degree = bounded_count(degrees[document])?; + variances.push( + proportions[..config.topic_count - 1] + .iter() + .map(|value| { + 1.0 / (token_count * value * (1.0 - value) + + 1.0 / config.prior_variance + + degree * config.relation_strength) + }) + .collect(), + ); + } + let dominant: Vec = theta.iter().map(|row| argmax(row)).collect(); + let mut sequence_edges = Vec::new(); + let mut connected = BTreeSet::new(); + let mut lineages = BTreeSet::new(); + for &(source, target) in &input.transition_pairs { + if dominant[source] != dominant[target] { + continue; + } + let topic = dominant[source]; + connected.insert(input.document_ids[source]); + connected.insert(input.document_ids[target]); + lineages.insert(topic); + sequence_edges.push(TopicSequenceEdge { + predecessor_document_id: input.document_ids[source], + successor_document_id: input.document_ids[target], + topic_index: topic, + association_strength: theta[source][topic].min(theta[target][topic]), + }); + } + Ok(ReferenceTopicModel { + seed: state.seed, + iterations: state.iterations, + objective: state.objective, + topic_term_probabilities: state.beta, + document_topic_proportions: theta, + document_coordinate_variances: variances, + prevalence_coefficients: state.coefficients, + prevalence_features: input.features.clone(), + sequence_edges, + connected_post_count: connected.len(), + lineage_count: lineages.len(), + }) +} + +#[allow(clippy::cast_precision_loss)] +fn standardize_event_time(times: &[EventTime]) -> Result, TopicMeasurementError> { + let origin = times[0].instant().as_nanosecond(); + let offsets: Vec = times + .iter() + .map(|time| (time.instant().as_nanosecond() - origin) as f64 / 1_000_000_000.0) + .collect(); + let mean = offsets.iter().sum::() / offsets.len() as f64; + let variance = offsets + .iter() + .map(|value| (value - mean).powi(2)) + .sum::() + / offsets.len() as f64; + if variance <= 0.0 { + return Err(TopicMeasurementError::InvalidModelInput); + } + let deviation = variance.sqrt(); + Ok(offsets + .iter() + .map(|value| (value - mean) / deviation) + .collect()) +} + +fn normalize(values: &mut [f64]) -> Result<(), TopicMeasurementError> { + let sum = values.iter().sum::(); + require_finite(sum.ln())?; + for value in values { + *value /= sum; + } + Ok(()) +} + +fn require_finite(value: f64) -> Result<(), TopicMeasurementError> { + value + .is_finite() + .then_some(()) + .ok_or(TopicMeasurementError::NonFiniteEstimate) +} + +fn bounded_count(value: usize) -> Result { + u32::try_from(value) + .map(f64::from) + .map_err(|_| TopicMeasurementError::InvalidModelInput) +} + +fn dot(left: &[f64], right: &[f64]) -> f64 { + left.iter().zip(right).map(|(a, b)| a * b).sum() +} + +fn argmax(values: &[f64]) -> usize { + values + .iter() + .enumerate() + .max_by(|left, right| left.1.total_cmp(right.1)) + .map_or(0, |(index, _)| index) +} + +fn next_unit(state: &mut u64) -> f64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + #[allow(clippy::cast_precision_loss)] + let value = (*state >> 11) as f64 / ((1_u64 << 53) as f64); + value.max(f64::EPSILON) +} + +#[cfg(test)] +mod tests { + use super::{ + FitState, PrevalenceFeature, ReferenceTopicInput, ReferenceTopicModelConfig, argmax, + bounded_count, build_result, dot, expectation, next_unit, normalize, objective, + require_finite, standardize_event_time, + }; + use crate::TopicMeasurementError; + use temporal_core::EventTime; + use uuid::Uuid; + + #[test] + fn numeric_helpers_are_deterministic_and_fail_closed() { + let mut seed = 1; + let first = next_unit(&mut seed); + assert!(first > 0.0); + assert!(first < 1.0); + assert!((dot(&[1.0, 2.0], &[3.0, 4.0]) - 11.0).abs() < f64::EPSILON); + assert_eq!(argmax(&[0.1, 0.8, 0.1]), 1); + assert_eq!(argmax(&[]), 0); + let mut values = [1.0, 3.0]; + normalize(&mut values).expect("normalize"); + assert!((values[0] - 0.25).abs() < f64::EPSILON); + assert!((values[1] - 0.75).abs() < f64::EPSILON); + assert_eq!( + normalize(&mut [0.0, 0.0]), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + assert_eq!( + normalize(&mut [f64::INFINITY]), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + assert_eq!(require_finite(1.0), Ok(())); + assert_eq!( + require_finite(f64::NAN), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + } + + #[test] + fn impossible_numeric_states_and_mixed_topic_edges_fail_closed() { + let input = ReferenceTopicInput { + document_ids: vec![Uuid::from_u128(1), Uuid::from_u128(2)], + term_rows: vec![vec![(0, f64::MAX)], vec![(0, f64::MAX)]], + vocabulary_size: 2, + design: vec![vec![1.0], vec![1.0]], + features: vec![PrevalenceFeature::Intercept], + transition_pairs: vec![(0, 1)], + }; + let theta = vec![vec![0.5, 0.5], vec![0.5, 0.5]]; + let zero_beta = vec![vec![0.0, 0.0], vec![0.0, 0.0]]; + assert_eq!( + expectation(&input, &theta, &zero_beta, 2), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + let infinite_beta = vec![vec![f64::INFINITY, 0.0], vec![0.0, 0.0]]; + assert_eq!( + expectation(&input, &theta, &infinite_beta, 2), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + let finite_beta = vec![vec![0.5, 0.5], vec![0.5, 0.5]]; + assert_eq!( + expectation(&input, &theta, &finite_beta, 2), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + let finite_input = ReferenceTopicInput { + term_rows: vec![vec![(0, 1.0)], vec![(0, 1.0)]], + ..input.clone() + }; + assert!(expectation(&finite_input, &theta, &finite_beta, 2).is_ok()); + + let config = ReferenceTopicModelConfig::new(2, vec![1], 10, 1e-6).expect("config"); + assert_eq!( + objective( + &input, + &config, + &theta, + &[vec![0.0], vec![0.0]], + &[vec![0.0], vec![0.0]], + &[vec![0.0]], + f64::INFINITY, + ), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + assert!( + objective( + &input, + &config, + &theta, + &[vec![0.0], vec![0.0]], + &[vec![0.0], vec![0.0]], + &[vec![0.0]], + 0.0, + ) + .is_ok() + ); + + let result = build_result( + &input, + &config, + FitState { + seed: 1, + iterations: 4, + objective: -1.0, + beta: finite_beta, + eta: vec![vec![10.0], vec![-10.0]], + coefficients: vec![vec![0.0]], + }, + ) + .expect("mixed-topic result"); + assert!(result.sequence_edges.is_empty()); + assert_eq!(result.connected_post_count, 0); + assert_eq!(result.lineage_count, 0); + + let same_time = EventTime::parse_rfc3339("2026-01-01T00:00:00Z").expect("time"); + assert_eq!( + standardize_event_time(&[same_time, same_time]), + Err(TopicMeasurementError::InvalidModelInput) + ); + #[cfg(target_pointer_width = "64")] + assert_eq!( + bounded_count(usize::try_from(u64::from(u32::MAX) + 1).expect("wide usize")), + Err(TopicMeasurementError::InvalidModelInput) + ); + } +} diff --git a/crates/topic_measurement/src/sparse.rs b/crates/topic_measurement/src/sparse.rs new file mode 100644 index 00000000..9749bd0a --- /dev/null +++ b/crates/topic_measurement/src/sparse.rs @@ -0,0 +1,222 @@ +//! Validated compressed sparse matrices used by the reference estimator. + +use crate::TopicMeasurementError; + +/// Whether compressed values are grouped by row or by column. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SparseOrientation { + /// Compressed sparse row storage. + Row, + /// Compressed sparse column storage. + Column, +} + +/// A finite numeric matrix in canonical CSR or CSC form. +#[derive(Clone, Debug, PartialEq)] +pub struct SparseMatrix { + rows: usize, + columns: usize, + offsets: Vec, + indices: Vec, + values: Vec, + orientation: SparseOrientation, +} + +impl SparseMatrix { + /// Construct and validate a compressed sparse row matrix. + /// + /// # Errors + /// + /// Returns [`TopicMeasurementError::InvalidSparseMatrix`] for zero + /// dimensions, malformed offsets, unsorted or repeated inner indices, + /// out-of-range indices, or non-finite values. + pub fn from_csr( + rows: usize, + columns: usize, + offsets: Vec, + indices: Vec, + values: Vec, + ) -> Result { + Self::new( + rows, + columns, + offsets, + indices, + values, + SparseOrientation::Row, + ) + } + + /// Construct and validate a compressed sparse column matrix. + /// + /// # Errors + /// + /// Returns [`TopicMeasurementError::InvalidSparseMatrix`] under the same + /// canonical-storage rules as [`Self::from_csr`]. + pub fn from_csc( + rows: usize, + columns: usize, + offsets: Vec, + indices: Vec, + values: Vec, + ) -> Result { + Self::new( + rows, + columns, + offsets, + indices, + values, + SparseOrientation::Column, + ) + } + + fn new( + rows: usize, + columns: usize, + offsets: Vec, + indices: Vec, + values: Vec, + orientation: SparseOrientation, + ) -> Result { + let outer = match orientation { + SparseOrientation::Row => rows, + SparseOrientation::Column => columns, + }; + let inner = match orientation { + SparseOrientation::Row => columns, + SparseOrientation::Column => rows, + }; + if rows == 0 + || columns == 0 + || offsets.len() != outer + 1 + || offsets.first() != Some(&0) + || offsets.last().copied() != Some(indices.len()) + || indices.len() != values.len() + || values.iter().any(|value| !value.is_finite()) + { + return Err(TopicMeasurementError::InvalidSparseMatrix); + } + for bounds in offsets.windows(2) { + if bounds[0] > bounds[1] || bounds[1] > indices.len() { + return Err(TopicMeasurementError::InvalidSparseMatrix); + } + let mut previous = None; + for &index in &indices[bounds[0]..bounds[1]] { + if index >= inner || previous.is_some_and(|value| index <= value) { + return Err(TopicMeasurementError::InvalidSparseMatrix); + } + previous = Some(index); + } + } + Ok(Self { + rows, + columns, + offsets, + indices, + values, + orientation, + }) + } + + /// Return the matrix row count. + #[must_use] + pub const fn rows(&self) -> usize { + self.rows + } + + /// Return the matrix column count. + #[must_use] + pub const fn columns(&self) -> usize { + self.columns + } + + /// Return the compressed orientation. + #[must_use] + pub const fn orientation(&self) -> SparseOrientation { + self.orientation + } + + pub(crate) fn row_entries(&self) -> Vec> { + let mut rows = vec![Vec::new(); self.rows]; + match self.orientation { + SparseOrientation::Row => { + for (row, bounds) in self.offsets.windows(2).enumerate() { + for index in bounds[0]..bounds[1] { + rows[row].push((self.indices[index], self.values[index])); + } + } + } + SparseOrientation::Column => { + for (column, bounds) in self.offsets.windows(2).enumerate() { + for index in bounds[0]..bounds[1] { + rows[self.indices[index]].push((column, self.values[index])); + } + } + } + } + rows + } +} + +#[cfg(test)] +mod tests { + use super::{SparseMatrix, SparseOrientation}; + use crate::TopicMeasurementError; + + #[test] + fn csr_and_csc_produce_the_same_rows() { + let csr = SparseMatrix::from_csr(2, 3, vec![0, 2, 3], vec![0, 2, 1], vec![1.0, 2.0, 3.0]) + .expect("csr"); + let csc = + SparseMatrix::from_csc(2, 3, vec![0, 1, 2, 3], vec![0, 1, 0], vec![1.0, 3.0, 2.0]) + .expect("csc"); + assert_eq!(csr.rows(), 2); + assert_eq!(csr.columns(), 3); + assert_eq!(csr.orientation(), SparseOrientation::Row); + assert_eq!(csc.orientation(), SparseOrientation::Column); + assert_eq!(csr.row_entries(), csc.row_entries()); + } + + #[test] + fn malformed_sparse_storage_fails_closed() { + let error = Err(TopicMeasurementError::InvalidSparseMatrix); + assert_eq!(SparseMatrix::from_csr(0, 1, vec![0], vec![], vec![]), error); + assert_eq!( + SparseMatrix::from_csr(1, 0, vec![0, 0], vec![], vec![]), + error + ); + assert_eq!(SparseMatrix::from_csr(1, 1, vec![0], vec![], vec![]), error); + assert_eq!( + SparseMatrix::from_csr(1, 1, vec![1, 1], vec![], vec![]), + error + ); + assert_eq!( + SparseMatrix::from_csr(1, 1, vec![0, 2], vec![0], vec![1.0]), + error + ); + assert_eq!( + SparseMatrix::from_csr(1, 1, vec![0, 1], vec![0], vec![]), + error + ); + assert_eq!( + SparseMatrix::from_csr(1, 1, vec![0, 1], vec![1], vec![1.0]), + error + ); + assert_eq!( + SparseMatrix::from_csr(1, 2, vec![0, 2], vec![1, 1], vec![1.0, 2.0]), + error + ); + assert_eq!( + SparseMatrix::from_csr(1, 1, vec![0, 1], vec![0], vec![f64::NAN]), + error + ); + assert_eq!( + SparseMatrix::from_csr(2, 1, vec![0, 2, 1], vec![0], vec![1.0]), + error + ); + assert_eq!( + SparseMatrix::from_csr(3, 2, vec![0, 2, 1, 3], vec![0, 1, 1], vec![1.0; 3]), + error + ); + } +} diff --git a/crates/topic_measurement/tests/reference_estimator_contract.rs b/crates/topic_measurement/tests/reference_estimator_contract.rs new file mode 100644 index 00000000..2aa6de87 --- /dev/null +++ b/crates/topic_measurement/tests/reference_estimator_contract.rs @@ -0,0 +1,516 @@ +//! Known-truth recovery contract for the CPU `f64` TRSL-TM reference. + +use corpus_split::{CorpusDocument, CorpusSnapshot}; +use membership_core::{ + GroupId, MemberId, MembershipAssignment, MembershipNetwork, MembershipRole, MembershipWeight, +}; +use relation_graph::{ + RelationEdge, RelationEndpointId, RelationEvidenceStatus, RelationGraph, RelationKind, +}; +use temporal_core::{ + AvailableTime, EventTime, KnowledgeCutoff, TemporalBoundary, TemporalInterval, + TemporalPrecision, +}; +use topic_measurement::{ + PrevalenceFeature, ReferenceTopicInput, ReferenceTopicModelConfig, SparseMatrix, + fit_reference_topic_model, +}; +use uuid::Uuid; +use validation_core::root_mean_square_error; + +fn event_time(day: u8) -> EventTime { + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T00:00:00Z")).expect("event time") +} + +fn relation(source: Uuid, target: Uuid, source_day: u8, target_day: u8) -> RelationEdge { + let interval = |day| { + TemporalInterval::bounded( + TemporalBoundary::Included(event_time(day)), + TemporalBoundary::Included( + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T12:00:00Z")) + .expect("interval end"), + ), + TemporalPrecision::Second, + ) + .expect("bounded interval") + }; + RelationEdge::new( + RelationKind::TransitionsTo, + RelationEndpointId::from_uuid(source), + RelationEndpointId::from_uuid(target), + RelationEvidenceStatus::Observed, + interval(source_day), + interval(target_day), + ) + .expect("forward relation") +} + +fn fixture() -> ( + CorpusSnapshot, + Vec, + Vec, + MembershipNetwork, + RelationGraph, +) { + let document_ids: Vec<_> = (1_u128..=6).map(Uuid::from_u128).collect(); + let times: Vec<_> = (1_u8..=6).map(event_time).collect(); + let available = AvailableTime::parse_rfc3339("2026-01-10T00:00:00Z").expect("available"); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"); + let mut snapshot = CorpusSnapshot::new(); + for id in &document_ids { + snapshot + .insert_if_eligible(CorpusDocument::new(*id, available), &cutoff) + .expect("eligible"); + } + + let organization = GroupId::from_uuid(Uuid::from_u128(100)); + let projects = [ + GroupId::from_uuid(Uuid::from_u128(101)), + GroupId::from_uuid(Uuid::from_u128(102)), + ]; + let validity_start = event_time(1); + let validity_end = event_time(9); + let mut memberships = MembershipNetwork::new(); + for (index, id) in document_ids.iter().enumerate() { + let member = MemberId::from_uuid(*id); + memberships + .insert( + MembershipAssignment::new( + member, + organization, + MembershipRole::Organization, + MembershipWeight::full().expect("full"), + validity_start, + validity_end, + ) + .expect("organization membership"), + ) + .expect("insert organization"); + memberships + .insert( + MembershipAssignment::new( + member, + projects[usize::from(index >= 3)], + MembershipRole::Project, + MembershipWeight::new(0.75).expect("partial"), + validity_start, + validity_end, + ) + .expect("project membership"), + ) + .expect("insert project"); + } + + let mut relations = RelationGraph::new(); + for (source, target, source_day, target_day) in [ + (0, 1, 1, 2), + (1, 2, 2, 3), + (2, 3, 3, 4), + (3, 4, 4, 5), + (4, 5, 5, 6), + ] { + relations + .insert(relation( + document_ids[source], + document_ids[target], + source_day, + target_day, + )) + .expect("insert relation"); + } + (snapshot, document_ids, times, memberships, relations) +} + +fn separated_counts() -> SparseMatrix { + SparseMatrix::from_csr( + 6, + 4, + vec![0, 2, 4, 6, 8, 10, 12], + vec![0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3], + vec![ + 90.0, 10.0, 85.0, 15.0, 80.0, 20.0, 10.0, 90.0, 15.0, 85.0, 20.0, 80.0, + ], + ) + .expect("counts") +} + +#[test] +fn separated_topics_recover_and_emit_predecessor_successor_counts() { + let (snapshot, document_ids, times, memberships, relations) = fixture(); + let counts = separated_counts(); + let input = ReferenceTopicInput::new( + &snapshot, + document_ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("input"); + assert_eq!(input.document_count(), 6); + assert_eq!(input.vocabulary_size(), 4); + assert!(matches!(input.features()[0], PrevalenceFeature::Intercept)); + assert!(matches!(input.features()[1], PrevalenceFeature::EventTime)); + + let config = ReferenceTopicModelConfig::new(2, vec![7, 11, 19], 2_000, 1e-5) + .expect("configuration") + .with_hyperparameters(1.0, 0.5, 0.01, 0.05, 0.2) + .expect("hyperparameters"); + let result = fit_reference_topic_model(&input, &config).expect("converged fit"); + assert!(result.objective.is_finite()); + assert!(result.iterations <= 2_000); + assert_eq!(result.connected_post_count, 6); + assert_eq!(result.lineage_count, 2); + assert_eq!(result.sequence_edges.len(), 4); + assert!( + result + .sequence_edges + .iter() + .all(|edge| edge.association_strength > 0.5) + ); + assert!( + result + .document_coordinate_variances + .iter() + .flatten() + .all(|value| *value > 0.0) + ); + + let recovered: Vec = result + .document_topic_proportions + .iter() + .map(|row| row[0]) + .collect(); + let truth_a = [0.9, 0.85, 0.8, 0.1, 0.15, 0.2]; + let truth_b = [0.1, 0.15, 0.2, 0.9, 0.85, 0.8]; + let rmse = root_mean_square_error(&truth_a, &recovered) + .expect("rmse") + .min(root_mean_square_error(&truth_b, &recovered).expect("label-swapped rmse")); + assert!(rmse < 0.25, "known-truth topic RMSE {rmse} exceeded 0.25"); +} + +#[test] +fn invalid_configuration_and_topic_dimension_fail_closed() { + let (snapshot, document_ids, times, memberships, relations) = fixture(); + let counts = SparseMatrix::from_csr( + 6, + 2, + vec![0, 1, 2, 3, 4, 5, 6], + vec![0, 0, 0, 1, 1, 1], + vec![1.0; 6], + ) + .expect("counts"); + let input = ReferenceTopicInput::new( + &snapshot, + document_ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("input"); + assert!(ReferenceTopicModelConfig::new(1, vec![1], 10, 1e-6).is_err()); + let too_many = ReferenceTopicModelConfig::new(3, vec![1], 10, 1e-6).expect("config"); + assert!(fit_reference_topic_model(&input, &too_many).is_err()); + + for (topics, seeds, iterations, tolerance) in [ + (2, vec![], 10, 1e-6), + (2, vec![1], 1, 1e-6), + (2, vec![1], 10, f64::NAN), + (2, vec![1], 10, 0.0), + ] { + assert!(ReferenceTopicModelConfig::new(topics, seeds, iterations, tolerance).is_err()); + } + let base = ReferenceTopicModelConfig::new(2, vec![1], 10, 1e-6).expect("base"); + for values in [ + (f64::NAN, 0.5, 0.01, 0.05, 0.2), + (0.0, 0.5, 0.01, 0.05, 0.2), + (1.0, f64::NAN, 0.01, 0.05, 0.2), + (1.0, -1.0, 0.01, 0.05, 0.2), + (1.0, 0.5, f64::NAN, 0.05, 0.2), + (1.0, 0.5, -1.0, 0.05, 0.2), + (1.0, 0.5, 0.01, f64::NAN, 0.2), + (1.0, 0.5, 0.01, 0.0, 0.2), + (1.0, 0.5, 0.01, 0.05, f64::NAN), + (1.0, 0.5, 0.01, 0.05, 0.0), + ] { + assert!( + base.clone() + .with_hyperparameters(values.0, values.1, values.2, values.3, values.4) + .is_err() + ); + } +} + +#[test] +#[allow(clippy::too_many_lines)] +fn invalid_structural_inputs_and_nonconvergence_fail_closed() { + let (snapshot, document_ids, times, memberships, relations) = fixture(); + let counts = separated_counts(); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids[..1].to_vec(), + &counts, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + let wrong_rows = + SparseMatrix::from_csr(2, 2, vec![0, 1, 2], vec![0, 1], vec![1.0; 2]).expect("wrong rows"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &wrong_rows, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + let one_column = + SparseMatrix::from_csr(6, 1, vec![0, 1, 2, 3, 4, 5, 6], vec![0; 6], vec![1.0; 6]) + .expect("one column"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &one_column, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + ×[..5], + None, + &memberships, + &relations, + ) + .is_err() + ); + let mut missing_snapshot = CorpusSnapshot::new(); + missing_snapshot + .insert_if_eligible( + CorpusDocument::new( + document_ids[0], + AvailableTime::parse_rfc3339("2026-01-10T00:00:00Z").expect("available"), + ), + &KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"), + ) + .expect("eligible"); + assert!( + ReferenceTopicInput::new( + &missing_snapshot, + document_ids.clone(), + &counts, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + let mut outside_relations = RelationGraph::new(); + outside_relations + .insert(relation(Uuid::from_u128(999), document_ids[0], 1, 2)) + .expect("outside source"); + outside_relations + .insert(relation(document_ids[0], Uuid::from_u128(998), 2, 3)) + .expect("outside target"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + ×, + None, + &memberships, + &outside_relations, + ) + .is_err() + ); + + let empty_row = SparseMatrix::from_csr( + 6, + 2, + vec![0, 0, 1, 2, 3, 4, 5], + vec![0, 0, 1, 1, 1], + vec![1.0; 5], + ) + .expect("empty row"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &empty_row, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + let zero_row = SparseMatrix::from_csr( + 6, + 2, + vec![0, 1, 2, 3, 4, 5, 6], + vec![0, 0, 0, 1, 1, 1], + vec![0.0, 1.0, 1.0, 1.0, 1.0, 1.0], + ) + .expect("zero row"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &zero_row, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + + let mut duplicate_ids = document_ids.clone(); + duplicate_ids[1] = duplicate_ids[0]; + assert!( + ReferenceTopicInput::new( + &snapshot, + duplicate_ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + + let negative = SparseMatrix::from_csr( + 6, + 2, + vec![0, 1, 2, 3, 4, 5, 6], + vec![0, 0, 0, 1, 1, 1], + vec![-1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + ) + .expect("finite sparse values"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &negative, + ×, + None, + &memberships, + &relations, + ) + .is_err() + ); + + let covariate = SparseMatrix::from_csc( + 6, + 1, + vec![0, 6], + vec![0, 1, 2, 3, 4, 5], + vec![-1.0, -0.5, 0.0, 0.0, 0.5, 1.0], + ) + .expect("covariate"); + let with_covariate = ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + ×, + Some(&covariate), + &memberships, + &relations, + ) + .expect("covariate input"); + assert!(matches!( + with_covariate.features()[2], + PrevalenceFeature::Covariate(0) + )); + let wrong_rows = + SparseMatrix::from_csr(2, 1, vec![0, 0, 0], vec![], vec![]).expect("covariate"); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + ×, + Some(&wrong_rows), + &memberships, + &relations, + ) + .is_err() + ); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + ×, + None, + &MembershipNetwork::new(), + &relations, + ) + .is_err() + ); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + ×, + None, + &memberships, + &RelationGraph::new(), + ) + .is_err() + ); + assert!( + ReferenceTopicInput::new( + &snapshot, + document_ids.clone(), + &counts, + &[event_time(1); 6], + None, + &memberships, + &relations, + ) + .is_err() + ); + + let input = ReferenceTopicInput::new( + &snapshot, + document_ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("valid input"); + let exhausted = ReferenceTopicModelConfig::new(2, vec![1], 2, 1e-12).expect("exhausted"); + assert!(fit_reference_topic_model(&input, &exhausted).is_err()); + let quick = ReferenceTopicModelConfig::new(2, vec![1], 10, f64::MAX).expect("quick"); + assert!(fit_reference_topic_model(&input, &quick).is_ok()); + let unstable = ReferenceTopicModelConfig::new(2, vec![1], 10, 1e-6) + .expect("unstable") + .with_hyperparameters(1.0, 0.5, 0.01, 0.05, f64::MAX) + .expect("finite hyperparameters"); + assert!(fit_reference_topic_model(&input, &unstable).is_err()); +} diff --git a/docs/API_CONTRACT.md b/docs/API_CONTRACT.md index 8980f1de..5c991888 100644 --- a/docs/API_CONTRACT.md +++ b/docs/API_CONTRACT.md @@ -1,13 +1,13 @@ # TEPP API and Modular Integration Contract **Status:** Accepted target contract; exact endpoints are introduced only with executable services. -**Last reviewed:** 2026-08-16 +**Last reviewed:** 2026-08-21 ## 1. Authority boundary TEPP must work both as a standalone product and as a modular CWL component. Integrations with `naruon`, `contextual-orchestrator`, `.github`, or other repositories use explicit versioned API/artifact contracts. Cross-service direct table access is prohibited. -Current protected main exposes Rust library/domain contracts. The active PR adds a loopback HTTP/1.1 listener for naruon analysis-run and export POSTs. That listener is not a production TLS/`$PORT` service. Endpoint examples below that are not covered by `NaruonLiveService` remain target interface shapes. +Current protected main exposes Rust library/domain contracts. The active stack adds a loopback HTTP/1.1 listener for naruon analysis-run, LineageWeave temporal-context, and export POSTs. `tepp-loopback` runs the shared consumer listener on `127.0.0.1:18081` by default; a caller may pass another loopback socket address and an optional maximum request count as its two arguments. The container is intended for a trusted same-host or shared-network-namespace sidecar, checks readiness through a synthetic bounded temporal-context request, and deliberately cannot bind a public or bridge address. It is not a production TLS/`$PORT` service. Endpoint examples below that are not covered by `NaruonLiveService` remain target interface shapes. ## 2. Contract families @@ -20,7 +20,9 @@ Current protected main exposes Rust library/domain contracts. The active PR adds | semantic/topic measurement API | future TEPP measurement service | naruon, batch jobs, visual analytics | accepted-target | | LLM interpretation provider port | `tepp_api` orchestration router + future HTTP gateway | contextual-orchestrator | partial | | model/artifact/export API | `tepp_api` export envelopes + future HTTP service | standalone UI/CWL consumers | partial | -| analysis-run request/accepted contracts | `tepp_api` v1 wire DTOs | naruon, orchestrator, UI | active-PR | +| analysis-run request/accepted/status/terminal-result contracts | `tepp_api` v1 wire DTOs | naruon, orchestrator, UI | active product branch | +| temporal-context ordering contract | `tepp_api` v1 wire DTOs | LineageWeave | active-PR | +| cutoff-safe analysis-run readiness execution | `analysis_engine` bounded Rust crate | `tepp_api`, future HTTP/service adapters | active product branch | ## 3. Versioning @@ -42,6 +44,7 @@ When the service layer is introduced, use resources such as: POST /v1/evidence-imports GET /v1/evidence-imports/{import_id} POST /v1/analysis-runs +POST /v1/temporal-context GET /v1/analysis-runs/{run_id} POST /v1/analysis-runs/{run_id}/cancel GET /v1/model-artifacts/{artifact_id} @@ -50,6 +53,30 @@ GET /v1/exports/{export_id} Long-running analysis is durable asynchronous work. `POST /v1/analysis-runs` accepts an idempotency key, immutable input snapshot identity, knowledge cutoff, versioned model contract/configuration, and requested output profile. A retry with the same principal/idempotency key and semantically identical request returns the same run identity; a conflicting body fails closed. +`POST /v1/temporal-context` is a bounded LineageWeave read contract. It accepts +only events whose availability time is at or before `knowledge_cutoff`, orders +them by event time and opaque event ID, and emits adjacent forward temporal +associations plus `candidate_not_causal` transition gaps. It does not infer +causality, mutate TEPP state, or return a completed psychometric result. + +The typed status/read contract returns `accepted`, `running`, `succeeded`, or +`failed`. Accepted and running statuses contain no measurement result. A +terminal status contains exactly one request-bound +`AnalysisRunTerminalResult`; consumers must validate its request, receipt, +snapshot, cutoff, model, profile, and idempotency bindings before treating the +run as measurement evidence. The Rust DTO is available before the future HTTP +service is deployed. + +The stacked `analysis_engine` slice provides the first executable service-side +path behind these DTOs. It consumes a bounded identity-free snapshot, excludes +evidence unavailable at the historical cutoff, preserves multiple-membership +counts, and emits a digest-bound terminal result or a redacted failure. For the +`trsl_topic_lineage_v1` profile it invokes the ADR-0012 `topic_measurement` +reference estimator and publishes validated fitted associations and counts in +`tepp.trsl_topic_lineage.v1`; it does not infer causality or replace production +`K` selection. This remains active product-branch evidence until its exact-head +checks and protected merge pass. + ## 5. Analysis request authority An analysis request cannot supply arbitrary facts that bypass validated domain state. The service resolves and validates: diff --git a/docs/DOCUMENTATION_ASSESSMENT.md b/docs/DOCUMENTATION_ASSESSMENT.md index ccbfb967..dbc6ed1b 100644 --- a/docs/DOCUMENTATION_ASSESSMENT.md +++ b/docs/DOCUMENTATION_ASSESSMENT.md @@ -28,7 +28,7 @@ Decision status and implementation maturity are separate. ADR `Accepted` means t | Architecture | PRESENT-CURRENT | root `ARCHITECTURE.md` owns service/crate boundaries and scientific/compute invariants | | UML / system flows | PRESENT-CURRENT | `docs/UML.md` covers component, sequence, clock state, relation authority, membership, compute and implementation lineage | | ERD / logical data model | PRESENT-CURRENT | `docs/ERD.md` distinguishes current domain objects from planned PostgreSQL entities and preserves uncertain time/membership/provenance | -| ADR index / core decisions | PRESENT-CURRENT | ADR 0001–0016 cover numerical authority, clocks, event/membership, multilingual semantics, ESEM/DSEM, GPU, quality, evidence, PII, LLM orchestration, MSA, topic measurement, persistence/manifests/splits, claim promotion/release, autonomous-development authority, and TDT/CHRONOS boundaries | +| ADR index / core decisions | PRESENT-CURRENT | Numbered ADRs in the canonical index cover numerical authority, clocks, event/membership, multilingual semantics, ESEM/DSEM, GPU, quality, evidence, PII, LLM orchestration, MSA, topic measurement, persistence/manifests/splits, claim promotion/release, autonomous-development authority, TDT/CHRONOS boundaries, and consumer-scoped modular ingress | | ADR status/maturity/supersession policy | PRESENT-CURRENT | `docs/adr/ADR_POLICY.md` makes `Accepted` vs implemented/released explicit and requires exact partial-supersession scope | | API / modular integration | PRESENT-CURRENT | `docs/API_CONTRACT.md` defines versioning, target async lifecycle, authority and naruon/contextual-orchestrator boundaries | | Security | PRESENT-CURRENT | `SECURITY.md` plus `docs/THREAT_MODEL.md` | diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index c47242b1..4e95f6bf 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -19,14 +19,15 @@ The full APA 7th standards/literature register remains `docs/research/standards- | recovery metrics (RMSE, bias, coverage, graph, temporal order, Monte Carlo SE gates) | PRD; Test Strategy; ADR 0007/0014 | `validation_core` on protected main (PR #19); SE-aware Monte Carlo gates included | implemented-main | | PostgreSQL bitemporal/lineage persistence | ADR 0013; Architecture/ERD | `persistence_postgres` migration contracts, in-memory adapters, live SQL session/document SQL port, tenant RLS (`0002` + session GUC/role helpers), `DATABASE_URL` SQLx gate, optional `live-sqlx` `PgPool` driver, exact-head live PostgreSQL CI with isolation proof, append-only immutability triggers (`0004`), temporal interval ordering CHECKs (`0005`), typed membership assignment (`0006` implemented-main), event-relation/mention/instance SQL (#37–#39 implemented-main), source-artifact SQL (#40 implemented-main), audit-event SQL (#41 implemented-main), concurrent document-write stress (#43 implemented-main), backup/restore integrity revalidation (active PR); remaining physical ERD constraints | partial | | known-truth temporal/event simulation manifests | PRD; TRD; Test Strategy | `tepp_simulation` on protected main; recovery metrics in `validation_core` | implemented-main | -| versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); HTTP service remaining accepted-target | partial | +| versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); LineageWeave loopback contracts and request-bound terminal result are composed on the active product branch; production TLS remaining | partial | +| executable cutoff-safe analysis runs | ADR 0012/0020; temporal research; API terminal-result contract | `analysis_engine` availability cutoff, snapshot binding, multiple-membership aggregation, digest-bound readiness artifact, and `tepp.trsl_topic_lineage.v1` execution through `topic_measurement`; synthetic recovery plus tamper/non-convergence tests and exact coverage on the active product branch | active-PR | | immutable split/run/reproducibility manifests | ADR 0013; ERD | `tepp_api` reproducibility manifest contract on protected main; `persistence_postgres` append-only SQL insert/lookup for `reproducibility_manifest`, `corpus_split_manifest`, `model_run`, and `model_artifact` (migration `0003`); full physical ERD constraints remaining | partial | | multilingual shared latent semantic space | PRD; ADR 0004 | future semantic/concept/topic crates | accepted-target | -| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR and sequential Egozcue ILR coordinates on the active PR; temporal STM backend remaining | partial | -| global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | future topic lineage/activity state | accepted-target | +| TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR/ILR coordinates and bounded CPU `f64` reference estimator on the active product branch; calibrated posterior promotion, method effects, persistence, and accelerated backends remaining | partial | +| global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | `topic_lineage` activity/dormancy/reactivation identity on the active product branch; birth/split/merge remain later extensions | partial | | no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `topic_measurement::refuse_lexical_inferential_weight` on the active PR; preprocessing pipeline remaining | partial | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; estimator-side method model remains future | partial | -| candidate K statistical/Pareto gates + blinded LLM review | ADR 0012; research | future `model_selection` | accepted-target | +| candidate K statistical/Pareto gates | ADR 0012; research | `model_selection` statistical/Pareto `K` gate on the active PR; candidate blinding, blinded LLM review, and backend comparison remain accepted-target | active-PR | | compositional topic correlation / stable clustering | ADR 0005/0012; research | future `network_analysis` | accepted-target | | posterior ESEM / longitudinal invariance / DSEM | ADR 0005 | future `psychometric_core` | accepted-target | | CPU bounded multithreading + GPU/VRAM streaming/parity | ADR 0001/0006 | future `compute_backend` | accepted-target | diff --git a/docs/adr/0011-standalone-modular-msa-boundary.md b/docs/adr/0011-standalone-modular-msa-boundary.md index 04181fb3..c7ff3edc 100644 --- a/docs/adr/0011-standalone-modular-msa-boundary.md +++ b/docs/adr/0011-standalone-modular-msa-boundary.md @@ -1,7 +1,7 @@ # ADR 0011 — Standalone operation and modular CWL MSA boundary **Decision status:** Accepted -**Implementation maturity:** partial — Rust crates are independently usable; naruon HTTP interchange and loopback live listener (`POST /v1/analysis-runs` and `/v1/exports`, fail-closed table-access, NIM/proxy headers, RFC 3339 cutoff, stream deadline) are on the active PR (not implemented-main); production TLS/`$PORT` and remaining persistence integrations remain accepted-target +**Implementation maturity:** partial — Rust crates are independently usable; naruon HTTP interchange is implemented-main at protected head `c45be17a9dbce95ef81cee230e9d128abc7160ac`, while the loopback consumer listener and terminal-result contract are composed on the active product branch; production TLS/`$PORT` and remaining persistence integrations remain accepted-target **Date:** 2026-08-10 **Supersedes:** The broad cross-service ownership wording in ADR 0001. ADR 0001 remains authoritative for Rust-first numerical architecture. diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index 40f338ba..3f114b0a 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -1,7 +1,7 @@ # ADR 0012 — Temporal Relational Shared-Latent Topic Measurement **Decision status:** Accepted -**Implementation maturity:** partial — logistic-normal additive log-ratio coordinates, sequential Egozcue isometric log-ratio coordinates, and lexical-weight refusal are implemented on the active PR and are not implemented-main until exact-head checks, review, and protected-main integration complete; temporal topic identity, backend STM estimator, method-effect model, and K-selection remain accepted-target +**Implementation maturity:** partial — logistic-normal ALR/ILR coordinates, lexical-weight refusal, statistical/Pareto candidate-`K` gates, stable active/dormant/reactivated topic identity, and the bounded CPU `f64` reference estimator are implemented on the active product branch; method effects, calibrated posterior acceptance, accelerated backends, and backend interchange remain accepted-target until implemented and protected-main integrated **Date:** 2026-08-12 **Supersedes:** None; refines ADR 0004 and ADR 0005 without replacing their multilingual and psychometric authorities. @@ -29,6 +29,54 @@ For the first production line: - model selection uses statistical/recovery/stability/alignment/fairness gates and a Pareto-style comparison before any blinded LLM review; - the LLM may recommend among statistically admissible candidates but never defines the numerical optimum or bypasses diagnostics. +### CPU `f64` reference estimand and inference + +The bounded reference estimator uses sparse document-by-term counts `C`, one +global `K`-topic word matrix `β`, and document logistic-normal coordinates +`η_d`, with `θ_d = softmax([η_d, 0])`. Its prevalence mean is the PRD-owned +structural equation + +\[ +m_d = x_d\Gamma + \sum_{g \in G_d} w_{dg}u_g, +\qquad +\eta_d \sim N(m_d, \sigma^2 I), +\] + +where `x_d` includes an intercept, standardized event time, and admitted +prevalence covariates, while the second term retains every active weighted +cross-classified/multiple-membership assignment. This is the logistic-normal +prevalence boundary of correlated/structural topic models, not a raw-simplex +regression (Blei & Lafferty, 2007; Roberts et al., 2019). + +For explicit observed predecessor/successor relations only, the reference +objective adds the harmonic network penalty + +\[ +R(\Theta,G)=\frac{1}{2}\sum_{(d,e)\in E}a_{de} +\lVert\theta_d-\theta_e\rVert_2^2, +\] + +so absent relations remain unobserved rather than negative. This follows the +document-network regularization estimand of Mei et al. (2008); it is not a +causal edge, an event-identity promotion, or an RTM link-probability claim. +The full bounded MAP objective is + +\[ +\sum_{d,v} C_{dv}\log\!\left(\sum_k\theta_{dk}\beta_{kv}\right) +-\frac{1}{2\sigma^2}\sum_d\lVert\eta_d-m_d\rVert_2^2 +-\lambda R(\Theta,G)-\frac{\rho}{2}\lVert\Gamma,u\rVert_2^2. +\] + +Production inference uses deterministic generalized EM: normalized latent +term-topic responsibilities, smoothed multinomial `β` updates, bounded +gradient updates for `η` and structural coefficients, and a diagonal Laplace +curvature approximation for document-coordinate uncertainty. Multiple seeded +initializations retain the best finite converged objective. A non-finite +intermediate, invalid sparse matrix, missing cutoff-safe document, reverse +transition, or exhausted iteration budget returns a typed failure; it never +emits a partial topic artifact. Recovery gates remain caller-owned promotion +criteria over completed validation evidence. + ## Non-goals This ADR does not select one neural architecture forever, claim a unique true topic count for every corpus, or authorize topic labels as causal constructs. It does not allow a fitted backend to redefine TEPP's temporal/event/membership or evidence semantics. @@ -67,3 +115,18 @@ Required evidence includes known-truth topic/covariate/covariance recovery, bias ## Rollback and supersession Rollback selects the last validated model/backend contract and immutable model artifact. Supersede only with evidence that the new model family preserves or explicitly and deliberately changes the estimand, with corresponding PRD/ADR and migration updates. + +## References + +Blei, D. M., & Lafferty, J. D. (2007). A correlated topic model of Science. +*The Annals of Applied Statistics, 1*(1), 17–35. +https://doi.org/10.1214/07-AOAS114 + +Mei, Q., Cai, D., Zhang, D., & Zhai, C. (2008). Topic modeling with network +regularization. In *Proceedings of the 17th International Conference on World +Wide Web* (pp. 101–110). Association for Computing Machinery. +https://doi.org/10.1145/1367497.1367512 + +Roberts, M. E., Stewart, B. M., & Tingley, D. (2019). stm: An R package for +structural topic models. *Journal of Statistical Software, 91*(2), 1–40. +https://doi.org/10.18637/jss.v091.i02 diff --git a/docs/adr/0017-consumer-scoped-analysis-run-ingress.md b/docs/adr/0017-consumer-scoped-analysis-run-ingress.md new file mode 100644 index 00000000..c190104b --- /dev/null +++ b/docs/adr/0017-consumer-scoped-analysis-run-ingress.md @@ -0,0 +1,104 @@ +# ADR 0017 — Consumer-scoped modular analysis-run ingress + +**Decision status:** Accepted +**Implementation maturity:** active-PR +**Date:** 2026-08-20 +**Supersedes:** None; narrows ADR 0011 for shared modular analysis-run ingress and leaves production TLS/deployment authority unchanged. + +## Context + +TEPP must operate standalone and as a modular CWL service. Its first live loopback analysis-run ingress admitted only `naruon`. LineageWeave already owns authorized source-post selection, lineage reconstruction, and Buyer navigation, and needs to submit a bounded TEPP analysis-run request without sharing application tables or forwarding browser, reviewer, model-provider, or database credentials. + +Admitting another consumer by duplicating the listener would create divergent validation, idempotency, error, and security behavior. Reusing one tenant-scoped idempotency namespace without the consumer identity would also allow two legitimate products to collide when they independently choose the same tenant and caller key. + +## Decision + +TEPP publishes one consumer-neutral `/v1/analysis-runs` ingress and a closed modular-consumer registry. The initial admitted identities are: + +- `naruon`; +- `lineageweave`. + +The request body remains the versioned `AnalysisRunRequest`. The transport requires a matching `idempotency-key`, `tepp-contract-version`, `tepp-consumer`, JSON content type, and a loopback host in the current live proof. Accepted-run replay identity is scoped by: + +```text +consumer_code + tenant_workspace_id + idempotency_key +``` + +A retry from the same consumer returns the original accepted run only when the complete validated request is semantically identical. The same tenant/key used by a different consumer has a separate namespace. A changed payload under the same consumer/tenant/key fails closed. + +Consumer-specific client builders may set only the published consumer identity. They reuse the shared request validation and must not add credentials. The Naruon compatibility listener remains available while new consumers use `AnalysisRunLiveService`. + +An HTTP `202 Accepted` response means only that TEPP accepted an analysis-run identity for later execution. In the current loopback proof the accepted-run registry is in-memory and is not yet durable across restarts; persistence remains separate work. The response is not a completed temporal model, calibrated score, theta estimate, uncertainty statement, or scientific claim. + +## Non-goals + +- This ADR does not authorize direct access to another product's tables or object store. +- It does not define production TLS termination, public routing, service discovery, or tenant authentication; those remain separate deployment/security work. +- It does not make arbitrary consumer strings self-registering. +- It does not authorize a consumer to submit raw credentials, prompt text, provider secrets, or unrestricted PII. +- It does not define the completed-result contract. + +## Alternatives considered + +1. **One listener per consumer** — rejected because validation and security behavior would drift and every new CWL product would require another transport implementation. +2. **Tenant plus caller key only** — rejected because distinct modular consumers can legitimately reuse a key and must not replay or conflict with each other's accepted run. +3. **Trust any `tepp-consumer` value** — rejected because an open consumer namespace defeats purpose-bound admission and weakens auditability. +4. **Forward the caller's bearer token or provider credential** — rejected because TEPP should receive a bounded service contract, not inherit browser, reviewer, or model-provider authority. +5. **Closed consumer registry plus shared ingress and consumer-scoped idempotency** — accepted. + +## Consequences + +- LineageWeave and Naruon can use one validated analysis-run boundary without sharing databases. +- Adding another consumer requires a reviewed code change, contract tests, and an ADR/index update when the authority boundary changes. +- Idempotent retries remain deterministic within one product while cross-product collisions are prevented. +- The accepted acknowledgement remains operational evidence only and cannot be promoted to a measurement result. +- The shared listener carries a larger compatibility responsibility and therefore must preserve the strictest existing size, header, host, timeout, and error-redaction behavior. + +## Failure and recovery + +Unknown consumers, credential-bearing headers, malformed or duplicate headers, transfer encoding, non-loopback hosts, invalid content length, oversized payloads, unsupported contract versions, idempotency mismatches, and changed replay payloads fail closed with a redacted versioned error envelope. + +Socket timeout or malformed I/O does not create an accepted run. A retry is safe when it reuses the same consumer, tenant, key, and semantically identical request. Recovery from a deployment outage replays the original bounded request; callers must not fabricate a succeeded run or infer that a missing acknowledgement means the computation failed after acceptance. + +## Security, privacy, scientific-integrity, and governance impact + +- No authorization, review, Copilot, NIM, OpenAI, database, or browser credential crosses the consumer boundary. +- The closed consumer registry is purpose-bound; consumer identity is included in replay/audit identity. +- Host validation, bounded header/body parsing, read/write deadlines, and content-redacting errors limit SSRF-style, request-smuggling, resource-exhaustion, and data-disclosure risks in the current loopback proof. +- Tenant/workspace and snapshot identities remain opaque service references. +- `202 Accepted` cannot be used as evidence of convergence, calibration, uncertainty, validity, or production release readiness. + +## Compatibility and migration + +The existing Naruon listener and `naruon_analysis_run_exchange` remain compatibility surfaces. LineageWeave uses `lineageweave_analysis_run_exchange`, which changes only the consumer header and preserves the shared payload contract. Existing Naruon idempotent retries retain their result within the new consumer-qualified namespace. + +Production HTTP/TLS adapters may replace the loopback transport while preserving the same consumer registry, request semantics, credential prohibition, idempotency namespace, and redacted error contract. Consumer removal requires a deprecation window and retained historical audit interpretation. + +## Verification + +The falsifiable acceptance evidence is: + +- LineageWeave receives HTTP `202` with a valid `AnalysisRunAccepted` response; +- Naruon remains accepted through the compatibility listener; +- same-consumer, semantically identical retries return the original run identity; +- Naruon and LineageWeave using the same tenant/key do not replay each other; +- a changed request under the same consumer/tenant/key is rejected; +- unpublished consumers are rejected; +- credential headers, non-loopback hosts, table-access-like hosts, malformed framing, unsupported versions, and oversized inputs are rejected; +- the LineageWeave exchange contains no credential header; +- formatting, Clippy with warnings denied, all-target Rust tests, public rustdoc, production line/branch coverage, documentation validation, and dependency policy pass on the exact PR head; +- independent current-head review remains required before merge. + +## Rollback and supersession + +Rollback returns callers to the last validated consumer-specific ingress while preserving accepted-run audit identities. It must not collapse existing consumer-qualified replay keys into a shared tenant/key namespace. + +A superseding ADR is required to open dynamic consumer registration, change idempotency identity, permit credential delegation, remove the closed registry, or promote the accepted acknowledgement into a completed-result claim. Production TLS/deployment changes may complement this ADR but must preserve its authority and credential boundaries unless explicitly superseded. + +## Related authority + +- ADR 0011 owns standalone/modular CWL service and persistence boundaries. +- ADR 0002 owns knowledge-cutoff temporal eligibility. +- ADR 0008 owns immutable evidence and strict wire reconstruction. +- ADR 0009 owns purpose-bound PII governance. +- ADR 0014 owns scientific claim and release promotion. diff --git a/docs/adr/0018-project-history-wire-size-symmetry.md b/docs/adr/0018-project-history-wire-size-symmetry.md new file mode 100644 index 00000000..70629731 --- /dev/null +++ b/docs/adr/0018-project-history-wire-size-symmetry.md @@ -0,0 +1,70 @@ +# ADR 0018 — Symmetric LineageWeave wire-size enforcement + +**Decision status:** Accepted +**Implementation maturity:** active-PR +**Date:** 2026-08-21 +**Supersedes:** None; narrows ADR 0008 for the project-history and temporal-context DTO boundaries. + +## Context + +The LineageWeave project-history and temporal-context request/response pairs +use symmetric wire-size ceilings when parsing JSON. A request can be valid and +close to its ceiling while its deterministic projection adds response +metadata. Without output guards, TEPP can construct a response that its own +parser rejects, leaving callers with an internally inconsistent success path. + +## Decision + +`ProjectHistoryRequest::to_json` and `ProjectHistoryProjection::to_json` both +enforce `DEFAULT_PROJECT_HISTORY_BYTE_LIMIT`. The +`project_history_projection` builder serializes and validates the generated +projection before returning it. A projection that cannot be represented by the +published wire contract fails closed with `ApiError::LimitExceeded`. + +`TemporalContextRequest::to_json` and `TemporalContextResponse::to_json` +likewise enforce `DEFAULT_TEMPORAL_CONTEXT_BYTE_LIMIT`. The live adapter cannot +return a success body that the published response parser rejects. + +## Alternatives considered + +1. **Only increase the response limit** — rejected because it silently changes + the published boundary and allows asymmetric resource consumption. +2. **Reserve an undocumented request headroom** — rejected because the + request-to-response size delta depends on event content and findings. +3. **Guard only the HTTP adapter** — rejected because callers can use the + standalone DTO builder and bypass that adapter. +4. **Validate every serialized request and generated projection at the shared + DTO boundary** — accepted. + +## Consequences and failure recovery + +Valid small projections are unchanged. Near-limit requests that would produce +an oversized response now fail deterministically before a success is exposed; +the caller can submit a smaller authorized evidence bundle. No event is +silently dropped and no truncation is introduced. + +## Security, privacy, and scientific integrity + +The shared bound limits memory and transport amplification without exposing +payload contents in errors. The complete explicit evidence set remains the +scientific input; rejecting an unrepresentable projection is safer than +silently changing temporal associations or findings. + +## Verification + +Contract tests construct project-history and temporal-context payloads whose +serialized forms exceed their response ceilings and assert `LimitExceeded`. +Existing round-trip, unknown-field, cutoff, ordering, and finding-invariant +tests remain required. + +## Rollback + +Rollback requires a superseding ADR because removing the guard would +reintroduce a self-rejecting success path. + +## Related authority + +- ADR 0008 owns strict versioned wire reconstruction and bounded evidence. +- ADR 0011 owns standalone and modular service boundaries. +- `docs/doctoring/LINEAGEWEAVE_PROJECT_HISTORY_REFERENCES.md` records the + standards and APA 7th sources for this contract. diff --git a/docs/adr/0019-lineageweave-project-history-boundary.md b/docs/adr/0019-lineageweave-project-history-boundary.md new file mode 100644 index 00000000..568b835e --- /dev/null +++ b/docs/adr/0019-lineageweave-project-history-boundary.md @@ -0,0 +1,91 @@ +# ADR 0019 — LineageWeave project-history service boundary + +**Decision status:** Accepted +**Implementation maturity:** active-PR +**Date:** 2026-08-21 +**Supersedes:** None; narrows ADR 0011 for the project-history projection. + +## Context + +LineageWeave owns authorization, source-post selection, and buyer navigation; +TEPP owns temporal eligibility and deterministic project-history projection. +The products need a versioned boundary that preserves this ownership split +without sharing application tables, provider credentials, or psychometric +claims. + +## Decision + +TEPP publishes the credential-free `POST /v1/project-histories` contract and +the `lineageweave_project_history_exchange` builder. LineageWeave supplies a +bounded, already-authorized set of explicit source events, an opaque tenant and +project identity, and a knowledge cutoff. TEPP validates the cutoff, orders +events deterministically, recomputes only explicit non-causal findings, and +returns a `temporal_association_only` projection. The contract is versioned, +strict JSON, bounded to 256 KiB, and contains no provider or caller +credentials. + +## Non-goals + +- TEPP does not authorize or discover LineageWeave source records. +- The projection is not a causal conclusion, psychometric score, theta, + confidence value, or completed model result. +- The boundary does not grant cross-service database access or production TLS + deployment authority. + +## Alternatives considered + +1. **Shared LineageWeave/TEPP tables** — rejected because it couples + authorization, migrations, retention, and service ownership. +2. **A TEPP endpoint that fetches LineageWeave records by name** — rejected + because authorization and evidence selection belong to LineageWeave. +3. **A credential-bearing provider request** — rejected because the boundary + needs only an evidence contract, not browser, reviewer, or model authority. +4. **A versioned bounded evidence-in/projection-out contract** — accepted. + +## Consequences + +The services can run independently and compose through a stable API. Every +event and finding remains traceable to opaque submitted identities. Consumers +must reduce the authorized evidence bundle when the bounded response cannot be +represented; TEPP never silently truncates evidence or upgrades temporal order +to causation. + +## Failure and recovery + +Malformed, future-leaking, duplicate, oversized, credential-bearing, or +unsupported payloads fail closed with content-redacting errors. A retry may +reuse the same validated evidence and cutoff. An unavailable TEPP service does +not become a fabricated buyer result; the consumer records deferred/unavailable +state and retries through its own controlled adapter. + +## Security, privacy, and scientific integrity + +Only authorized bounded evidence and opaque identities cross the boundary. +Purpose-bound identity disclosure remains governed by ADR 0009. Deterministic +ordering and explicit finding recomputation preserve the distinction between +observed evidence, temporal association, and scientific inference. + +## Verification + +The project-history contract tests cover strict JSON, unknown fields, cutoff +leakage, deterministic ordering, finding recomputation, credential-free +headers, request/response size limits, and the near-limit generated-response +failure path. Documentation validation, Rust quality gates, and independent +current-head review remain required before merge. + +## Rollback + +Disable the modular adapter while retaining standalone TEPP operation. Do not +replace the contract with direct table access or reinterpret historical +projection records. A changed endpoint, ownership boundary, evidence meaning, +or credential policy requires a superseding ADR. + +## Related authority + +- ADR 0002 owns knowledge-cutoff and temporal eligibility. +- ADR 0008 owns strict bounded wire reconstruction. +- ADR 0009 owns purpose-bound PII governance. +- ADR 0011 owns standalone and modular service authority. +- ADR 0018 owns symmetric project-history wire-size enforcement. +- `docs/doctoring/LINEAGEWEAVE_PROJECT_HISTORY_REFERENCES.md` records the + contract sources and APA 7th references. diff --git a/docs/adr/0020-deterministic-analysis-run-execution.md b/docs/adr/0020-deterministic-analysis-run-execution.md new file mode 100644 index 00000000..3626c8b8 --- /dev/null +++ b/docs/adr/0020-deterministic-analysis-run-execution.md @@ -0,0 +1,99 @@ +# ADR 0020 — Deterministic cutoff-safe analysis-run execution + +**Decision status:** Accepted +**Implementation maturity:** active-PR — composed on the active product branch; not implemented-main +**Date:** 2026-08-21 +**Supersedes:** None; complements ADR 0002, ADR 0003, ADR 0011, ADR 0013, and the terminal-result contract. +**Figma File ID:** N/A — this increment changes a Rust service crate and has no user-interface surface. +**Storybook inventory:** N/A — no reusable web object or interaction changed. + +## Context + +TEPP already accepts an analysis request and can describe a completed result, +but a consumer needs a demonstrable path between those contracts. Without one +bounded execution slice, an accepted run is only a receipt and consumers cannot +verify cutoff safety, multiple-membership preservation, or artifact identity. + +## Decision + +Add the standalone `analysis_engine` Rust crate as the first executable vertical +slice. It consumes a request, an accepted receipt, and a bounded identity-free +evidence snapshot. It: + +- excludes evidence whose `available_time` is later than the request's + `knowledge_cutoff`; +- preserves multiple-membership assignments by summing their counts rather than + reducing an evidence unit to one group; +- binds the result to the accepted run and source snapshot; +- verifies request/receipt idempotency identity before scanning the corpus; +- emits a canonical SHA-256-digested `AnalysisArtifact` and the versioned + `AnalysisRunTerminalResult` from `tepp_api`; +- returns a content-redacted failed terminal result when no evidence is + eligible; and +- remains a readiness/counting slice, not latent-variable, topic, or + psychometric estimator authority. + +The engine is deterministic, synchronous, bounded to `100_000` evidence units, +and CPU-only. Scientific estimators and their Rust CPU `f64`/GPU parity +contracts remain separate boundaries under ADR 0001 and ADR 0006. + +For the `trsl_topic_lineage_v1` output profile, the engine may invoke the +ADR-0012 `topic_measurement` CPU `f64` reference estimator through its validated +`ReferenceTopicInput`. The engine does not reimplement or reinterpret the +estimator. It binds the request snapshot and cutoff, then emits a canonical +`tepp.trsl_topic_lineage.v1` artifact containing only the selected seed, +iteration/objective evidence, topic count, evidence count, fitted +predecessor/successor topic edges, connectable-post count, and lineage count. +The artifact is bounded, digest-bound, and self-validating; invalid or +non-converged estimation returns no partial artifact. Production selection of +`K` remains governed by ADR 0012 and `model_selection`, outside this executor. + +## Alternatives considered + +1. Keep the API as contracts only — rejected because an accepted run would not +produce a consumer-verifiable terminal outcome. +2. Put execution into `tepp_api` — rejected because transport contracts and + scientific execution would become one service boundary. +3. Add a bounded standalone engine behind the existing contracts — accepted + because it is independently testable and composable without shared tables. + +## Consequences + +Consumers can run a reproducible readiness check while seeing only opaque +identifiers, bounded counts, temporal extrema, and a digest. The engine does +not expose source text or identity mappings and does not claim a psychometric +measurement. The initial linear scan is intentionally simple; a production +large-corpus adapter must stream snapshots and preserve the same artifact +semantics before raising the bound. + +LineageWeave may consume the topic-lineage artifact as completed model evidence +beside, but never inside, the project-history temporal-association claim. The +two contracts keep separate schema identities and inference-status copy. + +## Verification + +The stacked PR includes Rust unit and integration tests for cutoff exclusion, +multiple-membership summation, snapshot binding, duplicate identities, empty +eligibility, receipt validation, and package identity. Run: + +```text +cargo fmt --all -- --check +cargo test -p analysis_engine +cargo clippy -p analysis_engine --all-targets -- -D warnings +``` + +The topic-lineage execution contract additionally verifies a synthetic +known-topic corpus, exact request/snapshot/cutoff binding, canonical artifact +round-trip and digest stability, predecessor/successor count consistency, and +fail-closed tamper/non-convergence paths. + +The supporting research and APA 7th citations are recorded in +`docs/doctoring/analysis-engine-v1.md` and the standards register. + +## Rollback and supersession + +Rollback removes the `analysis_engine` workspace member and stops publishing +the readiness artifact while preserving the request and terminal-result DTOs. +No persisted schema migration is introduced. Supersession requires a new ADR +if execution changes cutoff semantics, artifact authority, privacy fields, or +scientific estimands. diff --git a/docs/adr/README.md b/docs/adr/README.md index 63c0d033..6ace0b49 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -17,11 +17,15 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0009](0009-purpose-bound-pii-governance.md) | Purpose-bound PII governance without blanket masking | Accepted | partial | Persistence retention/deletion/legal-hold (`0007`) is implemented-main; provider-payload minimization remains on the active PR until exact-head checks, review, and protected-main integration; deployment evidence remains accepted-target. | | [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding remain on the active PR until exact-head checks, review, and protected-main integration; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; no direct cross-service application-table coupling. | -| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR, sequential Egozcue ILR, and lexical-weight refusal are on the active PR; temporal topic identity, STM backend, method effects, and K-selection remain accepted-target. | +| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Logistic-normal ALR/ILR, lexical-weight refusal, statistical/Pareto candidate-`K` gates, and active/dormant/reactivated identity are on the active product branch; the estimator, method effects, and backend interchange remain accepted-target. | | [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; backup/restore integrity revalidation implemented-main; `0007` retention/deletion/legal-hold implemented-main; remaining physical ERD/backup accepted-target. | | [0014](0014-scientific-claim-promotion-and-release-evidence.md) | Scientific claim promotion and release evidence authority | Accepted | partial | Separates design, implementation, scientific/product claim, and release authority; repository SBOM/provenance generator implemented, full release bundle remaining. | | [0015](0015-autonomous-development-review-and-merge-authority.md) | Autonomous development, review, and merge authority separation | Accepted | active-PR | Separates model proposal, deterministic verification, publication, independent review, and merge/release authority. | | [0016](0016-tdt-chronos-event-intelligence-boundary.md) | TDT, CHRONOS, and Event Ontology intelligence boundary | Accepted | accepted-target | Separates observed evidence, detection/tracking, prediction/schema inference, temporal consistency, and promoted transition authority. | +| [0017](0017-consumer-scoped-analysis-run-ingress.md) | Consumer-scoped modular analysis-run ingress | Accepted | active-PR | Narrows ADR 0011 for the closed consumer registry, credential-free exchange, and consumer-qualified idempotency namespace; production TLS remains separate. | +| [0018](0018-project-history-wire-size-symmetry.md) | Symmetric project-history wire-size enforcement | Accepted | active-PR | Narrows ADR 0008 for request serialization and generated LineageWeave project-history projections. | +| [0019](0019-lineageweave-project-history-boundary.md) | LineageWeave project-history service boundary | Accepted | active-PR | Narrows ADR 0011 for the credential-free bounded project-history API and preserves LineageWeave authorization ownership. | +| [0020](0020-deterministic-analysis-run-execution.md) | Deterministic cutoff-safe analysis-run execution | Accepted | active-PR | Closes the first executable product path from accepted run to digest-bound terminal result without claiming estimator authority. | ## Decision ownership summary @@ -42,7 +46,11 @@ Use the narrowest owning ADR when decisions overlap: - **persistence / manifests / leakage-safe split:** ADR 0013; - **claim maturity / release evidence:** ADR 0014; - **autonomous development/review/merge authority:** ADR 0015; -- **TDT/CHRONOS event intelligence:** ADR 0016. +- **TDT/CHRONOS event intelligence:** ADR 0016; +- **modular consumer admission / replay identity:** ADR 0017; +- **project-history wire-size symmetry:** ADR 0018; +- **LineageWeave project-history service boundary:** ADR 0019; +- **accepted-run execution and terminal artifact production:** ADR 0020. ## Change and supersession rule diff --git a/docs/connectors/naruon-artifact-consumer.md b/docs/connectors/naruon-artifact-consumer.md index 5fe0424c..227c3d02 100644 --- a/docs/connectors/naruon-artifact-consumer.md +++ b/docs/connectors/naruon-artifact-consumer.md @@ -1,6 +1,6 @@ # naruon modular consumer contract for TEPP artifacts -**Status:** Partial — versioned DTO, HTTP interchange, and loopback live listener on the active PR; production TLS/`$PORT` remaining +**Status:** Partial — versioned DTO and HTTP interchange are implemented-main at protected head `c45be17a9dbce95ef81cee230e9d128abc7160ac`; the loopback listener and terminal-result contract are composed on the active product branch; production TLS/`$PORT` remaining **Last reviewed:** 2026-08-16 ## Boundary diff --git a/docs/doctoring/LINEAGEWEAVE_PROJECT_HISTORY_REFERENCES.md b/docs/doctoring/LINEAGEWEAVE_PROJECT_HISTORY_REFERENCES.md new file mode 100644 index 00000000..55c0c3d7 --- /dev/null +++ b/docs/doctoring/LINEAGEWEAVE_PROJECT_HISTORY_REFERENCES.md @@ -0,0 +1,46 @@ +# LineageWeave project-history contract references + +This doctoring record documents the authorities used by TEPP's versioned LineageWeave project-history projection. The projection validates and orders explicitly supplied evidence. It does not infer a missing event, identify a hidden actor, estimate theta, calculate confidence, or promote temporal order to causation. + +## Contract decisions + +| Authority | TEPP decision | +|---|---| +| ISO 8601-1:2019 and RFC 3339 | Parse event, availability, and knowledge-cutoff timestamps as absolute clocks and reject malformed or future-leaking evidence. | +| W3C Time Ontology in OWL and Allen interval algebra | Represent temporal relations separately from causal or psychometric authority. The response contract exposes `temporal_association_only`. | +| W3C PROV-O / PROV-DM | Preserve source identities and evidence references; findings may cite only event and post identities contained in the submitted authorized bundle. | +| RFC 8259 | Use strict versioned JSON DTOs with unknown-field rejection and bounded collections. | +| RFC 9110 | Publish an explicit POST resource path, media type, idempotency key, and fail-closed error behavior. | +| Allen (1983) | Apply deterministic qualitative temporal ordering without claiming that succession establishes cause. | + +## Invariants + +1. `available_at` must not exceed the request `knowledge_cutoff`. +2. Event identities must be unique and the focus event must belong to the request. +3. The response echoes the applied `knowledge_cutoff` and revalidates every event against it. +4. The response must preserve every submitted event and its evidence fields in deterministic occurrence-time/event-ID order. +5. Participant count must equal the distinct opaque actor identities present in the supplied events. +6. Findings are recomputed from explicit event types; fabricated causal or unsupported findings fail closed. +7. Findings may cite only supplied event IDs and source-post IDs. +8. LineageWeave and Naruon use consumer-scoped idempotency namespaces. +9. No caller credential or cross-service database access is part of the project-history contract. +10. Loopback HTTP is a local modular boundary; a non-loopback deployment requires HTTPS/TLS at the service edge. +11. Request serialization, projection serialization, and generated projections all enforce the same 256 KiB wire limit; TEPP never returns a projection that its own response parser must reject. + +## APA 7th references + +Allen, J. F. (1983). Maintaining knowledge about temporal intervals. *Communications of the ACM, 26*(11), 832–843. https://doi.org/10.1145/182.358434 + +Bray, T. (Ed.). (2017). *The JavaScript Object Notation (JSON) data interchange format* (RFC 8259). Internet Engineering Task Force. https://doi.org/10.17487/RFC8259 + +Cox, S., & Little, C. (Eds.). (2017). *Time ontology in OWL*. World Wide Web Consortium. https://www.w3.org/TR/owl-time/ + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). Internet Engineering Task Force. https://doi.org/10.17487/RFC9110 + +International Organization for Standardization. (2019). *Date and time—Representations for information interchange—Part 1: Basic rules* (ISO Standard No. 8601-1:2019). https://www.iso.org/standard/70907.html + +Klyne, G., & Newman, C. (2002). *Date and time on the Internet: Timestamps* (RFC 3339). Internet Engineering Task Force. https://doi.org/10.17487/RFC3339 + +Moreau, L., & Missier, P. (Eds.). (2013). *PROV-DM: The PROV data model*. World Wide Web Consortium. https://www.w3.org/TR/prov-dm/ + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ diff --git a/docs/doctoring/analysis-engine-gap-closure.md b/docs/doctoring/analysis-engine-gap-closure.md new file mode 100644 index 00000000..c9fef79d --- /dev/null +++ b/docs/doctoring/analysis-engine-gap-closure.md @@ -0,0 +1,46 @@ +# Analysis Engine v1 — Buyer Gap Closure + +**Review date:** 2026-08-21 +**Active slice:** PR #157 terminal-result contract → stacked analysis execution +engine for issue #166 + +The organization-wide buyer-gap register is maintained by a separate landing +vehicle. This document records only the analysis-engine slice so it can land +without competing with that register or requiring another PR to be present. + +## Buyer-visible gap + +An accepted analysis run previously had a durable receipt and a terminal-result +DTO, but no executable path that applied a historical availability cutoff and +returned a verifiable artifact. A buyer could submit work but could not yet +demonstrate that the result was complete, cutoff-safe, multiple-membership aware, +and unchanged after transport. + +## Bounded closure + +The stacked `analysis_engine` crate closes the first vertical slice with a +standalone Rust API. It accepts a bounded identity-free evidence snapshot, +rejects snapshot mismatches and duplicate opaque IDs, excludes future-available +evidence, preserves membership counts, and emits a digest-bound terminal result +or a redacted no-eligible-evidence failure. + +This is readiness evidence, not a psychometric estimate. Latent-variable, +multilingual, GPU, and HTTP service gaps remain separately governed by their +own ADRs and must not be implied by this slice. + +## Next leverage-ranked gaps + +1. Add a streaming snapshot adapter with the same digest and cutoff semantics. +2. Bind the engine to a versioned standalone HTTP port without cross-service + table access. +3. Add known-truth estimator execution with RMSE, bias, interval coverage, + multilevel/multiple-membership recovery, and CPU/GPU parity. +4. Add buyer-facing visual analytics only after the interaction contract is + stable; then create a Figma file and Storybook inventory and record its real + File ID in a new UI ADR. + +## Evidence boundary + +The current implementation is active-PR evidence only. Exact-head checks, +independent review, protected merge, release evidence, and deployment controls +are required before a capability is promoted to implemented-main. diff --git a/docs/doctoring/analysis-engine-v1.md b/docs/doctoring/analysis-engine-v1.md new file mode 100644 index 00000000..a54c5dbd --- /dev/null +++ b/docs/doctoring/analysis-engine-v1.md @@ -0,0 +1,55 @@ +# Analysis Engine v1 — Evidence Doctoring + +## Claim boundary + +The stacked PR proves one deterministic temporal-evidence-readiness execution +slice. It does not prove production psychometric estimation, topic validity, +GPU performance, HTTP deployment, certification, or customer-wide scale. + +## Decision-to-evidence mapping + +| Contract | Implementation evidence | Customer action enabled | +|---|---|---| +| Historical cutoff safety | `available_time <= knowledge_cutoff` filter in `analysis_engine` | Re-run a historical snapshot without future-availability leakage | +| Multiple membership | `membership_count` is summed for every eligible unit | Inspect inclusive counts without atomistic single-group collapse | +| Terminal completion | `AnalysisRunTerminalResult` is built from the accepted request and receipt | Poll one stable terminal contract instead of treating acceptance as completion | +| Artifact integrity | Canonical JSON and SHA-256 digest | Verify that a downloaded result matches the published artifact identity | +| Fitted topic lineage | `topic_measurement` reference fit projected as `tepp.trsl_topic_lineage.v1` | Read predecessor/successor-aware connectable-post and lineage counts without treating association as causation | +| Privacy boundary | Artifact contains opaque IDs, counts, and times only | Keep identity mapping in the authorized source boundary | + +## Scientific and standards basis + +The implementation preserves TEPP's distinct event and availability clocks and +does not infer an event time from availability time. The API payload is explicit +JSON, and the artifact digest is an integrity check rather than proof of origin +or scientific truth. These interpretations follow the existing temporal, +interchange, and hashing register entries (Bray, 2017; International +Organization for Standardization, 2012; National Institute of Standards and +Technology, 2015). + +## Verification record + +The local preflight for this slice passed with Rust 1.97.1: + +- `cargo fmt --all -- --check`; +- `cargo test -p analysis_engine` — 8 unit tests, 1 crate-contract test, 3 + readiness integration tests, 2 topic-lineage integration tests, and doctest + collection; +- `cargo clippy -p analysis_engine --all-targets -- -D warnings`. +- exact authored coverage — 148/148 lines and 74/74 branches. + +The protected-hosted exact-head checks and qualifying independent reviews are +still pending. This document must not be used as implemented-main or release +evidence before that merge. + +## APA 7th references + +Bray, T. (Ed.). (2017). *The JavaScript Object Notation (JSON) data interchange +format* (RFC 8259). RFC Editor. https://doi.org/10.17487/RFC8259 + +International Organization for Standardization. (2012). *Language resource +management—Semantic annotation framework (SemAF)—Part 1: Time and events +(SemAF-Time, ISO-TimeML)* (ISO Standard No. 24617-1:2012). + +National Institute of Standards and Technology. (2015). *Secure Hash Standard +(SHS)* (FIPS PUB 180-4). https://doi.org/10.6028/NIST.FIPS.180-4 diff --git a/docs/research/model-selection-pareto-gates.md b/docs/research/model-selection-pareto-gates.md new file mode 100644 index 00000000..f7da5c82 --- /dev/null +++ b/docs/research/model-selection-pareto-gates.md @@ -0,0 +1,43 @@ +# Candidate-K statistical and Pareto gates (doctoring) + +## Scope + +`model_selection` admits a topic count `K` only when it is statistically +supported (`K >= 2`, finite held-out log-likelihood, finite non-negative +complexity) and not Pareto-dominated on those two objectives. An LLM vote may +later recommend among admissible candidates. It cannot itself define the +numerical optimum or bypass diagnostics (ADR 0012). + +This slice does not fit a topic model, choose a neural architecture, or claim a +unique true `K` for every corpus. Known-truth recovery reports computed RMSE of +the selected `K` against the generating `K`. + +## Authority + +### Normative TEPP contract + +- `docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md` — + model selection uses statistical/recovery/stability/alignment/fairness gates + and a Pareto-style comparison before any future blinded LLM review; the LLM + never defines the numerical optimum. + +### Supporting model-selection literature + +Akaike (1974) and Burnham and Anderson (2002) provide background for +likelihood-and-complexity comparison of fitted candidates. Deb et al. (2002) +provides background for non-dominated (Pareto) filtering when two objectives +are compared simultaneously. Those sources do not by themselves validate the +exact TEPP thresholds, acceptance criteria, or orchestration boundary; ADR +0012 is normative for this repository. They do **not** authorize an LLM vote as +a statistical estimator. + +Akaike, H. (1974). A new look at the statistical model identification. *IEEE +Transactions on Automatic Control, 19*(6), 716–723. +https://doi.org/10.1109/TAC.1974.1100705 + +Burnham, K. P., & Anderson, D. R. (2002). *Model selection and multimodel +inference: A practical information-theoretic approach* (2nd ed.). Springer. + +Deb, K., Pratap, A., Agarwal, S., & Meyarivan, T. (2002). A fast and elitist +multiobjective genetic algorithm: NSGA-II. *IEEE Transactions on Evolutionary +Computation, 6*(2), 182–197. https://doi.org/10.1109/4235.996017 diff --git a/docs/research/rust-quality-tooling.md b/docs/research/rust-quality-tooling.md index e2d06ff6..429705e2 100644 --- a/docs/research/rust-quality-tooling.md +++ b/docs/research/rust-quality-tooling.md @@ -25,9 +25,9 @@ surface. - `cargo-nextest` 0.9.140 runs process-isolated tests without retries. - Doctests run separately because nextest does not currently execute doctests. - `cargo-llvm-cov` 0.8.6 produces stable line coverage. -- Branch coverage uses the same tool on `nightly-2026-08-01` because the +- Branch coverage uses the same tool on `nightly-2026-08-21` because the upstream project identifies Rust branch coverage as unstable and - nightly-only. + nightly-only (Endo, 2026). - Coverage thresholds are evaluated from LLVM JSON totals. A nonzero line or branch denominator passes only when all units are covered. - Coverage.py 7.15.2 measures the repository-quality Python scripts at 100% diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index ccbde909..f7e4e5ac 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -32,7 +32,7 @@ Bianchi, F., Terragni, S., Hovy, D., Nozza, D., & Fersini, E. (2021). Cross-ling Nguyen, T. P., Minh, N. V., Nguyen, T., Van, L. N., Nguyen, D. A., Sang, D. V., & Le, T. (2025). XTRA: Cross-lingual topic modeling with topic and representation alignments. In *Findings of the Association for Computational Linguistics: EMNLP 2025*. Association for Computational Linguistics. -TEPP retains a logistic-normal CPU reference while allowing adapter backends that satisfy shared-latent, posterior, temporal, relational, and measurement-invariance contracts. +TEPP retains a logistic-normal CPU reference while allowing adapter backends that satisfy shared-latent, posterior, temporal, relational, and measurement-invariance contracts. `topic_lineage` keeps one global topic identity when activity becomes dormant or reactivated. ## Topic-model evaluation and LLM judges @@ -44,7 +44,18 @@ Stammbach, D., Zouhar, V., Hoyle, A., Sachan, M., & Ash, E. (2023). Revisiting a Yang, X., Zhao, H., Phung, D., Buntine, W., & Du, L. (2025). LLM reading tea leaves: Automatically evaluating topic models with large language models. *Transactions of the Association for Computational Linguistics, 13*. -LLM evaluation complements but never replaces predictive, posterior, stability, alignment, fairness, recovery, and human-validation evidence. Candidates are blinded and statistically gated before LLM review. +Akaike, H. (1974). A new look at the statistical model identification. *IEEE Transactions on Automatic Control, 19*(6), 716–723. https://doi.org/10.1109/TAC.1974.1100705 + +Burnham, K. P., & Anderson, D. R. (2002). *Model selection and multimodel inference: A practical information-theoretic approach* (2nd ed.). Springer. + +Deb, K., Pratap, A., Agarwal, S., & Meyarivan, T. (2002). A fast and elitist multiobjective genetic algorithm: NSGA-II. *IEEE Transactions on Evolutionary Computation, 6*(2), 182–197. https://doi.org/10.1109/4235.996017 + +LLM evaluation complements but never replaces predictive, posterior, stability, +alignment, fairness, recovery, and human-validation evidence. The current +`model_selection` crate performs statistical/Pareto gating; candidate blinding +and blinded LLM review remain accepted-target extensions and are not executed +by this crate. Pareto-filtered held-out log-likelihood and complexity admit a +candidate `K`; an LLM vote cannot define the numerical optimum. ## Compositional data, correlation, and clusters @@ -66,10 +77,14 @@ International Organization for Standardization. (2012). *Language resource manag Hobbs, J. R., & Pan, F. (2017). *Time ontology in OWL* (W3C Recommendation). World Wide Web Consortium. https://www.w3.org/TR/owl-time/ +Allen, J. F. (1983). Maintaining knowledge about temporal intervals. *Communications of the ACM, 26*(11), 832–843. https://doi.org/10.1145/182.358434 + Allan, J. (Ed.). (2002). *Topic detection and tracking: Event-based information organization*. Kluwer Academic Publishers. Anagnostopoulos, E., Batsakis, S., & Petrakis, E. G. M. (2013). CHRONOS: A reasoning engine for qualitative temporal information in OWL. *Procedia Computer Science, 22*, 70–77. https://doi.org/10.1016/j.procs.2013.09.082 +International Organization for Standardization. (2019). *Date and time—Representations for information interchange—Part 1: Basic rules* (ISO Standard No. 8601-1:2019). https://www.iso.org/standard/70907.html + TEPP uses interval and partial-order reasoning, bitemporal availability, leakage-safe cutoffs, TDT segmentation/link/detection/first-story/tracking tasks, and separate neural/symbolic event-schema and temporal-consistency layers. ## Unicode, language tags, and multilingual structure diff --git a/docs/research/topic-activity-identity.md b/docs/research/topic-activity-identity.md new file mode 100644 index 00000000..e47dc67d --- /dev/null +++ b/docs/research/topic-activity-identity.md @@ -0,0 +1,34 @@ +# Global topic activity identity (doctoring) + +## Scope + +`topic_lineage` keeps one P0 topic identity across `active`, `dormant`, and +`reactivated` states. Reactivation cannot mint a new identity. Identity +recovery is the computed share of recovered identities that match known truth. + +This slice does not fit a topic model, implement birth/split/merge/retirement, +or treat activity change as a new latent construct. + +## Authority + +### Normative TEPP contract + +- `docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md` — + one global topic identity set is selected across the modeled period; topics + may be active, dormant, or reactivated without losing identity; birth, split, + merge, and retirement are a later explicit lineage extension. + +### Supporting literature + +Blei and Lafferty (2006) and Roberts, Stewart, and Tingley (2019) motivate +temporal topic prevalence that can rise and fall. They do **not** authorize +minting a new topic identity merely because prevalence returns after a quiet +period. + +Blei, D. M., & Lafferty, J. D. (2006). Dynamic topic models. In *Proceedings +of the 23rd International Conference on Machine Learning* (pp. 113–120). +Association for Computing Machinery. https://doi.org/10.1145/1143844.1143859 + +Roberts, M. E., Stewart, B. M., & Tingley, D. (2019). stm: An R package for +structural topic models. *Journal of Statistical Software, 91*(2), 1–40. +https://doi.org/10.18637/jss.v091.i02 diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index c3936755..78a1ca0d 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -23,9 +23,11 @@ This report tracks exact-head scientific and engineering evidence required befor | Truth corpora / manifests | `tepp_simulation` | implemented-main | — | deterministic generator tests | Task 10 / PR #18 | | Recovery metrics | `validation_core` | implemented-main | — | RMSE/bias/coverage/MC gates | Task 11 / PR #19 | | Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining | +| Candidate-K statistical/Pareto gates | `model_selection` | active-PR | this PR | known-K RMSE + LLM-vote refusal | ADR 0012; estimator/backend remaining | +| Global topic activity identity | `topic_lineage` | active-PR | this PR | dormancy/reactivation identity recovery | ADR 0012; birth/split/merge remaining | | Purpose-bound provider payloads | `tepp_api` | implemented-main | provider-payload minimization | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | | Adaptive orchestration router | `tepp_api` | implemented-main | router + ablation | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | -| Logistic-normal topic coordinates | `topic_measurement` | active-PR | stable ALR + sequential ILR + lexical refusal | known-simplex ALR/ILR RMSE, Aitchison-distance ILR isometry | ADR 0012; `docs/research/topic-logratio-coordinates.md` | +| Logistic-normal topic coordinates and CPU reference estimator | `topic_measurement` | active-PR | stable ALR + sequential ILR + lexical refusal + bounded sparse TRSL-TM fit | known-simplex ALR/ILR RMSE, Aitchison-distance ILR isometry, known-topic RMSE, exact line/branch coverage | ADR 0012; calibrated posterior, method effects, persistence, and accelerated backends remaining | | CWL modular connectors | `docs/connectors/*` | implemented-main | — | contract docs + examples | PR #22; live HTTP ports remaining | | Release SBOM/provenance generator | `scripts/release_evidence.py` | partial | — | generate+validate in CI | Task 13 partial / PR #28 | diff --git a/docs/verification/consumer-ingress-main-base.md b/docs/verification/consumer-ingress-main-base.md new file mode 100644 index 00000000..fe4df8a0 --- /dev/null +++ b/docs/verification/consumer-ingress-main-base.md @@ -0,0 +1,30 @@ +# Consumer ingress base stabilization + +## Scope + +The modular LineageWeave consumer-admission change is reviewed and merged as a direct successor to the already-merged loopback ingress in PR #107. + +## Exact base correction + +- Protected target branch: `main` +- `main` at the correction point: `c45be17a9dbce95ef81cee230e9d128abc7160ac` +- Product head before this evidence-only commit: `17f06e814e943ebd9bf592549e2d218a4efed112` +- Superseded target: the historical PR #107 feature branch + +Retargeting does not remove or reimplement any admitted-consumer behavior. It prevents a successful merge from updating only the already-consumed feature branch instead of advancing TEPP `main`. + +## Preserved product boundary + +The change continues to preserve: + +- one consumer-neutral `/v1/analysis-runs` ingress; +- a closed `naruon` and `lineageweave` consumer registry; +- credential-free request construction; +- consumer-qualified tenant/idempotency namespaces; +- deterministic replay and changed-payload rejection; +- the existing Naruon compatibility listener; +- the distinction between `202 Accepted` and a completed psychometric result. + +## Merge evidence rule + +Retargeting invalidates remembered branch-base assumptions. Merge requires fresh terminal checks and an independent approval on the exact current head. Cancelled, predecessor-head, child-PR, or author-only evidence is not transferable. diff --git a/docs/verification/project-history-parent-restack.md b/docs/verification/project-history-parent-restack.md new file mode 100644 index 00000000..af05d743 --- /dev/null +++ b/docs/verification/project-history-parent-restack.md @@ -0,0 +1,22 @@ +# Project-history parent restack evidence + +## Exact parents + +This stacked branch preserves both reviewed lines through an ordinary two-parent merge commit: + +- project-history child before restack: `855c6c7153c2f66a1c14e842ad700f571592dd35`; +- current modular-consumer parent: `cbb3dc0aa657c8d95f18be512ae33d0a1263f2ca`. +- resulting merge commit: `c9103d1e4becb597af98470dfe54ec7d1603762c`, with parents `855c6c7153c2f66a1c14e842ad700f571592dd35` and `cbb3dc0aa657c8d95f18be512ae33d0a1263f2ca`; +- current PR head at verification: `18c0fcd43a35fa934505065f1465c5b825f30e84`; +- protected remote `main` head at verification: `c45be17a9dbce95ef81cee230e9d128abc7160ac` (the local `main` ref was `3810bb73e3606431e1e19497b9746a8335e5d379`). + +`git merge-base --is-ancestor` confirms the merge commit is an ancestor of this +branch head, but not of either observed local or protected-remote `main` ref. + +The merge retains the parent’s current analysis-run parsing, consumer-aware live ingress, wire validation, and regression tests. It retains the child’s `/v1/project-histories` DTOs, credential-free LineageWeave exchange, deterministic projection logic, scientific-claim boundary, and contract tests. + +## Conflict disposition + +The two branches both touched the crate root and root changelog. The crate root keeps the child’s superset exports, including all parent analysis-run exports plus the project-history module. The root changelog keeps the parent entry, while the child release note is retained as `CHANGELOG.d/lineageweave-project-history.md` rather than deleting the parent record. + +No branch history is rewritten, no force update is used, and no product capability is removed. Fresh exact-head Rust, documentation, security, SAST, coverage, and independent-review evidence remains mandatory after this merge. diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..a635c5c0 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +pythonpath = . diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index 65e140a7..bec9ce26 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -10,16 +10,68 @@ def load_totals(path: Path) -> Mapping[str, Any]: - """Load the single-report totals mapping from LLVM coverage JSON.""" + """Load exact totals from LLVM coverage JSON. + + Full LLVM branch exports can contain several instrumented copies of the + same source file when unit and integration test binaries are merged. The + source-level contract is the union of each branch coordinate's true and + false outcomes, so those copies are merged before the branch gate runs. + Summary-only reports retain the original LLVM totals fallback. + """ payload = json.loads(path.read_text(encoding="utf-8")) data = payload.get("data") if not isinstance(data, list) or len(data) != 1: raise ValueError("coverage JSON must contain exactly one data entry") - totals = data[0].get("totals") + report = data[0] + totals = report.get("totals") if not isinstance(totals, Mapping): raise ValueError("coverage JSON data entry must contain totals") - return totals + files = report.get("files") + if not isinstance(files, list) or not any( + isinstance(record, Mapping) and "branches" in record for record in files + ): + return totals + return {**totals, "branches": load_union_branch_totals(files)} + + +def load_union_branch_totals(files: Sequence[object]) -> Mapping[str, int | float]: + """Merge LLVM branch outcomes by source coordinate across test binaries.""" + + outcomes: dict[tuple[str, int, int, int, int], list[int]] = {} + for file_record in files: + if not isinstance(file_record, Mapping): + raise ValueError("coverage file record must be an object") + filename = file_record.get("filename") + if "branches" not in file_record: + raise ValueError("coverage file record must contain branches") + branches = file_record["branches"] + if not isinstance(filename, str) or not filename: + raise ValueError("coverage file record must contain a filename") + if not isinstance(branches, list): + raise ValueError("coverage branches must be a list") + for branch in branches: + if not isinstance(branch, list) or len(branch) < 6: + raise ValueError("coverage branch record is malformed") + coordinates = branch[:4] + counts = branch[4:6] + if not all( + isinstance(value, int) and not isinstance(value, bool) and value >= 0 + for value in coordinates + ): + raise ValueError("coverage branch coordinates are invalid") + if not all( + isinstance(value, int) and not isinstance(value, bool) and value >= 0 + for value in counts + ): + raise ValueError("coverage branch counts are invalid") + key = (filename, *coordinates) + outcome = outcomes.setdefault(key, [0, 0]) + outcome[0] += counts[0] + outcome[1] += counts[1] + count = len(outcomes) * 2 + covered = sum(outcome > 0 for counts in outcomes.values() for outcome in counts) + return {"count": count, "covered": covered} def resolve_repository_source_path(source_path: str, repository_root: Path) -> Path: @@ -85,10 +137,26 @@ def is_executable_source_line( return False if text.startswith("#[") or text.startswith("#!["): return False - if text in {"{", "}", "},", ")", ");", "];", "();", "};"}: + if text in { + "{", + "}", + "(", + ")", + "},", + ");", + "];", + "();", + "};", + "});", + "Ok(())", + }: return False if _is_standalone_string_literal(text) or text.startswith("} else"): return False + if text.endswith(" {"): + type_name = text[:-2] + if type_name and all(character.isalnum() or character in "_:" for character in type_name): + return False if text.startswith("use ") or text.startswith("pub use "): return False if text.startswith("mod ") or text.startswith("pub mod "): @@ -99,6 +167,14 @@ def is_executable_source_line( return False if text.startswith("pub fn ") or text.startswith("fn "): return False + if text.startswith("pub(crate) fn "): + return False + # Keep guarded match arms in the authored-line denominator: the guard + # executes even though the arm label itself is structural. + if text.endswith("=> {") and " if " not in text: + if text.startswith("if ") or text.startswith("if("): + return True + return _is_multiline_match_guard(lines, line_number) if text.startswith("pub struct ") or text.startswith("struct "): return False if text.startswith("pub enum ") or text.startswith("enum "): @@ -110,6 +186,31 @@ def is_executable_source_line( return True +def _is_multiline_match_guard(lines: list[str], line_number: int) -> bool: + """Recognize a guard continued onto the lines immediately before an arm.""" + + target_prefix = lines[line_number - 1].strip().partition("=>")[0] + brace_depth = target_prefix.count("}") - target_prefix.count("{") + guard_found = False + boundary_candidate = False + for candidate in reversed(lines[: line_number - 1]): + stripped = candidate.strip() + if brace_depth == 0 and "=>" in stripped: + return guard_found and not boundary_candidate + if brace_depth == 1 and stripped.endswith("=> {"): + boundary_candidate = True + brace_depth += stripped.count("}") - stripped.count("{") + if ( + (stripped.startswith("if ") or stripped.startswith("if(")) + and not stripped.endswith(("}", ";")) + and brace_depth == 0 + ): + guard_found = True + if stripped.startswith("match "): + return guard_found and not boundary_candidate + return guard_found and not boundary_candidate + + def _cfg_test_module_line_numbers(lines: list[str]) -> set[int]: """Return line numbers belonging to any ``#[cfg(test)] mod ... { ... }`` block.""" diff --git a/scripts/check_workspace_contract.py b/scripts/check_workspace_contract.py index 616f8d07..d596e13c 100644 --- a/scripts/check_workspace_contract.py +++ b/scripts/check_workspace_contract.py @@ -24,6 +24,9 @@ "validation_core", "tepp_api", "topic_measurement", + "model_selection", + "topic_lineage", + "analysis_engine", ) REQUIRED_CI_SNIPPETS: tuple[str, ...] = ( diff --git a/scripts/validate_documentation.py b/scripts/validate_documentation.py index c0603c4d..ebcce251 100644 --- a/scripts/validate_documentation.py +++ b/scripts/validate_documentation.py @@ -40,6 +40,7 @@ "docs/adr/0014-scientific-claim-promotion-and-release-evidence.md", "docs/adr/0015-autonomous-development-review-and-merge-authority.md", "docs/adr/0016-tdt-chronos-event-intelligence-boundary.md", + "docs/adr/0017-consumer-scoped-analysis-run-ingress.md", "docs/product/prd-v0.4-approved.md", "docs/roadmaps/2026-08-05-tepp-delivery-roadmap.md", "docs/superpowers/plans/2026-08-05-temporal-event-foundation.md", diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index 45433b2b..23c708e2 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -116,6 +116,70 @@ def test_report_shape_validation(self) -> None: with self.assertRaisesRegex(ValueError, "contain totals"): coverage_contract.load_totals(path) + def test_full_branch_reports_merge_duplicate_instrumented_copies(self) -> None: + """A source branch passes when either test binary covers each outcome.""" + + payload = self.payload(branch_count=4, branch_covered=2) + payload["data"][0]["files"] = [ # type: ignore[index] + { + "filename": "src/live.rs", + "branches": [[10, 4, 10, 12, 1, 0, 0, 0, 4]], + }, + { + "filename": "src/live.rs", + "branches": [[10, 4, 10, 12, 0, 1, 0, 0, 4]], + }, + ] + with tempfile.TemporaryDirectory() as temporary: + path = self.write_report(temporary, payload) + self.assertEqual( + coverage_contract.load_totals(path)["branches"], + {"count": 2, "covered": 2}, + ) + self.assertEqual( + coverage_contract.validate_report(path, ["branches"]), + ["branches coverage: PASS (2/2, 100%)"], + ) + + def test_full_branch_reports_fail_closed_on_malformed_records(self) -> None: + """Malformed branch exports cannot weaken the coverage gate.""" + + malformed_reports = ( + ([None], "file record must be an object"), + ([{"filename": "", "branches": []}], "must contain a filename"), + ([{"filename": "src.rs"}], "must contain branches"), + ([{"filename": "src.rs", "branches": {}}], "branches must be a list"), + ([{"filename": "src.rs", "branches": [[1, 2]]}], "record is malformed"), + ( + [{"filename": "src.rs", "branches": [[True, 2, 3, 4, 1, 0]]}], + "coordinates are invalid", + ), + ( + [{"filename": "src.rs", "branches": [[1.5, 2, 3, 4, 1, 0]]}], + "coordinates are invalid", + ), + ( + [{"filename": "src.rs", "branches": [[1, 2, 3, 4, True, 0]]}], + "counts are invalid", + ), + ( + [{"filename": "src.rs", "branches": [[-1, 2, 3, 4, 1, 0]]}], + "coordinates are invalid", + ), + ( + [{"filename": "src.rs", "branches": [[1, 2, 3, 4, 0.5, 0]]}], + "counts are invalid", + ), + ( + [{"filename": "src.rs", "branches": [[1, 2, 3, 4, -1, 0]]}], + "counts are invalid", + ), + ) + for files, message in malformed_reports: + with self.subTest(message=message): + with self.assertRaisesRegex(ValueError, message): + coverage_contract.load_union_branch_totals(files) + def test_lcov_authored_line_totals_and_incomplete_detection(self) -> None: """LCOV counts unique authored source lines and exposes zero-hit lines.""" @@ -324,6 +388,20 @@ def test_executable_source_line_filters_noise_records(self) -> None: " }", # 58 "}", # 59 " executable_statement();", # 60 executable + " append_value(", # 61 multiline call opener + " value,", # 62 trailing comma noise + " );", # 63 call close + " values", # 64 + " .iter()", # 65 method-chain continuation + " .collect::>()", # 66 method-chain continuation + " });", # 67 closure call close + "(", # 68 structural call opener + ")", # 69 structural call close + " Ok(())", # 70 structural unit result + " NaruonLiveResponse {", # 71 structural struct literal + "pub(crate) fn crate_visible() {", # 72 visibility-qualified fn + "State::Accepted => {", # 73 match-arm structure + "State::Guarded(value) if valid(value) => {", # 74 guarded arm is executable ] source.write_text("\n".join(source_lines) + "\n", encoding="utf-8") path = str(source) @@ -338,7 +416,7 @@ def test_executable_source_line_filters_noise_records(self) -> None: coverage_contract.is_executable_source_line(path, len(source_lines) + 5) ) - expected_executable = {13, 40, 47, 60} + expected_executable = {13, 40, 47, 60, 61, 64, 65, 66, 74} for line_number in range(1, len(source_lines) + 1): is_exec = coverage_contract.is_executable_source_line(path, line_number) if line_number in expected_executable: @@ -358,6 +436,9 @@ def test_executable_source_line_filters_noise_records(self) -> None: [ f"SF:{path}", "DA:60,1", + "DA:61,0", + "DA:65,0", + "DA:66,0", "DA:1,0", "DA:2,0", "DA:51,0", @@ -370,7 +451,7 @@ def test_executable_source_line_filters_noise_records(self) -> None: coverage_contract.load_lcov_line_totals( lcov, repository_root=Path(temporary) ), - {"lines": {"count": 1, "covered": 1}}, + {"lines": {"count": 4, "covered": 1}}, ) def test_lcov_rejects_source_paths_outside_repository(self) -> None: @@ -405,6 +486,174 @@ def test_lcov_rejects_source_paths_outside_repository(self) -> None: if outside.exists(): outside.unlink() + def test_multiline_guard_arm_is_executable(self) -> None: + """Retain the final expression of a multiline Rust match guard.""" + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "multiline_guard.rs" + source.write_text( + "match state {\n" + " State::Ready(value)\n" + " if value.is_valid()\n" + " && value.is_fresh() => {\n" + " consume(value);\n" + " }\n" + " _ => {\n" + " ignore(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + + self.assertTrue(coverage_contract.is_executable_source_line(str(source), 4)) + self.assertFalse(coverage_contract.is_executable_source_line(str(source), 7)) + + def test_guard_after_brace_closing_pattern_is_executable(self) -> None: + """Count a guard after a destructuring pattern that closes with a brace.""" + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "destructured_guard.rs" + source.write_text( + "match state {\n" + " State::Ready { value }\n" + " if value.is_valid() => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + + self.assertTrue(coverage_contract.is_executable_source_line(str(source), 3)) + + def test_long_and_nested_match_guards_are_executable(self) -> None: + """Track guard boundaries beyond the old scan window and nested arms.""" + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "complex_guard.rs" + long_guard = [ + "match state {", + " State::Ready(value)", + " if value.is_valid()", + *[f" && value.part_{index}()" for index in range(40)], + " && value.is_fresh() => {", + " consume(value);", + " }", + "}", + ] + source.write_text("\n".join(long_guard) + "\n", encoding="utf-8") + self.assertTrue( + coverage_contract.is_executable_source_line( + str(source), len(long_guard) - 3 + ) + ) + + source.write_text( + "match state {\n" + " State::Ready(value)\n" + " if match value {\n" + " 0 => true,\n" + " _ => false,\n" + " } && value.is_fresh() => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + self.assertTrue(coverage_contract.is_executable_source_line(str(source), 6)) + + source.write_text( + "match state {\n" + " State::Ready(value)\n" + " if match value {\n" + " 0 => true,\n" + " _ => false,\n" + " }\n" + " && value.is_fresh() => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + self.assertTrue(coverage_contract.is_executable_source_line(str(source), 7)) + + def test_previous_arm_body_does_not_make_next_label_executable(self) -> None: + """Do not treat an ``if`` inside the preceding arm as a guard.""" + + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "previous_arm.rs" + source.write_text( + "match state {\n" + " State::Previous => {\n" + " if value.is_valid() {\n" + " consume(value);\n" + " }\n" + " }\n" + " State::Current => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + + self.assertFalse(coverage_contract.is_executable_source_line(str(source), 7)) + + one_line_previous = Path(temporary) / "one_line_previous.rs" + one_line_previous.write_text( + "match state {\n" + " State::Previous => value,\n" + " State::Current => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + self.assertFalse( + coverage_contract.is_executable_source_line( + str(one_line_previous), 3 + ) + ) + + second_guard = Path(temporary) / "second_guard.rs" + second_guard.write_text( + "match state {\n" + " State::First => value,\n" + " State::Ready(value)\n" + " if value.is_valid()\n" + " && value.is_fresh() => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + self.assertTrue( + coverage_contract.is_executable_source_line(str(second_guard), 5) + ) + + first_arm = Path(temporary) / "first_arm.rs" + first_arm.write_text( + "match state {\n" + " State::Current => {\n" + " consume(value);\n" + " }\n" + "}\n", + encoding="utf-8", + ) + self.assertFalse( + coverage_contract.is_executable_source_line(str(first_arm), 2) + ) + self.assertFalse( + coverage_contract._is_multiline_match_guard( # noqa: SLF001 + [" if value.is_valid() { consume(value); }", "State::Current => {"], + 2, + ) + ) + self.assertFalse( + coverage_contract._is_multiline_match_guard( + ["State::Current", "State::Current => {"], + 2, + ) + ) + def test_cfg_test_and_not_feature_block_helpers(self) -> None: """cfg(test) modules and cfg(not(feature)) blocks are fully recognized.""" diff --git a/tests/quality/test_check_docstrings.py b/tests/quality/test_check_docstrings.py index b99537c5..a280df3a 100644 --- a/tests/quality/test_check_docstrings.py +++ b/tests/quality/test_check_docstrings.py @@ -24,7 +24,7 @@ def test_live_repository_is_documented(self) -> None: sources = docstrings.rust_sources(REPOSITORY_ROOT) crate_roots = sorted(REPOSITORY_ROOT.glob("crates/*/src/lib.rs")) - self.assertEqual(len(crate_roots), 11) + self.assertEqual(len(crate_roots), 14) self.assertTrue(set(crate_roots).issubset(sources)) self.assertGreaterEqual(len(sources), len(crate_roots)) self.assertEqual(docstrings.validate_repository(REPOSITORY_ROOT), []) diff --git a/tests/quality/test_ci_coverage_diagnostics.py b/tests/quality/test_ci_coverage_diagnostics.py index eaa1b6b8..825c268e 100644 --- a/tests/quality/test_ci_coverage_diagnostics.py +++ b/tests/quality/test_ci_coverage_diagnostics.py @@ -28,7 +28,7 @@ def test_line_and_branch_failures_print_exact_missing_locations(self) -> None: self.assertIn("steps.line-report.outcome == 'success'", workflow) self.assertIn("id: branch-report", workflow) self.assertIn( - "cargo +nightly-2026-08-01 llvm-cov report --branch --text --show-missing-lines", + "cargo +nightly-2026-08-21 llvm-cov report --branch --text --show-missing-lines", workflow, ) self.assertIn("steps.branch-report.outcome == 'success'", workflow) diff --git a/tests/quality/test_hourly_nim_product_development.py b/tests/quality/test_hourly_nim_product_development.py index 7c56183c..04f965fe 100644 --- a/tests/quality/test_hourly_nim_product_development.py +++ b/tests/quality/test_hourly_nim_product_development.py @@ -182,7 +182,9 @@ def test_hourly_prompt_and_verifier_keep_commercial_quality_gates(self) -> None: "cargo deny check", 'line_coverage="$RUNNER_TEMP/coverage.lcov"', 'branch_coverage="$RUNNER_TEMP/coverage-branches.json"', + "cargo llvm-cov --workspace --all-features --lcov --output-path \"$line_coverage\" --ignore-filename-regex 'sqlx_live\\.rs'", 'python3 scripts/check_coverage.py "$line_coverage" --kind lines --format lcov', + "cargo +nightly-2026-08-21 llvm-cov --branch --workspace --all-features --json --output-path \"$branch_coverage\" --ignore-filename-regex 'sqlx_live\\.rs'", 'python3 scripts/check_coverage.py "$branch_coverage" --kind branches', ): self.assertIn(command, verifier) From 6110d3660607ba46b312b4d76f048f1bcc4f3bc5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 02:16:37 +0900 Subject: [PATCH 35/38] fix: fail closed on log-ratio division underflow --- CHANGELOG.md | 3 ++ Dockerfile | 4 +-- crates/topic_measurement/src/coordinates.rs | 29 ++++++++++--------- .../tests/ilr_recovery_contract.rs | 11 +++++++ .../tests/logratio_recovery_contract.rs | 5 ++++ 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f85fb5d4..aef4bae3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,9 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Changed +- ALR and ILR inverse normalization now fails closed when division would turn + a representable subnormal weight into a zero simplex part; runtime images + are pinned to the reviewed multi-platform Rust and Debian OCI digests. - The LineageWeave temporal-context read exchange no longer emits a fabricated `idempotency-key`; that header remains reserved for retryable write/export operations with a caller-owned operation key. diff --git a/Dockerfile b/Dockerfile index ce294a14..c129ed18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM rust:1.97.1-bookworm AS build +FROM rust:1.97.1-bookworm@sha256:0e2bcaef56d041a486784e54104a81aebe0da44bd03019bd70bc0401e42e4a97 AS build WORKDIR /src COPY . . RUN cargo build --locked --release -p tepp_api --bin tepp-loopback -FROM debian:bookworm-slim +FROM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 RUN apt-get update \ && apt-get install --yes --no-install-recommends ca-certificates curl \ && rm -rf /var/lib/apt/lists/* diff --git a/crates/topic_measurement/src/coordinates.rs b/crates/topic_measurement/src/coordinates.rs index ba1d6a99..aa4922a7 100644 --- a/crates/topic_measurement/src/coordinates.rs +++ b/crates/topic_measurement/src/coordinates.rs @@ -49,23 +49,16 @@ pub fn from_additive_log_ratio(coordinates: &[f64]) -> Result, TopicMea if reference_weight == 0.0 { return Err(TopicMeasurementError::InvalidLogRatioDimension); } - let mut shifted_weights = Vec::with_capacity(coordinates.len()); - let mut denominator = reference_weight; + let mut shifted_weights = Vec::with_capacity(coordinates.len() + 1); for &value in coordinates { let weight = (value - maximum).exp(); if weight == 0.0 { return Err(TopicMeasurementError::InvalidLogRatioDimension); } - denominator += weight; shifted_weights.push(weight); } - - let mut simplex = Vec::with_capacity(coordinates.len() + 1); - for weight in shifted_weights { - simplex.push(weight / denominator); - } - simplex.push(reference_weight / denominator); - Ok(simplex) + shifted_weights.push(reference_weight); + normalize_positive_weights(shifted_weights) } /// Map a strictly positive unit simplex vector to isometric log-ratio coordinates. @@ -141,16 +134,26 @@ pub fn from_isometric_log_ratio(coordinates: &[f64]) -> Result, TopicMe } let mut weights = Vec::with_capacity(dimension); - let mut denominator = 0.0_f64; for &value in ¢ered_logs { let weight = (value - maximum).exp(); if weight == 0.0 { return Err(TopicMeasurementError::InvalidLogRatioDimension); } - denominator += weight; weights.push(weight); } - Ok(weights.iter().map(|weight| weight / denominator).collect()) + normalize_positive_weights(weights) +} + +fn normalize_positive_weights(weights: Vec) -> Result, TopicMeasurementError> { + let denominator: f64 = weights.iter().sum(); + let simplex: Vec = weights + .into_iter() + .map(|weight| weight / denominator) + .collect(); + if simplex.iter().any(|part| *part <= 0.0) { + return Err(TopicMeasurementError::InvalidLogRatioDimension); + } + Ok(simplex) } /// Aitchison distance between two strictly positive unit simplex vectors. diff --git a/crates/topic_measurement/tests/ilr_recovery_contract.rs b/crates/topic_measurement/tests/ilr_recovery_contract.rs index a97dd153..74b94d46 100644 --- a/crates/topic_measurement/tests/ilr_recovery_contract.rs +++ b/crates/topic_measurement/tests/ilr_recovery_contract.rs @@ -146,6 +146,17 @@ fn large_finite_ilr_coordinates_round_trip_or_fail_closed() { Err(TopicMeasurementError::InvalidLogRatioDimension), "overflowing CLR reconstruction must fail closed" ); + let gap = 745.0_f64; + let division_underflow = [ + (3.0_f64 / 4.0).sqrt() * gap / 3.0, + (2.0_f64 / 3.0).sqrt() * gap / 2.0, + (1.0_f64 / 2.0).sqrt() * gap, + ]; + assert_eq!( + from_isometric_log_ratio(&division_underflow), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "normalization must not underflow a nonzero weight to a zero simplex part" + ); } #[test] diff --git a/crates/topic_measurement/tests/logratio_recovery_contract.rs b/crates/topic_measurement/tests/logratio_recovery_contract.rs index 2b50250d..e08a497e 100644 --- a/crates/topic_measurement/tests/logratio_recovery_contract.rs +++ b/crates/topic_measurement/tests/logratio_recovery_contract.rs @@ -112,6 +112,11 @@ fn invalid_compositions_and_lexical_weights_fail_closed() { Err(TopicMeasurementError::InvalidLogRatioDimension), "inverse must not return a zero simplex part after underflow" ); + assert_eq!( + from_additive_log_ratio(&[0.0, 0.0, 0.0, -744.0]), + Err(TopicMeasurementError::InvalidLogRatioDimension), + "normalization must not underflow a nonzero weight to a zero simplex part" + ); assert_eq!( additive_log_ratio(&[f64::MAX, f64::MAX]), Err(TopicMeasurementError::InvalidComposition), From 96e556fa12d11418d34194b975294ebdc491d3de Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 25 Aug 2026 02:05:58 +0000 Subject: [PATCH 36/38] test(quality): cover blank-predecessor commas and escaped char literals Restore the 100% Python branch-coverage gate after the main rebase by exercising empty-predecessor structural commas, escaped Rust character literals, and the past-EOF fail-closed scanner path. Co-authored-by: Seongho Bae --- CHANGELOG.md | 1 + tests/quality/test_check_coverage.py | 68 ++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d08f74ea..852acb97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang - `topic_measurement` logistic-normal additive log-ratio and sequential Egozcue isometric log-ratio coordinates: fail-closed simplex validation, max-shifted stable ALR/ILR inverses with true-parameter RMSE, pairwise CLR Aitchison distance recovered by ILR Euclidean isometry for valid composition pairs, and refusal of TF-IDF/BM25/keyword scores as inferential topic coordinates (ADR 0012 first production slice; no new migration). - Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. - Coverage source classification now scans Rust normal/raw/byte strings, comments, and character literals with escape-aware state, preserving executable string method calls and ignoring quoted comments. +- Quality-gate coverage tests now exercise blank-predecessor structural commas and escaped character literals in the authored-line scanner, including the past-EOF fail-closed path. - Registered the analysis-engine gap-closure doctoring in the canonical documentation map so its product and scientific traceability record is discoverable. - Authored Rust coverage classification now ignores standalone structural closing parentheses, preventing formatting-only LCOV rows from appearing as uncovered production behavior. - `analysis_engine` vertical slice (ADR 0021): bounded Rust execution from an accepted analysis run to either a cutoff-safe readiness result or a validated `tepp.trsl_topic_lineage.v1` artifact from the ADR-0012 estimator. Topic artifacts preserve fitted predecessor/successor edges, connectable-post and lineage counts, request/snapshot/cutoff bindings, SHA-256 identity, and fail-closed non-convergence/tamper behavior with exact line/branch coverage. This remains active-PR evidence and does not claim causal or psychometric authority. diff --git a/tests/quality/test_check_coverage.py b/tests/quality/test_check_coverage.py index fde6689b..3a9bb147 100644 --- a/tests/quality/test_check_coverage.py +++ b/tests/quality/test_check_coverage.py @@ -951,6 +951,74 @@ def test_multiline_scanner_ignores_comments_char_literals_and_raw_strings(self) self.assertTrue(coverage_contract.is_executable_source_line(path, 8)) self.assertFalse(coverage_contract.is_executable_source_line(path, 7)) + def test_structural_comma_skips_blank_predecessors(self) -> None: + """Blank predecessors do not invent a call opener for a trailing comma.""" + + lines = [ + "", + " ", + " field_name,", + ] + self.assertFalse( + coverage_contract._is_structural_comma_continuation( + lines, 3, "field_name," + ) + ) + self.assertFalse( + coverage_contract._is_structural_comma_continuation( + ["field_name,"], 1, "field_name," + ) + ) + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "blank_predecessor.rs" + source.write_text("\n".join(lines) + "\n", encoding="utf-8") + self.assertTrue( + coverage_contract.is_executable_source_line(str(source), 3) + ) + + def test_structural_comma_after_blank_lines_still_sees_call_opener(self) -> None: + """Empty lines between a call opener and an argument remain structural.""" + + lines = [ + "record_value(", + "", + " field_name,", + ")", + ] + self.assertTrue( + coverage_contract._is_structural_comma_continuation( + lines, 3, "field_name," + ) + ) + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "call_opener.rs" + source.write_text("\n".join(lines) + "\n", encoding="utf-8") + self.assertFalse( + coverage_contract.is_executable_source_line(str(source), 3) + ) + + def test_multiline_string_scanner_handles_escaped_char_and_past_eof(self) -> None: + """Escaped char literals keep later lines classified; past-EOF is closed.""" + + lines = [ + "fn query() {", + r" let quote = '\'';", + r" let slash = '\\';", + r" let newline = '\n';", + " execute();", + "}", + ] + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "escaped_char.rs" + source.write_text("\n".join(lines) + "\n", encoding="utf-8") + path = str(source) + self.assertFalse(coverage_contract._line_in_multiline_string(lines, 2)) + self.assertFalse(coverage_contract._line_in_multiline_string(lines, 5)) + self.assertTrue(coverage_contract.is_executable_source_line(path, 5)) + self.assertFalse( + coverage_contract._line_in_multiline_string(lines, len(lines) + 1) + ) + if __name__ == "__main__": # pragma: no cover unittest.main() From c879a893672afb8778aa3982f09eedce390ca5ba Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 25 Aug 2026 02:13:41 +0000 Subject: [PATCH 37/38] docs(research): restore Graham Neubig initial in Liu 2023 The protected-main rebase changed Neubig, G. to Neubig, P. in the APA register. Keep the published ACM Computing Surveys author initial and pin it with a quality-gate regression. Co-authored-by: Seongho Bae --- CHANGELOG.md | 1 + docs/research/standards-and-literature.md | 2 +- tests/quality/test_check_workspace_contract.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 852acb97..e1457d21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang - Coverage contract now excludes Rust multiline string continuation records emitted by LLVM LCOV, keeping the 100% authored-line gate focused on executable production lines. - Coverage source classification now scans Rust normal/raw/byte strings, comments, and character literals with escape-aware state, preserving executable string method calls and ignoring quoted comments. - Quality-gate coverage tests now exercise blank-predecessor structural commas and escaped character literals in the authored-line scanner, including the past-EOF fail-closed path. +- Restored Graham Neubig's correct APA 7 initial in the Liu et al. (2023) prompting-survey register entry after the protected-main rebase. - Registered the analysis-engine gap-closure doctoring in the canonical documentation map so its product and scientific traceability record is discoverable. - Authored Rust coverage classification now ignores standalone structural closing parentheses, preventing formatting-only LCOV rows from appearing as uncovered production behavior. - `analysis_engine` vertical slice (ADR 0021): bounded Rust execution from an accepted analysis run to either a cutoff-safe readiness result or a validated `tepp.trsl_topic_lineage.v1` artifact from the ADR-0012 estimator. Topic artifacts preserve fitted predecessor/successor edges, connectable-post and lineage counts, request/snapshot/cutoff bindings, SHA-256 identity, and fail-closed non-convergence/tamper behavior with exact line/branch coverage. This remains active-PR evidence and does not claim causal or psychometric authority. diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index a10060b9..377ebe8c 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -46,7 +46,7 @@ Brown, T. B., Mann, B., Ryder, N., Subbiah, M., Kaplan, J. D., Dhariwal, P., Nee Reynolds, L., & McDonell, K. (2021). Prompt programming for large language models: Beyond the few-shot paradigm. In *Extended abstracts of the 2021 CHI conference on human factors in computing systems*. Association for Computing Machinery. https://doi.org/10.1145/3411763.3451760 -Liu, P., Yuan, W., Fu, J., Jiang, Z., Hayashi, H., & Neubig, P. (2023). Pre-train, prompt, and predict: A systematic survey of prompting methods in natural language processing. *ACM Computing Surveys, 55*(9), Article 195. https://doi.org/10.1145/3560815 +Liu, P., Yuan, W., Fu, J., Jiang, Z., Hayashi, H., & Neubig, G. (2023). Pre-train, prompt, and predict: A systematic survey of prompting methods in natural language processing. *ACM Computing Surveys, 55*(9), Article 195. https://doi.org/10.1145/3560815 TEPP retains a logistic-normal CPU reference while allowing adapter backends that satisfy shared-latent, posterior, temporal, relational, and measurement-invariance contracts. Brown et al. (2020) and Reynolds and McDonell (2021) provide primary research context for prompts as task-conditioning and prompt-programming mechanisms; they do not define TEPP's latent-content labels. As a normative ADR 0004/0012 contract, instruction and prompt boilerplate is therefore modeled as explicit method structure, not unique latent content and not a stopword deletion. Liu et al. (2023) is secondary survey background only and is not evidence for that repository-specific classification. `topic_lineage` keeps one global topic identity when activity becomes dormant or reactivated. diff --git a/tests/quality/test_check_workspace_contract.py b/tests/quality/test_check_workspace_contract.py index 906d8a27..352058bf 100644 --- a/tests/quality/test_check_workspace_contract.py +++ b/tests/quality/test_check_workspace_contract.py @@ -47,6 +47,18 @@ def test_standards_register_cites_rfc_5646_once(self) -> None: ).read_text(encoding="utf-8") self.assertEqual(text.count("RFC 5646"), 1) + def test_liu_2023_register_cites_graham_neubig(self) -> None: + """The Liu et al. (2023) survey must keep Graham Neubig's initial.""" + + text = ( + REPOSITORY_ROOT / "docs" / "research" / "standards-and-literature.md" + ).read_text(encoding="utf-8") + self.assertIn( + "Liu, P., Yuan, W., Fu, J., Jiang, Z., Hayashi, H., & Neubig, G. (2023).", + text, + ) + self.assertNotIn("Neubig, P.", text) + def test_member_paths_match_expected_crates(self) -> None: """Workspace members resolve to the approved crate roots by name.""" From 2084c84b917ad439d3465a0b1cb39c92bfffcd99 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 25 Aug 2026 02:41:26 +0000 Subject: [PATCH 38/38] docs(adr): remap analysis-run execution to ADR 0022 Protected-main consolidation #215 already owns ADR 0020 span-grounded units and ADR 0021 LineageWeave project-history. Move this PR's analysis-run decision to ADR 0022 and restore a single balanced README crate list that includes topic_measurement and analysis_engine. Co-authored-by: Seongho Bae --- CHANGELOG.md | 3 +- README.md | 84 ++++--------------- docs/TRACEABILITY.md | 2 +- ...2-deterministic-analysis-run-execution.md} | 2 +- docs/adr/README.md | 8 +- scripts/validate_documentation.py | 4 +- 6 files changed, 27 insertions(+), 76 deletions(-) rename docs/adr/{0021-deterministic-analysis-run-execution.md => 0022-deterministic-analysis-run-execution.md} (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ba29204..bd4f0fb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,10 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang - Coverage source classification now scans Rust normal/raw/byte strings, comments, and character literals with escape-aware state, preserving executable string method calls and ignoring quoted comments. - Quality-gate coverage tests now exercise blank-predecessor structural commas and escaped character literals in the authored-line scanner, including the past-EOF fail-closed path. - Restored Graham Neubig's correct APA 7 initial in the Liu et al. (2023) prompting-survey register entry after the protected-main rebase. +- After protected-main consolidation #215, the analysis-run execution decision is recorded as ADR 0022 so it does not collide with ADR 0021 LineageWeave project-history. - Registered the analysis-engine gap-closure doctoring in the canonical documentation map so its product and scientific traceability record is discoverable. - Authored Rust coverage classification now ignores standalone structural closing parentheses, preventing formatting-only LCOV rows from appearing as uncovered production behavior. -- `analysis_engine` vertical slice (ADR 0021): bounded Rust execution from an accepted analysis run to either a cutoff-safe readiness result or a validated `tepp.trsl_topic_lineage.v1` artifact from the ADR-0012 estimator. Topic artifacts preserve fitted predecessor/successor edges, connectable-post and lineage counts, request/snapshot/cutoff bindings, SHA-256 identity, and fail-closed non-convergence/tamper behavior with exact line/branch coverage. This remains active-PR evidence and does not claim causal or psychometric authority. +- `analysis_engine` vertical slice (ADR 0022): bounded Rust execution from an accepted analysis run to either a cutoff-safe readiness result or a validated `tepp.trsl_topic_lineage.v1` artifact from the ADR-0012 estimator. Topic artifacts preserve fitted predecessor/successor edges, connectable-post and lineage counts, request/snapshot/cutoff bindings, SHA-256 identity, and fail-closed non-convergence/tamper behavior with exact line/branch coverage. This remains active-PR evidence and does not claim causal or psychometric authority. - Coverage classification preserves the final expression line of multiline Rust `match` guards while respecting preceding-arm boundaries, keeping the 100% authored-line gate conservative. - `tepp_api` fail-closed analysis-result boundaries: status constructors reject terminal envelopes that cannot fit the default 64 KiB status limit, and diff --git a/README.md b/README.md index e1458f63..fa090fb5 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,21 @@ implemented in Rust. ## Current implementation state -This branch establishes the Rust workspace and quality-gate foundation. The -bounded crates compile independently; domain behavior includes immutable -evidence records, topic measurement, and cutoff-safe analysis execution. +The repository currently implements 53 independently documented crates rather +than a full commercial release. The implemented crates include topic measurement +and the analysis engine; they do not claim a complete commercial estimator, +operator workspace, or supported release. + +- `topic_measurement`: the first production topic-measurement crate. It + estimates topic proportions from observed counts, maps those proportions into + additive log-ratio coordinates, and keeps posterior uncertainty attached so + later psychometric models do not treat raw topic proportions as ordinary + Euclidean indicators. +- `analysis_engine`: the first production analysis-run crate. It assembles one + cutoff-safe run from a validated design, documented evidence graph, and + estimator contract; persists the run with the six TEPP clocks; and emits a + typed terminal result. The crate does not claim buyer-visible product + completeness. ```text crates/analysis_engine @@ -16,61 +28,7 @@ crates/assertion_clock crates/available_clock crates/checkpoint_authority crates/citation_edge -The current workspace contains 50 independently documented Rust crates. Each -crate exposes a bounded, tested contract for evidence, temporal semantics, -event and relation reasoning, membership, persistence, simulation, validation, -API exchange, compute planning, or evidence-grounded interpretation. Numerical -and psychometric authority remains on the CPU `f64` reference path; streamed -accelerator plans must preserve the full observation set and fail closed to the -reference path when resources or validation are insufficient. - -These are production contracts, not a claim that the complete commercial -estimator, operator workspace, or supported release already exists. Read the -[product and technical gap baseline](docs/product-technical-gap-baseline.md) -before treating a crate as a shipped product capability. -This branch keeps the Rust workspace quality foundation and the bounded -foundation crates. Domain crates expose only tested contracts: immutable -evidence, six-clock temporal values, event mentions/instances, relations, -membership, persistence, splits, simulation, validation, API DTOs, and the -predicted-versus-observed promotion gate. -This branch establishes the Rust workspace, quality-gate foundation, and the -longitudinal within/between decomposition capability. The eleven bounded crates -compile independently. `longitudinal_core` exposes within/between decomposition -and component RMSE APIs; the remaining crates expose no placeholder production -APIs, and domain behavior for them begins in Task 2 with immutable evidence -identifiers and source records. -This branch establishes the Task 1 Rust workspace and quality-gate foundation. -The twelve bounded crates compile independently but intentionally expose no -The eleven bounded crates compile independently; Task 1 includes the -implemented `encrypted_mapping` crate with AES-256-GCM sealing and -purpose-bound opening, while the remaining domain behavior begins in Task 2 -with immutable evidence identifiers and source records. -The eleven bounded crates compile independently. `derived_sensitivity` inherits -source Restricted/Internal classes onto topic, factor, and relation artifacts -and fails closed on unknown kinds; derivation and blanket PII masking are not -declassification. Other crates still begin domain behavior in Task 2 with -immutable evidence identifiers and source records. - -The eleven bounded crates compile independently but intentionally expose no -placeholder production APIs. Domain behavior begins in Task 2 with immutable -evidence identifiers and source records. - -```text -crates/evidence_core -crates/semantic_core -crates/temporal_core -crates/event_core -crates/relation_graph -crates/membership_core -crates/persistence_postgres -crates/corpus_split -crates/tepp_simulation -crates/validation_core -crates/tepp_api -crates/location_membership -crates/prompt_source -crates/corpus_background -crates/modality_source +crates/compute_backend crates/copied_text crates/copy_identity crates/corpus_background @@ -79,6 +37,7 @@ crates/cutoff_clock crates/derived_sensitivity crates/document_clocks crates/encrypted_mapping +crates/episode_membership crates/event_clock crates/event_core crates/evidence_core @@ -104,7 +63,6 @@ crates/relation_graph crates/retrospective_edge crates/revision_order crates/semantic_core -crates/operational_log crates/service_tls crates/stopword_deletion crates/style_source @@ -118,14 +76,6 @@ crates/tepp_simulation crates/topic_lineage crates/topic_measurement crates/validation_core -crates/network_analysis -crates/interpretation_gateway -crates/model_selection -crates/checkpoint_authority -crates/compute_backend -crates/episode_membership -crates/membership_target - ``` ## Local verification diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index 43bd961a..8ae19fa5 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -47,7 +47,7 @@ The full APA 7th standards/literature register remains `docs/research/standards- | PostgreSQL bitemporal/lineage persistence | ADR 0013; Architecture/ERD | `persistence_postgres` migration contracts, in-memory adapters, live SQL session/document SQL port, tenant RLS (`0002` + session GUC/role helpers), `DATABASE_URL` SQLx gate, optional session-affine `live-sqlx` `PgPool` driver, exact-head live PostgreSQL CI with isolation proof, append-only immutability triggers (`0004`), temporal interval ordering CHECKs (`0005`), typed membership assignment (`0006` implemented-main), event-relation/mention/instance SQL (#37–#39 implemented-main), source-artifact SQL (#40 implemented-main), audit-event SQL (#41 implemented-main), concurrent document-write stress (#43 implemented-main), backup/restore integrity revalidation (#44 implemented-main), `revision_order` later-revision system-time ordering implemented-main, entity/project target SQL on PR #131; remaining physical ERD constraints | partial | | known-truth temporal/event simulation manifests | PRD; TRD; Test Strategy | `tepp_simulation` on protected main; recovery metrics in `validation_core` | implemented-main | | versioned service/API contracts and exports | PRD; API contract; ADR 0011/0013 | `tepp_api` analysis-run/export/JSON-LD/GraphML contracts on protected main (PR #21); LineageWeave loopback contracts and request-bound terminal result are composed on the active product branch; production TLS remaining | partial | -| executable cutoff-safe analysis runs | ADR 0012/0021; temporal research; API terminal-result contract | `analysis_engine` availability cutoff, snapshot binding, multiple-membership aggregation, digest-bound readiness artifact, and `tepp.trsl_topic_lineage.v1` execution through `topic_measurement`; synthetic recovery plus tamper/non-convergence tests and exact coverage on the active product branch | active-PR | +| executable cutoff-safe analysis runs | ADR 0012/0022; temporal research; API terminal-result contract | `analysis_engine` availability cutoff, snapshot binding, multiple-membership aggregation, digest-bound readiness artifact, and `tepp.trsl_topic_lineage.v1` execution through `topic_measurement`; synthetic recovery plus tamper/non-convergence tests and exact coverage on the active product branch | active-PR | | immutable split/run/reproducibility manifests | ADR 0013; ERD | `tepp_api` reproducibility manifest contract on protected main; `persistence_postgres` append-only SQL insert/lookup for `reproducibility_manifest`, `corpus_split_manifest`, `model_run`, and `model_artifact` (migration `0003`); full physical ERD constraints remaining | partial | | multilingual shared latent semantic space | PRD; ADR 0004; ADR 0020 | `semantic_core` span-grounded units (active-PR); concept dictionary and shared latent estimator remaining | active-PR | | TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | `topic_measurement` stable ALR/ILR coordinates and bounded CPU `f64` reference estimator on the active product branch; calibrated posterior promotion, method effects, persistence, and accelerated backends remaining | partial | diff --git a/docs/adr/0021-deterministic-analysis-run-execution.md b/docs/adr/0022-deterministic-analysis-run-execution.md similarity index 98% rename from docs/adr/0021-deterministic-analysis-run-execution.md rename to docs/adr/0022-deterministic-analysis-run-execution.md index 123902df..cb7fe646 100644 --- a/docs/adr/0021-deterministic-analysis-run-execution.md +++ b/docs/adr/0022-deterministic-analysis-run-execution.md @@ -1,4 +1,4 @@ -# ADR 0021 — Deterministic cutoff-safe analysis-run execution +# ADR 0022 — Deterministic cutoff-safe analysis-run execution **Decision status:** Accepted **Implementation maturity:** active-PR — composed on the active product branch; not implemented-main diff --git a/docs/adr/README.md b/docs/adr/README.md index 534c9d3f..7b3a3798 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -27,6 +27,7 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0019](0019-project-history-wire-size-symmetry.md) | Symmetric LineageWeave project-history wire-size enforcement | Accepted | active-PR | Request serialization and generated project-history projections share bounded size rules. | | [0020](0020-span-grounded-semantic-units.md) | Span-grounded semantic units; language tags are not identity | Accepted | active-PR | First ADR 0004 production slice; concept alignment, invariance, and topic estimation are not claimed. | | [0021](0021-lineageweave-project-history-boundary.md) | LineageWeave project-history service boundary | Accepted | active-PR | Credential-free bounded project-history API preserves LineageWeave authorization ownership. | +| [0022](0022-deterministic-analysis-run-execution.md) | Deterministic cutoff-safe analysis-run execution | Accepted | active-PR | Closes the first executable product path from accepted run to digest-bound terminal result without claiming estimator authority. | | [0001](0001-rust-first-modular-msa.md) | Rust-first numerical core and CPU `f64` reference | Accepted | partial | ADR 0011 owns cross-service/MSA authority; 0001 retains numerical/backend authority. | | [0002](0002-six-clock-temporal-semantics.md) | Six-clock temporal semantics and fail-closed historical leakage prevention | Accepted | partial | Typed clocks/intervals (merged PR #8), Allen/path-consistency (merged PR #9), the clock-identity/revision-order/document-completeness gates (`system_clock`, `event_clock`, `assertion_clock`, `cutoff_clock`, `available_clock`, `document_clocks`, `revision_order`), and the provenance/ordering gates (`citation_edge`, `support_edge`, `retrospective_edge`) are implemented-main; superseded PRs #5/#6 are historical lineage only; remaining graph/split enforcement stays accepted-target. | | [0003](0003-relational-event-multiple-membership.md) | Relational event ontology and time-varying cross-classified multiple membership | Accepted | partial | Membership network/roles, Kish ESS, nested ICC, subevent parent-window containment (`subevent_containment`), the forward-only relation graph, evidential-vs-transition identity (`support_edge`), inferred-versus-observed identity (`inferred_status`), retrospective-reporting identity (`retrospective_edge`), summary-versus-source identity (`summarizes_edge`), copy-versus-source identity (`copy_identity`), location-versus-entity/language identity (`location_membership`), and IPO event-time order (`outcome_order`) are implemented-main; typed target-kind identity in `membership_target` is on PR #131; full multilevel estimators and persistence remain accepted-target. ADR 0016 owns event-intelligence tasks. | @@ -59,8 +60,8 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0017](0017-hourly-contextual-orchestrator-gateway.md) | Hourly contextual-orchestrator gateway and all-provider model discovery | Accepted | active-PR | Keeps proposal-model execution behind a pinned loopback gateway while preserving independent verifier, publisher, reviewer, and merge authority. | | [0018](0018-consumer-scoped-analysis-run-ingress.md) | Consumer-scoped modular analysis-run ingress | Accepted | active-PR | Narrows ADR 0011 for the closed consumer registry, credential-free exchange, and consumer-qualified idempotency namespace; production TLS remains separate. | | [0019](0019-project-history-wire-size-symmetry.md) | Symmetric project-history wire-size enforcement | Accepted | active-PR | Narrows ADR 0008 for request serialization and generated LineageWeave project-history projections. | -| [0020](0020-lineageweave-project-history-boundary.md) | LineageWeave project-history service boundary | Accepted | active-PR | Narrows ADR 0011 for the credential-free bounded project-history API and preserves LineageWeave authorization ownership. | -| [0021](0021-deterministic-analysis-run-execution.md) | Deterministic cutoff-safe analysis-run execution | Accepted | active-PR | Closes the first executable product path from accepted run to digest-bound terminal result without claiming estimator authority. | +| [0021](0021-lineageweave-project-history-boundary.md) | LineageWeave project-history service boundary | Accepted | active-PR | Narrows ADR 0011 for the credential-free bounded project-history API and preserves LineageWeave authorization ownership. | +| [0022](0022-deterministic-analysis-run-execution.md) | Deterministic cutoff-safe analysis-run execution | Accepted | active-PR | Closes the first executable product path from accepted run to digest-bound terminal result without claiming estimator authority. | ## Decision ownership summary @@ -85,9 +86,8 @@ Use the narrowest owning ADR when decisions overlap: - **hourly proposal gateway and provider discovery:** ADR 0017. - **modular consumer admission / replay identity:** ADR 0018. - **project-history wire-size symmetry:** ADR 0019. -- **LineageWeave project-history service boundary:** ADR 0020. -- **accepted-run execution and terminal artifact production:** ADR 0021. - **LineageWeave project-history service boundary:** ADR 0021. +- **accepted-run execution and terminal artifact production:** ADR 0022. ## Change and supersession rule diff --git a/scripts/validate_documentation.py b/scripts/validate_documentation.py index c00fcebb..7b7364c9 100644 --- a/scripts/validate_documentation.py +++ b/scripts/validate_documentation.py @@ -44,9 +44,9 @@ "docs/adr/0017-hourly-contextual-orchestrator-gateway.md", "docs/adr/0018-consumer-scoped-analysis-run-ingress.md", "docs/adr/0019-project-history-wire-size-symmetry.md", - "docs/adr/0020-lineageweave-project-history-boundary.md", - "docs/adr/0021-deterministic-analysis-run-execution.md", + "docs/adr/0020-span-grounded-semantic-units.md", "docs/adr/0021-lineageweave-project-history-boundary.md", + "docs/adr/0022-deterministic-analysis-run-execution.md", "docs/product/prd-v0.4-approved.md", PRODUCT_TECHNICAL_GAP_BASELINE, "docs/roadmaps/2026-08-05-tepp-delivery-roadmap.md",