diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 393e6f59..37a25c17 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,6 +61,7 @@ boundaries above remain the target modular MSA architecture. | `tepp_simulation` | known-truth temporal/event data generation | | `validation_core` | RMSE, bias, coverage, graph, and Monte Carlo metrics | | `tepp_api` | versioned DTO, schema, and export contracts | +| `stopword_deletion` | default stopword deletion is not a valid method for repeated report language | | `copy_identity` | a template copy is not the source document and not a state transition | | `intake_authorization` | untrusted intake fails closed without a grant; bounds are not authorization | | `summarizes_edge` | a summary is not a state transition and not the source document | diff --git a/CHANGELOG.md b/CHANGELOG.md index fe4fdd78..18ee773b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added +- `stopword_deletion` method gate: a default or global stopword list cannot erase repeated report language; recovered deletion kinds match known truth at a higher computed rate than collapsing every token treatment to stopword deletion (ADR 0004/0012). +- `persistence_postgres` retention/deletion/legal-hold (migration `0007`): policy rows, legal holds that block completed deletion, evidence tombstones without raw-source restore, analysis exclusion only for `logical_revocation`/`identity_tombstone` (not `cache_export_removal`), and deletion requests bound to the cited retention policy's tenant/class/purpose. - `copy_identity` identity gate: a template or pasted copy cannot reuse the source document identity or become a state transition; recovered copy kinds match known truth at a higher computed rate than collapsing every copy to the source (ADR 0003). - `persistence_postgres` retention/deletion/legal-hold (migration `0007`): policy rows, legal holds that block completed deletion, evidence tombstones without raw-source restore, analysis exclusion only for `logical_revocation`/`identity_tombstone` (not `cache_export_removal`), and deletion requests bound to the cited retention policy's tenant/class/purpose. - `provider_receipt` disclosure receipt: records provider field codes and diff --git a/Cargo.lock b/Cargo.lock index 18f8df11..ee47318d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1462,6 +1462,10 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stopword_deletion" +version = "0.1.0" + [[package]] name = "stringprep" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index 34010482..6add5026 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "crates/tepp_simulation", "crates/validation_core", "crates/tepp_api", + "crates/stopword_deletion", "crates/copy_identity", "crates/provider_receipt", "crates/intake_authorization", @@ -54,6 +55,7 @@ default-members = [ "crates/tepp_simulation", "crates/validation_core", "crates/tepp_api", + "crates/stopword_deletion", "crates/copy_identity", "crates/provider_receipt", "crates/intake_authorization", diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index cea9708b..a9c90e02 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -38,6 +38,7 @@ TEPP's approved PRD v0.4 and implementation plan are the primary product baselin | Mention-confidence Brier doctoring | [`docs/research/mention-confidence-brier.md`](docs/research/mention-confidence-brier.md) | | Event-intelligence status-gate doctoring | [`docs/research/event-intelligence-status-gates.md`](docs/research/event-intelligence-status-gates.md) | | Retention/deletion/legal-hold doctoring | [`docs/research/retention-deletion-legal-hold.md`](docs/research/retention-deletion-legal-hold.md) | +| Stopword-deletion doctoring | [`docs/research/stopword-deletion.md`](docs/research/stopword-deletion.md) | | Provider-payload minimization doctoring | [`docs/research/provider-payload-minimization.md`](docs/research/provider-payload-minimization.md) | | Adaptive orchestration router doctoring | [`docs/research/adaptive-orchestration-router.md`](docs/research/adaptive-orchestration-router.md) | | Hourly NIM OpenCode doctoring | [`docs/doctoring/hourly-nim-opencode-development.md`](docs/doctoring/hourly-nim-opencode-development.md) | diff --git a/README.md b/README.md index 77dcd0ed..281ce609 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ crates/corpus_split crates/tepp_simulation crates/validation_core crates/tepp_api +crates/stopword_deletion crates/copy_identity crates/provider_receipt crates/intake_authorization diff --git a/crates/stopword_deletion/Cargo.toml b/crates/stopword_deletion/Cargo.toml new file mode 100644 index 00000000..a75246f2 --- /dev/null +++ b/crates/stopword_deletion/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "stopword_deletion" +description = "Default stopword deletion is not a valid method for repeated report language." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[lints] +workspace = true diff --git a/crates/stopword_deletion/src/error.rs b/crates/stopword_deletion/src/error.rs new file mode 100644 index 00000000..d3c334da --- /dev/null +++ b/crates/stopword_deletion/src/error.rs @@ -0,0 +1,48 @@ +//! Fail-closed stopword-deletion errors. + +use std::fmt; + +/// A fail-closed stopword-deletion error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum StopwordDeletionError { + /// A default or global stopword list was used as deletion. + DefaultStopwordDeletion, + /// A recovery slice was empty or length-mismatched. + InvalidDeletionPayload, +} + +impl fmt::Display for StopwordDeletionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::DefaultStopwordDeletion => { + "default stopword deletion is not a valid method for repeated report language" + } + Self::InvalidDeletionPayload => "invalid stopword-deletion payload", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for StopwordDeletionError {} + +#[cfg(test)] +mod tests { + use super::StopwordDeletionError; + + #[test] + fn error_messages_are_stable() { + for (error, message) in [ + ( + StopwordDeletionError::DefaultStopwordDeletion, + "default stopword deletion is not a valid method for repeated report language", + ), + ( + StopwordDeletionError::InvalidDeletionPayload, + "invalid stopword-deletion payload", + ), + ] { + assert_eq!(error.to_string(), message); + } + } +} diff --git a/crates/stopword_deletion/src/kind.rs b/crates/stopword_deletion/src/kind.rs new file mode 100644 index 00000000..f3bbd76e --- /dev/null +++ b/crates/stopword_deletion/src/kind.rs @@ -0,0 +1,114 @@ +//! Deletion methods that cannot silently erase repeated report language. + +use crate::StopwordDeletionError; + +/// Closed vocabulary of deletion versus explicit method-source treatments. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DeletionKind { + /// A default or global stopword list applied as deletion. + DefaultStopwordList, + /// Repeated language kept as explicit method/background structure. + ExplicitMethodSource, +} + +impl DeletionKind { + /// Return the stable wire kind name. + #[must_use] + pub const fn wire_name(self) -> &'static str { + match self { + Self::DefaultStopwordList => "default_stopword_list", + Self::ExplicitMethodSource => "explicit_method_source", + } + } + + /// Parse a stable wire kind name. + /// + /// # Errors + /// + /// Returns [`StopwordDeletionError::InvalidDeletionPayload`] for unrecognized + /// names. + pub fn from_wire_name(name: &str) -> Result { + match name { + "default_stopword_list" => Ok(Self::DefaultStopwordList), + "explicit_method_source" => Ok(Self::ExplicitMethodSource), + _ => Err(StopwordDeletionError::InvalidDeletionPayload), + } + } +} + +/// Refuse to treat a default stopword list as a valid deletion method. +/// +/// # Errors +/// +/// Returns [`StopwordDeletionError::DefaultStopwordDeletion`] when `kind` is +/// [`DeletionKind::DefaultStopwordList`]. +pub fn refuse_default_stopword_deletion(kind: DeletionKind) -> Result<(), StopwordDeletionError> { + match kind { + DeletionKind::DefaultStopwordList => Err(StopwordDeletionError::DefaultStopwordDeletion), + DeletionKind::ExplicitMethodSource => Ok(()), + } +} + +/// Fraction of recovered deletion kinds that match known truth. +/// +/// # Errors +/// +/// Returns [`StopwordDeletionError::InvalidDeletionPayload`] when either slice +/// is empty or the lengths differ. +pub fn identity_recovery_rate( + truth: &[DeletionKind], + decided: &[DeletionKind], +) -> Result { + if truth.is_empty() || truth.len() != decided.len() { + return Err(StopwordDeletionError::InvalidDeletionPayload); + } + let mut matches = 0_u32; + for (truth_kind, decided_kind) in truth.iter().zip(decided) { + if truth_kind == decided_kind { + matches += 1; + } + } + Ok(f64::from(matches) / truth.len() as f64) +} + +#[cfg(test)] +mod tests { + use super::{DeletionKind, identity_recovery_rate, refuse_default_stopword_deletion}; + use crate::StopwordDeletionError; + + #[test] + fn local_branches_cover_kinds_payloads_and_wire_names() { + assert_eq!( + refuse_default_stopword_deletion(DeletionKind::DefaultStopwordList), + Err(StopwordDeletionError::DefaultStopwordDeletion) + ); + refuse_default_stopword_deletion(DeletionKind::ExplicitMethodSource).expect("source"); + for kind in [ + DeletionKind::DefaultStopwordList, + DeletionKind::ExplicitMethodSource, + ] { + assert_eq!( + DeletionKind::from_wire_name(kind.wire_name()).expect("round-trip"), + kind + ); + } + assert_eq!( + DeletionKind::from_wire_name("tfidf_weight"), + Err(StopwordDeletionError::InvalidDeletionPayload) + ); + let matched = identity_recovery_rate( + &[DeletionKind::ExplicitMethodSource], + &[DeletionKind::ExplicitMethodSource], + ) + .expect("rate"); + assert!((matched - 1.0).abs() < f64::EPSILON); + assert_eq!( + identity_recovery_rate(&[], &[]), + Err(StopwordDeletionError::InvalidDeletionPayload) + ); + assert_eq!( + identity_recovery_rate(&[DeletionKind::DefaultStopwordList], &[]), + Err(StopwordDeletionError::InvalidDeletionPayload) + ); + } +} diff --git a/crates/stopword_deletion/src/lib.rs b/crates/stopword_deletion/src/lib.rs new file mode 100644 index 00000000..58fb6702 --- /dev/null +++ b/crates/stopword_deletion/src/lib.rs @@ -0,0 +1,20 @@ +#![forbid(unsafe_code)] +#![deny(missing_docs)] +#![allow(clippy::cast_precision_loss)] +//! Default stopword deletion is not a valid method for repeated report language. +//! +//! A global stopword list cannot erase boilerplate. Repeated template, section, +//! copied-text, style, modality, and corpus-background wording stays explicit +//! method/background structure (ADR 0004/0012). + +mod error; +mod kind; + +/// Fail-closed stopword-deletion errors. +pub use error::StopwordDeletionError; +/// Closed vocabulary of deletion versus explicit method-source treatments. +pub use kind::DeletionKind; +/// Fraction of recovered deletion kinds that match known truth. +pub use kind::identity_recovery_rate; +/// Refuse to treat a default stopword list as a valid deletion method. +pub use kind::refuse_default_stopword_deletion; diff --git a/crates/stopword_deletion/tests/crate_contract.rs b/crates/stopword_deletion/tests/crate_contract.rs new file mode 100644 index 00000000..f5d80d24 --- /dev/null +++ b/crates/stopword_deletion/tests/crate_contract.rs @@ -0,0 +1,7 @@ +//! Integration contract for the `stopword_deletion` package identity. + +#[test] +fn package_identity_is_stable() { + let observed = std::hint::black_box(env!("CARGO_PKG_NAME")); + assert_eq!(observed, "stopword_deletion"); +} diff --git a/crates/stopword_deletion/tests/stopword_deletion_contract.rs b/crates/stopword_deletion/tests/stopword_deletion_contract.rs new file mode 100644 index 00000000..cf451352 --- /dev/null +++ b/crates/stopword_deletion/tests/stopword_deletion_contract.rs @@ -0,0 +1,64 @@ +//! Default stopword deletion cannot erase repeated report language. + +use stopword_deletion::{ + DeletionKind, StopwordDeletionError, identity_recovery_rate, refuse_default_stopword_deletion, +}; + +#[test] +fn a_default_stopword_list_cannot_delete_repeated_report_language() { + assert_eq!( + refuse_default_stopword_deletion(DeletionKind::DefaultStopwordList), + Err(StopwordDeletionError::DefaultStopwordDeletion) + ); + refuse_default_stopword_deletion(DeletionKind::ExplicitMethodSource).expect("source"); +} + +#[test] +fn recovered_kinds_match_known_truth_better_than_a_stopword_collapse() { + let truth = [ + DeletionKind::ExplicitMethodSource, + DeletionKind::ExplicitMethodSource, + DeletionKind::DefaultStopwordList, + ]; + let recovered = truth; + let collapsed = [ + DeletionKind::DefaultStopwordList, + DeletionKind::DefaultStopwordList, + DeletionKind::DefaultStopwordList, + ]; + let recovered_rate = identity_recovery_rate(&truth, &recovered).expect("recovered"); + let collapsed_rate = identity_recovery_rate(&truth, &collapsed).expect("collapsed"); + let expected = { + let mut matches = 0_u32; + for (truth_kind, decided_kind) in truth.iter().zip(recovered.iter()) { + if truth_kind == decided_kind { + matches += 1; + } + } + f64::from(matches) / f64::from(u32::try_from(truth.len()).expect("len")) + }; + assert!((recovered_rate - expected).abs() < f64::EPSILON); + assert!(recovered_rate > collapsed_rate); +} + +#[test] +fn empty_or_mismatched_kind_payloads_fail_closed() { + assert_eq!( + identity_recovery_rate(&[], &[]), + Err(StopwordDeletionError::InvalidDeletionPayload) + ); + assert_eq!( + identity_recovery_rate(&[DeletionKind::DefaultStopwordList], &[]), + Err(StopwordDeletionError::InvalidDeletionPayload) + ); + assert_eq!( + identity_recovery_rate( + &[ + DeletionKind::DefaultStopwordList, + DeletionKind::ExplicitMethodSource + ], + &[DeletionKind::DefaultStopwordList] + ), + Err(StopwordDeletionError::InvalidDeletionPayload) + ); +} diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index 169b7c80..9fd291e7 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -23,8 +23,8 @@ The full APA 7th standards/literature register remains `docs/research/standards- | immutable split/run/reproducibility manifests | ADR 0013; ERD | `tepp_api` reproducibility manifest and corpus-split leakage-audit wire (`CorpusSplitManifest` v1) on this PR; `persistence_postgres` append-only SQL insert/lookup for `reproducibility_manifest`, `corpus_split_manifest`, `model_run`, and `model_artifact` (migration `0003`); full physical ERD constraints remaining | partial | | multilingual shared latent semantic space | PRD; ADR 0004 | future semantic/concept/topic crates | accepted-target | | TRSL-TM temporal/relational topic posterior and backend compatibility | ADR 0012; ADR 0004 | future `topic_measurement` | accepted-target | -| global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | `topic_lineage` activity/dormancy/reactivation identity on the active PR; birth/split/merge remaining | active-PR | -| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | future semantic/method-source model | accepted-target | +| global P0 topic identity with activity/dormancy/reactivation | ADR 0012 | future topic lineage/activity state | accepted-target | +| no default stopword deletion / no TF-IDF-BM25 inferential weighting | ADR 0004/0012; PRD/TRD | `stopword_deletion` default-list refusal on the active PR; TF-IDF/BM25 inferential-weight refusal remains accepted-target | partial | | report template/section/copied/style/modality method effects | ADR 0004/0012; PRD/TRD | simulation truth factors implemented; estimator-side method model remains future | partial | | candidate K statistical/Pareto gates + blinded LLM review | ADR 0012; research | future `model_selection` | accepted-target | | compositional topic correlation / stable clustering | ADR 0005/0012; research | future `network_analysis` | accepted-target | diff --git a/docs/adr/0004-shared-multilingual-latent-space.md b/docs/adr/0004-shared-multilingual-latent-space.md index f5cdc730..06f42ada 100644 --- a/docs/adr/0004-shared-multilingual-latent-space.md +++ b/docs/adr/0004-shared-multilingual-latent-space.md @@ -1,7 +1,7 @@ # ADR 0004 — Shared multilingual latent semantic space **Decision status:** Accepted -**Implementation maturity:** accepted-target +**Implementation maturity:** partial — default stopword-deletion refusal is `stopword_deletion` on the active PR; shared-space estimators, language profiles, and TF-IDF/BM25 inferential-weight refusal remain accepted-target **Date:** 2026-08-05 **Supersedes:** None. ADR 0012 governs the complete topic-estimator/backend/global-topic contract built on this multilingual measurement decision. diff --git a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md index 53303e1f..e3b70c71 100644 --- a/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md +++ b/docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md @@ -1,6 +1,7 @@ # ADR 0012 — Temporal Relational Shared-Latent Topic Measurement **Decision status:** Accepted +**Implementation maturity:** partial — default stopword-deletion refusal is `stopword_deletion` on the active PR; topic estimator, global topic identity, method-effect model, and TF-IDF/BM25 inferential-weight refusal remain accepted-target **Implementation maturity:** active-PR — `topic_lineage` keeps one P0 identity across active/dormant/reactivated states; remaining TRSL-TM estimator, method effects, and backend interchange remain accepted-target **Implementation maturity:** active-PR — `network_analysis` refuses raw-simplex Euclidean geometry and scores cluster pair precision/recall; remaining TRSL-TM estimator, global topic identity, method effects, and backend interchange remain accepted-target **Implementation maturity:** active-PR — `model_selection` statistical/Pareto candidate-`K` gates and known-`K` RMSE live in the new crate; remaining TRSL-TM estimator, global topic identity, method effects, and backend interchange remain accepted-target diff --git a/docs/adr/README.md b/docs/adr/README.md index 1e385e42..058f5472 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -25,6 +25,8 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0002](0002-six-clock-temporal-semantics.md) | Six-clock temporal semantics and fail-closed historical leakage prevention | Accepted | active-PR | Typed clocks/intervals are implemented-main; `document_clocks` refuses omitted assertion/document time on the active PR. Later graph/split enforcement remains target work. | | [0002](0002-six-clock-temporal-semantics.md) | Six-clock temporal semantics and fail-closed historical leakage prevention | Accepted | active-PR | Provenance-vs-transition gate in `citation_edge` on the active PR; remaining graph/split enforcement stays accepted-target. | | [0003](0003-relational-event-multiple-membership.md) | Relational event ontology and time-varying cross-classified multiple membership | Accepted | partial | Weighted time-varying membership network/roles are active-PR (PR #12); full multilevel estimators, graph ontology, and persistence remain accepted-target. ADR 0016 owns event-intelligence tasks. | +| [0004](0004-shared-multilingual-latent-space.md) | One shared multilingual latent space with explicit invariance status | Accepted | partial | Default stopword-deletion refusal is `stopword_deletion` on the active PR; ADR 0012 owns the full topic-estimator/backend/global-topic contract. | +| [0005](0005-posterior-esem-dsem.md) | Posterior-aware ESEM/DSEM and valid compositional coordinates | Accepted | accepted-target | Downstream psychometric authority; upstream topic/network model is clarified by ADR 0012. | | [0002](0002-six-clock-temporal-semantics.md) | Six-clock temporal semantics and fail-closed historical leakage prevention | Accepted | partial | Typed clocks/intervals (merged PR #8) and Allen/path-consistency (merged PR #9) are implemented-main; superseded PRs #5/#6 are historical lineage only. Downstream estimator and remaining persistence-policy uses stay with their owning ADRs. | | [0003](0003-relational-event-multiple-membership.md) | Relational event ontology and time-varying cross-classified multiple membership | Accepted | partial | Membership network/roles and forward-only relation graph are implemented-main; multilevel estimators remain accepted-target. This is an ontology/membership contract, not a statistical REM paper. ADR 0016 owns event-intelligence tasks. | | [0001](0001-rust-first-modular-msa.md) | Rust-first numerical core and CPU `f64` reference | Accepted | partial | Workspace foundation is implemented-main; checkpoint-versus-estimator authority is `checkpoint_authority` on the active PR. ADR 0011 owns cross-service/MSA authority; 0001 retains numerical/backend authority. | @@ -51,6 +53,8 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding implemented-main; evidence-bounded `interpretation_gateway` is on the active PR; live NIM execution and production ablation evidence remain accepted-target. | | [0010](0010-adaptive-llm-orchestration.md) | Adaptive LLM orchestration and test-time compute | Accepted | partial | `tepp_api` router/ablation/orchestrator binding implemented-main; live NIM execution and production ablation evidence remain accepted-target. | | [0011](0011-standalone-modular-msa-boundary.md) | Standalone operation and modular CWL MSA boundary | Accepted | partial | Owns cross-service persistence/credential/API authority; no direct cross-service application-table coupling. | +| [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | partial | Default stopword-deletion refusal is `stopword_deletion` on the active PR; topic backend, global topic identity, method effects, K/model-selection, and TF-IDF/BM25 inferential-weight refusal remain accepted-target. | +| [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold on the active PR; remaining physical ERD/backup accepted-target. | | [0012](0012-temporal-relational-shared-latent-topic-measurement.md) | Temporal Relational Shared-Latent Topic Measurement (TRSL-TM) | Accepted | accepted-target | Owns topic backend compatibility, global topic identity, method effects, K/model-selection prerequisites, and compositional topic coordinates. | | [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, and `0006` membership implemented-main; `0007` retention/deletion/legal-hold implemented-main; document revision system-time order in `revision_order` is active on this PR; remaining physical ERD/backup accepted-target. | | [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, tenant RLS, `0006` membership, `0007` retention/deletion/legal-hold, and backup/restore integrity revalidation implemented-main; remaining physical ERD/DR-runbook depth accepted-target. | diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index 1766a42c..36dffe08 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -36,7 +36,9 @@ Bianchi, F., Terragni, S., Hovy, D., Nozza, D., & Fersini, E. (2021). Cross-ling Nguyen, T. P., Minh, N. V., Nguyen, T., Van, L. N., Nguyen, D. A., Sang, D. V., & Le, T. (2025). XTRA: Cross-lingual topic modeling with topic and representation alignments. In *Findings of the Association for Computational Linguistics: EMNLP 2025*. Association for Computational Linguistics. -TEPP retains a logistic-normal CPU reference while allowing adapter backends that satisfy shared-latent, posterior, temporal, relational, and measurement-invariance contracts. `topic_lineage` keeps one global topic identity when activity becomes dormant or reactivated. +Schofield, A., Magnusson, M., & Mimno, D. (2017). Pulling out the stops: Rethinking stopword removal for topic models. In *Proceedings of the 15th Conference of the European Chapter of the Association for Computational Linguistics: Volume 2, Short Papers* (pp. 432–436). Association for Computational Linguistics. https://doi.org/10.18653/v1/E17-2069 + +TEPP retains a logistic-normal CPU reference while allowing adapter backends that satisfy shared-latent, posterior, temporal, relational, and measurement-invariance contracts. Default or global stopword deletion is not a valid method for removing repeated report language; `stopword_deletion` refuses that treatment so boilerplate stays explicit method/background structure (Schofield, Magnusson, & Mimno, 2017). ## Topic-model evaluation and LLM judges diff --git a/docs/research/stopword-deletion.md b/docs/research/stopword-deletion.md new file mode 100644 index 00000000..dd0a6003 --- /dev/null +++ b/docs/research/stopword-deletion.md @@ -0,0 +1,33 @@ +# Default stopword deletion is not a valid method (doctoring) + +## Scope + +`stopword_deletion` keeps a default or global stopword list from erasing +repeated report language. Recovery is the computed share of recovered +deletion kinds that match known truth. + +This slice does not persist tokens, allocate migration `0008`, apply +TF-IDF/BM25 inferential weights, or replace `method_effects`. + +## Authority + +### Normative TEPP contract + +- `docs/adr/0004-shared-multilingual-latent-space.md` — + stopword deletion is not the default; repeated template/section/copied + wording is modeled as method/background structure. +- `docs/adr/0012-temporal-relational-shared-latent-topic-measurement.md` — + stopword deletion is not the default preprocessing rule. + +### Supporting literature + +Schofield, Magnusson, and Mimno (2017) show that stopword removal is not +a harmless default for topic models. A global list can remove +substantive terms and hide the method-source structure TEPP must keep +explicit. + +Schofield, A., Magnusson, M., & Mimno, D. (2017). Pulling out the stops: +Rethinking stopword removal for topic models. In *Proceedings of the 15th +Conference of the European Chapter of the Association for Computational +Linguistics: Volume 2, Short Papers* (pp. 432–436). Association for +Computational Linguistics. https://doi.org/10.18653/v1/E17-2069 diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index 3a002d32..c5de91ed 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -67,6 +67,7 @@ This report tracks exact-head scientific and engineering evidence required befor | Adaptive orchestration router | `tepp_api` | partial | — | mode selection, document-control denial, ablation, credential-free bind; live NIM execution remains future | ADR 0010; `docs/research/adaptive-orchestration-router.md` | | CWL modular connectors | `docs/connectors/*` | implemented-main | — | contract docs + examples | PR #22; live HTTP ports remaining | | Release SBOM/provenance generator | `scripts/release_evidence.py` | partial | — | generate+validate in CI | Task 13 partial / PR #28 | +| Default stopword deletion refusal | `stopword_deletion` | accepted-target | active PR | refuse default/global stopword lists + recovery vs stopword collapse | ADR 0004/0012 | ## Scientific acceptance checklist (foundation) diff --git a/scripts/check_workspace_contract.py b/scripts/check_workspace_contract.py index a32d6f4b..e3dfb8db 100644 --- a/scripts/check_workspace_contract.py +++ b/scripts/check_workspace_contract.py @@ -23,6 +23,7 @@ "tepp_simulation", "validation_core", "tepp_api", + "stopword_deletion", "copy_identity", "provider_receipt", "intake_authorization",