From b056ac97e8110b271b8e4db73ac5c11e34334919 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 02:28:13 +0900 Subject: [PATCH 1/2] feat(event): score mention confidence with a known-truth Brier rule Perfect forecasts recover Brier 0 and constant 0.5 recovers 0.25. Empty or mismatched streams fail closed. No new migration. --- CHANGELOG.md | 1 + DOCUMENTATION.md | 1 + crates/event_core/src/confidence.rs | 27 ++++++++++++++ crates/event_core/src/lib.rs | 2 + .../tests/confidence_calibration_contract.rs | 37 +++++++++++++++++++ docs/TRACEABILITY.md | 2 +- docs/research/mention-confidence-brier.md | 27 ++++++++++++++ docs/validation/temporal-event-foundation.md | 1 + 8 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 crates/event_core/tests/confidence_calibration_contract.rs create mode 100644 docs/research/mention-confidence-brier.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 9abfea7e..97bca31c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added +- `event_core` mention-confidence Brier score: known-truth binary outcomes recover a computed Brier of 0 for perfect forecasts and 0.25 for constant 0.5, with empty or mismatched streams failing closed. - `persistence_postgres` typed membership assignment (migration `0006`): `entity_record`, `project_record`, and `text_segment` plus exactly-one observed-unit and target constraints that replace the polymorphic `membership_target_id` stub, with SQL insert/lookup, fail-closed inverted-window and backslash-label refusal, and live proof that one document persists two entity memberships and one project membership. - Actions workflow fleet auditor (`scripts/actions_workflow_fleet.py`): paginated registry inventory bound to the exact default-branch SHA/tree, classification of present/orphan/disabled/GitHub-dynamic identities, and fail-closed orphan disable that confirms GitHub's official `disabled_manually` state. - `persistence_postgres` temporal interval ordering migration (`0005`): multi-word CHECK constraints on `document_record`, `event_instance`, and `membership_assignment` that reject inverted valid/system windows and non-positive document revisions while preserving open-ended NULL upper bounds and equal point bounds; catalog validation and live inverted-window proof. diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 230c5abe..d98384e0 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -33,6 +33,7 @@ TEPP's approved PRD v0.4 and implementation plan are the primary product baselin | Hourly NIM product-development operations | [`docs/operations/HOURLY_NIM_PRODUCT_DEVELOPMENT.md`](docs/operations/HOURLY_NIM_PRODUCT_DEVELOPMENT.md) | | Actions workflow fleet audit | [`docs/operations/ACTIONS_WORKFLOW_FLEET.md`](docs/operations/ACTIONS_WORKFLOW_FLEET.md) | | Actions fleet research doctoring | [`docs/research/actions-workflow-fleet.md`](docs/research/actions-workflow-fleet.md) | +| Mention-confidence Brier doctoring | [`docs/research/mention-confidence-brier.md`](docs/research/mention-confidence-brier.md) | | Hourly NIM OpenCode doctoring | [`docs/doctoring/hourly-nim-opencode-development.md`](docs/doctoring/hourly-nim-opencode-development.md) | | Change history | [`CHANGELOG.md`](CHANGELOG.md) | diff --git a/crates/event_core/src/confidence.rs b/crates/event_core/src/confidence.rs index d16597eb..f2bf42bb 100644 --- a/crates/event_core/src/confidence.rs +++ b/crates/event_core/src/confidence.rs @@ -39,6 +39,29 @@ impl EventConfidence { } } +/// Mean squared error of mention probabilities against binary truth. +/// +/// # Errors +/// +/// Returns [`EventError::InvalidWirePayload`] when the slices are empty or +/// have unequal length. +pub fn mention_brier_score( + forecasts: &[EventConfidence], + outcomes: &[bool], +) -> Result { + if forecasts.is_empty() || forecasts.len() != outcomes.len() { + return Err(EventError::InvalidWirePayload); + } + let mut square_sum = 0.0_f64; + for (forecast, outcome) in forecasts.iter().zip(outcomes) { + let target = if *outcome { 1.0 } else { 0.0 }; + let residual = forecast.value() - target; + square_sum += residual * residual; + } + #[allow(clippy::cast_precision_loss)] + Ok(square_sum / forecasts.len() as f64) +} + #[cfg(test)] mod tests { use super::EventConfidence; @@ -56,5 +79,9 @@ mod tests { EventConfidence::new(f64::NAN), Err(EventError::InvalidEventConfidence) ); + let one = EventConfidence::certain().expect("certain"); + assert!((one.value() - 1.0).abs() < 1e-15); + let miss = super::mention_brier_score(&[one], &[false]).expect("miss"); + assert!((miss - 1.0).abs() < 1e-15); } } diff --git a/crates/event_core/src/lib.rs b/crates/event_core/src/lib.rs index 25fd1022..eb5595c8 100644 --- a/crates/event_core/src/lib.rs +++ b/crates/event_core/src/lib.rs @@ -16,6 +16,8 @@ mod role; /// Finite confidence on the closed unit interval. pub use confidence::EventConfidence; +/// Mean squared error of mention probabilities against binary truth. +pub use confidence::mention_brier_score; /// Fail-closed event-ontology errors. pub use error::EventError; /// Opaque event-instance identifier. diff --git a/crates/event_core/tests/confidence_calibration_contract.rs b/crates/event_core/tests/confidence_calibration_contract.rs new file mode 100644 index 00000000..631e1468 --- /dev/null +++ b/crates/event_core/tests/confidence_calibration_contract.rs @@ -0,0 +1,37 @@ +//! Mention confidence recovers known Brier scores against binary truth. + +use event_core::{EventConfidence, EventError, mention_brier_score}; + +#[test] +fn perfectly_calibrated_forecasts_recover_zero_brier() { + let forecasts = [ + EventConfidence::new(0.0).expect("0"), + EventConfidence::new(1.0).expect("1"), + EventConfidence::new(0.0).expect("0"), + EventConfidence::new(1.0).expect("1"), + ]; + let outcomes = [false, true, false, true]; + let score = mention_brier_score(&forecasts, &outcomes).expect("brier"); + assert!(score.abs() < 1e-15, "perfect Brier {score}"); +} + +#[test] +fn constant_half_recovers_quarter_and_mismatches_fail_closed() { + let forecasts = [ + EventConfidence::new(0.5).expect("half"), + EventConfidence::new(0.5).expect("half"), + ]; + let outcomes = [false, true]; + let score = mention_brier_score(&forecasts, &outcomes).expect("half"); + let residual = score - 0.25; + let rmse = (residual * residual).sqrt(); + assert!(rmse < 1e-15, "Brier RMSE {rmse}"); + assert_eq!( + mention_brier_score(&forecasts, &[true]), + Err(EventError::InvalidWirePayload) + ); + assert_eq!( + mention_brier_score(&[], &[]), + Err(EventError::InvalidWirePayload) + ); +} diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index 051062ea..1f2516af 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -13,7 +13,7 @@ The full APA 7th standards/literature register remains `docs/research/standards- | six distinct clocks and uncertain intervals | PRD; ADR 0002 | PR #8 `temporal_core` on protected main; PR #5 historical only | implemented-main | | Allen relation algebra/bounded closure | ADR 0002; temporal research | PR #9 `temporal_core` path-consistency on protected main | implemented-main | | forward-only transition subgraph | PRD; ADR 0002/0003 | `relation_graph` on protected main | implemented-main | -| event ontology/evidence mentions | PRD; ADR 0003 | `event_core` mention/instance separation on protected main; full intelligence stack remaining | partial | +| event ontology/evidence mentions | PRD; ADR 0003 | `event_core` mention/instance separation on protected main; Brier calibration on the active PR; full intelligence stack remaining | partial | | time-varying cross-classified multiple membership | PRD; ADR 0003 | `membership_core` network on protected main; multilevel estimators remaining | partial | | leakage-safe availability/cutoff snapshots | PRD; ADR 0002/0013 | `corpus_split` on protected main | implemented-main | | recovery metrics (RMSE, bias, coverage, graph, temporal order, Monte Carlo SE gates) | PRD; Test Strategy; ADR 0007/0014 | `validation_core` on protected main (PR #19); SE-aware Monte Carlo gates included | implemented-main | diff --git a/docs/research/mention-confidence-brier.md b/docs/research/mention-confidence-brier.md new file mode 100644 index 00000000..afb7382b --- /dev/null +++ b/docs/research/mention-confidence-brier.md @@ -0,0 +1,27 @@ +# Mention-confidence Brier score + +## Scope + +This note doctors the `event_core` calibration contract for fallible event mentions: + +1. mention confidence is a probability on `[0, 1]`; +2. `mention_brier_score` is the mean squared error against binary truth; +3. empty or length-mismatched streams fail closed. + +TDT/CHRONOS promotion remains on the event-intelligence active PR. No database migration is allocated. + +## Authoritative sources + +Brier, G. W. (1950). Verification of forecasts expressed in terms of probability. *Monthly Weather Review, 78*(1), 1–3. https://doi.org/10.1175/1520-0493(1950)078<0001:VOFEIT>2.0.CO;2 + +Gneiting, T., & Raftery, A. E. (2007). Strictly proper scoring rules, prediction, and estimation. *Journal of the American Statistical Association, 102*(477), 359–378. https://doi.org/10.1198/016214506000001437 + +## Application + +Brier (1950) defines the mean squared error of a probability forecast. Gneiting and Raftery (2007) treat the Brier score as a strictly proper scoring rule, so a mention that is certain when true and impossible when false is uniquely optimal. TEPP therefore scores mention confidence against known binary outcomes rather than treating a high score as an event instance (Brier, 1950; Gneiting & Raftery, 2007). + +## Verification + +- forecasts `(0,1,0,1)` against outcomes `(false,true,false,true)` recover Brier `0`; +- constant `0.5` against mixed outcomes recovers `0.25` with computed residual RMSE; +- empty and mismatched streams return `InvalidWirePayload`. diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index 984d329c..16f91ffd 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -23,6 +23,7 @@ This report tracks exact-head scientific and engineering evidence required befor | Truth corpora / manifests | `tepp_simulation` | implemented-main | — | deterministic generator tests | Task 10 / PR #18 | | Recovery metrics | `validation_core` | implemented-main | — | RMSE/bias/coverage/MC gates | Task 11 / PR #19 | | Versioned API/export contracts | `tepp_api` | implemented-main | — | unknown-field/version/limit tests | Task 12 / PR #21; HTTP service remaining | +| Mention-confidence Brier score | `event_core` | active-PR | calibration vs binary truth | perfect 0 / half 0.25 RMSE | ADR 0003; `docs/research/mention-confidence-brier.md` | | CWL modular connectors | `docs/connectors/*` | implemented-main | — | contract docs + examples | PR #22; live HTTP ports remaining | | Release SBOM/provenance generator | `scripts/release_evidence.py` | partial | — | generate+validate in CI | Task 13 partial / PR #28 | From 22a5955a0ba23a65ee597ff5f630eadb389e7446 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 02:03:02 +0900 Subject: [PATCH 2/2] docs: deduplicate event validation ledger --- docs/validation/temporal-event-foundation.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index ce6b0d61..09fcb32d 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -22,12 +22,10 @@ This report tracks exact-head scientific and engineering evidence required befor | Leakage-safe splits | `corpus_split` | implemented-main | — | cutoff + co-partition tests | Task 9 / PR #17 | | Truth corpora / manifests | `tepp_simulation` | implemented-main | — | deterministic generator tests | Task 10 / PR #18 | | Recovery metrics | `validation_core` | implemented-main | — | RMSE/bias/coverage/MC gates | Task 11 / PR #19 | -| Versioned API/export contracts | `tepp_api` | implemented-main | — | unknown-field/version/limit tests | Task 12 / PR #21; HTTP service remaining | | Mention-confidence Brier score | `event_core` | active-PR | calibration vs binary truth | perfect 0 / half 0.25 RMSE | ADR 0003; `docs/research/mention-confidence-brier.md` | | Versioned API/export contracts | `tepp_api` | implemented-main | naruon HTTP interchange | unknown-field/version/limit + naruon HTTPS interchange tests | Task 12 / PR #21; live HTTP service remaining | | Purpose-bound provider payloads | `tepp_api` | implemented-main | provider-payload minimization | expired/not-yet-valid/inverted/cross-tenant/impossible-calendar grant, mapping refusal, audited elevated re-id replay | ADR 0009; `docs/research/provider-payload-minimization.md` | | Adaptive orchestration router | `tepp_api` | accepted-target | active PR | mode selection, document-control denial, ablation, credential-free bind | ADR 0010; `docs/research/adaptive-orchestration-router.md` | -| Mention-confidence Brier score | `event_core` | active-PR | calibration vs binary truth | perfect 0 / half 0.25 RMSE | ADR 0003; `docs/research/mention-confidence-brier.md` | | CWL modular connectors | `docs/connectors/*` | implemented-main | — | contract docs + examples | PR #22; live HTTP ports remaining | | Release SBOM/provenance generator | `scripts/release_evidence.py` | partial | — | generate+validate in CI | Task 13 partial / PR #28 |