From 366ca724462a34e081d3e4325ee0d60026c997d4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 03:23:30 +0900 Subject: [PATCH 1/2] feat(event): score first-story detections with FAR and miss rates Keep TDT first-story labels distinct from promoted event instances and require computed false-alarm, miss, and RMSE evidence against known truth. --- CHANGELOG.md | 1 + DOCUMENTATION.md | 1 + crates/event_core/src/error.rs | 14 ++ crates/event_core/src/first_story.rs | 175 ++++++++++++++++++ crates/event_core/src/lib.rs | 14 +- .../event_core/tests/first_story_contract.rs | 137 ++++++++++++++ docs/TRACEABILITY.md | 2 +- ...tdt-chronos-event-intelligence-boundary.md | 2 +- docs/adr/README.md | 2 +- .../first-story-detection-calibration.md | 30 +++ docs/research/standards-and-literature.md | 2 + docs/validation/temporal-event-foundation.md | 1 + 12 files changed, 377 insertions(+), 4 deletions(-) create mode 100644 crates/event_core/src/first_story.rs create mode 100644 crates/event_core/tests/first_story_contract.rs create mode 100644 docs/research/first-story-detection-calibration.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cea0ee2..572f9e4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to TEPP are documented here. The format follows Keep a Chang ### Added +- `event_core` first-story detection gate: first-story versus follow-up labels stay distinct from promoted instances, false-alarm and miss rates are computed from known truth, and calibrated detection scores recover the binary first-story target with lower RMSE than an always-first detector. - `persistence_postgres` event-relation SQL contracts: closed ERD transition/provenance vocabulary bound to `transition_edge`, fail-closed unknown types and transition self-loops, live insert of `causes`/`references`. - `persistence_postgres` typed membership assignment (migration `0006`): `entity_record`, `project_record`, and `text_segment` plus exactly-one observed-unit and target constraints that replace the polymorphic `membership_target_id` stub, with SQL insert/lookup, fail-closed inverted-window and backslash-label refusal, and live proof that one document persists two entity memberships and one project membership. - Actions workflow fleet auditor (`scripts/actions_workflow_fleet.py`): paginated registry inventory bound to the exact default-branch SHA/tree, classification of present/orphan/disabled/GitHub-dynamic identities, and fail-closed orphan disable that confirms GitHub's official `disabled_manually` state. diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 230c5abe..e83ec3a9 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -33,6 +33,7 @@ TEPP's approved PRD v0.4 and implementation plan are the primary product baselin | Hourly NIM product-development operations | [`docs/operations/HOURLY_NIM_PRODUCT_DEVELOPMENT.md`](docs/operations/HOURLY_NIM_PRODUCT_DEVELOPMENT.md) | | Actions workflow fleet audit | [`docs/operations/ACTIONS_WORKFLOW_FLEET.md`](docs/operations/ACTIONS_WORKFLOW_FLEET.md) | | Actions fleet research doctoring | [`docs/research/actions-workflow-fleet.md`](docs/research/actions-workflow-fleet.md) | +| First-story detection FAR/miss doctoring | [`docs/research/first-story-detection-calibration.md`](docs/research/first-story-detection-calibration.md) | | Hourly NIM OpenCode doctoring | [`docs/doctoring/hourly-nim-opencode-development.md`](docs/doctoring/hourly-nim-opencode-development.md) | | Change history | [`CHANGELOG.md`](CHANGELOG.md) | diff --git a/crates/event_core/src/error.rs b/crates/event_core/src/error.rs index 6c795fef..5d1e2066 100644 --- a/crates/event_core/src/error.rs +++ b/crates/event_core/src/error.rs @@ -20,6 +20,10 @@ pub enum EventError { UnsupportedWireVersion, /// An unknown event-role name was supplied. UnknownEventRole, + /// A first-story detection was treated as an event instance. + FirstStoryIsNotEventInstance, + /// An unknown first-story label name was supplied. + UnknownFirstStoryLabel, } impl fmt::Display for EventError { @@ -32,6 +36,8 @@ impl fmt::Display for EventError { Self::InvalidWirePayload => "invalid event wire payload", Self::UnsupportedWireVersion => "unsupported event wire version", Self::UnknownEventRole => "unknown event role", + Self::FirstStoryIsNotEventInstance => "first-story detection is not an event instance", + Self::UnknownFirstStoryLabel => "unknown first-story label", }; formatter.write_str(message) } @@ -65,6 +71,14 @@ mod tests { "unsupported event wire version", ), (EventError::UnknownEventRole, "unknown event role"), + ( + EventError::FirstStoryIsNotEventInstance, + "first-story detection is not an event instance", + ), + ( + EventError::UnknownFirstStoryLabel, + "unknown first-story label", + ), ] { assert_eq!(error.to_string(), message); } diff --git a/crates/event_core/src/first_story.rs b/crates/event_core/src/first_story.rs new file mode 100644 index 00000000..0e36a42f --- /dev/null +++ b/crates/event_core/src/first_story.rs @@ -0,0 +1,175 @@ +//! First-story detection scores stay distinct from promoted instances. + +use crate::{EventConfidence, EventError, EventInstanceId, EventMentionId}; + +/// TDT first-story versus follow-up label. +/// +/// A first-story decision is detection evidence. It is never a promoted event +/// instance and cannot create a forward state transition by itself. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FirstStoryLabel { + /// The mention is scored as the onset of a new story. + FirstStory, + /// The mention is scored as a continuation of an earlier story. + FollowUp, +} + +impl FirstStoryLabel { + /// Return the stable wire label name. + #[must_use] + pub const fn wire_name(self) -> &'static str { + match self { + Self::FirstStory => "first_story", + Self::FollowUp => "follow_up", + } + } + + /// Parse a stable wire first-story label. + /// + /// # Errors + /// + /// Returns [`EventError::UnknownFirstStoryLabel`] for unrecognized names. + pub fn from_wire_name(name: &str) -> Result { + match name { + "first_story" => Ok(Self::FirstStory), + "follow_up" => Ok(Self::FollowUp), + _ => Err(EventError::UnknownFirstStoryLabel), + } + } + + /// Return whether this label is a first-story detection. + #[must_use] + pub const fn is_first_story(self) -> bool { + matches!(self, Self::FirstStory) + } + + /// Return the binary probability target used for RMSE. + /// + /// First-story truth is `1.0`; follow-up truth is `0.0`. + #[must_use] + pub const fn as_probability_target(self) -> f64 { + match self { + Self::FirstStory => 1.0, + Self::FollowUp => 0.0, + } + } +} + +/// Threshold a first-story probability into a detection label. +/// +/// The threshold is inclusive: `probability >= threshold` is a first story. +#[must_use] +pub fn decide_first_story( + probability: EventConfidence, + threshold: EventConfidence, +) -> FirstStoryLabel { + if probability.value() >= threshold.value() { + FirstStoryLabel::FirstStory + } else { + FirstStoryLabel::FollowUp + } +} + +/// Explicit refusal to treat a first-story detection as an event instance. +/// +/// # Errors +/// +/// Always returns [`EventError::FirstStoryIsNotEventInstance`]. +pub fn refuse_first_story_as_instance( + _mention_id: EventMentionId, +) -> Result { + Err(EventError::FirstStoryIsNotEventInstance) +} + +/// False-alarm rate: follow-ups labeled first story, over follow-up truth. +/// +/// # Errors +/// +/// Returns [`EventError::InvalidWirePayload`] when lengths differ, either +/// slice is empty, or the truth stream contains no follow-up. +pub fn first_story_false_alarm_rate( + truth: &[FirstStoryLabel], + decided: &[FirstStoryLabel], +) -> Result { + rate_over_class( + truth, + decided, + FirstStoryLabel::FollowUp, + FirstStoryLabel::FirstStory, + ) +} + +/// Miss rate: first stories labeled follow-up, over first-story truth. +/// +/// # Errors +/// +/// Returns [`EventError::InvalidWirePayload`] when lengths differ, either +/// slice is empty, or the truth stream contains no first story. +pub fn first_story_miss_rate( + truth: &[FirstStoryLabel], + decided: &[FirstStoryLabel], +) -> Result { + rate_over_class( + truth, + decided, + FirstStoryLabel::FirstStory, + FirstStoryLabel::FollowUp, + ) +} + +fn rate_over_class( + truth: &[FirstStoryLabel], + decided: &[FirstStoryLabel], + class: FirstStoryLabel, + error_label: FirstStoryLabel, +) -> Result { + if truth.is_empty() || truth.len() != decided.len() { + return Err(EventError::InvalidWirePayload); + } + let mut class_count = 0_u32; + let mut error_count = 0_u32; + for (truth_label, decided_label) in truth.iter().zip(decided) { + if *truth_label == class { + class_count += 1; + if *decided_label == error_label { + error_count += 1; + } + } + } + if class_count == 0 { + return Err(EventError::InvalidWirePayload); + } + Ok(f64::from(error_count) / f64::from(class_count)) +} + +#[cfg(test)] +mod tests { + use super::{ + FirstStoryLabel, decide_first_story, first_story_false_alarm_rate, first_story_miss_rate, + refuse_first_story_as_instance, + }; + use crate::{EventConfidence, EventError, EventMentionId}; + + #[test] + fn first_story_helpers_cover_local_branches() { + let mention = EventMentionId::new(); + assert_eq!( + refuse_first_story_as_instance(mention), + Err(EventError::FirstStoryIsNotEventInstance) + ); + let high = EventConfidence::new(0.8).expect("high"); + let low = EventConfidence::new(0.2).expect("low"); + assert_eq!(decide_first_story(high, low), FirstStoryLabel::FirstStory); + assert_eq!(decide_first_story(low, high), FirstStoryLabel::FollowUp); + let mixed_truth = [FirstStoryLabel::FirstStory, FirstStoryLabel::FollowUp]; + let mixed_decided = [FirstStoryLabel::FollowUp, FirstStoryLabel::FirstStory]; + assert!( + (first_story_false_alarm_rate(&mixed_truth, &mixed_decided).expect("far") - 1.0).abs() + < f64::EPSILON + ); + assert!( + (first_story_miss_rate(&mixed_truth, &mixed_decided).expect("miss") - 1.0).abs() + < f64::EPSILON + ); + } +} diff --git a/crates/event_core/src/lib.rs b/crates/event_core/src/lib.rs index 25fd1022..027b4f9b 100644 --- a/crates/event_core/src/lib.rs +++ b/crates/event_core/src/lib.rs @@ -4,10 +4,12 @@ //! //! TEPP separates **fallible event mentions** grounded in evidence from //! **versioned event instances** used for temporal state, multilevel membership, -//! and scientific estimation. Mentions never silently become instances. +//! and scientific estimation. Mentions and first-story detections never +//! silently become instances. mod confidence; mod error; +mod first_story; mod identifier; mod instance; mod mention; @@ -18,6 +20,16 @@ mod role; pub use confidence::EventConfidence; /// Fail-closed event-ontology errors. pub use error::EventError; +/// First-story versus follow-up detection label. +pub use first_story::FirstStoryLabel; +/// Threshold a first-story probability into a detection label. +pub use first_story::decide_first_story; +/// False-alarm rate for first-story detections. +pub use first_story::first_story_false_alarm_rate; +/// Miss rate for first-story detections. +pub use first_story::first_story_miss_rate; +/// Explicit refusal to treat a first-story detection as an instance. +pub use first_story::refuse_first_story_as_instance; /// Opaque event-instance identifier. pub use identifier::EventInstanceId; /// Opaque event-mention identifier. diff --git a/crates/event_core/tests/first_story_contract.rs b/crates/event_core/tests/first_story_contract.rs new file mode 100644 index 00000000..19cc05f4 --- /dev/null +++ b/crates/event_core/tests/first_story_contract.rs @@ -0,0 +1,137 @@ +//! First-story detections are not instances; FAR/miss are computed from truth. + +use event_core::{ + EventConfidence, EventError, EventMentionId, FirstStoryLabel, decide_first_story, + first_story_false_alarm_rate, first_story_miss_rate, refuse_first_story_as_instance, +}; + +fn computed_rmse(truth: &[f64], recovered: &[f64]) -> f64 { + assert_eq!(truth.len(), recovered.len()); + let n = f64::from(u32::try_from(truth.len()).expect("tiny fixture")); + let sse: f64 = truth + .iter() + .zip(recovered) + .map(|(truth_value, recovered_value)| { + let residual = truth_value - recovered_value; + residual * residual + }) + .sum(); + (sse / n).sqrt() +} + +fn decide_all(scores: &[f64], threshold: f64) -> Vec { + let cut = EventConfidence::new(threshold).expect("threshold"); + scores + .iter() + .map(|score| decide_first_story(EventConfidence::new(*score).expect("score"), cut)) + .collect() +} + +#[test] +fn first_story_detection_cannot_be_cast_to_an_instance() { + assert_eq!( + refuse_first_story_as_instance(EventMentionId::new()), + Err(EventError::FirstStoryIsNotEventInstance) + ); +} + +#[test] +fn false_alarm_and_miss_rates_are_computed_from_known_truth() { + let truth = [ + FirstStoryLabel::FirstStory, + FirstStoryLabel::FollowUp, + FirstStoryLabel::FollowUp, + FirstStoryLabel::FirstStory, + FirstStoryLabel::FollowUp, + FirstStoryLabel::FollowUp, + ]; + let calibrated = decide_all(&[0.90, 0.10, 0.15, 0.85, 0.20, 0.05], 0.50); + let always_first = decide_all(&[1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 0.50); + + let calibrated_far = first_story_false_alarm_rate(&truth, &calibrated).expect("far"); + let naive_far = first_story_false_alarm_rate(&truth, &always_first).expect("naive far"); + let calibrated_miss = first_story_miss_rate(&truth, &calibrated).expect("miss"); + let naive_miss = first_story_miss_rate(&truth, &always_first).expect("naive miss"); + + assert!( + calibrated_far < naive_far, + "computed FAR {calibrated_far} must be below always-first FAR {naive_far}" + ); + assert!(calibrated_miss <= naive_miss); +} + +#[test] +fn calibrated_first_story_scores_have_lower_rmse_than_always_first() { + let truth_labels = [ + FirstStoryLabel::FirstStory, + FirstStoryLabel::FollowUp, + FirstStoryLabel::FollowUp, + FirstStoryLabel::FirstStory, + FirstStoryLabel::FollowUp, + FirstStoryLabel::FollowUp, + ]; + let truth: Vec = truth_labels + .iter() + .copied() + .map(FirstStoryLabel::as_probability_target) + .collect(); + let calibrated = [0.90_f64, 0.10, 0.15, 0.85, 0.20, 0.05]; + let always_first = [1.0_f64, 1.0, 1.0, 1.0, 1.0, 1.0]; + let calibrated_rmse = computed_rmse(&truth, &calibrated); + let naive_rmse = computed_rmse(&truth, &always_first); + assert!( + calibrated_rmse < naive_rmse, + "computed calibrated RMSE {calibrated_rmse} must be below always-first RMSE {naive_rmse}" + ); +} + +#[test] +fn rate_helpers_fail_closed_on_empty_mismatch_and_missing_class() { + let first = [FirstStoryLabel::FirstStory]; + let follow = [FirstStoryLabel::FollowUp]; + assert_eq!( + first_story_false_alarm_rate(&[], &[]), + Err(EventError::InvalidWirePayload) + ); + assert_eq!( + first_story_miss_rate(&first, &[]), + Err(EventError::InvalidWirePayload) + ); + assert_eq!( + first_story_false_alarm_rate(&first, &first), + Err(EventError::InvalidWirePayload) + ); + assert_eq!( + first_story_miss_rate(&follow, &follow), + Err(EventError::InvalidWirePayload) + ); +} + +#[test] +fn labels_round_trip_and_threshold_is_inclusive() { + assert_eq!(FirstStoryLabel::FirstStory.wire_name(), "first_story"); + assert_eq!(FirstStoryLabel::FollowUp.wire_name(), "follow_up"); + assert_eq!( + FirstStoryLabel::from_wire_name("first_story").expect("parse"), + FirstStoryLabel::FirstStory + ); + assert_eq!( + FirstStoryLabel::from_wire_name("follow_up").expect("parse"), + FirstStoryLabel::FollowUp + ); + assert_eq!( + FirstStoryLabel::from_wire_name("maybe_new"), + Err(EventError::UnknownFirstStoryLabel) + ); + assert!(FirstStoryLabel::FirstStory.is_first_story()); + assert!(!FirstStoryLabel::FollowUp.is_first_story()); + assert!((FirstStoryLabel::FirstStory.as_probability_target() - 1.0).abs() < f64::EPSILON); + assert!((FirstStoryLabel::FollowUp.as_probability_target() - 0.0).abs() < f64::EPSILON); + + let half = EventConfidence::new(0.5).expect("half"); + assert_eq!(decide_first_story(half, half), FirstStoryLabel::FirstStory); + assert_eq!( + decide_first_story(EventConfidence::new(0.49).expect("below"), half), + FirstStoryLabel::FollowUp + ); +} diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index 6fad26a1..70c519bb 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -31,7 +31,7 @@ The full APA 7th standards/literature register remains `docs/research/standards- | compositional topic correlation / stable clustering | ADR 0005/0012; research | future `network_analysis` | accepted-target | | posterior ESEM / longitudinal invariance / DSEM | ADR 0005 | future `psychometric_core` | accepted-target | | CPU bounded multithreading + GPU/VRAM streaming/parity | ADR 0001/0006 | future `compute_backend` | accepted-target | -| TDT detection/tracking vs CHRONOS schema/prediction/temporal consistency | ADR 0016; PRD/research | future `event_intelligence` | accepted-target | +| TDT detection/tracking vs CHRONOS schema/prediction/temporal consistency | ADR 0016; PRD/research | `event_core` first-story FAR/miss on the active PR; remaining TDT/CHRONOS stack and any future `event_intelligence` crate remain accepted-target | active-PR | | evidence-bounded LLM interpretation | ADR 0010/0012; PRD | future `interpretation_gateway` | accepted-target | | adaptive direct/verify/committee/conductor test-time compute | ADR 0010; `docs/LLM_ORCHESTRATION.md` | future contextual-orchestrator integration + ablation evidence | accepted-target | | purpose-bound PII handling without blanket masking | ADR 0009; `docs/PRIVACY_DATA_GOVERNANCE.md` | future authorization/persistence/export/provider adapters | accepted-target | diff --git a/docs/adr/0016-tdt-chronos-event-intelligence-boundary.md b/docs/adr/0016-tdt-chronos-event-intelligence-boundary.md index b85ee0b4..09c33ddb 100644 --- a/docs/adr/0016-tdt-chronos-event-intelligence-boundary.md +++ b/docs/adr/0016-tdt-chronos-event-intelligence-boundary.md @@ -1,7 +1,7 @@ # ADR 0016 — TDT, CHRONOS, and Event Ontology intelligence boundary **Decision status:** Accepted -**Implementation maturity:** accepted-target +**Implementation maturity:** active-PR — first-story false-alarm/miss rates and detection-versus-instance refusal live in existing `event_core`; remaining TDT segmentation/tracking and CHRONOS schema/prediction layers remain accepted-target **Date:** 2026-08-12 **Supersedes:** None; complements ADR 0002 temporal semantics and ADR 0003 event ontology/membership. diff --git a/docs/adr/README.md b/docs/adr/README.md index 1a9a7b31..afeb02c5 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -21,7 +21,7 @@ Read [`ADR_POLICY.md`](ADR_POLICY.md) first. **Decision status and implementatio | [0013](0013-bitemporal-persistence-reproducibility-and-split-authority.md) | Bitemporal persistence, reproducibility manifests, and relation-aware split authority | Accepted | partial | Owns PostgreSQL adapter semantics, immutable run/split manifests, leakage-safe partitions, and recovery identity; optional `live-sqlx` `PgPool`, live PG CI, and tenant RLS implemented; full physical ERD remaining. | | [0014](0014-scientific-claim-promotion-and-release-evidence.md) | Scientific claim promotion and release evidence authority | Accepted | partial | Separates design, implementation, scientific/product claim, and release authority; repository SBOM/provenance generator implemented, full release bundle remaining. | | [0015](0015-autonomous-development-review-and-merge-authority.md) | Autonomous development, review, and merge authority separation | Accepted | active-PR | Separates model proposal, deterministic verification, publication, independent review, and merge/release authority. | -| [0016](0016-tdt-chronos-event-intelligence-boundary.md) | TDT, CHRONOS, and Event Ontology intelligence boundary | Accepted | accepted-target | Separates observed evidence, detection/tracking, prediction/schema inference, temporal consistency, and promoted transition authority. | +| [0016](0016-tdt-chronos-event-intelligence-boundary.md) | TDT, CHRONOS, and Event Ontology intelligence boundary | Accepted | active-PR | First-story FAR/miss in existing `event_core`; remaining TDT/CHRONOS stack remains accepted-target. | ## Decision ownership summary diff --git a/docs/research/first-story-detection-calibration.md b/docs/research/first-story-detection-calibration.md new file mode 100644 index 00000000..eabb8332 --- /dev/null +++ b/docs/research/first-story-detection-calibration.md @@ -0,0 +1,30 @@ +# First-story detection calibration + +## Scope + +This note doctors the `event_core` gate that keeps TDT first-story detection distinct from event-instance promotion: + +1. a first-story versus follow-up label is detection evidence, not a promoted instance; +2. false-alarm and miss rates are computed from known-truth labels; +3. calibrated first-story probabilities recover the binary onset target with lower RMSE than an always-first detector. + +No database migration is allocated. A later TDT tracker may consume these scores as measurement evidence only. + +## Authoritative sources + +Allan, J., Carbonell, J., Doddington, G., Yamron, J., & Yang, Y. (1998). Topic detection and tracking pilot study: Final report. In *Proceedings of the DARPA Broadcast News Transcription and Understanding Workshop* (pp. 194–218). + +Allan, J. (Ed.). (2002). *Topic detection and tracking: Event-based information organization*. Kluwer Academic Publishers. + +Fiscus, J. G., & Doddington, G. R. (2002). Topic detection and tracking evaluation overview. In J. Allan (Ed.), *Topic detection and tracking: Event-based information organization* (pp. 17–31). Kluwer Academic Publishers. + +## Application + +Allan et al. (1998) and Allan (2002) define first-story detection as a *new-event onset* task whose official evaluation reports false-alarm and miss rates, not instance identity. Fiscus and Doddington (2002) keep those detection scores in the measurement layer. TEPP therefore refuses to cast a first-story label as an event instance and requires computed FAR, miss, and RMSE against known truth (Allan et al., 1998; Allan, 2002; Fiscus & Doddington, 2002). + +## Verification + +- `refuse_first_story_as_instance` always returns `FirstStoryIsNotEventInstance`; +- `decide_first_story` uses an inclusive probability threshold; +- `first_story_false_alarm_rate` and `first_story_miss_rate` fail closed on empty, mismatched, or single-class streams; +- computed RMSE of known first-story targets is lower under calibrated probabilities than under an always-first detector. diff --git a/docs/research/standards-and-literature.md b/docs/research/standards-and-literature.md index b4b14468..ee60cf39 100644 --- a/docs/research/standards-and-literature.md +++ b/docs/research/standards-and-literature.md @@ -62,6 +62,8 @@ International Organization for Standardization. (2012). *Language resource manag Hobbs, J. R., & Pan, F. (2017). *Time ontology in OWL* (W3C Recommendation). World Wide Web Consortium. https://www.w3.org/TR/owl-time/ +Allan, J., Carbonell, J., Doddington, G., Yamron, J., & Yang, Y. (1998). Topic detection and tracking pilot study: Final report. In *Proceedings of the DARPA Broadcast News Transcription and Understanding Workshop* (pp. 194–218). + Allan, J. (Ed.). (2002). *Topic detection and tracking: Event-based information organization*. Kluwer Academic Publishers. Anagnostopoulos, E., Batsakis, S., & Petrakis, E. G. M. (2013). CHRONOS: A reasoning engine for qualitative temporal information in OWL. *Procedia Computer Science, 22*, 70–77. https://doi.org/10.1016/j.procs.2013.09.082 diff --git a/docs/validation/temporal-event-foundation.md b/docs/validation/temporal-event-foundation.md index 984d329c..31ea16ec 100644 --- a/docs/validation/temporal-event-foundation.md +++ b/docs/validation/temporal-event-foundation.md @@ -16,6 +16,7 @@ This report tracks exact-head scientific and engineering evidence required befor | Six-clock temporal | `temporal_core` | implemented-main | — | unit + wire | Task 3 / PR #8 | | Allen path-consistency | `temporal_core` | implemented-main | — | unit + budget tests | Task 4 / PR #9 | | Event mention/instance | `event_core` | partial | — | unit + fail-closed promotion | Task 5 / PR #13 | +| First-story FAR/miss | `event_core` | active-PR | this PR | computed FAR/miss + RMSE vs always-first | ADR 0016; `docs/research/first-story-detection-calibration.md` | | Multiple membership | `membership_core` | partial | — | unit + ESS weights | Task 7 / PR #12 + #25 | | Forward transition DAG | `relation_graph` | implemented-main | — | unit + cycle rejection | Task 6 / PR #14 | | Bitemporal persistence + live SQL port | `persistence_postgres` | partial | #36 typed membership | migration contracts + recording transport + optional PgPool + live CI + tenant RLS + `0005` interval CHECKs (implemented-main via #35) + `0006` typed membership (active PR) | Task 8 / PR #16 + #23 + #26 + #27 + #29 + #30–#35 + `0006` | From 74351f398d67df590019049ab85a7f649495b860 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 22:19:47 +0900 Subject: [PATCH 2/2] test(event): recover known-truth FAR and miss of 1/2 Drive first_story_false_alarm_rate and first_story_miss_rate with a four-mention mixed stream whose class-conditional error rates are exactly one half, and require residual RMSE below machine scale. --- .../event_core/tests/first_story_contract.rs | 29 +++++++++++++++++++ .../first-story-detection-calibration.md | 3 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/event_core/tests/first_story_contract.rs b/crates/event_core/tests/first_story_contract.rs index 19cc05f4..4a189c6e 100644 --- a/crates/event_core/tests/first_story_contract.rs +++ b/crates/event_core/tests/first_story_contract.rs @@ -58,6 +58,35 @@ fn false_alarm_and_miss_rates_are_computed_from_known_truth() { "computed FAR {calibrated_far} must be below always-first FAR {naive_far}" ); assert!(calibrated_miss <= naive_miss); + assert!( + calibrated_far.abs() < 1e-15 && calibrated_miss.abs() < 1e-15, + "calibrated stream must recover FAR 0 and miss 0; far={calibrated_far} miss={calibrated_miss}" + ); +} + +#[test] +fn mixed_detection_errors_recover_half_far_and_half_miss() { + let truth = [ + FirstStoryLabel::FirstStory, + FirstStoryLabel::FollowUp, + FirstStoryLabel::FollowUp, + FirstStoryLabel::FirstStory, + ]; + let decided = [ + FirstStoryLabel::FirstStory, + FirstStoryLabel::FirstStory, + FirstStoryLabel::FollowUp, + FirstStoryLabel::FollowUp, + ]; + let far = first_story_false_alarm_rate(&truth, &decided).expect("far"); + let miss = first_story_miss_rate(&truth, &decided).expect("miss"); + let recovered = [far, miss]; + let truth_rates = [0.5_f64, 0.5]; + let rmse = computed_rmse(&truth_rates, &recovered); + assert!( + rmse < 1e-15, + "known-truth FAR/miss RMSE {rmse} (far={far} miss={miss})" + ); } #[test] diff --git a/docs/research/first-story-detection-calibration.md b/docs/research/first-story-detection-calibration.md index eabb8332..9e03f798 100644 --- a/docs/research/first-story-detection-calibration.md +++ b/docs/research/first-story-detection-calibration.md @@ -27,4 +27,5 @@ Allan et al. (1998) and Allan (2002) define first-story detection as a *new-even - `refuse_first_story_as_instance` always returns `FirstStoryIsNotEventInstance`; - `decide_first_story` uses an inclusive probability threshold; - `first_story_false_alarm_rate` and `first_story_miss_rate` fail closed on empty, mismatched, or single-class streams; -- computed RMSE of known first-story targets is lower under calibrated probabilities than under an always-first detector. +- computed RMSE of known first-story targets is lower under calibrated probabilities than under an always-first detector; +- a mixed stream with one false alarm and one miss recovers FAR `0.5` and miss `0.5` with residual RMSE below `1e-15`.