-
Notifications
You must be signed in to change notification settings - Fork 0
feat(event): score TDT mention links with precision and recall #66
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
f8d432b
633d41c
380be3f
0fd2e20
0b18c99
07fdb02
80592e3
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -24,6 +24,12 @@ pub enum EventError { | |||||||||||||||||
| DetectionIsNotTransition, | ||||||||||||||||||
| /// A CHRONOS prediction was treated as an observed or promoted fact. | ||||||||||||||||||
| PredictionIsNotFact, | ||||||||||||||||||
| /// A TDT link detection was treated as an event instance. | ||||||||||||||||||
| EventLinkIsNotEventInstance, | ||||||||||||||||||
| /// A TDT link detection was treated as a state transition. | ||||||||||||||||||
| EventLinkIsNotStateTransition, | ||||||||||||||||||
| /// An unknown event-link label was supplied. | ||||||||||||||||||
| UnknownEventLinkLabel, | ||||||||||||||||||
| /// A first-story detection was treated as an event instance. | ||||||||||||||||||
| FirstStoryIsNotEventInstance, | ||||||||||||||||||
| /// An unknown first-story label name was supplied. | ||||||||||||||||||
|
|
@@ -64,6 +70,9 @@ impl fmt::Display for EventError { | |||||||||||||||||
| Self::UnknownEventRole => "unknown event role", | ||||||||||||||||||
| Self::DetectionIsNotTransition => "detection is not a state transition", | ||||||||||||||||||
| Self::PredictionIsNotFact => "prediction is not an observed fact", | ||||||||||||||||||
| Self::EventLinkIsNotEventInstance => "event link is not an event instance", | ||||||||||||||||||
| Self::EventLinkIsNotStateTransition => "event link is not a state transition", | ||||||||||||||||||
| Self::UnknownEventLinkLabel => "unknown event link label", | ||||||||||||||||||
| Self::FirstStoryIsNotEventInstance => "first-story detection is not an event instance", | ||||||||||||||||||
| Self::UnknownFirstStoryLabel => "unknown first-story label", | ||||||||||||||||||
| Self::EventTrackIsNotEventInstance => "event track is not an event instance", | ||||||||||||||||||
|
|
@@ -127,6 +136,16 @@ mod tests { | |||||||||||||||||
| "prediction is not an observed fact", | ||||||||||||||||||
| ), | ||||||||||||||||||
| ( | ||||||||||||||||||
| EventError::EventLinkIsNotEventInstance, | ||||||||||||||||||
| "event link is not an event instance", | ||||||||||||||||||
| ), | ||||||||||||||||||
| ( | ||||||||||||||||||
| EventError::EventLinkIsNotStateTransition, | ||||||||||||||||||
| "event link is not a state transition", | ||||||||||||||||||
| ), | ||||||||||||||||||
| ( | ||||||||||||||||||
| EventError::UnknownEventLinkLabel, | ||||||||||||||||||
| "unknown event link label", | ||||||||||||||||||
|
Comment on lines
+146
to
+148
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔴 Link-label error test fused with first-story case The added
Suggested change
Was this helpful? React with 👍 or 👎 to provide feedback. |
||||||||||||||||||
| EventError::FirstStoryIsNotEventInstance, | ||||||||||||||||||
| "first-story detection is not an event instance", | ||||||||||||||||||
| ), | ||||||||||||||||||
|
|
||||||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,223 @@ | ||
| //! TDT link-detection scores stay distinct from instances and transitions. | ||
|
|
||
| use crate::{EventConfidence, EventError, EventInstanceId, EventMentionId}; | ||
| use std::collections::BTreeSet; | ||
|
|
||
| /// TDT same-event versus distinct-event link label. | ||
| /// | ||
| /// A link decision is detection evidence. It is never a promoted event instance | ||
| /// and cannot create a forward state transition by itself. | ||
| #[derive(Clone, Copy, Debug, Eq, PartialEq)] | ||
| pub enum EventLinkLabel { | ||
| /// The mention pair is scored as the same event or story. | ||
| Linked, | ||
| /// The mention pair is scored as distinct events. | ||
| Unlinked, | ||
| } | ||
|
|
||
| impl EventLinkLabel { | ||
| /// Return the stable wire label name. | ||
| #[must_use] | ||
| pub const fn wire_name(self) -> &'static str { | ||
| match self { | ||
| Self::Linked => "linked", | ||
| Self::Unlinked => "unlinked", | ||
| } | ||
| } | ||
|
|
||
| /// Parse a stable wire link label. | ||
| /// | ||
| /// # Errors | ||
| /// | ||
| /// Returns [`EventError::UnknownEventLinkLabel`] for unrecognized names. | ||
| pub fn from_wire_name(name: &str) -> Result<Self, EventError> { | ||
| match name { | ||
| "linked" => Ok(Self::Linked), | ||
| "unlinked" => Ok(Self::Unlinked), | ||
| _ => Err(EventError::UnknownEventLinkLabel), | ||
| } | ||
| } | ||
|
|
||
| /// Return whether this label is a positive link detection. | ||
| #[must_use] | ||
| pub const fn is_linked(self) -> bool { | ||
| matches!(self, Self::Linked) | ||
| } | ||
|
|
||
| /// Return the binary probability target used for RMSE. | ||
| /// | ||
| /// Linked truth is `1.0`; unlinked truth is `0.0`. | ||
| #[must_use] | ||
| pub const fn as_probability_target(self) -> f64 { | ||
| match self { | ||
| Self::Linked => 1.0, | ||
| Self::Unlinked => 0.0, | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /// An undirected TDT link hypothesis between two mentions. | ||
| #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] | ||
| pub struct EventLinkPair { | ||
| left: EventMentionId, | ||
| right: EventMentionId, | ||
| } | ||
|
|
||
| impl EventLinkPair { | ||
| /// Construct a normalized undirected mention pair. | ||
| /// | ||
| /// # Errors | ||
| /// | ||
| /// Returns [`EventError::InvalidWirePayload`] when both mentions are the | ||
| /// same identity. A mention cannot link to itself. | ||
| pub fn new(left: EventMentionId, right: EventMentionId) -> Result<Self, EventError> { | ||
| if left == right { | ||
| return Err(EventError::InvalidWirePayload); | ||
| } | ||
| if left <= right { | ||
| Ok(Self { left, right }) | ||
| } else { | ||
| Ok(Self { | ||
| left: right, | ||
| right: left, | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| /// Return the lexicographically smaller mention identifier. | ||
| #[must_use] | ||
| pub const fn left(self) -> EventMentionId { | ||
| self.left | ||
| } | ||
|
|
||
| /// Return the lexicographically larger mention identifier. | ||
| #[must_use] | ||
| pub const fn right(self) -> EventMentionId { | ||
| self.right | ||
| } | ||
| } | ||
|
|
||
| /// Threshold a link probability into a detection label. | ||
| /// | ||
| /// The threshold is inclusive: `probability >= threshold` is linked. | ||
| #[must_use] | ||
| pub fn decide_event_link( | ||
| probability: EventConfidence, | ||
| threshold: EventConfidence, | ||
| ) -> EventLinkLabel { | ||
| if probability.value() >= threshold.value() { | ||
| EventLinkLabel::Linked | ||
| } else { | ||
| EventLinkLabel::Unlinked | ||
| } | ||
| } | ||
|
|
||
| /// Explicit refusal to treat a TDT link as an event instance. | ||
| /// | ||
| /// # Errors | ||
| /// | ||
| /// Always returns [`EventError::EventLinkIsNotEventInstance`]. | ||
| pub fn refuse_event_link_as_instance(_link: EventLinkPair) -> Result<EventInstanceId, EventError> { | ||
| Err(EventError::EventLinkIsNotEventInstance) | ||
| } | ||
|
|
||
| /// Explicit refusal to treat a TDT link as a state transition. | ||
| /// | ||
| /// # Errors | ||
| /// | ||
| /// Always returns [`EventError::EventLinkIsNotStateTransition`]. | ||
| pub fn refuse_event_link_as_transition(_link: EventLinkPair) -> Result<(), EventError> { | ||
| Err(EventError::EventLinkIsNotStateTransition) | ||
| } | ||
|
|
||
| /// Precision of recovered TDT links against the known-truth pair set. | ||
| /// | ||
| /// # Errors | ||
| /// | ||
| /// Returns [`EventError::InvalidWirePayload`] when the recovered set is empty. | ||
| pub fn event_link_precision( | ||
| truth: &[EventLinkPair], | ||
| recovered: &[EventLinkPair], | ||
| ) -> Result<f64, EventError> { | ||
| let truth_set: BTreeSet<_> = truth.iter().copied().collect(); | ||
| let recovered_set: BTreeSet<_> = recovered.iter().copied().collect(); | ||
| counted_rate( | ||
| recovered_set.intersection(&truth_set).count(), | ||
| recovered_set.len(), | ||
| ) | ||
| } | ||
|
seonghobae marked this conversation as resolved.
|
||
|
|
||
| /// Recall of recovered TDT links against the known-truth pair set. | ||
| /// | ||
| /// # Errors | ||
| /// | ||
| /// Returns [`EventError::InvalidWirePayload`] when the truth set is empty. | ||
| pub fn event_link_recall( | ||
| truth: &[EventLinkPair], | ||
| recovered: &[EventLinkPair], | ||
| ) -> Result<f64, EventError> { | ||
| let truth_set: BTreeSet<_> = truth.iter().copied().collect(); | ||
| let recovered_set: BTreeSet<_> = recovered.iter().copied().collect(); | ||
| counted_rate( | ||
| recovered_set.intersection(&truth_set).count(), | ||
| truth_set.len(), | ||
| ) | ||
|
seonghobae marked this conversation as resolved.
|
||
| } | ||
|
|
||
| fn counted_rate(numerator: usize, denominator: usize) -> Result<f64, EventError> { | ||
| let numerator = u32::try_from(numerator).map_err(|_| EventError::InvalidWirePayload)?; | ||
| let denominator = u32::try_from(denominator).map_err(|_| EventError::InvalidWirePayload)?; | ||
| if denominator == 0 { | ||
| return Err(EventError::InvalidWirePayload); | ||
| } | ||
| Ok(f64::from(numerator) / f64::from(denominator)) | ||
| } | ||
|
seonghobae marked this conversation as resolved.
Comment on lines
+167
to
+174
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 📝 Info: Empty-set precision/recall fails closed by design
Was this helpful? React with 👍 or 👎 to provide feedback. |
||
|
|
||
| #[cfg(test)] | ||
| mod tests { | ||
| use super::{ | ||
| EventLinkLabel, EventLinkPair, counted_rate, decide_event_link, event_link_precision, | ||
| event_link_recall, refuse_event_link_as_instance, refuse_event_link_as_transition, | ||
| }; | ||
| use crate::{EventConfidence, EventError, EventMentionId}; | ||
|
|
||
| #[test] | ||
| fn link_helpers_cover_local_branches() { | ||
| let left = EventMentionId::new(); | ||
| let right = EventMentionId::new(); | ||
| let link = EventLinkPair::new(left, right).expect("pair"); | ||
| assert_eq!( | ||
| refuse_event_link_as_instance(link), | ||
| Err(EventError::EventLinkIsNotEventInstance) | ||
| ); | ||
| assert_eq!( | ||
| refuse_event_link_as_transition(link), | ||
| Err(EventError::EventLinkIsNotStateTransition) | ||
| ); | ||
| let high = EventConfidence::new(0.8).expect("high"); | ||
| let low = EventConfidence::new(0.2).expect("low"); | ||
| assert_eq!(decide_event_link(high, low), EventLinkLabel::Linked); | ||
| assert_eq!(decide_event_link(low, high), EventLinkLabel::Unlinked); | ||
| let truth = [link]; | ||
| let recovered = [link]; | ||
| assert!((event_link_precision(&truth, &recovered).expect("p") - 1.0).abs() < f64::EPSILON); | ||
| assert!((event_link_recall(&truth, &recovered).expect("r") - 1.0).abs() < f64::EPSILON); | ||
| assert_eq!( | ||
| EventLinkPair::new(left, left), | ||
| Err(EventError::InvalidWirePayload) | ||
| ); | ||
| assert_eq!( | ||
| event_link_precision(&truth, &[]), | ||
| Err(EventError::InvalidWirePayload) | ||
| ); | ||
| assert_eq!( | ||
| event_link_recall(&[], &recovered), | ||
| Err(EventError::InvalidWirePayload) | ||
| ); | ||
| assert_eq!( | ||
| counted_rate(0, usize::MAX), | ||
| Err(EventError::InvalidWirePayload) | ||
| ); | ||
| assert_eq!(counted_rate(1, 0), Err(EventError::InvalidWirePayload)); | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🔍 Pre-existing fused error-message tuple
The
UnknownFirstStoryLabelentry is already fused withEventTrackIsNotEventInstanceinto a four-element tuple in the merge-base, the same defect as the newly introduced one. The test array was already structurally broken; fixing it in the same pass would restore both message assertions.(Refers to this code)
Was this helpful? React with 👍 or 👎 to provide feedback.