fix: improve discovery anonymity gate — token-level entropy, address NER, phone dot separator

src/db_operations/native_index/anonymity.rs

@@ -24,7 +24,7 @@ pub enum FragmentDecision {
 }
 
 /// Minimum Shannon entropy (bits per character) for a fragment to be publishable.
-const MIN_ENTROPY_BITS: f64 = 2.0;
+const MIN_ENTROPY_BITS: f64 = 1.5;
 /// Minimum word count for a fragment to be publishable.
 const MIN_WORD_COUNT: usize = 3;
 
@@ -120,7 +120,7 @@ pub fn default_privacy_class(field_name: &str) -> FieldPrivacyClass {
 /// Check whether text contains named entities (PII patterns).
 /// Returns true if any PII pattern is detected.
 pub fn contains_named_entities(text: &str) -> bool {
-    has_email(text) || has_phone(text) || has_url(text) || has_id_pattern(text)
+    has_email(text) || has_phone(text) || has_url(text) || has_id_pattern(text) || has_address(text)
 }
 
 fn has_email(text: &str) -> bool {
@@ -139,7 +139,13 @@ fn has_phone(text: &str) -> bool {
         // Check for phone-like patterns: sequences of digits with separators
         let mut consecutive_phone_chars = 0;
         for ch in text.chars() {
-            if ch.is_ascii_digit() || ch == '-' || ch == '(' || ch == ')' || ch == ' ' || ch == '+'
+            if ch.is_ascii_digit()
+                || ch == '-'
+                || ch == '('
+                || ch == ')'
+                || ch == ' '
+                || ch == '+'
+                || ch == '.'
             {
                 consecutive_phone_chars += 1;
             } else {
@@ -174,17 +180,86 @@ fn has_id_pattern(text: &str) -> bool {
     false
 }
 
-/// Calculate Shannon entropy (bits per character) of text.
+fn has_address(text: &str) -> bool {
+    // Look for patterns like "123 Main St" or "456 Oak Avenue"
+    let street_suffixes = [
+        " st",
+        " st.",
+        " street",
+        " ave",
+        " ave.",
+        " avenue",
+        " blvd",
+        " blvd.",
+        " boulevard",
+        " dr",
+        " dr.",
+        " drive",
+        " rd",
+        " rd.",
+        " road",
+        " ln",
+        " ln.",
+        " lane",
+        " ct",
+        " ct.",
+        " court",
+        " pl",
+        " pl.",
+        " place",
+        " way",
+        " cir",
+        " circle",
+        " pkwy",
+        " parkway",
+    ];
+    let lower = text.to_lowercase();
+    // Check every occurrence of each street suffix for a preceding digit
+    for suffix in &street_suffixes {
+        let mut search_from = 0;
+        loop {
+            // Ensure search_from is on a char boundary
+            if search_from >= lower.len() || !lower.is_char_boundary(search_from) {
+                break;
+            }
+            let Some(rel) = lower[search_from..].find(suffix) else {
+                break;
+            };
+            let pos = search_from + rel;
+            // Walk backwards up to ~30 chars, staying on a char boundary
+            let start = lower[..pos]
+                .char_indices()
+                .rev()
+                .nth(30)
+                .map_or(0, |(i, _)| i);
+            let preceding = &lower[start..pos];
+            if preceding.chars().any(|c| c.is_ascii_digit()) {
+                return true;
+            }
+            let next = pos + suffix.len();
+            // Advance to the next char boundary after the suffix
+            search_from = lower[next..]
+                .char_indices()
+                .next()
+                .map_or(lower.len(), |(i, _)| next + i);
+        }
+    }
+    false
+}
+
+/// Calculate Shannon entropy (bits per token) of text.
 pub fn token_entropy(text: &str) -> f64 {
-    if text.is_empty() {
+    let tokens: Vec<&str> = text.split_whitespace().collect();
+    if tokens.is_empty() {
         return 0.0;
     }
 
     let mut freq = std::collections::HashMap::new();
-    let total = text.len() as f64;
+    let total = tokens.len() as f64;
 
-    for byte in text.bytes() {
-        *freq.entry(byte).or_insert(0u64) += 1;
+    for token in &tokens {
+        let lower = token.to_lowercase();
+        *freq.entry(lower).or_insert(0u64) += 1;
     }
 
     freq.values()
@@ -336,8 +411,8 @@ mod tests {
 
     #[test]
     fn test_entropy_too_low() {
-        // Very short, repetitive text has low entropy
-        let entropy = token_entropy("aaa");
+        // Repetitive tokens have low entropy
+        let entropy = token_entropy("the the the");
         assert!(entropy < MIN_ENTROPY_BITS, "entropy was {}", entropy);
     }
 
@@ -422,4 +497,38 @@ mod tests {
             FragmentDecision::Reject("field name suggests PII")
         );
     }
+
+    #[test]
+    fn test_entropy_token_level_unicode() {
+        // Unicode text should get same entropy as ASCII with same token diversity
+        let ascii_entropy = token_entropy("hello world foo bar baz");
+        let unicode_entropy = token_entropy("café résumé naïve über straße");
+        // Both have 5 unique tokens, so entropy should be similar
+        assert!(
+            (ascii_entropy - unicode_entropy).abs() < 0.1,
+            "ascii={}, unicode={} — should be similar for same token count",
+            ascii_entropy,
+            unicode_entropy
+        );
+    }
+
+    #[test]
+    fn test_ner_detects_address() {
+        assert!(contains_named_entities("lives at 123 Main St in town"));
+        assert!(contains_named_entities("office is 456 Oak Avenue"));
+        assert!(contains_named_entities("send to 789 Elm Blvd."));
+        assert!(!contains_named_entities("no address information here"));
+        // Second occurrence has the digit (first "Main St" has no number)
+        assert!(contains_named_entities(
+            "Main St is nice but 123 Elm St is better"
+        ));
+        // Unicode preceding the suffix must not panic
+        assert!(contains_named_entities("café résumé 42 Oak Dr in town"));
+        assert!(!contains_named_entities("café résumé naïve über straße"));
+    }
+
+    #[test]
+    fn test_ner_detects_phone_with_dots() {
+        assert!(contains_named_entities("call 555.123.4567 for info"));
+    }
 }