From 1b256d411b3839580270a14a5a53c1ad222da7a1 Mon Sep 17 00:00:00 2001 From: Tom Tang <4220945+shiba4life@users.noreply.github.com> Date: Mon, 30 Mar 2026 09:06:51 -0700 Subject: [PATCH 1/4] =?UTF-8?q?fix:=20improve=20discovery=20anonymity=20ga?= =?UTF-8?q?te=20=E2=80=94=20token-level=20entropy,=20address=20NER,=20phon?= =?UTF-8?q?e=20dot=20separator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Change token_entropy() from byte-level to token-level Shannon entropy (measures word diversity, not character diversity) - Recalibrate MIN_ENTROPY_BITS from 2.0 to 1.5 for token-level metric - Add has_address() NER detection for street address patterns - Add '.' as valid phone number separator - Add tests for all new detection patterns Co-Authored-By: Claude Opus 4.6 (1M context) --- src/db_operations/native_index/anonymity.rs | 72 ++++++++++++++++++--- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/src/db_operations/native_index/anonymity.rs b/src/db_operations/native_index/anonymity.rs index 05c03276f..1d96caa2c 100644 --- a/src/db_operations/native_index/anonymity.rs +++ b/src/db_operations/native_index/anonymity.rs @@ -24,7 +24,7 @@ pub enum FragmentDecision { } /// Minimum Shannon entropy (bits per character) for a fragment to be publishable. -const MIN_ENTROPY_BITS: f64 = 2.0; +const MIN_ENTROPY_BITS: f64 = 1.5; /// Minimum word count for a fragment to be publishable. const MIN_WORD_COUNT: usize = 3; @@ -120,7 +120,7 @@ pub fn default_privacy_class(field_name: &str) -> FieldPrivacyClass { /// Check whether text contains named entities (PII patterns). /// Returns true if any PII pattern is detected. pub fn contains_named_entities(text: &str) -> bool { - has_email(text) || has_phone(text) || has_url(text) || has_id_pattern(text) + has_email(text) || has_phone(text) || has_url(text) || has_id_pattern(text) || has_address(text) } fn has_email(text: &str) -> bool { @@ -139,7 +139,7 @@ fn has_phone(text: &str) -> bool { // Check for phone-like patterns: sequences of digits with separators let mut consecutive_phone_chars = 0; for ch in text.chars() { - if ch.is_ascii_digit() || ch == '-' || ch == '(' || ch == ')' || ch == ' ' || ch == '+' + if ch.is_ascii_digit() || ch == '-' || ch == '(' || ch == ')' || ch == ' ' || ch == '+' || ch == '.' { consecutive_phone_chars += 1; } else { @@ -174,17 +174,42 @@ fn has_id_pattern(text: &str) -> bool { false } -/// Calculate Shannon entropy (bits per character) of text. +fn has_address(text: &str) -> bool { + // Look for patterns like "123 Main St" or "456 Oak Avenue" + let street_suffixes = [ + " st", " st.", " street", " ave", " ave.", " avenue", " blvd", " blvd.", + " boulevard", " dr", " dr.", " drive", " rd", " rd.", " road", " ln", + " ln.", " lane", " ct", " ct.", " court", " pl", " pl.", " place", + " way", " cir", " circle", " pkwy", " parkway", + ]; + let lower = text.to_lowercase(); + // Also do a simpler check: number followed by street suffix anywhere + for suffix in &street_suffixes { + if let Some(pos) = lower.find(suffix) { + // Check if there's a number before this suffix (within ~30 chars) + let start = pos.saturating_sub(30); + let preceding = &lower[start..pos]; + if preceding.chars().any(|c| c.is_ascii_digit()) { + return true; + } + } + } + false +} + +/// Calculate Shannon entropy (bits per token) of text. pub fn token_entropy(text: &str) -> f64 { - if text.is_empty() { + let tokens: Vec<&str> = text.split_whitespace().collect(); + if tokens.is_empty() { return 0.0; } let mut freq = std::collections::HashMap::new(); - let total = text.len() as f64; + let total = tokens.len() as f64; - for byte in text.bytes() { - *freq.entry(byte).or_insert(0u64) += 1; + for token in &tokens { + let lower = token.to_lowercase(); + *freq.entry(lower).or_insert(0u64) += 1; } freq.values() @@ -336,8 +361,8 @@ mod tests { #[test] fn test_entropy_too_low() { - // Very short, repetitive text has low entropy - let entropy = token_entropy("aaa"); + // Repetitive tokens have low entropy + let entropy = token_entropy("the the the"); assert!(entropy < MIN_ENTROPY_BITS, "entropy was {}", entropy); } @@ -422,4 +447,31 @@ mod tests { FragmentDecision::Reject("field name suggests PII") ); } + + #[test] + fn test_entropy_token_level_unicode() { + // Unicode text should get same entropy as ASCII with same token diversity + let ascii_entropy = token_entropy("hello world foo bar baz"); + let unicode_entropy = token_entropy("café résumé naïve über straße"); + // Both have 5 unique tokens, so entropy should be similar + assert!( + (ascii_entropy - unicode_entropy).abs() < 0.1, + "ascii={}, unicode={} — should be similar for same token count", + ascii_entropy, + unicode_entropy + ); + } + + #[test] + fn test_ner_detects_address() { + assert!(contains_named_entities("lives at 123 Main St in town")); + assert!(contains_named_entities("office is 456 Oak Avenue")); + assert!(contains_named_entities("send to 789 Elm Blvd.")); + assert!(!contains_named_entities("no address information here")); + } + + #[test] + fn test_ner_detects_phone_with_dots() { + assert!(contains_named_entities("call 555.123.4567 for info")); + } } From 2e945726ec6f2bcb27cf2e9d2f9df493ddee5d5a Mon Sep 17 00:00:00 2001 From: Tom Tang <4220945+shiba4life@users.noreply.github.com> Date: Mon, 30 Mar 2026 09:10:26 -0700 Subject: [PATCH 2/4] style: cargo fmt Co-Authored-By: Claude Opus 4.6 (1M context) --- src/db_operations/native_index/anonymity.rs | 41 ++++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/src/db_operations/native_index/anonymity.rs b/src/db_operations/native_index/anonymity.rs index 1d96caa2c..b598ba395 100644 --- a/src/db_operations/native_index/anonymity.rs +++ b/src/db_operations/native_index/anonymity.rs @@ -139,7 +139,13 @@ fn has_phone(text: &str) -> bool { // Check for phone-like patterns: sequences of digits with separators let mut consecutive_phone_chars = 0; for ch in text.chars() { - if ch.is_ascii_digit() || ch == '-' || ch == '(' || ch == ')' || ch == ' ' || ch == '+' || ch == '.' + if ch.is_ascii_digit() + || ch == '-' + || ch == '(' + || ch == ')' + || ch == ' ' + || ch == '+' + || ch == '.' { consecutive_phone_chars += 1; } else { @@ -177,10 +183,35 @@ fn has_id_pattern(text: &str) -> bool { fn has_address(text: &str) -> bool { // Look for patterns like "123 Main St" or "456 Oak Avenue" let street_suffixes = [ - " st", " st.", " street", " ave", " ave.", " avenue", " blvd", " blvd.", - " boulevard", " dr", " dr.", " drive", " rd", " rd.", " road", " ln", - " ln.", " lane", " ct", " ct.", " court", " pl", " pl.", " place", - " way", " cir", " circle", " pkwy", " parkway", + " st", + " st.", + " street", + " ave", + " ave.", + " avenue", + " blvd", + " blvd.", + " boulevard", + " dr", + " dr.", + " drive", + " rd", + " rd.", + " road", + " ln", + " ln.", + " lane", + " ct", + " ct.", + " court", + " pl", + " pl.", + " place", + " way", + " cir", + " circle", + " pkwy", + " parkway", ]; let lower = text.to_lowercase(); // Also do a simpler check: number followed by street suffix anywhere From 4145814e7535485a99e93a45878c580af8e09130 Mon Sep 17 00:00:00 2001 From: Tom Tang <4220945+shiba4life@users.noreply.github.com> Date: Mon, 30 Mar 2026 09:19:07 -0700 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20UTF-8?= =?UTF-8?q?=20safe=20slicing=20and=20check=20all=20suffix=20occurrences?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Use char_indices().rev() instead of byte offset for preceding-text window, preventing panic on multi-byte UTF-8 codepoints - Loop over all occurrences of each street suffix (not just first match) to avoid false negatives when the first occurrence lacks a number - Add tests for Unicode input and second-occurrence address patterns Co-Authored-By: Claude Opus 4.6 (1M context) --- src/db_operations/native_index/anonymity.rs | 22 +++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/db_operations/native_index/anonymity.rs b/src/db_operations/native_index/anonymity.rs index b598ba395..7e344a1ae 100644 --- a/src/db_operations/native_index/anonymity.rs +++ b/src/db_operations/native_index/anonymity.rs @@ -214,15 +214,22 @@ fn has_address(text: &str) -> bool { " parkway", ]; let lower = text.to_lowercase(); - // Also do a simpler check: number followed by street suffix anywhere + // Check every occurrence of each street suffix for a preceding digit for suffix in &street_suffixes { - if let Some(pos) = lower.find(suffix) { - // Check if there's a number before this suffix (within ~30 chars) - let start = pos.saturating_sub(30); + let mut search_from = 0; + while let Some(rel) = lower[search_from..].find(suffix) { + let pos = search_from + rel; + // Walk backwards up to ~30 chars, staying on a char boundary + let start = lower[..pos] + .char_indices() + .rev() + .nth(30) + .map_or(0, |(i, _)| i); let preceding = &lower[start..pos]; if preceding.chars().any(|c| c.is_ascii_digit()) { return true; } + search_from = pos + suffix.len(); } } false @@ -499,6 +506,13 @@ mod tests { assert!(contains_named_entities("office is 456 Oak Avenue")); assert!(contains_named_entities("send to 789 Elm Blvd.")); assert!(!contains_named_entities("no address information here")); + // Second occurrence has the digit (first "Main St" has no number) + assert!(contains_named_entities( + "Main St is nice but 123 Elm St is better" + )); + // Unicode preceding the suffix must not panic + assert!(contains_named_entities("café résumé 42 Oak Dr in town")); + assert!(!contains_named_entities("café résumé naïve über straße")); } #[test] From b64d6fb7acb3bfdb914c389ed222eae9a72f0cdd Mon Sep 17 00:00:00 2001 From: Tom Tang <4220945+shiba4life@users.noreply.github.com> Date: Mon, 30 Mar 2026 09:19:57 -0700 Subject: [PATCH 4/4] fix: additional char-boundary safety for search_from advancement Co-Authored-By: Claude Opus 4.6 (1M context) --- src/db_operations/native_index/anonymity.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/db_operations/native_index/anonymity.rs b/src/db_operations/native_index/anonymity.rs index 7e344a1ae..02324d71b 100644 --- a/src/db_operations/native_index/anonymity.rs +++ b/src/db_operations/native_index/anonymity.rs @@ -217,7 +217,14 @@ fn has_address(text: &str) -> bool { // Check every occurrence of each street suffix for a preceding digit for suffix in &street_suffixes { let mut search_from = 0; - while let Some(rel) = lower[search_from..].find(suffix) { + loop { + // Ensure search_from is on a char boundary + if search_from >= lower.len() || !lower.is_char_boundary(search_from) { + break; + } + let Some(rel) = lower[search_from..].find(suffix) else { + break; + }; let pos = search_from + rel; // Walk backwards up to ~30 chars, staying on a char boundary let start = lower[..pos] @@ -229,7 +236,12 @@ fn has_address(text: &str) -> bool { if preceding.chars().any(|c| c.is_ascii_digit()) { return true; } - search_from = pos + suffix.len(); + let next = pos + suffix.len(); + // Advance to the next char boundary after the suffix + search_from = lower[next..] + .char_indices() + .next() + .map_or(lower.len(), |(i, _)| next + i); } } false