Introduce a token efficiency checker

LICENSE-3rdparty.csv

@@ -13,6 +13,8 @@ base58ck,https://github.com/rust-bitcoin/rust-bitcoin,CC0-1.0,Andrew Poelstra <a
 base62,https://github.com/fbernier/base62,MIT,"François Bernier <frankbernier@gmail.com>, Chai T. Rex <ChaiTRex@users.noreply.github.com>"
 base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,Marshall Pierce <marshall@mpierce.org>
 bech32,https://github.com/rust-bitcoin/rust-bech32,MIT,"Clark Moody, Andrew Poelstra, Tobin Harding"
+bit-set,https://github.com/contain-rs/bit-set,Apache-2.0 OR MIT,Alexis Beingessner <a.beingessner@gmail.com>
+bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessner <a.beingessner@gmail.com>
 bitcoin,https://github.com/rust-bitcoin/rust-bitcoin,CC0-1.0,Andrew Poelstra <apoelstra@wpsoftware.net>
 bitcoin-internals,https://github.com/rust-bitcoin/rust-bitcoin,CC0-1.0,"Andrew Poelstra <apoelstra@wpsoftware.net>, The Rust Bitcoin developers"
 bitcoin-io,https://github.com/rust-bitcoin/rust-bitcoin,CC0-1.0,Matt Corallo <birchneutea@mattcorallo.com>
@@ -21,6 +23,7 @@ bitcoin_hashes,https://github.com/rust-bitcoin/rust-bitcoin,CC0-1.0,Andrew Poels
 bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers
 block-buffer,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers
 bs58,https://github.com/Nullus157/bs58-rs,MIT OR Apache-2.0,The bs58 Authors
+bstr,https://github.com/BurntSushi/bstr,MIT OR Apache-2.0,Andrew Gallant <jamslam@gmail.com>
 bumpalo,https://github.com/fitzgen/bumpalo,MIT OR Apache-2.0,Nick Fitzgerald <fitzgen@gmail.com>
 byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant <jamslam@gmail.com>
 bytes,https://github.com/tokio-rs/bytes,MIT,"Carl Lerche <me@carllerche.com>, Sean McArthur <sean@seanmonstar.com>"
@@ -53,6 +56,7 @@ endian-type,https://github.com/Lolirofle/endian-type,MIT,Lolirofle <lolipopple@h
 equivalent,https://github.com/indexmap-rs/equivalent,Apache-2.0 OR MIT,The equivalent Authors
 errno,https://github.com/lambda-fairy/rust-errno,MIT OR Apache-2.0,"Chris Wong <lambda.fairy@gmail.com>, Dan Gohman <dev@sunfishcode.online>"
 ethaddr,https://github.com/nlordell/ethprim-rs,MIT OR Apache-2.0,Nicholas Rodrigues Lordello <nlordell@gmail.com>
+fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien <raph@google.com>, Robin Stocker <robin@nibor.org>, Keith Hall <keith.hall@available.systems>"
 farmhash2,https://github.com/seiflotfy/rust-farmhash,MIT,Seif Lotfy <seif.lotfy@gmail.com>
 fiat-crypto,https://github.com/mit-plv/fiat-crypto,MIT OR Apache-2.0 OR BSD-1-Clause,Fiat Crypto library authors <jgross@mit.edu>
 fixed-hash,https://github.com/paritytech/parity-common,MIT OR Apache-2.0,Parity Technologies <admin@parity.io>
@@ -170,6 +174,7 @@ regex-syntax,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Proj
 regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant <jamslam@gmail.com>"
 reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur <sean@seanmonstar.com>
 ring,https://github.com/briansmith/ring,ISC OR Apache License v2,The ring Authors
+rustc-hash,https://github.com/rust-lang-nursery/rustc-hash,Apache-2.0 OR MIT,The Rust Project Developers
 rustc-hash,https://github.com/rust-lang/rustc-hash,Apache-2.0 OR MIT,The Rust Project Developers
 rustc-hex,https://github.com/debris/rustc-hex,MIT OR Apache-2.0,"The Rust Project Developers, debris <marek.kotewicz@gmail.com>, Robert Habermeier"
 rustc_version,https://github.com/djc/rustc-version-rs,MIT OR Apache-2.0,The rustc_version Authors
@@ -216,6 +221,7 @@ synstructure,https://github.com/mystor/synstructure,MIT,Nika Layzell <nika@thela
 tagptr,https://github.com/oliver-giersch/tagptr,MIT OR Apache-2.0,Oliver Giersch
 thiserror,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
 thiserror-impl,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
+tiktoken-rs,https://github.com/zurawiki/tiktoken-rs,MIT,Roger Zurawicki <roger@zura.wiki>
 time,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt <open-source@jhpratt.dev>, Time contributors"
 time-core,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt <open-source@jhpratt.dev>, Time contributors"
 time-macros,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt <open-source@jhpratt.dev>, Time contributors"

sds/Cargo.toml

@@ -72,6 +72,7 @@ rand = { version = "0.8.5", optional = true }
 slab = "0.4.11"
 farmhash2 = "1.1.5"
 regex-pool = "0.1.1"
+tiktoken-rs = "0.11.0"
 
 # Optional fuzzing dependency.
 hyperscan = { version = "0.3.2", features = ["static"], optional = true }

sds/benches/bench.rs

@@ -3,7 +3,7 @@ use dd_sds::{
     ContentVisitor, ExclusionCheck, Path, PathSegment, ProximityKeywordsConfig, RegexRuleConfig,
     RootRuleConfig, RuleIndexVisitor, ScannerError, Scope, ScopedRuleSet,
 };
-use dd_sds::{LuhnChecksum, Validator};
+use dd_sds::{EntropyCheck, LuhnChecksum, TokenEfficiencyCheck, Validator};
 use dd_sds::{Scanner, SimpleEvent};
 use std::collections::BTreeMap;
 
@@ -117,6 +117,35 @@ pub fn luhn_checksum(c: &mut Criterion) {
     });
 }
 
+/// `TokenEfficiencyCheck` runs a full BPE tokenization, so it is far more expensive than the
+/// other statistical validator. Benchmark it against `EntropyCheck` on identical inputs to keep
+/// that gap visible.
+pub fn token_efficiency_vs_entropy(c: &mut Criterion) {
+    let values = vec![
+        "SOKXxs00k30PUuH4KLoDPNmwlQ4EwXKw",
+        "LibraryWebpageUploadUr1RowStatus",
+        "d41d8cd98f00b204e9800998ecf8427e",
+        "this is a normal sentence with common words",
+    ];
+
+    c.bench_function("entropy-check", |b| {
+        b.iter(|| {
+            for value in &values {
+                EntropyCheck.is_valid_match(value);
+            }
+        })
+    });
+
+    let token_efficiency = TokenEfficiencyCheck::new();
+    c.bench_function("token-efficiency-check", |b| {
+        b.iter(|| {
+            for value in &values {
+                token_efficiency.is_valid_match(value);
+            }
+        })
+    });
+}
+
 pub fn included_keywords(c: &mut Criterion) {
     let scanner = Scanner::builder(&[RootRuleConfig::new(
         RegexRuleConfig::new("[a-zA-z0-9]{4,25}")
@@ -318,6 +347,7 @@ criterion::criterion_group!(
     benches,
     scoped_ruleset,
     luhn_checksum,
+    token_efficiency_vs_entropy,
     included_keywords,
     included_keywords_on_path,
     multipass_excluded_scan

sds/src/lib.rs

@@ -131,5 +131,5 @@ pub use scanner::debug_scan::debug_scan;
 #[cfg(all(feature = "dd-sds", any(feature = "testing", feature = "bench")))]
 pub use crate::{
     scoped_ruleset::{ContentVisitor, RuleIndexVisitor, ScopedRuleSet},
-    secondary_validation::{LuhnChecksum, Validator},
+    secondary_validation::{EntropyCheck, LuhnChecksum, TokenEfficiencyCheck, Validator},
 };

sds/src/scanner/regex_rule/config.rs

@@ -277,6 +277,7 @@ pub enum SecondaryValidator {
     SpanishDniChecksum,
     SpanishNussChecksum,
     SwedenPINChecksum,
+    TokenEfficiencyCheck,
     UkNinoFormatCheck,
     UkTrnChecksum,
     UsDeaChecksum,

sds/src/scanner/test/validators.rs

@@ -1,6 +1,6 @@
 use crate::SecondaryValidator::{
     ChineseIdChecksum, GithubTokenChecksum, IbanChecker, JwtExpirationChecker, NhsCheckDigit,
-    NonHexChecker,
+    NonHexChecker, TokenEfficiencyCheck,
 };
 use crate::scanner::RootRuleConfig;
 use crate::{MatchAction, RegexRuleConfig, ScannerBuilder, SecondaryValidator};
@@ -200,3 +200,34 @@ fn test_non_hex_checker_filters_pure_hex() {
     assert_eq!(scanner_with.scan(&mut with_prefix).unwrap().len(), 1);
     assert_eq!(with_prefix, "[token]");
 }
+
+#[test]
+fn test_token_efficiency_check_filters_identifiers() {
+    let rule = RegexRuleConfig::new("[A-Za-z0-9]{16,}");
+    let match_action = MatchAction::Redact {
+        replacement: "[secret]".to_string(),
+    };
+
+    let rule_with_validator = RootRuleConfig::new(
+        rule.clone()
+            .with_validator(Some(TokenEfficiencyCheck))
+            .build(),
+    )
+    .match_action(match_action.clone());
+
+    // Both values are 32 mixed-case alphanumerics, so the regex alone cannot tell them apart.
+    let mut content =
+        "SOKXxs00k30PUuH4KLoDPNmwlQ4EwXKw LibraryWebpageUploadUr1RowStatus".to_string();
+    let scanner_without =
+        ScannerBuilder::new(&[RootRuleConfig::new(rule.build()).match_action(match_action)])
+            .build()
+            .unwrap();
+    assert_eq!(scanner_without.scan(&mut content).unwrap().len(), 2);
+    assert_eq!(content, "[secret] [secret]");
+
+    let scanner_with = ScannerBuilder::new(&[rule_with_validator]).build().unwrap();
+    let mut content =
+        "SOKXxs00k30PUuH4KLoDPNmwlQ4EwXKw LibraryWebpageUploadUr1RowStatus".to_string();
+    assert_eq!(scanner_with.scan(&mut content).unwrap().len(), 1);
+    assert_eq!(content, "[secret] LibraryWebpageUploadUr1RowStatus");
+}

sds/src/secondary_validation/mod.rs

@@ -50,6 +50,7 @@ mod slovenian_pin_checksum;
 mod spain_dni_checksum;
 mod spanish_nuss_checksum;
 mod sweden_pin_checksum;
+mod token_efficiency;
 mod uk_nino_format_check;
 mod uk_trn_checksum;
 mod us_dea_checksum;
@@ -115,6 +116,7 @@ pub use crate::secondary_validation::slovenian_pin_checksum::SlovenianPINChecksu
 pub use crate::secondary_validation::spain_dni_checksum::SpanishDniChecksum;
 pub use crate::secondary_validation::spanish_nuss_checksum::SpanishNussChecksum;
 pub use crate::secondary_validation::sweden_pin_checksum::SwedenPINChecksum;
+pub use crate::secondary_validation::token_efficiency::TokenEfficiencyCheck;
 pub use crate::secondary_validation::uk_nino_format_check::UkNinoFormatCheck;
 pub use crate::secondary_validation::uk_trn_checksum::UkTrnChecksum;
 pub use crate::secondary_validation::us_dea_checksum::UsDeaChecksum;
@@ -272,6 +274,7 @@ impl SecondaryValidator {
             SecondaryValidator::SpanishDniChecksum => Arc::new(SpanishDniChecksum),
             SecondaryValidator::SpanishNussChecksum => Arc::new(SpanishNussChecksum),
             SecondaryValidator::SwedenPINChecksum => Arc::new(SwedenPINChecksum),
+            SecondaryValidator::TokenEfficiencyCheck => Arc::new(TokenEfficiencyCheck::new()),
             SecondaryValidator::UkNinoFormatCheck => Arc::new(UkNinoFormatCheck),
             SecondaryValidator::UkTrnChecksum => Arc::new(UkTrnChecksum),
             SecondaryValidator::UsDeaChecksum => Arc::new(UsDeaChecksum),