diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index e4754330..fd4ea702 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -13,6 +13,8 @@ base58ck,https://github.com/rust-bitcoin/rust-bitcoin,CC0-1.0,Andrew Poelstra , Chai T. Rex " base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,Marshall Pierce bech32,https://github.com/rust-bitcoin/rust-bech32,MIT,"Clark Moody, Andrew Poelstra, Tobin Harding" +bit-set,https://github.com/contain-rs/bit-set,Apache-2.0 OR MIT,Alexis Beingessner +bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessner bitcoin,https://github.com/rust-bitcoin/rust-bitcoin,CC0-1.0,Andrew Poelstra bitcoin-internals,https://github.com/rust-bitcoin/rust-bitcoin,CC0-1.0,"Andrew Poelstra , The Rust Bitcoin developers" bitcoin-io,https://github.com/rust-bitcoin/rust-bitcoin,CC0-1.0,Matt Corallo @@ -21,6 +23,7 @@ bitcoin_hashes,https://github.com/rust-bitcoin/rust-bitcoin,CC0-1.0,Andrew Poels bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers block-buffer,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers bs58,https://github.com/Nullus157/bs58-rs,MIT OR Apache-2.0,The bs58 Authors +bstr,https://github.com/BurntSushi/bstr,MIT OR Apache-2.0,Andrew Gallant bumpalo,https://github.com/fitzgen/bumpalo,MIT OR Apache-2.0,Nick Fitzgerald byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant bytes,https://github.com/tokio-rs/bytes,MIT,"Carl Lerche , Sean McArthur " @@ -53,6 +56,7 @@ endian-type,https://github.com/Lolirofle/endian-type,MIT,Lolirofle , Dan Gohman " ethaddr,https://github.com/nlordell/ethprim-rs,MIT OR Apache-2.0,Nicholas Rodrigues Lordello +fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker , Keith Hall " farmhash2,https://github.com/seiflotfy/rust-farmhash,MIT,Seif Lotfy fiat-crypto,https://github.com/mit-plv/fiat-crypto,MIT OR Apache-2.0 OR BSD-1-Clause,Fiat Crypto library authors fixed-hash,https://github.com/paritytech/parity-common,MIT OR Apache-2.0,Parity Technologies @@ -170,6 +174,7 @@ regex-syntax,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Proj regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur ring,https://github.com/briansmith/ring,ISC OR Apache License v2,The ring Authors +rustc-hash,https://github.com/rust-lang-nursery/rustc-hash,Apache-2.0 OR MIT,The Rust Project Developers rustc-hash,https://github.com/rust-lang/rustc-hash,Apache-2.0 OR MIT,The Rust Project Developers rustc-hex,https://github.com/debris/rustc-hex,MIT OR Apache-2.0,"The Rust Project Developers, debris , Robert Habermeier" rustc_version,https://github.com/djc/rustc-version-rs,MIT OR Apache-2.0,The rustc_version Authors @@ -216,6 +221,7 @@ synstructure,https://github.com/mystor/synstructure,MIT,Nika Layzell thiserror-impl,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay +tiktoken-rs,https://github.com/zurawiki/tiktoken-rs,MIT,Roger Zurawicki time,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" time-core,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" time-macros,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" diff --git a/sds/Cargo.lock b/sds/Cargo.lock index d030930e..d5990cfb 100644 --- a/sds/Cargo.lock +++ b/sds/Cargo.lock @@ -356,7 +356,16 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" dependencies = [ - "bit-vec", + "bit-vec 0.6.3", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", ] [[package]] @@ -365,6 +374,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitcoin" version = "0.32.9" @@ -456,6 +471,17 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -816,6 +842,7 @@ dependencies = [ "strum", "thiserror 1.0.69", "threadpool", + "tiktoken-rs", "tokio", ] @@ -972,6 +999,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fancy-regex" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" +dependencies = [ + "bit-set 0.8.0", + "regex-automata", + "regex-syntax 0.8.10", +] + [[package]] name = "farmhash2" version = "1.1.5" @@ -1785,7 +1823,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cb077ad656299f160924eb2912aa147d7339ea7d69e1b5517326fdcec3c1ca" dependencies = [ "ascii-canvas", - "bit-set", + "bit-set 0.5.3", "ena", "itertools 0.11.0", "lalrpop-util", @@ -2266,7 +2304,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.2", "rustls", "socket2 0.6.3", "thiserror 2.0.18", @@ -2286,7 +2324,7 @@ dependencies = [ "lru-slab", "rand 0.9.4", "ring", - "rustc-hash", + "rustc-hash 2.1.2", "rustls", "rustls-pki-types", "slab", @@ -2576,6 +2614,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.2" @@ -3172,6 +3216,21 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "tiktoken-rs" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fac4a168cfc1d8ed65bf17a6ee0843ad9a68f863c63c0fb2fa7eab67838782ee" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bstr", + "fancy-regex", + "lazy_static", + "regex", + "rustc-hash 1.1.0", +] + [[package]] name = "time" version = "0.3.47" diff --git a/sds/Cargo.toml b/sds/Cargo.toml index 5eb0436a..1fd6e98d 100644 --- a/sds/Cargo.toml +++ b/sds/Cargo.toml @@ -72,6 +72,7 @@ rand = { version = "0.8.5", optional = true } slab = "0.4.11" farmhash2 = "1.1.5" regex-pool = "0.1.1" +tiktoken-rs = "0.11.0" # Optional fuzzing dependency. hyperscan = { version = "0.3.2", features = ["static"], optional = true } diff --git a/sds/benches/bench.rs b/sds/benches/bench.rs index 169e3c81..3c6f4227 100644 --- a/sds/benches/bench.rs +++ b/sds/benches/bench.rs @@ -3,7 +3,7 @@ use dd_sds::{ ContentVisitor, ExclusionCheck, Path, PathSegment, ProximityKeywordsConfig, RegexRuleConfig, RootRuleConfig, RuleIndexVisitor, ScannerError, Scope, ScopedRuleSet, }; -use dd_sds::{LuhnChecksum, Validator}; +use dd_sds::{EntropyCheck, LuhnChecksum, TokenEfficiencyCheck, Validator}; use dd_sds::{Scanner, SimpleEvent}; use std::collections::BTreeMap; @@ -117,6 +117,35 @@ pub fn luhn_checksum(c: &mut Criterion) { }); } +/// `TokenEfficiencyCheck` runs a full BPE tokenization, so it is far more expensive than the +/// other statistical validator. Benchmark it against `EntropyCheck` on identical inputs to keep +/// that gap visible. +pub fn token_efficiency_vs_entropy(c: &mut Criterion) { + let values = vec![ + "SOKXxs00k30PUuH4KLoDPNmwlQ4EwXKw", + "LibraryWebpageUploadUr1RowStatus", + "d41d8cd98f00b204e9800998ecf8427e", + "this is a normal sentence with common words", + ]; + + c.bench_function("entropy-check", |b| { + b.iter(|| { + for value in &values { + EntropyCheck.is_valid_match(value); + } + }) + }); + + let token_efficiency = TokenEfficiencyCheck::new(); + c.bench_function("token-efficiency-check", |b| { + b.iter(|| { + for value in &values { + token_efficiency.is_valid_match(value); + } + }) + }); +} + pub fn included_keywords(c: &mut Criterion) { let scanner = Scanner::builder(&[RootRuleConfig::new( RegexRuleConfig::new("[a-zA-z0-9]{4,25}") @@ -318,6 +347,7 @@ criterion::criterion_group!( benches, scoped_ruleset, luhn_checksum, + token_efficiency_vs_entropy, included_keywords, included_keywords_on_path, multipass_excluded_scan diff --git a/sds/src/lib.rs b/sds/src/lib.rs index e08abf80..fb45c512 100644 --- a/sds/src/lib.rs +++ b/sds/src/lib.rs @@ -131,5 +131,5 @@ pub use scanner::debug_scan::debug_scan; #[cfg(all(feature = "dd-sds", any(feature = "testing", feature = "bench")))] pub use crate::{ scoped_ruleset::{ContentVisitor, RuleIndexVisitor, ScopedRuleSet}, - secondary_validation::{LuhnChecksum, Validator}, + secondary_validation::{EntropyCheck, LuhnChecksum, TokenEfficiencyCheck, Validator}, }; diff --git a/sds/src/scanner/regex_rule/config.rs b/sds/src/scanner/regex_rule/config.rs index 5fd9e669..d37f971b 100644 --- a/sds/src/scanner/regex_rule/config.rs +++ b/sds/src/scanner/regex_rule/config.rs @@ -277,6 +277,7 @@ pub enum SecondaryValidator { SpanishDniChecksum, SpanishNussChecksum, SwedenPINChecksum, + TokenEfficiencyCheck, UkNinoFormatCheck, UkTrnChecksum, UsDeaChecksum, diff --git a/sds/src/scanner/test/validators.rs b/sds/src/scanner/test/validators.rs index 1d50fd01..10bc7aff 100644 --- a/sds/src/scanner/test/validators.rs +++ b/sds/src/scanner/test/validators.rs @@ -1,6 +1,6 @@ use crate::SecondaryValidator::{ ChineseIdChecksum, GithubTokenChecksum, IbanChecker, JwtExpirationChecker, NhsCheckDigit, - NonHexChecker, + NonHexChecker, TokenEfficiencyCheck, }; use crate::scanner::RootRuleConfig; use crate::{MatchAction, RegexRuleConfig, ScannerBuilder, SecondaryValidator}; @@ -200,3 +200,34 @@ fn test_non_hex_checker_filters_pure_hex() { assert_eq!(scanner_with.scan(&mut with_prefix).unwrap().len(), 1); assert_eq!(with_prefix, "[token]"); } + +#[test] +fn test_token_efficiency_check_filters_identifiers() { + let rule = RegexRuleConfig::new("[A-Za-z0-9]{16,}"); + let match_action = MatchAction::Redact { + replacement: "[secret]".to_string(), + }; + + let rule_with_validator = RootRuleConfig::new( + rule.clone() + .with_validator(Some(TokenEfficiencyCheck)) + .build(), + ) + .match_action(match_action.clone()); + + // Both values are 32 mixed-case alphanumerics, so the regex alone cannot tell them apart. + let mut content = + "SOKXxs00k30PUuH4KLoDPNmwlQ4EwXKw LibraryWebpageUploadUr1RowStatus".to_string(); + let scanner_without = + ScannerBuilder::new(&[RootRuleConfig::new(rule.build()).match_action(match_action)]) + .build() + .unwrap(); + assert_eq!(scanner_without.scan(&mut content).unwrap().len(), 2); + assert_eq!(content, "[secret] [secret]"); + + let scanner_with = ScannerBuilder::new(&[rule_with_validator]).build().unwrap(); + let mut content = + "SOKXxs00k30PUuH4KLoDPNmwlQ4EwXKw LibraryWebpageUploadUr1RowStatus".to_string(); + assert_eq!(scanner_with.scan(&mut content).unwrap().len(), 1); + assert_eq!(content, "[secret] LibraryWebpageUploadUr1RowStatus"); +} diff --git a/sds/src/secondary_validation/mod.rs b/sds/src/secondary_validation/mod.rs index a8091e5c..1bcf86b1 100644 --- a/sds/src/secondary_validation/mod.rs +++ b/sds/src/secondary_validation/mod.rs @@ -50,6 +50,7 @@ mod slovenian_pin_checksum; mod spain_dni_checksum; mod spanish_nuss_checksum; mod sweden_pin_checksum; +mod token_efficiency; mod uk_nino_format_check; mod uk_trn_checksum; mod us_dea_checksum; @@ -115,6 +116,7 @@ pub use crate::secondary_validation::slovenian_pin_checksum::SlovenianPINChecksu pub use crate::secondary_validation::spain_dni_checksum::SpanishDniChecksum; pub use crate::secondary_validation::spanish_nuss_checksum::SpanishNussChecksum; pub use crate::secondary_validation::sweden_pin_checksum::SwedenPINChecksum; +pub use crate::secondary_validation::token_efficiency::TokenEfficiencyCheck; pub use crate::secondary_validation::uk_nino_format_check::UkNinoFormatCheck; pub use crate::secondary_validation::uk_trn_checksum::UkTrnChecksum; pub use crate::secondary_validation::us_dea_checksum::UsDeaChecksum; @@ -272,6 +274,7 @@ impl SecondaryValidator { SecondaryValidator::SpanishDniChecksum => Arc::new(SpanishDniChecksum), SecondaryValidator::SpanishNussChecksum => Arc::new(SpanishNussChecksum), SecondaryValidator::SwedenPINChecksum => Arc::new(SwedenPINChecksum), + SecondaryValidator::TokenEfficiencyCheck => Arc::new(TokenEfficiencyCheck::new()), SecondaryValidator::UkNinoFormatCheck => Arc::new(UkNinoFormatCheck), SecondaryValidator::UkTrnChecksum => Arc::new(UkTrnChecksum), SecondaryValidator::UsDeaChecksum => Arc::new(UsDeaChecksum), diff --git a/sds/src/secondary_validation/token_efficiency.rs b/sds/src/secondary_validation/token_efficiency.rs new file mode 100644 index 00000000..2f22263c --- /dev/null +++ b/sds/src/secondary_validation/token_efficiency.rs @@ -0,0 +1,98 @@ +use crate::secondary_validation::Validator; +use tiktoken_rs::cl100k_base_singleton; + +// Prose and identifiers merge into long BPE tokens; random secrets do not. Below this many +// characters per token, the value is too incompressible to be natural language. +const MAX_CHARS_PER_TOKEN: f64 = 2.5; + +pub struct TokenEfficiencyCheck; + +impl TokenEfficiencyCheck { + pub fn new() -> Self { + // Build the ~100k-entry cl100k vocabulary now, during scanner construction, so that the + // first scanned event does not absorb the ~40ms of table building. Rules that do not use + // this validator never construct one, and so never pay for the vocabulary at all. + let _ = cl100k_base_singleton(); + Self + } +} + +impl Default for TokenEfficiencyCheck { + fn default() -> Self { + Self::new() + } +} + +impl Validator for TokenEfficiencyCheck { + fn is_valid_match(&self, regex_match: &str) -> bool { + let value = regex_match.trim(); + let token_count = cl100k_base_singleton().encode_ordinary(value).len(); + if token_count == 0 { + return false; + } + (value.chars().count() as f64 / token_count as f64) < MAX_CHARS_PER_TOKEN + } +} + +#[cfg(test)] +mod test { + use crate::secondary_validation::Validator; + use crate::secondary_validation::token_efficiency::TokenEfficiencyCheck; + + /// Both strings are 32 mixed-case alphanumerics, so a character-frequency measure such as + /// `EntropyCheck` barely separates them. The BPE vocabulary does: the identifier splits into + /// a handful of English word pieces, while the random token shatters into many short ones. + /// This pair is the reason this validator exists - keep it as an explicit test. + #[test] + fn separates_random_token_from_camel_case_identifier() { + assert!(TokenEfficiencyCheck.is_valid_match("SOKXxs00k30PUuH4KLoDPNmwlQ4EwXKw")); + assert!(!TokenEfficiencyCheck.is_valid_match("LibraryWebpageUploadUr1RowStatus")); + } + + #[test] + fn accepts_secret_shaped_values() { + let valid_inputs = vec![ + "k9V@x2L#q7R!m3T$w8Z%h4N^p1D&y6", + "d41d8cd98f00b204e9800998ecf8427e", + ]; + + for input in valid_inputs { + assert!( + TokenEfficiencyCheck.is_valid_match(input), + "expected a match for {input}" + ); + } + } + + #[test] + fn rejects_natural_language_and_placeholders() { + let invalid_inputs = vec![ + "this is a normal sentence with common words", + "YOUR_API_KEY_GOES_HERE", + // Short natural input stays rejected without a minimum-length guard, because it is a + // single token: 3 chars / 1 token is well above the threshold. + "abc", + ]; + + for input in invalid_inputs { + assert!( + !TokenEfficiencyCheck.is_valid_match(input), + "expected no match for {input}" + ); + } + } + + #[test] + fn ignores_surrounding_whitespace() { + assert_eq!( + TokenEfficiencyCheck.is_valid_match(" SOKXxs00k30PUuH4KLoDPNmwlQ4EwXKw "), + TokenEfficiencyCheck.is_valid_match("SOKXxs00k30PUuH4KLoDPNmwlQ4EwXKw") + ); + } + + #[test] + fn rejects_input_without_tokens() { + assert!(!TokenEfficiencyCheck.is_valid_match("")); + assert!(!TokenEfficiencyCheck.is_valid_match(" ")); + } +}