From b23896ad1102bd8a73379158aeb7df5429a89042 Mon Sep 17 00:00:00 2001 From: Matt Brown Date: Thu, 17 Sep 2026 23:56:55 -0400 Subject: [PATCH] fix: filter structured-text false positives from the generic-secret detector The generic-secret rule matches a keyword (password/secret/api_key/...) plus a high-entropy value, but its value alphabet and gates (length + entropy + a small wordlist) did not tell an opaque credential apart from structured text. On firmware with a web UI it produced almost entirely false positives: printf/URL format strings, JavaScript object access, and localization strings, all of which are long and diverse enough to clear the entropy gate. Add a value-shape filter (generic_value_is_noise) that rejects a value which is (1) a format string ('%' + a printf conversion), (2) an assignment/query shape (an interior '=' or any '&'; a trailing '=' run is treated as base64 padding), or (3) a no-digit word/identifier chain (after trimming non-alphanumeric ends, the core is only letters and '.'/'_'/'-' separators). Real credential values carry a digit or base64 density and are kept. Measured across the corpus: false positives from web/source/localization drop to zero (e.g. a router web UI 7 -> 0, an NVR web UI 32 -> 0), while real values are retained (a device's base64-encoded config credentials 29 -> 29, a JWT-shaped access token kept). The one accepted false negative, documented in the code, is a purely-alphabetic hardcoded password with no digit, at this pattern tier. Add test_generic_secret_shape covering the three reject shapes (including the leading/trailing-punctuation and consecutive-separator variants seen in minified JS) and two must-keep positives. --- src/rules_builtin.cpp | 60 ++++++++++++++++++++++++++++++++++++++- tests/unit/unit_tests.cpp | 27 ++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/src/rules_builtin.cpp b/src/rules_builtin.cpp index 9afa746..8f8cb00 100644 --- a/src/rules_builtin.cpp +++ b/src/rules_builtin.cpp @@ -1,6 +1,7 @@ #include "rules_builtin.hpp" #include +#include #include #include "validators.hpp" @@ -148,8 +149,64 @@ std::optional match_pem_private(const Reader& r, size_t off) { return m; } +// Generic-secret precision: reject "values" that are structured text, not an +// opaque credential token. Real hardcoded secrets carry a digit / base64 +// density; the corpus false positives are all one of three shapes: +// (1) format strings -- "...%s&mac=%s..." (printf/URL templates) +// (2) assignment/query -- "a=b&c=d" (query strings, not a field) +// (3) no-digit word/identifier -- "document.getElementsByName", +// "createInputPseudo", "Passwortwiederherstellung" (code / minified JS / +// localization strings) +// These three drop the code/web-UI/i18n FPs while keeping digit-bearing tokens +// (e.g. a JWT/base64 access token). Precision-first at the pattern tier; a +// purely-alphabetic hardcoded password is the one accepted false-negative. +bool generic_value_is_noise(const std::string& v) { + // (1) printf/format specifier: '%' + optional flags/width/precision + conv. + for (size_t i = 0; i + 1 < v.size(); ++i) { + if (v[i] != '%') continue; + size_t j = i + 1; + while (j < v.size() && (std::strchr("-+ 0#.", v[j]) != nullptr || (v[j] >= '0' && v[j] <= '9'))) + ++j; + if (j < v.size() && std::strchr("sdiouxXeEfgGcpaAn@", v[j]) != nullptr) return true; + } + // (2) assignment / query shape: an interior '=' or any '&' (a trailing run of + // '=' is base64 padding and is ignored). + size_t end = v.size(); + while (end > 0 && v[end - 1] == '=') --end; + for (size_t i = 0; i < end; ++i) + if (v[i] == '=' || v[i] == '&') return true; + // (3) no-digit word / identifier chain: letters joined by single '.', '_' or + // '-' separators, no digit -> code / prose, not a random token. Trim + // leading/trailing non-alphanumerics first (stray quotes, operators, and + // sentence punctuation carried in from source/markup, e.g. "+this.x.y" or + // "Wachtwoord-formatteerfout." or "this._szAuth="). + auto is_alnum = [](char c) { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'); + }; + size_t a = 0, b = v.size(); + while (a < b && !is_alnum(v[a])) ++a; + while (b > a && !is_alnum(v[b - 1])) --b; + if (b > a) { + bool has_digit = false; + for (size_t i = a; i < b; ++i) + if (v[i] >= '0' && v[i] <= '9') { has_digit = true; break; } + if (!has_digit) { + // The trimmed core (alnum at both ends, no digit) is word/identifier + // shaped when every char is a letter or a '.'/'_'/'-' separator -- + // dotted member access, snake/camel identifiers, hyphenated words. + auto is_alpha = [](char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); }; + auto is_sep = [](char c) { return c == '.' || c == '_' || c == '-'; }; + bool shape = true; + for (size_t i = a; shape && i < b; ++i) + if (!is_alpha(v[i]) && !is_sep(v[i])) shape = false; + if (shape) return true; + } + } + return false; +} + // Generic assignment: KEY . Noisy -> hard entropy gate -// applied by the engine (min_entropy on the rule). +// applied by the engine (min_entropy on the rule), plus a value-shape filter. std::optional match_generic(const Reader& r, size_t off) { size_t p = off; p += run(r, p, 32, c_alnum_us); // the keyword @@ -171,6 +228,7 @@ std::optional match_generic(const Reader& r, size_t off) { } size_t v = run(r, p, 200, c_secretval); if (v < 16) return std::nullopt; + if (generic_value_is_noise(token_str(r, p, v))) return std::nullopt; return Match{(p + v) - off, Confidence::Pattern, "", "", "", ""}; } diff --git a/tests/unit/unit_tests.cpp b/tests/unit/unit_tests.cpp index e3f803e..bd6f28f 100644 --- a/tests/unit/unit_tests.cpp +++ b/tests/unit/unit_tests.cpp @@ -201,6 +201,32 @@ static void test_false_positives() { CHECK(!has_type(scan("secret = abc"), "generic-secret")); } +// generic-secret value-shape filter: structured text (format strings, query +// shapes, code/identifier/localization words) must not be reported, while a real +// digit-bearing opaque token still is. Shapes drawn from real minified-JS web +// bundles and localization files, plus a JWT-shaped access token to keep. +static void test_generic_secret_shape() { + // (1) format strings. + CHECK(!has_type(scan("password=%s&mac=%s&firmware=%s&serial=%s"), "generic-secret")); + CHECK(!has_type(scan("secret = value_is_%s_formatted_padding"), "generic-secret")); + // (2) assignment / query shapes. + CHECK(!has_type(scan("password=user=admin&role=root&scope=all"), "generic-secret")); + // (3) no-digit code / identifier / localization words. + CHECK(!has_type(scan("password = document.getElementsByName sysDNSPassword"), "generic-secret")); + CHECK(!has_type(scan("password: cf.sysDNSPassword_Netgear.value"), "generic-secret")); + CHECK(!has_type(scan("passwd = Passwortwiederherstellungxx"), "generic-secret")); + CHECK(!has_type(scan("secretKey = createInputPseudoElement"), "generic-secret")); + // real minified-JS / localization shapes with stray leading/trailing punct and + // consecutive separators (drawn from the corpus residuals). + CHECK(!has_type(scan("password:this._szSecondAuthValue=\"x"), "generic-secret")); + CHECK(!has_type(scan("password = Wachtwoord-formatteerfoutmelding."), "generic-secret")); + CHECK(!has_type(scan("password = +this.oSecondAuthentication.value"), "generic-secret")); + // Real digit-bearing tokens still match: a JWT-shaped access token (the one + // corpus true positive) and a base64-ish credential. + CHECK(has_type(scan("access_token = eyJhbGciOiJIUzI1NiIsImtpZCI6Im45aXV9x"), "generic-secret")); + CHECK(has_type(scan("password = S3cr3t_Pa55w0rd_9xQ7zLmNq end"), "generic-secret")); +} + static void test_encoded() { std::string enc = b64("cred AKIAJ2K3L4M5N6P7Q8R9 tail"); auto fs = scan("data: " + enc + " done"); @@ -1811,6 +1837,7 @@ int main() { test_jsonparse(); test_detectors(); test_false_positives(); + test_generic_secret_shape(); test_encoded(); test_validators_github_crc(); test_validators_jwt();