diff --git a/sds-go/go/regex_rule.go b/sds-go/go/regex_rule.go index 05ab71ad..a25d9528 100644 --- a/sds-go/go/regex_rule.go +++ b/sds-go/go/regex_rule.go @@ -18,6 +18,7 @@ type RegexRuleConfig struct { Pattern string `json:"pattern"` MatchAction MatchAction `json:"match_action"` ProximityKeywords *ProximityKeywordsConfig `json:"proximity_keywords,omitempty"` + Suppressions Suppressions `json:"suppressions,omitempty"` SecondaryValidator *SecondaryValidator `json:"validator,omitempty"` ThirdPartyActiveChecker ThirdPartyActiveChecker `json:"third_party_active_checker,omitempty"` PatternCaptureGroups []string `json:"pattern_capture_groups,omitempty"` @@ -126,6 +127,7 @@ const ( // ExtraConfig is used to provide more configuration while creating the rules. type ExtraConfig struct { ProximityKeywords *ProximityKeywordsConfig + Suppressions Suppressions SecondaryValidator *SecondaryValidator ThirdPartyActiveChecker ThirdPartyActiveChecker PatternCaptureGroups []string @@ -155,6 +157,13 @@ type ProximityKeywordsConfig struct { ExcludedKeywords []string `json:"excluded_keywords"` } +// Suppressions holds a configuration to suppress matches. +type Suppressions struct { + StartsWith []string `json:"starts_with,omitempty"` + EndsWith []string `json:"ends_with,omitempty"` + ExactMatch []string `json:"exact_match,omitempty"` +} + type MatchStatus string const ( @@ -197,6 +206,7 @@ func NewMatchingRule(id string, pattern string, extraConfig ExtraConfig) RegexRu Type: MatchActionNone, }, ProximityKeywords: extraConfig.ProximityKeywords, + Suppressions: extraConfig.Suppressions, SecondaryValidator: extraConfig.SecondaryValidator, ThirdPartyActiveChecker: extraConfig.ThirdPartyActiveChecker, PatternCaptureGroups: extraConfig.PatternCaptureGroups, @@ -231,6 +241,7 @@ func NewRedactingRule(id string, pattern string, redactionValue string, extraCon RedactionValue: redactionValue, }, ProximityKeywords: extraConfig.ProximityKeywords, + Suppressions: extraConfig.Suppressions, SecondaryValidator: extraConfig.SecondaryValidator, ThirdPartyActiveChecker: extraConfig.ThirdPartyActiveChecker, PatternCaptureGroups: extraConfig.PatternCaptureGroups, @@ -247,6 +258,7 @@ func NewHashRule(id string, pattern string, extraConfig ExtraConfig) RegexRuleCo Type: MatchActionHash, }, ProximityKeywords: extraConfig.ProximityKeywords, + Suppressions: extraConfig.Suppressions, SecondaryValidator: extraConfig.SecondaryValidator, ThirdPartyActiveChecker: extraConfig.ThirdPartyActiveChecker, PatternCaptureGroups: extraConfig.PatternCaptureGroups, @@ -265,6 +277,7 @@ func NewPartialRedactRule(id string, pattern string, characterCount uint32, dire Direction: direction, }, ProximityKeywords: extraConfig.ProximityKeywords, + Suppressions: extraConfig.Suppressions, SecondaryValidator: extraConfig.SecondaryValidator, ThirdPartyActiveChecker: extraConfig.ThirdPartyActiveChecker, IsSupportingRule: extraConfig.IsSupportingRule, diff --git a/sds-go/go/scanner.go b/sds-go/go/scanner.go index 42cc57b6..78f45813 100644 --- a/sds-go/go/scanner.go +++ b/sds-go/go/scanner.go @@ -22,6 +22,7 @@ var ( ErrInvalidRegex error = fmt.Errorf("invalid regex") ErrInvalidKeywords error = fmt.Errorf("invalid keywords") ErrInvalidMatchAction error = fmt.Errorf("invalid match action") + ErrInvalidSuppressions error = fmt.Errorf("invalid suppressions") ErrSupportingRuleHasMatchAction error = fmt.Errorf("supporting rules cannot have a match action other than None") ) @@ -115,6 +116,8 @@ func CreateScannerWithOptions(ruleConfigs []RuleConfig, options ScannerOptions) } else { return nil, fmt.Errorf("internal panic") } + case -6: // rust: CreateScannerError::InvalidSuppressions + return nil, ErrInvalidSuppressions case -8: // rust: CreateScannerError::SupportingRuleHasMatchAction return nil, ErrSupportingRuleHasMatchAction } diff --git a/sds-go/go/scanner_test.go b/sds-go/go/scanner_test.go index bc1f253b..d1573651 100644 --- a/sds-go/go/scanner_test.go +++ b/sds-go/go/scanner_test.go @@ -445,6 +445,53 @@ func TestProximityKeywords(t *testing.T) { runTest(t, scanner, testData, false) } +func TestScanStringWithSuppressions(t *testing.T) { + rules := []RuleConfig{ + NewRedactingRule("rule_email", `[a-z]+@[a-z.]+`, "[REDACTED]", ExtraConfig{ + Suppressions: Suppressions{ + StartsWith: []string{"admin"}, + EndsWith: []string{"@datadoghq.com"}, + ExactMatch: []string{"oli@oli.com"}, + }, + }), + } + + scanner, err := CreateScanner(rules) + if err != nil { + t.Fatal("failed to create the scanner:", err.Error()) + } + defer scanner.Delete() + + testData := map[string]bool{ + "arthur@datadoghq.com": false, + "admin@google.com": false, + "oli@oli.com": false, + "arthur@yahoo.com": true, + } + + for input, shouldBeRedacted := range testData { + result, err := scanner.Scan([]byte(input)) + if err != nil { + t.Fatal("failed to scan the event:", err.Error()) + } + if shouldBeRedacted { + if string(result.Event) != "[REDACTED]" { + t.Fatalf("match %q should have been redacted, got %q", input, result.Event) + } + if len(result.Matches) == 0 { + t.Fatalf("match %q should have been reported", input) + } + } else { + if string(result.Event) != input { + t.Fatalf("match %q should have been suppressed, got %q", input, result.Event) + } + if len(result.Matches) != 0 { + t.Fatalf("match %q should not have been reported, got %d matches", input, len(result.Matches)) + } + } + } +} + func TestSecondaryValidator(t *testing.T) { scannerWithoutChecksum, err := CreateScanner([]RuleConfig{ NewRedactingRule("rule_card", @@ -1086,6 +1133,29 @@ func TestCreateScannerFailsOnSupportingRuleWithMatchAction(t *testing.T) { } } +func TestCreateScannerFailsOnInvalidSuppressions(t *testing.T) { + cases := []struct { + name string + suppressions Suppressions + }{ + {"empty string", Suppressions{StartsWith: []string{""}}}, + {"duplicate", Suppressions{ExactMatch: []string{"foo", "foo"}}}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + scanner, err := CreateScanner([]RuleConfig{ + NewMatchingRule("rule", `\w+`, ExtraConfig{Suppressions: tt.suppressions}), + }) + if err != ErrInvalidSuppressions { + t.Fatalf("err = %v, want ErrInvalidSuppressions", err) + } + if scanner != nil { + t.Fatal("on failed creation, the returned scanner should be nil") + } + }) + } +} + func TestScanWithOptions(t *testing.T) { scanner, err := CreateScanner([]RuleConfig{ NewMatchingRule("digits", `\d{16}`, ExtraConfig{