From 2fc43a176251ca6a40d790f8a672dc9fd2e725b6 Mon Sep 17 00:00:00 2001 From: Nathaniel Leonardjoi Date: Tue, 7 Jul 2026 09:01:10 -0700 Subject: [PATCH] fix(file): prefilter must enumerate overlapping literal hits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit relevantSourcesBytes selected candidate templates per line with a non-overlapping leftmost-first Find loop (start = hit.Start + 1). When one template's literal (e.g. "begin rsa private key") shares its start with — or is otherwise shadowed by — a shorter literal contributed by another loaded template (e.g. "begin"), the automaton never reported the longer literal, so that template was silently dropped from the scan. On a real multi-secret file this made only ~5 of 156 loaded rules fire (private-key, jwt, stripe, database ... all missed) while each fired correctly when loaded alone. Enumerate every literal occurrence with FindAllOverlapping, which reports all matches regardless of the automaton's leftmost match kind. Add TestScanner_Prefilter_OverlappingLiterals: shorter shadowing literals are listed before the private-key template so they win leftmost-first tie-breaking — the exact condition that used to hide the finding. Fails on the old loop, passes now. Co-Authored-By: Claude Opus 4.8 (1M context) --- proton/file/prefilter_overlap_test.go | 74 +++++++++++++++++++++++++++ proton/file/unified.go | 15 +++--- 2 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 proton/file/prefilter_overlap_test.go diff --git a/proton/file/prefilter_overlap_test.go b/proton/file/prefilter_overlap_test.go new file mode 100644 index 0000000..28e9150 --- /dev/null +++ b/proton/file/prefilter_overlap_test.go @@ -0,0 +1,74 @@ +package file + +import ( + "sort" + "testing" + + "github.com/chainreactors/neutron/operators" + "github.com/stretchr/testify/require" +) + +// TestScanner_Prefilter_OverlappingLiterals is a regression test for a +// prefilter bug where a template whose literal (e.g. "BEGIN RSA PRIVATE KEY") +// is shadowed by a shorter literal from another loaded template (e.g. "BEGIN") +// was silently dropped from the scan. The line prefilter enumerated candidate +// templates with a non-overlapping leftmost-first Find loop, so a short literal +// sharing the start position masked the longer one and its template never ran. +// +// The shadowing templates are listed BEFORE the private-key template so their +// shorter literals win leftmost-first tie-breaking, which is exactly the +// condition that used to hide the private-key finding. +func TestScanner_Prefilter_OverlappingLiterals(t *testing.T) { + dir := writeTempFiles(t, map[string]string{ + "key.txt": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEA...\n-----END RSA PRIVATE KEY-----\n", + }) + + shadowBegin := compileRequest(t, &Request{ + ID: "kw-begin", + MaxSize: "1Gb", + Extensions: []string{"all"}, + Operators: operators.Operators{ + Extractors: []*operators.Extractor{{ + Part: "raw", Type: "regex", Regex: []string{`BEGIN`}, + }}, + }, + }, nil) + + shadowPrivate := compileRequest(t, &Request{ + ID: "kw-private", + MaxSize: "1Gb", + Extensions: []string{"all"}, + Operators: operators.Operators{ + Extractors: []*operators.Extractor{{ + Part: "raw", Type: "regex", Regex: []string{`PRIVATE`}, + }}, + }, + }, nil) + + privateKey := compileRequest(t, &Request{ + ID: "private-key", + MaxSize: "1Gb", + Extensions: []string{"all"}, + Operators: operators.Operators{ + Extractors: []*operators.Extractor{{ + Part: "raw", Type: "regex", Regex: []string{`BEGIN RSA PRIVATE KEY`}, + }}, + }, + }, nil) + + scanner := NewScanner([]Rule{ + makeRule("kw-begin", "Keyword BEGIN", "info", shadowBegin), + makeRule("kw-private", "Keyword PRIVATE", "info", shadowPrivate), + makeRule("private-key", "Private Key Detect", "high", privateKey), + }, newTestOptions()) + + findings := collectFindings(t, scanner, dir) + + ids := make([]string, 0, len(findings)) + for _, f := range findings { + ids = append(ids, f.TemplateID) + } + sort.Strings(ids) + require.Contains(t, ids, "private-key", + "private-key template must fire even when its literal is shadowed by shorter literals; got %v", ids) +} diff --git a/proton/file/unified.go b/proton/file/unified.go index 5bfcae6..61b1f29 100644 --- a/proton/file/unified.go +++ b/proton/file/unified.go @@ -336,19 +336,20 @@ func (idx *patternIndex) relevantSourcesBytes(lower []byte, buf *[]int, seen []b return idx.fallbackSources } result := (*buf)[:0] - start := 0 - for { - hit := idx.ac.Find(lower, start) - if hit == nil { - break - } + // The prefilter must enumerate ALL literal occurrences, including + // overlapping ones. A non-overlapping Find loop (leftmost-first) drops a + // literal whose span shares a start with — or is otherwise shadowed by — a + // different literal (e.g. "begin rsa private key" is masked by "begin"), + // which silently excludes that template from the scan whenever another + // loaded template contributes the shadowing literal. FindAllOverlapping + // reports every occurrence regardless of the automaton's match kind. + for _, hit := range idx.ac.FindAllOverlapping(lower) { for _, srcIdx := range idx.literalToSources[hit.PatternID] { if !seen[srcIdx] { seen[srcIdx] = true result = append(result, srcIdx) } } - start = hit.Start + 1 } if len(result) == 0 && len(idx.fallbackSources) == 0 { *buf = result