From b32b560529be9962dc3ba1a8f526eb59c80d1820 Mon Sep 17 00:00:00 2001 From: Eric Barkie Date: Wed, 21 Oct 2026 10:41:20 -0400 Subject: [PATCH] Skip to the bytes a pattern's first item can match before entering the matcher find, match, gsub and gmatch tried the matcher at every position of the subject. A pattern whose first item is a literal, a class or a set, and must take a byte, can only match at a byte that item accepts; most bytes of a text are not one. The set of such bytes is built once per pattern, cached in the executor, and a scan skips to the next candidate with a byte loop, or IndexByte for a set of one byte. gsub copies the bytes it skips in one piece. Skipped bytes and the build are charged to the step budget, so Limits.Steps bounds a scan as before. The scan of a set for its ']' moves into setEnd, which neither charges nor raises, so that the build has no side effects; classEnd does both as before. Class and set scans over a megabyte go from 8 to 15 ms to under 1 ms, several times faster than C Lua 5.4; dense matches and anchored patterns are unchanged. --- exec.go | 5 ++ lib_string.go | 46 +++++++++++ lib_string_test.go | 141 ++++++++++++++++++++++++++++++++++ pattern.go | 186 +++++++++++++++++++++++++++++++++++++++------ 4 files changed, 356 insertions(+), 22 deletions(-) diff --git a/exec.go b/exec.go index 7c6433e..b4e64d5 100644 --- a/exec.go +++ b/exec.go @@ -134,6 +134,8 @@ type executor struct { nextAt int // the node index of the key the last inline next returned, for the next step coStacks []coStack // stacks of coroutines that returned in this entry, for the next ones + + firstSets map[string]*firstSet // the bytes a pattern's matches can begin with, by pattern } // write sends text from print or warn to the output. @@ -283,6 +285,9 @@ func (ex *executor) reset() { clear(ex.coStacks) ex.coStacks = ex.coStacks[:0] } + if len(ex.firstSets) > 0 { // the patterns of one script are not kept for the next + clear(ex.firstSets) + } } // release returns an executor to the pool, unless it ended abnormally or has grown large. diff --git a/lib_string.go b/lib_string.go index 2e3db00..f71dff1 100644 --- a/lib_string.go +++ b/lib_string.go @@ -229,10 +229,20 @@ func strFindAux(c *Call, find bool) error { ms := &matchState{src: s, pat: p, c: c} anchor := len(p) > 0 && p[0] == '^' pi := 0 + var fs *firstSet if anchor { pi = 1 + } else if fs = ms.firstSet(); ms.err != nil { + return ms.err } for s1 := start; ; s1++ { + if fs != nil { + next := ms.skip(fs, s1) + if next < 0 { + break + } + s1 = next + } ms.level, ms.depth = 0, 0 if e := ms.match(s1, pi); e != -1 { if ms.err != nil { @@ -255,6 +265,9 @@ func strFindAux(c *Call, find bool) error { break } } + if ms.err != nil { + return ms.err + } return c.ReturnValues(Value{}) } @@ -276,10 +289,26 @@ func strGmatch(c *Call) error { pos = len(s) + 1 } lastMatch := -1 + ms := &matchState{src: s, pat: p, c: c} + fs := ms.firstSet() // nil for a '^', a literal in gmatch + if ms.err != nil { + return ms.err + } it := &goFunc{name: "gmatch iterator", stateful: true} it.f = func(c *Call) error { ms := &matchState{src: s, pat: p, c: c} for ; pos <= len(s); pos++ { + if fs != nil { + next := ms.skip(fs, pos) + if next < 0 { + if ms.err != nil { + return ms.err // pos stays: a closer that calls the iterator again finds it sound + } + pos = len(s) + 1 + break + } + pos = next + } ms.level, ms.depth = 0, 0 if e := ms.match(pos, 0); e != -1 && e != lastMatch { if ms.err != nil { @@ -327,10 +356,27 @@ func strGsub(c *Call) error { pi = 1 } ms := &matchState{src: src, pat: p, c: c} + var fs *firstSet + if !anchor { + if fs = ms.firstSet(); ms.err != nil { + return ms.err + } + } var b strings.Builder charge := building{c: c} s, n, lastMatch, changed := 0, int64(0), -1, false for n < maxN { + if fs != nil { // copy the bytes no match can begin at in one piece + next := ms.skip(fs, s) + if ms.err != nil { + return ms.err + } + if next < 0 { + goto done + } + b.WriteString(src[s:next]) + s = next + } ms.level, ms.depth = 0, 0 e := ms.match(s, pi) if ms.err != nil { diff --git a/lib_string_test.go b/lib_string_test.go index cdf8d55..ba552f1 100644 --- a/lib_string_test.go +++ b/lib_string_test.go @@ -1,6 +1,7 @@ package glua import ( + "errors" "fmt" "strings" "testing" @@ -61,3 +62,143 @@ func TestStringFormat(t *testing.T) { } } } + +// A scan for a match skips to the bytes its first item can take (pattern.go's firstSet), +// with the results C Lua gives: for the first items that take a byte, those that may take +// none or any, and the anchors, captures, balances, frontiers and back-references that are +// left to the matcher. Malformed patterns still raise, and the bytes passed over are +// charged as steps. +func TestPatternFirstSet(t *testing.T) { + src := ` + local out = {} + local function r(...) + local t = table.pack(...) + for i = 1, t.n do t[i] = tostring(t[i]) end + out[#out + 1] = table.concat(t, ",", 1, t.n) + end + r(("hello world"):find("o")) -- one byte: memchr + r(("hello world"):find("o", 6)) + r(("hello world"):find("o", 12)) + r(("abc"):find("d")) + r((""):find("a")) + r(("a1b22c333"):gsub("%d+", "#")) -- a class + r(("x1y2"):gsub("%d", { ["1"] = "one" })) -- the run between matches is copied once + r(("a\0b"):gsub("%z", "Z")) + r(("abc"):find("[^a]")) -- a set + r(("ab-c]d"):find("[%]-]")) + r(("aaa"):gsub("a", "b", 2)) -- the limit + r(("abc"):gsub("x*", "-")) -- may take no byte: every position + r(("abc"):gsub("(x?)", "-")) + r(("abc"):gsub("x-", "-")) + r(("ab"):find("()b")) -- a position capture, then b + r(("abab"):gsub("(ab)%1", "X")) -- a capture's first item + r(("xaby"):find("((a)b)")) + r(("foo(bar)"):find("%b()")) + r(("THE (quick) fox"):find("%f[%a]%a+")) + r(("a$b"):find("%$")) + r(("ab"):find("$")) + r(("ab"):gsub("$", "!")) + r(("a$b"):find("$b")) + r(("a^b"):find("^b")) + r(("a^b"):find("b^")) + r(("a.b"):find("%.")) + r(("abc"):find(".")) + local w = {} + for n in ("1/100 101 2/200"):gmatch("%d+") do w[#w + 1] = n end + r(table.concat(w, " ")) + local c = 0 + for _ in ("a^b^c"):gmatch("^.") do c = c + 1 end -- '^' is a literal in gmatch + r(c) + c = 0 + for _ in ("aaa"):gmatch("a*") do c = c + 1 end -- not the empty match right after + r(c) + r(pcall(string.find, "abc", "[a")) + r(pcall(string.find, "abc", "a%")) + r(pcall(string.gsub, "abc", "[", "")) + local ok, err = pcall(function() for _ in ("abc"):gmatch("[a") do end end) -- raised by the iterator + r(ok, err:match("malformed.*$")) + for i = 0, 99 do -- past the cache's size, with and without a set + local p = string.char(97 + i % 26) .. (i % 2 == 0 and "%d" or "") + if ("q7"):find(p) ~= (i % 26 == 16 and 1 or nil) then error(p) end + end + r(#out) + r((("a"):rep(400)):find(("a"):rep(300))) -- too long for the cache + return table.concat(out, "|")` + want := []string{ + "5,5", "8,8", "nil", "nil", "nil", + "a#b#c#,3", "xoney2,2", "aZb,1", + "2,2", "3,3", + "bba,2", + "-a-b-c-,4", "-a-b-c-,4", "-a-b-c-,4", + "2,2,2", "X,1", "2,3,ab,a", + "4,8", "1,3", + "2,2", "3,2", "ab!,1", "2,3", "nil", "nil", "2,2", "1,1", + "1 100 101 2 200", "2", "1", + "false,malformed pattern (missing ']')", + "false,malformed pattern (ends with '%')", + "false,malformed pattern (missing ']')", + "false,malformed pattern (missing ']')", + "34", "1,300", + } + for _, checked := range []bool{true, false} { + res, err := runInEnv(t, MustNew(withLoop(checked)), src) + if err != nil || len(res) != 1 { + t.Fatalf("checked=%v: %v %v", checked, res, err) + } + got := strings.Split(res[0].String(), "|") + for i := range max(len(got), len(want)) { + var g, w string + if i < len(got) { + g = got[i] + } + if i < len(want) { + w = want[i] + } + if g != w { + t.Errorf("checked=%v: case %d: got %q, want %q", checked, i, g, w) + } + } + } + // the bytes skipped are charged, found or not: a megabyte scanned is 16K steps, over + // the 7K left after the subject is made + for _, src := range []string{ + `return (("x"):rep(1 << 20)):match("[yz]")`, + `return (("x"):rep(1 << 20)):match("y")`, + `return (("x"):rep(1 << 20) .. "y"):match("[yz]")`, + `return (("x"):rep(1 << 20)):gsub("[yz]", "")`, + `for _ in (("x"):rep(1 << 20)):gmatch("%d") do end`, + } { + _, err := runInEnv(t, MustNew(Limits{Steps: 40_000, AllocBytes: NoLimit}), src) + if !errors.Is(err, ErrSteps) { + t.Errorf("%s: %v, want the step limit", src, err) + } + } + if _, err := runInEnv(t, MustNew(Limits{Steps: 40_000, AllocBytes: NoLimit}), `return #(("x"):rep(1 << 20))`); err != nil { + t.Errorf("the subject alone: %v", err) // so the scans above were what ran out + } + // building a set is charged too, and a long set is left to the matcher + _, err := runInEnv(t, MustNew(Limits{Steps: 2_000, AllocBytes: NoLimit}), + `local p = "[" .. ("a"):rep(1 << 16) .. "]" for i = 1, 200 do ("x"):find(p) end`) + if !errors.Is(err, ErrSteps) { + t.Errorf("a long set: %v, want the step limit", err) + } + _, err = runInEnv(t, MustNew(Limits{Steps: 2_000, AllocBytes: NoLimit}), + `for i = 1, 100 do ("x"):find("[" .. ("a"):rep(i % 30 + 1) .. "]") end`) + if !errors.Is(err, ErrSteps) { + t.Errorf("sets built each call: %v, want the step limit", err) + } + // the matcher's errors before the first item are not lost + res, err := runInEnv(t, MustNew(), `return pcall(string.find, "xyz", ("("):rep(33) .. "a")`) + if err != nil || len(res) != 2 || res[0].Truthy() || !strings.Contains(res[1].String(), "too many captures") { + t.Errorf("33 captures: %v %v", res, err) + } + // an iterator the budget stopped mid-scan can be called again by a closer + _, err = runInEnv(t, MustNew(Limits{Steps: 40_000, AllocBytes: NoLimit}), ` + local s = ("x"):rep(1 << 20) .. "y" + local it = s:gmatch("yq") + local t = setmetatable({}, { __close = function() pcall(it) end }) + return it()`) + if !errors.Is(err, ErrSteps) { + t.Errorf("gmatch stopped by the budget: %v, want the step limit", err) + } +} diff --git a/pattern.go b/pattern.go index 65346da..5086f55 100644 --- a/pattern.go +++ b/pattern.go @@ -1,5 +1,7 @@ package glua +import "strings" + // Lua patterns, ported from C Lua 5.4's lstrlib.c: a direct interpreter over the pattern // text, with no compile step. Backtracking, scanning sets and comparing back-references // charge steps, and recursion is capped at 200 levels ("pattern too complex"). @@ -99,33 +101,42 @@ func (ms *matchState) classEnd(p int) int { return p + 1 } if c == '[' { - start := p - if p < len(pat) && pat[p] == '^' { - p++ + end := setEnd(pat, p) + if end < 0 { + return ms.fail("malformed pattern (missing ']')") } - for { // look for a ']' - if p >= len(pat) { - return ms.fail("malformed pattern (missing ']')") - } - cc := pat[p] - p++ - if cc == luaEsc && p < len(pat) { - p++ // skip escapes, such as '%]' - } - if p < len(pat) && pat[p] == ']' { - if !ms.work(p - start) { - return -1 - } - return p + 1 - } - if p >= len(pat) { - return ms.fail("malformed pattern (missing ']')") - } + if !ms.work(end - p - 1) { // the set's length + return -1 } + return end } return p } +// setEnd returns the index just past the ']' of the set whose '[' is at p-1 of pat, or -1 +// when none closes it. +func setEnd(pat string, p int) int { + if p < len(pat) && pat[p] == '^' { + p++ + } + for { // look for a ']' + if p >= len(pat) { + return -1 + } + cc := pat[p] + p++ + if cc == luaEsc && p < len(pat) { + p++ // skip escapes, such as '%]' + } + if p < len(pat) && pat[p] == ']' { + return p + 1 + } + if p >= len(pat) { + return -1 + } + } +} + func isAlpha(c byte) bool { return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' } func isDigitB(c byte) bool { return c >= '0' && c <= '9' } func isLower(c byte) bool { return c >= 'a' && c <= 'z' } @@ -207,7 +218,11 @@ func (ms *matchState) singleMatch(s, p, ep int) bool { if s >= len(ms.src) { return false } - c := ms.src[s] + return ms.matchByte(ms.src[s], p, ep) +} + +// matchByte reports whether the byte c matches the class from p to ep. +func (ms *matchState) matchByte(c byte, p, ep int) bool { switch ms.pat[p] { case '.': return true @@ -470,3 +485,130 @@ func (ms *matchState) captures(dst []Value, s, e int, whole bool) []Value { } return dst } + +// firstSet is the bytes a match of an unanchored pattern can begin with, when its first +// item must take one: a literal, a class or a set, not followed by a quantifier that +// accepts none. A scan for a match skips the bytes outside it without entering the matcher, +// as most of a text is outside it: find, gsub and gmatch then look at each byte of the +// subject once, and the matcher sees only the bytes a match could start at. +type firstSet struct { + one int // the one byte of the set, searched for as memchr does, or -1 + set [256]bool +} + +// maxCachedPattern is the longest pattern whose first set is kept in the executor's cache, +// and maxFirstItem the longest first item a set is built for: the build tests 256 bytes +// against the item, and a long set is left to the matcher. +const ( + maxCachedPattern = 256 + maxFirstItem = 64 +) + +// firstSet returns the set for ms's pattern, or nil when its first item may match nothing +// or anything, or is not a single class, or on an error of the budget, which is in ms.err. +// The sets are cached by pattern in the executor, as a set's 256 tests cost more than a +// short subject's scan; the cache is cleared when it fills with patterns built at run +// time, and long patterns are not kept. +func (ms *matchState) firstSet() *firstSet { + ex := ms.c.ex + if fs, ok := ex.firstSets[ms.pat]; ok { + return fs + } + fs := ms.buildFirstSet() + if ms.err != nil || len(ms.pat) > maxCachedPattern { + return fs + } + if ex.firstSets == nil { + ex.firstSets = make(map[string]*firstSet) + } else if len(ex.firstSets) >= 64 { + clear(ex.firstSets) + } + ex.firstSets[strings.Clone(ms.pat)] = fs // its own bytes: the pattern may be part of a large string + return fs +} + +// buildFirstSet reads the first item of the pattern and tests every byte against it, +// charging the tests as the matcher charges a scan of the item. A malformed pattern, or +// one the matcher fails on before its first item, gives no set, and the matcher reports it. +func (ms *matchState) buildFirstSet() *firstSet { + pat := ms.pat + p := 0 + for p < len(pat) && pat[p] == '(' && !(p+1 < len(pat) && pat[p+1] == ')') { + p++ // a capture begins with its first item; a position capture takes no byte + } + if p >= len(pat) || p > maxCaptures { // past maxCaptures the matcher fails: "too many captures" + return nil + } + var ep int + switch pat[p] { + case '^', '.', '(', ')': + return nil // a literal '^' here would be one, but is left to the matcher + case '$': + if p+1 == len(pat) { + return nil // the end anchor takes no byte + } + ep = p + 1 + case luaEsc: + if p+1 >= len(pat) { + return nil + } + if c := pat[p+1]; c == 'b' || c == 'f' || c >= '0' && c <= '9' { + return nil // a balance, a frontier or a back-reference + } + ep = p + 2 + case '[': + if ep = setEnd(pat, p+1); ep < 0 || ep-p > maxFirstItem { + return nil + } + default: + ep = p + 1 + } + if ep < len(pat) && (pat[ep] == '*' || pat[ep] == '?' || pat[ep] == '-') { + return nil // may take no byte + } + if !ms.work(256 * (ep - p)) { + return nil + } + fs := &firstSet{one: -1} + n := 0 + for c := range 256 { + if ms.matchByte(byte(c), p, ep) { + fs.set[c] = true + n, fs.one = n+1, c + } + } + if n > 1 { + fs.one = -1 + } + return fs +} + +// next returns the first position at or after s where a match can begin, or -1. +func (fs *firstSet) next(src string, s int) int { + if fs.one >= 0 { + if i := strings.IndexByte(src[s:], byte(fs.one)); i >= 0 { + return s + i + } + return -1 + } + for ; s < len(src); s++ { + if fs.set[src[s]] { + return s + } + } + return -1 +} + +// skip returns the next position at or after s a match can begin at, charging for the +// bytes passed over, or -1 when there is none, or the budget is spent (ms.err). +func (ms *matchState) skip(fs *firstSet, s int) int { + n := fs.next(ms.src, s) + if n < 0 { + ms.work(len(ms.src) - s) // the whole rest was scanned + return -1 + } + if n > s && !ms.work(n-s) { + return -1 + } + return n +}