Skip to content

Commit b3a606d

Browse files
committed
Add 'vm/sre_engine/' from commit '21fc2059b70ebd5bf4a7c524c40e7d4347e065dc'
git-subtree-dir: vm/sre_engine git-subtree-mainline: 426e582 git-subtree-split: 21fc205
2 parents 426e582 + 21fc205 commit b3a606d

11 files changed

Lines changed: 2346 additions & 0 deletions

File tree

‎vm/sre_engine/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/target
2+
Cargo.lock

‎vm/sre_engine/.vscode/launch.json‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"version": "0.2.0",
3+
"configurations": [
4+
{
5+
"type": "lldb",
6+
"request": "launch",
7+
"name": "Debug Unit Test",
8+
"cargo": {
9+
"args": [
10+
"test",
11+
"--no-run"
12+
],
13+
"filter": {
14+
"kind": "test"
15+
}
16+
},
17+
"args": [],
18+
"cwd": "${workspaceFolder}"
19+
}
20+
]
21+
}

‎vm/sre_engine/Cargo.toml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[package]
2+
name = "sre-engine"
3+
version = "0.6.0"
4+
authors = ["Kangzhi Shi <shikangzhi@gmail.com>", "RustPython Team"]
5+
description = "A low-level implementation of Python's SRE regex engine"
6+
repository = "https://github.com/RustPython/sre-engine"
7+
license = "MIT"
8+
edition = "2021"
9+
keywords = ["regex"]
10+
include = ["LICENSE", "src/**/*.rs"]
11+
12+
[dependencies]
13+
num_enum = "0.7"
14+
bitflags = "2"
15+
optional = "0.5"

‎vm/sre_engine/LICENSE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2020 RustPython Team
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎vm/sre_engine/benches/benches.rs‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#![feature(test)]
2+
3+
extern crate test;
4+
use test::Bencher;
5+
6+
use sre_engine::{Request, State, StrDrive};
7+
8+
struct Pattern {
9+
code: &'static [u32],
10+
}
11+
12+
impl Pattern {
13+
fn state<'a, S: StrDrive>(&self, string: S) -> (Request<'a, S>, State) {
14+
self.state_range(string, 0..usize::MAX)
15+
}
16+
17+
fn state_range<'a, S: StrDrive>(
18+
&self,
19+
string: S,
20+
range: std::ops::Range<usize>,
21+
) -> (Request<'a, S>, State) {
22+
let req = Request::new(string, range.start, range.end, self.code, false);
23+
let state = State::default();
24+
(req, state)
25+
}
26+
}
27+
28+
#[bench]
29+
fn benchmarks(b: &mut Bencher) {
30+
// # test common prefix
31+
// pattern p1 = re.compile('Python|Perl') # , 'Perl'), # Alternation
32+
// START GENERATED by generate_tests.py
33+
#[rustfmt::skip] let p1 = Pattern { code: &[14, 8, 1, 4, 6, 1, 1, 80, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 1] };
34+
// END GENERATED
35+
// pattern p2 = re.compile('(Python|Perl)') #, 'Perl'), # Grouped alternation
36+
// START GENERATED by generate_tests.py
37+
#[rustfmt::skip] let p2 = Pattern { code: &[14, 8, 1, 4, 6, 1, 0, 80, 0, 17, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 17, 1, 1] };
38+
// END GENERATED
39+
// pattern p3 = re.compile('Python|Perl|Tcl') #, 'Perl'), # Alternation
40+
// START GENERATED by generate_tests.py
41+
#[rustfmt::skip] let p3 = Pattern { code: &[14, 9, 4, 3, 6, 16, 80, 16, 84, 0, 7, 15, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 22, 11, 16, 80, 16, 101, 16, 114, 16, 108, 15, 11, 9, 16, 84, 16, 99, 16, 108, 15, 2, 0, 1] };
42+
// END GENERATED
43+
// pattern p4 = re.compile('(Python|Perl|Tcl)') #, 'Perl'), # Grouped alternation
44+
// START GENERATED by generate_tests.py
45+
#[rustfmt::skip] let p4 = Pattern { code: &[14, 9, 4, 3, 6, 16, 80, 16, 84, 0, 17, 0, 7, 15, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 22, 11, 16, 80, 16, 101, 16, 114, 16, 108, 15, 11, 9, 16, 84, 16, 99, 16, 108, 15, 2, 0, 17, 1, 1] };
46+
// END GENERATED
47+
// pattern p5 = re.compile('(Python)\\1') #, 'PythonPython'), # Backreference
48+
// START GENERATED by generate_tests.py
49+
#[rustfmt::skip] let p5 = Pattern { code: &[14, 18, 1, 12, 12, 6, 0, 80, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 17, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 11, 0, 1] };
50+
// END GENERATED
51+
// pattern p6 = re.compile('([0a-z][a-z0-9]*,)+') #, 'a5,b7,c9,'), # Disable the fastmap optimization
52+
// START GENERATED by generate_tests.py
53+
#[rustfmt::skip] let p6 = Pattern { code: &[14, 4, 0, 2, 4294967295, 23, 31, 1, 4294967295, 17, 0, 13, 7, 16, 48, 22, 97, 122, 0, 24, 13, 0, 4294967295, 13, 8, 22, 97, 122, 22, 48, 57, 0, 1, 16, 44, 17, 1, 18, 1] };
54+
// END GENERATED
55+
// pattern p7 = re.compile('([a-z][a-z0-9]*,)+') #, 'a5,b7,c9,'), # A few sets
56+
// START GENERATED by generate_tests.py
57+
#[rustfmt::skip] let p7 = Pattern { code: &[14, 4, 0, 2, 4294967295, 23, 29, 1, 4294967295, 17, 0, 13, 5, 22, 97, 122, 0, 24, 13, 0, 4294967295, 13, 8, 22, 97, 122, 22, 48, 57, 0, 1, 16, 44, 17, 1, 18, 1] };
58+
// END GENERATED
59+
// pattern p8 = re.compile('Python') #, 'Python'), # Simple text literal
60+
// START GENERATED by generate_tests.py
61+
#[rustfmt::skip] let p8 = Pattern { code: &[14, 18, 3, 6, 6, 6, 6, 80, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 1] };
62+
// END GENERATED
63+
// pattern p9 = re.compile('.*Python') #, 'Python'), # Bad text literal
64+
// START GENERATED by generate_tests.py
65+
#[rustfmt::skip] let p9 = Pattern { code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 1] };
66+
// END GENERATED
67+
// pattern p10 = re.compile('.*Python.*') #, 'Python'), # Worse text literal
68+
// START GENERATED by generate_tests.py
69+
#[rustfmt::skip] let p10 = Pattern { code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 24, 5, 0, 4294967295, 2, 1, 1] };
70+
// END GENERATED
71+
// pattern p11 = re.compile('.*(Python)') #, 'Python'), # Bad text literal with grouping
72+
// START GENERATED by generate_tests.py
73+
#[rustfmt::skip] let p11 = Pattern { code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 17, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 1] };
74+
// END GENERATED
75+
76+
let tests = [
77+
(p1, "Perl"),
78+
(p2, "Perl"),
79+
(p3, "Perl"),
80+
(p4, "Perl"),
81+
(p5, "PythonPython"),
82+
(p6, "a5,b7,c9,"),
83+
(p7, "a5,b7,c9,"),
84+
(p8, "Python"),
85+
(p9, "Python"),
86+
(p10, "Python"),
87+
(p11, "Python"),
88+
];
89+
90+
b.iter(move || {
91+
for (p, s) in &tests {
92+
let (req, mut state) = p.state(s.clone());
93+
assert!(state.search(req));
94+
let (req, mut state) = p.state(s.clone());
95+
assert!(state.pymatch(&req));
96+
let (mut req, mut state) = p.state(s.clone());
97+
req.match_all = true;
98+
assert!(state.pymatch(&req));
99+
let s2 = format!("{}{}{}", " ".repeat(10000), s, " ".repeat(10000));
100+
let (req, mut state) = p.state_range(s2.as_str(), 0..usize::MAX);
101+
assert!(state.search(req));
102+
let (req, mut state) = p.state_range(s2.as_str(), 10000..usize::MAX);
103+
assert!(state.pymatch(&req));
104+
let (req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len());
105+
assert!(state.pymatch(&req));
106+
let (mut req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len());
107+
req.match_all = true;
108+
assert!(state.pymatch(&req));
109+
}
110+
})
111+
}

‎vm/sre_engine/generate_tests.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import os
2+
from pathlib import Path
3+
import re
4+
import sre_constants
5+
import sre_compile
6+
import sre_parse
7+
import json
8+
from itertools import chain
9+
10+
m = re.search(r"const SRE_MAGIC: usize = (\d+);", open("src/constants.rs").read())
11+
sre_engine_magic = int(m.group(1))
12+
del m
13+
14+
assert sre_constants.MAGIC == sre_engine_magic
15+
16+
class CompiledPattern:
17+
@classmethod
18+
def compile(cls, pattern, flags=0):
19+
p = sre_parse.parse(pattern)
20+
code = sre_compile._code(p, flags)
21+
self = cls()
22+
self.pattern = pattern
23+
self.code = code
24+
self.flags = re.RegexFlag(flags | p.state.flags)
25+
return self
26+
27+
for k, v in re.RegexFlag.__members__.items():
28+
setattr(CompiledPattern, k, v)
29+
30+
31+
# matches `// pattern {varname} = re.compile(...)`
32+
pattern_pattern = re.compile(r"^((\s*)\/\/\s*pattern\s+(\w+)\s+=\s+(.+?))$(?:.+?END GENERATED)?", re.M | re.S)
33+
def replace_compiled(m):
34+
line, indent, varname, pattern = m.groups()
35+
pattern = eval(pattern, {"re": CompiledPattern})
36+
pattern = f"Pattern {{ code: &{json.dumps(pattern.code)} }}"
37+
return f'''{line}
38+
{indent}// START GENERATED by generate_tests.py
39+
{indent}#[rustfmt::skip] let {varname} = {pattern};
40+
{indent}// END GENERATED'''
41+
42+
with os.scandir("tests") as t, os.scandir("benches") as b:
43+
for f in chain(t, b):
44+
path = Path(f.path)
45+
if path.suffix == ".rs":
46+
replaced = pattern_pattern.sub(replace_compiled, path.read_text())
47+
path.write_text(replaced)

‎vm/sre_engine/src/constants.rs‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
/*
2+
* Secret Labs' Regular Expression Engine
3+
*
4+
* regular expression matching engine
5+
*
6+
* NOTE: This file is generated by sre_constants.py. If you need
7+
* to change anything in here, edit sre_constants.py and run it.
8+
*
9+
* Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved.
10+
*
11+
* See the _sre.c file for information on usage and redistribution.
12+
*/
13+
14+
use bitflags::bitflags;
15+
16+
pub const SRE_MAGIC: usize = 20221023;
17+
#[derive(num_enum::TryFromPrimitive, Debug, PartialEq, Eq)]
18+
#[repr(u32)]
19+
#[allow(non_camel_case_types, clippy::upper_case_acronyms)]
20+
pub enum SreOpcode {
21+
FAILURE = 0,
22+
SUCCESS = 1,
23+
ANY = 2,
24+
ANY_ALL = 3,
25+
ASSERT = 4,
26+
ASSERT_NOT = 5,
27+
AT = 6,
28+
BRANCH = 7,
29+
CATEGORY = 8,
30+
CHARSET = 9,
31+
BIGCHARSET = 10,
32+
GROUPREF = 11,
33+
GROUPREF_EXISTS = 12,
34+
IN = 13,
35+
INFO = 14,
36+
JUMP = 15,
37+
LITERAL = 16,
38+
MARK = 17,
39+
MAX_UNTIL = 18,
40+
MIN_UNTIL = 19,
41+
NOT_LITERAL = 20,
42+
NEGATE = 21,
43+
RANGE = 22,
44+
REPEAT = 23,
45+
REPEAT_ONE = 24,
46+
SUBPATTERN = 25,
47+
MIN_REPEAT_ONE = 26,
48+
ATOMIC_GROUP = 27,
49+
POSSESSIVE_REPEAT = 28,
50+
POSSESSIVE_REPEAT_ONE = 29,
51+
GROUPREF_IGNORE = 30,
52+
IN_IGNORE = 31,
53+
LITERAL_IGNORE = 32,
54+
NOT_LITERAL_IGNORE = 33,
55+
GROUPREF_LOC_IGNORE = 34,
56+
IN_LOC_IGNORE = 35,
57+
LITERAL_LOC_IGNORE = 36,
58+
NOT_LITERAL_LOC_IGNORE = 37,
59+
GROUPREF_UNI_IGNORE = 38,
60+
IN_UNI_IGNORE = 39,
61+
LITERAL_UNI_IGNORE = 40,
62+
NOT_LITERAL_UNI_IGNORE = 41,
63+
RANGE_UNI_IGNORE = 42,
64+
}
65+
#[derive(num_enum::TryFromPrimitive, Debug, PartialEq, Eq)]
66+
#[repr(u32)]
67+
#[allow(non_camel_case_types, clippy::upper_case_acronyms)]
68+
pub enum SreAtCode {
69+
BEGINNING = 0,
70+
BEGINNING_LINE = 1,
71+
BEGINNING_STRING = 2,
72+
BOUNDARY = 3,
73+
NON_BOUNDARY = 4,
74+
END = 5,
75+
END_LINE = 6,
76+
END_STRING = 7,
77+
LOC_BOUNDARY = 8,
78+
LOC_NON_BOUNDARY = 9,
79+
UNI_BOUNDARY = 10,
80+
UNI_NON_BOUNDARY = 11,
81+
}
82+
#[derive(num_enum::TryFromPrimitive, Debug)]
83+
#[repr(u32)]
84+
#[allow(non_camel_case_types, clippy::upper_case_acronyms)]
85+
pub enum SreCatCode {
86+
DIGIT = 0,
87+
NOT_DIGIT = 1,
88+
SPACE = 2,
89+
NOT_SPACE = 3,
90+
WORD = 4,
91+
NOT_WORD = 5,
92+
LINEBREAK = 6,
93+
NOT_LINEBREAK = 7,
94+
LOC_WORD = 8,
95+
LOC_NOT_WORD = 9,
96+
UNI_DIGIT = 10,
97+
UNI_NOT_DIGIT = 11,
98+
UNI_SPACE = 12,
99+
UNI_NOT_SPACE = 13,
100+
UNI_WORD = 14,
101+
UNI_NOT_WORD = 15,
102+
UNI_LINEBREAK = 16,
103+
UNI_NOT_LINEBREAK = 17,
104+
}
105+
bitflags! {
106+
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
107+
pub struct SreFlag: u16 {
108+
const TEMPLATE = 1;
109+
const IGNORECASE = 2;
110+
const LOCALE = 4;
111+
const MULTILINE = 8;
112+
const DOTALL = 16;
113+
const UNICODE = 32;
114+
const VERBOSE = 64;
115+
const DEBUG = 128;
116+
const ASCII = 256;
117+
}
118+
}
119+
bitflags! {
120+
pub struct SreInfo: u32 {
121+
const PREFIX = 1;
122+
const LITERAL = 2;
123+
const CHARSET = 4;
124+
}
125+
}

0 commit comments

Comments
 (0)