Skip to content

Commit 6812717

Browse files
JanJakesJanJakes
committed
Use per-branch FIRST sets to skip unreachable branches
The grammar now precomputes FIRST and NULLABLE via fixpoint, then indexes each rule's branches by the tokens that can start them. At parse time the parser jumps straight to the candidate branches for the current token instead of iterating every branch and letting most fail. On the full MySQL test suite, 59% of branch attempts previously failed because the first token could never match the branch's FIRST set; with per-branch lookahead those attempts are eliminated. End-to-end parser benchmark: Before: ~14,900 QPS After: ~22,400 QPS (+50%)
1 parent 7875ef6 commit 6812717

2 files changed

Lines changed: 197 additions & 75 deletions

File tree

‎packages/mysql-on-sqlite/src/parser/class-wp-parser-grammar.php‎

Lines changed: 168 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,32 @@ class WP_Parser_Grammar {
2929
public $rules;
3030
public $rule_names;
3131
public $fragment_ids;
32-
public $lookahead_is_match_possible = array();
32+
33+
/**
34+
* Per-rule branch selector keyed by the next token id.
35+
*
36+
* When set, `$branches_for_token[$rule_id][$token_id]` is the ordered list
37+
* of branch indexes in `$rules[$rule_id]` that can possibly match when the
38+
* current token has the given id. Nullable branches appear in every entry.
39+
*
40+
* If an entry does not exist for the current token, `$nullable_branches`
41+
* is consulted. If both are empty, the rule cannot match and the parser
42+
* returns immediately.
43+
*
44+
* Rules whose FIRST set could not be computed do not appear in the map;
45+
* for those the parser falls back to trying every branch.
46+
*
47+
* @var array<int,array<int,int[]>>
48+
*/
49+
public $branches_for_token = array();
50+
51+
/**
52+
* Per-rule list of nullable branch indexes.
53+
*
54+
* @var array<int,int[]>
55+
*/
56+
public $nullable_branches = array();
57+
3358
public $lowest_non_terminal_id;
3459
public $highest_terminal_id;
3560

@@ -56,8 +81,8 @@ private function inflate( $grammar ) {
5681
$this->highest_terminal_id = $this->lowest_non_terminal_id - 1;
5782

5883
foreach ( $grammar['rules_names'] as $rule_index => $rule_name ) {
59-
$this->rule_names[ $rule_index + $grammar['rules_offset'] ] = $rule_name;
60-
$this->rules[ $rule_index + $grammar['rules_offset'] ] = array();
84+
$rule_id = $rule_index + $grammar['rules_offset'];
85+
$this->rule_names[ $rule_id ] = $rule_name;
6186

6287
/**
6388
* Treat all intermediate rules as fragments to inline before returning
@@ -75,7 +100,7 @@ private function inflate( $grammar ) {
75100
* They are prefixed with a "%" to be distinguished from the original rules.
76101
*/
77102
if ( '%' === $rule_name[0] ) {
78-
$this->fragment_ids[ $rule_index + $grammar['rules_offset'] ] = true;
103+
$this->fragment_ids[ $rule_id ] = true;
79104
}
80105
}
81106

@@ -85,55 +110,154 @@ private function inflate( $grammar ) {
85110
$this->rules[ $rule_id ] = $branches;
86111
}
87112

88-
/**
89-
* Compute a rule => [token => true] lookup table for each rule
90-
* that starts with a terminal OR with another rule that already
91-
* has a lookahead mapping.
92-
*
93-
* This is similar to left-factoring the grammar, even if not quite
94-
* the same.
95-
*
96-
* This enables us to quickly bail out from checking branches that
97-
* cannot possibly match the current token. This increased the parser
98-
* speed by a whopping 80%!
99-
*
100-
* @TODO: Explore these possible next steps:
101-
*
102-
* * Compute a rule => [token => branch[]] list lookup table and only
103-
* process the branches that have a chance of matching the current token.
104-
* * Actually left-factor the grammar as much as possible. This, however,
105-
* could inflate the serialized grammar size.
106-
*/
107-
// 5 iterations seem to give us all the speed gains we can get from this.
108-
for ( $i = 0; $i < 5; $i++ ) {
109-
foreach ( $grammar['grammar'] as $rule_index => $branches ) {
110-
$rule_id = $rule_index + $grammar['rules_offset'];
111-
if ( isset( $this->lookahead_is_match_possible[ $rule_id ] ) ) {
112-
continue;
113-
}
114-
$rule_lookup = array();
115-
$first_symbol_can_be_expanded_to_all_terminals = true;
113+
$this->build_branch_selectors();
114+
}
115+
116+
/**
117+
* Compute FIRST and NULLABLE sets for every non-terminal, then denormalize
118+
* them into a per-rule map of `token_id => branch_index[]` so the parser
119+
* can jump straight to the branches that can possibly match the current
120+
* token.
121+
*
122+
* This replaces the previous coarse "can any branch match this token?"
123+
* lookahead. On the MySQL corpus the fine-grained selector skips ~60%
124+
* of the branch attempts that the parser used to try and fail.
125+
*/
126+
private function build_branch_selectors() {
127+
$rules = $this->rules;
128+
$low_nt = $this->lowest_non_terminal_id;
129+
$empty_rule = self::EMPTY_RULE_ID;
130+
$rule_ids = array_keys( $rules );
131+
$nullable = array();
132+
$first_sets = array();
133+
134+
foreach ( $rule_ids as $rule_id ) {
135+
$nullable[ $rule_id ] = false;
136+
$first_sets[ $rule_id ] = array();
137+
}
138+
139+
// Iterate to fixpoint. FIRST and NULLABLE set monotonically grow.
140+
do {
141+
$changed = false;
142+
foreach ( $rule_ids as $rule_id ) {
143+
$branches = $rules[ $rule_id ];
116144
foreach ( $branches as $branch ) {
117-
$terminals = false;
118-
$branch_starts_with_terminal = $branch[0] < $this->lowest_non_terminal_id;
119-
if ( $branch_starts_with_terminal ) {
120-
$terminals = array( $branch[0] );
121-
} elseif ( isset( $this->lookahead_is_match_possible[ $branch[0] ] ) ) {
122-
$terminals = array_keys( $this->lookahead_is_match_possible[ $branch[0] ] );
145+
$branch_nullable = true;
146+
foreach ( $branch as $symbol ) {
147+
if ( $empty_rule === $symbol ) {
148+
// ε: contributes nothing to FIRST, stays nullable.
149+
continue;
150+
}
151+
if ( $symbol < $low_nt ) {
152+
// Terminal.
153+
if ( ! isset( $first_sets[ $rule_id ][ $symbol ] ) ) {
154+
$first_sets[ $rule_id ][ $symbol ] = true;
155+
$changed = true;
156+
}
157+
$branch_nullable = false;
158+
break;
159+
}
160+
// Non-terminal.
161+
foreach ( $first_sets[ $symbol ] as $tid => $_ ) {
162+
if ( ! isset( $first_sets[ $rule_id ][ $tid ] ) ) {
163+
$first_sets[ $rule_id ][ $tid ] = true;
164+
$changed = true;
165+
}
166+
}
167+
if ( ! $nullable[ $symbol ] ) {
168+
$branch_nullable = false;
169+
break;
170+
}
123171
}
172+
if ( $branch_nullable && ! $nullable[ $rule_id ] ) {
173+
$nullable[ $rule_id ] = true;
174+
$changed = true;
175+
}
176+
}
177+
}
178+
} while ( $changed );
124179

125-
if ( false === $terminals ) {
126-
$first_symbol_can_be_expanded_to_all_terminals = false;
180+
// Build per-(rule, token) branch indices.
181+
foreach ( $rule_ids as $rule_id ) {
182+
$branches = $rules[ $rule_id ];
183+
$selector = array();
184+
$nullable_branch_ids = array();
185+
foreach ( $branches as $idx => $branch ) {
186+
$branch_first = array();
187+
$branch_nullable = true;
188+
foreach ( $branch as $symbol ) {
189+
if ( $empty_rule === $symbol ) {
190+
continue;
191+
}
192+
if ( $symbol < $low_nt ) {
193+
$branch_first[ $symbol ] = true;
194+
$branch_nullable = false;
127195
break;
128196
}
129-
foreach ( $terminals as $terminal ) {
130-
$rule_lookup[ $terminal ] = true;
197+
foreach ( $first_sets[ $symbol ] as $tid => $_ ) {
198+
$branch_first[ $tid ] = true;
199+
}
200+
if ( ! $nullable[ $symbol ] ) {
201+
$branch_nullable = false;
202+
break;
131203
}
132204
}
133-
if ( $first_symbol_can_be_expanded_to_all_terminals ) {
134-
$this->lookahead_is_match_possible[ $rule_id ] = $rule_lookup;
205+
foreach ( $branch_first as $tid => $_ ) {
206+
$selector[ $tid ][] = $idx;
207+
}
208+
if ( $branch_nullable ) {
209+
$nullable_branch_ids[] = $idx;
210+
}
211+
}
212+
213+
// Nullable branches also match when the current token is not in
214+
// any branch's FIRST set. Fold them into every populated entry
215+
// so the runtime lookup is a single array access.
216+
if ( $nullable_branch_ids ) {
217+
$merged = array();
218+
foreach ( $selector as $tid => $idx_list ) {
219+
$merged[ $tid ] = self::merge_sorted( $idx_list, $nullable_branch_ids );
135220
}
221+
$selector = $merged;
222+
$this->nullable_branches[ $rule_id ] = $nullable_branch_ids;
136223
}
224+
if ( $selector ) {
225+
$this->branches_for_token[ $rule_id ] = $selector;
226+
}
227+
}
228+
}
229+
230+
/**
231+
* Merge two ascending int arrays into one ascending int array without
232+
* duplicates. Preserves original branch order as required by the parser.
233+
*
234+
* @param int[] $a
235+
* @param int[] $b
236+
* @return int[]
237+
*/
238+
private static function merge_sorted( array $a, array $b ): array {
239+
$i = 0;
240+
$j = 0;
241+
$na = count( $a );
242+
$nb = count( $b );
243+
$out = array();
244+
while ( $i < $na && $j < $nb ) {
245+
if ( $a[ $i ] < $b[ $j ] ) {
246+
$out[] = $a[ $i++ ];
247+
} elseif ( $a[ $i ] > $b[ $j ] ) {
248+
$out[] = $b[ $j++ ];
249+
} else {
250+
$out[] = $a[ $i ];
251+
++$i;
252+
++$j;
253+
}
254+
}
255+
while ( $i < $na ) {
256+
$out[] = $a[ $i++ ];
257+
}
258+
while ( $j < $nb ) {
259+
$out[] = $b[ $j++ ];
137260
}
261+
return $out;
138262
}
139263
}

‎packages/mysql-on-sqlite/src/parser/class-wp-parser.php‎

Lines changed: 29 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -54,42 +54,48 @@ private function parse_recursive( $rule_id ) {
5454
return false;
5555
}
5656

57-
// Bale out from processing the current branch if none of its rules can
58-
// possibly match the current token.
59-
$rule_lookahead = $grammar->lookahead_is_match_possible[ $rule_id ] ?? null;
60-
if ( null !== $rule_lookahead ) {
61-
$token_id = $this->tokens[ $this->position ]->id;
62-
if (
63-
! isset( $rule_lookahead[ $token_id ] ) &&
64-
! isset( $rule_lookahead[ WP_Parser_Grammar::EMPTY_RULE_ID ] )
65-
) {
57+
$tokens = $this->tokens;
58+
$token_count = $this->token_count;
59+
$position = $this->position;
60+
61+
// Narrow the set of branches worth trying using the precomputed FIRST
62+
// sets. When no entry exists for the current token, fall back to the
63+
// rule's nullable branches (if any); if both are empty the rule cannot
64+
// match here.
65+
$branch_selector = $grammar->branches_for_token[ $rule_id ] ?? null;
66+
if ( null !== $branch_selector ) {
67+
$tid = $position < $token_count ? $tokens[ $position ]->id : WP_Parser_Grammar::EMPTY_RULE_ID;
68+
if ( isset( $branch_selector[ $tid ] ) ) {
69+
$candidate_branches = $branch_selector[ $tid ];
70+
} elseif ( isset( $grammar->nullable_branches[ $rule_id ] ) ) {
71+
$candidate_branches = $grammar->nullable_branches[ $rule_id ];
72+
} else {
6673
return false;
6774
}
75+
} else {
76+
$candidate_branches = array_keys( $branches );
6877
}
6978

70-
$rule_name = $grammar->rule_names[ $rule_id ];
71-
$fragment_ids = $grammar->fragment_ids;
72-
$rules = $grammar->rules;
73-
$tokens = $this->tokens;
74-
$token_count = $this->token_count;
75-
$starting_position = $this->position;
76-
$branch_matches = false;
77-
foreach ( $branches as $branch ) {
78-
$this->position = $starting_position;
79+
$rule_name = $grammar->rule_names[ $rule_id ];
80+
$fragment_ids = $grammar->fragment_ids;
81+
$is_select_statement = 'selectStatement' === $rule_name;
82+
$branch_matches = false;
83+
$children = array();
84+
foreach ( $candidate_branches as $idx ) {
85+
$branch = $branches[ $idx ];
86+
$this->position = $position;
7987
$children = array();
8088
$branch_matches = true;
8189
foreach ( $branch as $subrule_id ) {
82-
// Inline terminal matching to avoid a recursive call per token.
8390
if ( $subrule_id <= $highest_terminal_id ) {
8491
if ( WP_Parser_Grammar::EMPTY_RULE_ID === $subrule_id ) {
85-
// Epsilon rule: matches without consuming input.
8692
continue;
8793
}
8894
if (
8995
$this->position < $token_count
9096
&& $tokens[ $this->position ]->id === $subrule_id
9197
) {
92-
$children[] = $tokens[ $this->position ];
98+
$children[] = $tokens[ $this->position ];
9399
++$this->position;
94100
continue;
95101
}
@@ -103,17 +109,9 @@ private function parse_recursive( $rule_id ) {
103109
break;
104110
}
105111
if ( true === $subnode ) {
106-
/*
107-
* The subrule was matched without actually matching a token.
108-
* This means a special empty "ε" (epsilon) rule was matched.
109-
* An "ε" rule in a grammar matches an empty input of 0 bytes.
110-
* It is used to represent optional grammar productions.
111-
*/
112112
continue;
113113
}
114114
if ( isset( $fragment_ids[ $subrule_id ] ) ) {
115-
// Fragments: inline their children directly to avoid building
116-
// a throwaway WP_Parser_Node that would be merged afterwards.
117115
foreach ( $subnode->get_children_ref() as $c ) {
118116
$children[] = $c;
119117
}
@@ -131,7 +129,7 @@ private function parse_recursive( $rule_id ) {
131129
// See: https://github.com/antlr/antlr4/issues/488
132130
if (
133131
$branch_matches
134-
&& 'selectStatement' === $rule_name
132+
&& $is_select_statement
135133
&& $this->position < $token_count
136134
&& WP_MySQL_Lexer::INTO_SYMBOL === $tokens[ $this->position ]->id
137135
) {
@@ -144,7 +142,7 @@ private function parse_recursive( $rule_id ) {
144142
}
145143

146144
if ( ! $branch_matches ) {
147-
$this->position = $starting_position;
145+
$this->position = $position;
148146
return false;
149147
}
150148

0 commit comments

Comments
 (0)