Skip to content

Commit e657ad0

Browse files
JanJakesJanJakes
committed
Pass input in tokens and get token values lazily
1 parent 0283884 commit e657ad0

5 files changed

Lines changed: 42 additions & 33 deletions

File tree

‎wp-includes/mysql/class-wp-mysql-lexer.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2249,9 +2249,9 @@ public function get_token(): ?WP_MySQL_Token {
22492249
}
22502250
return new WP_MySQL_Token(
22512251
$this->token_type,
2252-
$this->get_current_token_bytes(),
22532252
$this->token_starts_at,
2254-
$this->bytes_already_read - $this->token_starts_at
2253+
$this->bytes_already_read - $this->token_starts_at,
2254+
$this->sql
22552255
);
22562256
}
22572257

‎wp-includes/mysql/class-wp-mysql-token.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,6 @@ public function get_name(): string {
3434
* @return string
3535
*/
3636
public function __toString(): string {
37-
return $this->value . '<' . $this->id . ',' . $this->get_name() . '>';
37+
return $this->get_value() . '<' . $this->id . ',' . $this->get_name() . '>';
3838
}
3939
}

‎wp-includes/parser/class-wp-parser-token.php‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,6 @@ class WP_Parser_Token {
1616
*/
1717
public $id;
1818

19-
/**
20-
* Token value in its original raw form.
21-
*
22-
* @var string
23-
*/
24-
public $value;
25-
2619
/**
2720
* Byte offset in the input where the token begins.
2821
*
@@ -37,23 +30,39 @@ class WP_Parser_Token {
3730
*/
3831
public $length;
3932

33+
/**
34+
* Input bytes from which the token was parsed.
35+
*
36+
* @var string
37+
*/
38+
private $input;
39+
4040
/**
4141
* Constructor.
4242
*
4343
* @param int $id Token type.
44-
* @param string $value Token value.
4544
* @param int $start Byte offset in the input where the token begins.
4645
* @param int $length Byte length of the token in the input.
46+
* @param string $input Input bytes from which the token was parsed.
4747
*/
4848
public function __construct(
4949
int $id,
50-
string $value,
5150
int $start,
52-
int $length
51+
int $length,
52+
string $input
5353
) {
5454
$this->id = $id;
55-
$this->value = $value;
5655
$this->start = $start;
5756
$this->length = $length;
57+
$this->input = $input;
58+
}
59+
60+
/**
61+
* Get the token value as raw bytes from the input.
62+
*
63+
* @return string The token value.
64+
*/
65+
public function get_value(): string {
66+
return substr( $this->input, $this->start, $this->length );
5867
}
5968
}

‎wp-includes/sqlite-ast/class-wp-sqlite-driver.php‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1525,7 +1525,7 @@ private function execute_show_statement( WP_Parser_Node $node ): void {
15251525
sprintf(
15261526
'statement type: "%s" > "%s"',
15271527
$node->rule_name,
1528-
$keyword1->value
1528+
$keyword1->get_value()
15291529
)
15301530
);
15311531
}
@@ -2045,7 +2045,7 @@ private function execute_administration_statement( WP_Parser_Node $node ): void
20452045
sprintf(
20462046
'statement type: "%s" > "%s"',
20472047
$node->rule_name,
2048-
$first_token->value
2048+
$first_token->get_value()
20492049
)
20502050
);
20512051
}
@@ -2057,7 +2057,7 @@ private function execute_administration_statement( WP_Parser_Node $node ): void
20572057
}
20582058
}
20592059

2060-
$operation = strtolower( $first_token->value );
2060+
$operation = strtolower( $first_token->get_value() );
20612061
foreach ( $errors as $error ) {
20622062
$results[] = (object) array(
20632063
'Table' => $this->db_name . '.' . $table_name,
@@ -2181,7 +2181,7 @@ private function translate( $node ): ?string {
21812181

21822182
// @TODO: Handle SET and JSON.
21832183
throw $this->new_not_supported_exception(
2184-
sprintf( 'data type: %s', $child->value )
2184+
sprintf( 'data type: %s', $child->get_value() )
21852185
);
21862186
case 'fromClause':
21872187
// FROM DUAL is MySQL-specific syntax that means "FROM no tables"
@@ -2239,7 +2239,7 @@ private function translate( $node ): ?string {
22392239
'%s AS %s',
22402240
$value,
22412241
$this->quote_sqlite_identifier(
2242-
'@@' . ( $type_token ? "$type_token->value." : '' ) . $original_name
2242+
'@@' . ( $type_token ? "{$type_token->get_value()}." : '' ) . $original_name
22432243
)
22442244
);
22452245
case 'castType':
@@ -2290,7 +2290,7 @@ private function translate_token( WP_MySQL_Token $token ): ?string {
22902290
*/
22912291
return null;
22922292
default:
2293-
return $token->value;
2293+
return $token->get_value();
22942294
}
22952295
}
22962296

@@ -2333,8 +2333,8 @@ private function translate_string_literal( WP_Parser_Node $node ): string {
23332333
/*
23342334
* 1. Remove bounding quotes.
23352335
*/
2336-
$quote = $token->value[0];
2337-
$value = substr( $token->value, 1, -1 );
2336+
$quote = $token->get_value()[0];
2337+
$value = substr( $token->get_value(), 1, -1 );
23382338

23392339
/*
23402340
* 2. Normalize escaping of "%" and "_" characters.
@@ -2420,13 +2420,13 @@ private function translate_pure_identifier( WP_Parser_Node $node ): string {
24202420
$token = $node->get_first_child_token();
24212421

24222422
if ( WP_MySQL_Lexer::DOUBLE_QUOTED_TEXT === $token->id ) {
2423-
$value = substr( $token->value, 1, -1 );
2423+
$value = substr( $token->get_value(), 1, -1 );
24242424
$value = str_replace( '""', '"', $value );
24252425
} elseif ( WP_MySQL_Lexer::BACK_TICK_QUOTED_ID === $token->id ) {
2426-
$value = substr( $token->value, 1, -1 );
2426+
$value = substr( $token->get_value(), 1, -1 );
24272427
$value = str_replace( '``', '`', $value );
24282428
} else {
2429-
$value = $token->value;
2429+
$value = $token->get_value();
24302430
}
24312431

24322432
return '`' . str_replace( '`', '``', $value ) . '`';

‎wp-includes/sqlite-ast/class-wp-sqlite-information-schema-builder.php‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -568,7 +568,7 @@ public function record_alter_table( WP_Parser_Node $node ): void {
568568
continue;
569569
}
570570

571-
throw new \Exception( sprintf( 'Unsupported ALTER TABLE ADD action: %s', $first_token->value ) );
571+
throw new \Exception( sprintf( 'Unsupported ALTER TABLE ADD action: %s', $first_token->get_value() ) );
572572
}
573573

574574
// CHANGE [COLUMN]
@@ -1353,7 +1353,7 @@ private function get_column_data_types( WP_Parser_Node $node ): array {
13531353
) {
13541354
$type = 'mediumtext';
13551355
} else {
1356-
throw new \RuntimeException( 'Unknown data type: ' . $token->value );
1356+
throw new \RuntimeException( 'Unknown data type: ' . $token->get_value() );
13571357
}
13581358

13591359
// Get full type.
@@ -1619,8 +1619,8 @@ private function get_column_numeric_attributes( WP_Parser_Node $node, string $da
16191619
$precision_node = $node->get_first_descendant_node( 'precision' );
16201620
if ( null !== $precision_node ) {
16211621
$values = $precision_node->get_descendant_tokens( WP_MySQL_Lexer::INT_NUMBER );
1622-
$precision = (int) $values[0]->value;
1623-
$scale = (int) $values[1]->value;
1622+
$precision = (int) $values[0]->get_value();
1623+
$scale = (int) $values[1]->get_value();
16241624
}
16251625

16261626
if ( 'float' === $data_type ) {
@@ -1865,20 +1865,20 @@ private function get_value( WP_Parser_Node $node ): string {
18651865
if ( $child instanceof WP_Parser_Node ) {
18661866
$value = $this->get_value( $child );
18671867
} elseif ( WP_MySQL_Lexer::BACK_TICK_QUOTED_ID === $child->id ) {
1868-
$value = substr( $child->value, 1, -1 );
1868+
$value = substr( $child->get_value(), 1, -1 );
18691869
$value = str_replace( '``', '`', $value );
18701870
} elseif ( WP_MySQL_Lexer::SINGLE_QUOTED_TEXT === $child->id ) {
1871-
$value = $child->value;
1871+
$value = $child->get_value();
18721872
$value = substr( $value, 1, -1 );
18731873
$value = str_replace( '\"', '"', $value );
18741874
$value = str_replace( '""', '"', $value );
18751875
} elseif ( WP_MySQL_Lexer::DOUBLE_QUOTED_TEXT === $child->id ) {
1876-
$value = $child->value;
1876+
$value = $child->get_value();
18771877
$value = substr( $value, 1, -1 );
18781878
$value = str_replace( '\"', '"', $value );
18791879
$value = str_replace( '""', '"', $value );
18801880
} else {
1881-
$value = $child->value;
1881+
$value = $child->get_value();
18821882
}
18831883
$full_value .= $value;
18841884
}

0 commit comments

Comments
 (0)