Speed up MySQL lexer and parser

Author: adamziel

packages/mysql-on-sqlite/src/mysql/class-wp-mysql-lexer.php

@@ -2111,6 +2111,13 @@ class WP_MySQL_Lexer {
 	 */
 	private $sql;
 
+	/**
+	 * Byte length of the SQL payload.
+	 *
+	 * @var int
+	 */
+	private $sql_length;
+
 	/**
 	 * The version of the MySQL server that the SQL payload is intended for.
 	 *
@@ -2189,6 +2196,7 @@ public function __construct(
 		array $sql_modes = array()
 	) {
 		$this->sql           = $sql;
+		$this->sql_length    = strlen( $sql );
 		$this->mysql_version = $mysql_version;
 
 		foreach ( $sql_modes as $sql_mode ) {
@@ -2284,10 +2292,46 @@ public function get_token(): ?WP_MySQL_Token {
 	 * @return WP_MySQL_Token[] An array of token objects representing the remaining tokens.
 	 */
 	public function remaining_tokens(): array {
-		$tokens = array();
-		while ( true === $this->next_token() ) {
-			$token    = $this->get_token();
-			$tokens[] = $token;
+		$tokens                            = array();
+		$no_backslash_escapes_sql_mode_set = $this->is_sql_mode_active(
+			self::SQL_MODE_NO_BACKSLASH_ESCAPES
+		);
+
+		while ( true ) {
+			if (
+				self::EOF === $this->token_type
+				|| ( null === $this->token_type && $this->bytes_already_read > 0 )
+			) {
+				$this->token_type = null;
+				break;
+			}
+
+			do {
+				$this->token_starts_at = $this->bytes_already_read;
+				$this->token_type      = $this->read_next_token();
+			} while (
+				self::WHITESPACE === $this->token_type
+				|| self::COMMENT === $this->token_type
+				|| self::MYSQL_COMMENT_START === $this->token_type
+				|| self::MYSQL_COMMENT_END === $this->token_type
+			);
+
+			if ( null === $this->token_type ) {
+				break;
+			}
+
+			$tokens[] = new WP_MySQL_Token(
+				$this->token_type,
+				$this->token_starts_at,
+				$this->bytes_already_read - $this->token_starts_at,
+				$this->sql,
+				$no_backslash_escapes_sql_mode_set
+			);
+
+			if ( self::EOF === $this->token_type ) {
+				$this->token_type = null;
+				break;
+			}
 		}
 		return $tokens;
 	}
@@ -2356,10 +2400,10 @@ private function read_next_token(): ?int {
 
 		if ( "'" === $byte || '"' === $byte || '`' === $byte ) {
 			$type = $this->read_quoted_text();
-		} elseif ( null !== $byte && strspn( $byte, self::DIGIT_MASK ) > 0 ) {
+		} elseif ( null !== $byte && $byte >= '0' && $byte <= '9' ) {
 			$type = $this->read_number();
 		} elseif ( '.' === $byte ) {
-			if ( null !== $next_byte && strspn( $next_byte, self::DIGIT_MASK ) > 0 ) {
+			if ( null !== $next_byte && $next_byte >= '0' && $next_byte <= '9' ) {
 				$type = $this->read_number();
 			} else {
 				$this->bytes_already_read += 1;
@@ -2420,8 +2464,8 @@ private function read_next_token(): ?int {
 		} elseif ( '-' === $byte ) {
 			if (
 				'-' === $next_byte
-				&& $this->bytes_already_read + 2 < strlen( $this->sql )
-				&& strspn( $this->sql[ $this->bytes_already_read + 2 ], self::WHITESPACE_MASK ) > 0
+				&& $this->bytes_already_read + 2 < $this->sql_length
+				&& false !== strpos( self::WHITESPACE_MASK, $this->sql[ $this->bytes_already_read + 2 ] )
 			) {
 				$type = $this->read_line_comment();
 			} elseif ( '>' === $next_byte ) {
@@ -2547,7 +2591,13 @@ private function read_next_token(): ?int {
 			}
 		} elseif ( '#' === $byte ) {
 			$type = $this->read_line_comment();
-		} elseif ( null !== $byte && strspn( $byte, self::WHITESPACE_MASK ) > 0 ) {
+		} elseif (
+			' ' === $byte
+			|| "\t" === $byte
+			|| "\n" === $byte
+			|| "\r" === $byte
+			|| "\f" === $byte
+		) {
 			$this->bytes_already_read += strspn( $this->sql, self::WHITESPACE_MASK, $this->bytes_already_read );
 			$type                      = self::WHITESPACE;
 		} elseif ( ( 'x' === $byte || 'X' === $byte || 'b' === $byte || 'B' === $byte ) && "'" === $next_byte ) {
@@ -2675,7 +2725,7 @@ private function read_number(): ?int {
 				'0' === $byte
 				&& 'x' === $next_byte
 				&& null !== $third_byte
-				&& strspn( $third_byte, self::HEX_DIGIT_MASK ) > 0
+				&& false !== strpos( self::HEX_DIGIT_MASK, $third_byte )
 			)
 			// HEX number in the form of x'N' or X'N'.
 			|| ( ( 'x' === $byte || 'X' === $byte ) && "'" === $next_byte )
@@ -2685,7 +2735,7 @@ private function read_number(): ?int {
 			$this->bytes_already_read += strspn( $this->sql, self::HEX_DIGIT_MASK, $this->bytes_already_read );
 			if ( $is_quoted ) {
 				if (
-					$this->bytes_already_read >= strlen( $this->sql )
+					$this->bytes_already_read >= $this->sql_length
 					|| "'" !== $this->sql[ $this->bytes_already_read ]
 				) {
 					return null; // Invalid input.
@@ -2708,7 +2758,7 @@ private function read_number(): ?int {
 			$this->bytes_already_read += strspn( $this->sql, '01', $this->bytes_already_read );
 			if ( $is_quoted ) {
 				if (
-					$this->bytes_already_read >= strlen( $this->sql )
+					$this->bytes_already_read >= $this->sql_length
 					|| "'" !== $this->sql[ $this->bytes_already_read ]
 				) {
 					return null; // Invalid input.
@@ -2737,11 +2787,12 @@ private function read_number(): ?int {
 				( 'e' === $byte || 'E' === $byte )
 				&& null !== $next_byte
 				&& (
-					strspn( $next_byte, self::DIGIT_MASK ) > 0
+					( $next_byte >= '0' && $next_byte <= '9' )
 					|| (
 						( '+' === $next_byte || '-' === $next_byte )
-						&& $this->bytes_already_read + 2 < strlen( $this->sql )
-						&& strspn( $this->sql[ $this->bytes_already_read + 2 ], self::DIGIT_MASK ) > 0
+						&& $this->bytes_already_read + 2 < $this->sql_length
+						&& $this->sql[ $this->bytes_already_read + 2 ] >= '0'
+						&& $this->sql[ $this->bytes_already_read + 2 ] <= '9'
 					)
 				);
 			if ( $has_exponent ) {
@@ -2840,7 +2891,11 @@ private function read_quoted_text(): ?int {
 		// in which case the escape sequence is consumed and the loop continues.
 		$at = $this->bytes_already_read;
 		while ( true ) {
-			$at += strcspn( $this->sql, $quote, $at );
+			$quote_at = strpos( $this->sql, $quote, $at );
+			if ( false === $quote_at ) {
+				return null; // Invalid input.
+			}
+			$at = $quote_at;
 
 			/*
 			 * By default, quotes can be escaped with a "\".
@@ -2852,18 +2907,16 @@ private function read_quoted_text(): ?int {
 			 * "\\\" is an escaped backslash and an escape sequence, and so on.
 			 */
 			if ( ! $no_backslash_escapes ) {
-				for ($i = 0; '\\' === $this->sql[ $at - $i - 1 ]; $i += 1);
+				$i = 0;
+				while ( '\\' === ( $this->sql[ $at - $i - 1 ] ?? null ) ) {
+					$i += 1;
+				}
 				if ( 1 === $i % 2 ) {
 					$at += 1;
 					continue;
 				}
 			}
 
-			// Unclosed string - unexpected EOF.
-			if ( ( $this->sql[ $at ] ?? null ) !== $quote ) {
-				return null; // Invalid input.
-			}
-
 			// Check if the quote is doubled.
 			if ( ( $this->sql[ $at + 1 ] ?? null ) === $quote ) {
 				$at += 2;
@@ -2922,17 +2975,11 @@ private function read_mysql_comment(): int {
 	}
 
 	private function read_comment_content(): void {
-		while ( true ) {
-			$this->bytes_already_read += strcspn( $this->sql, '*', $this->bytes_already_read );
-			$this->bytes_already_read += 1; // Consume the '*'.
-			$byte                      = $this->sql[ $this->bytes_already_read ] ?? null;
-			if ( null === $byte ) {
-				break;
-			}
-			if ( '/' === $byte ) {
-				$this->bytes_already_read += 1; // Consume the '/'.
-				break;
-			}
+		$comment_end = strpos( $this->sql, '*/', $this->bytes_already_read );
+		if ( false === $comment_end ) {
+			$this->bytes_already_read = $this->sql_length;
+		} else {
+			$this->bytes_already_read = $comment_end + 2;
 		}
 	}
 

packages/mysql-on-sqlite/src/mysql/class-wp-mysql-token.php

@@ -30,7 +30,10 @@ public function __construct(
 		string $input,
 		bool $sql_mode_no_backslash_escapes_enabled
 	) {
-		parent::__construct( $id, $start, $length, $input );
+		$this->id     = $id;
+		$this->start  = $start;
+		$this->length = $length;
+		$this->input  = $input;
 		$this->sql_mode_no_backslash_escapes_enabled = $sql_mode_no_backslash_escapes_enabled;
 	}