Speed up the lexer with cheaper byte checks

Apply lexer optimisations from PR #375:

- Cache `strlen($sql)` once in `$sql_length` instead of recomputing on each
  EOF check.
- Replace `strspn($byte, MASK) > 0` with direct byte comparisons
  (`$byte >= '0' && $byte <= '9'`, `false !== strpos(MASK, $byte)`,
  unrolled whitespace check).
- Use `strpos($sql, '*/', $pos)` instead of a manual scan loop in
  `read_comment_content()`.
- In `read_quoted_text()`, use `strpos()` to find the next quote, eliminating
  the separate end-of-input check that follows the `strcspn()` scan.
- Inline `next_token()` + `get_token()` in `remaining_tokens()` so the hot
  loop builds tokens directly.

Co-authored-by: Adam Zieliński <adam@adamziel.com>

Adapted from #375

Author: JanJakes

packages/mysql-on-sqlite/src/mysql/class-wp-mysql-lexer.php

@@ -2111,6 +2111,13 @@ class WP_MySQL_Lexer {
 	 */
 	private $sql;
 
+	/**
+	 * Byte length of the SQL payload.
+	 *
+	 * @var int
+	 */
+	private $sql_length;
+
 	/**
 	 * The version of the MySQL server that the SQL payload is intended for.
 	 *
@@ -2189,6 +2196,7 @@ public function __construct(
 		array $sql_modes = array()
 	) {
 		$this->sql           = $sql;
+		$this->sql_length    = strlen( $sql );
 		$this->mysql_version = $mysql_version;
 
 		foreach ( $sql_modes as $sql_mode ) {
@@ -2284,10 +2292,46 @@ public function get_token(): ?WP_MySQL_Token {
 	 * @return WP_MySQL_Token[] An array of token objects representing the remaining tokens.
 	 */
 	public function remaining_tokens(): array {
-		$tokens = array();
-		while ( true === $this->next_token() ) {
-			$token    = $this->get_token();
-			$tokens[] = $token;
+		$tokens                            = array();
+		$no_backslash_escapes_sql_mode_set = $this->is_sql_mode_active(
+			self::SQL_MODE_NO_BACKSLASH_ESCAPES
+		);
+
+		while ( true ) {
+			if (
+				self::EOF === $this->token_type
+				|| ( null === $this->token_type && $this->bytes_already_read > 0 )
+			) {
+				$this->token_type = null;
+				break;
+			}
+
+			do {
+				$this->token_starts_at = $this->bytes_already_read;
+				$this->token_type      = $this->read_next_token();
+			} while (
+				self::WHITESPACE === $this->token_type
+				|| self::COMMENT === $this->token_type
+				|| self::MYSQL_COMMENT_START === $this->token_type
+				|| self::MYSQL_COMMENT_END === $this->token_type
+			);
+
+			if ( null === $this->token_type ) {
+				break;
+			}
+
+			$tokens[] = new WP_MySQL_Token(
+				$this->token_type,
+				$this->token_starts_at,
+				$this->bytes_already_read - $this->token_starts_at,
+				$this->sql,
+				$no_backslash_escapes_sql_mode_set
+			);
+
+			if ( self::EOF === $this->token_type ) {
+				$this->token_type = null;
+				break;
+			}
 		}
 		return $tokens;
 	}
@@ -2356,10 +2400,10 @@ private function read_next_token(): ?int {
 
 		if ( "'" === $byte || '"' === $byte || '`' === $byte ) {
 			$type = $this->read_quoted_text();
-		} elseif ( null !== $byte && strspn( $byte, self::DIGIT_MASK ) > 0 ) {
+		} elseif ( null !== $byte && $byte >= '0' && $byte <= '9' ) {
 			$type = $this->read_number();
 		} elseif ( '.' === $byte ) {
-			if ( null !== $next_byte && strspn( $next_byte, self::DIGIT_MASK ) > 0 ) {
+			if ( null !== $next_byte && $next_byte >= '0' && $next_byte <= '9' ) {
 				$type = $this->read_number();
 			} else {
 				$this->bytes_already_read += 1;
@@ -2420,8 +2464,8 @@ private function read_next_token(): ?int {
 		} elseif ( '-' === $byte ) {
 			if (
 				'-' === $next_byte
-				&& $this->bytes_already_read + 2 < strlen( $this->sql )
-				&& strspn( $this->sql[ $this->bytes_already_read + 2 ], self::WHITESPACE_MASK ) > 0
+				&& $this->bytes_already_read + 2 < $this->sql_length
+				&& false !== strpos( self::WHITESPACE_MASK, $this->sql[ $this->bytes_already_read + 2 ] )
 			) {
 				$type = $this->read_line_comment();
 			} elseif ( '>' === $next_byte ) {
@@ -2547,7 +2591,13 @@ private function read_next_token(): ?int {
 			}
 		} elseif ( '#' === $byte ) {
 			$type = $this->read_line_comment();
-		} elseif ( null !== $byte && strspn( $byte, self::WHITESPACE_MASK ) > 0 ) {
+		} elseif (
+			' ' === $byte
+			|| "\t" === $byte
+			|| "\n" === $byte
+			|| "\r" === $byte
+			|| "\f" === $byte
+		) {
 			$this->bytes_already_read += strspn( $this->sql, self::WHITESPACE_MASK, $this->bytes_already_read );
 			$type                      = self::WHITESPACE;
 		} elseif ( ( 'x' === $byte || 'X' === $byte || 'b' === $byte || 'B' === $byte ) && "'" === $next_byte ) {
@@ -2675,7 +2725,7 @@ private function read_number(): ?int {
 				'0' === $byte
 				&& 'x' === $next_byte
 				&& null !== $third_byte
-				&& strspn( $third_byte, self::HEX_DIGIT_MASK ) > 0
+				&& false !== strpos( self::HEX_DIGIT_MASK, $third_byte )
 			)
 			// HEX number in the form of x'N' or X'N'.
 			|| ( ( 'x' === $byte || 'X' === $byte ) && "'" === $next_byte )
@@ -2685,7 +2735,7 @@ private function read_number(): ?int {
 			$this->bytes_already_read += strspn( $this->sql, self::HEX_DIGIT_MASK, $this->bytes_already_read );
 			if ( $is_quoted ) {
 				if (
-					$this->bytes_already_read >= strlen( $this->sql )
+					$this->bytes_already_read >= $this->sql_length
 					|| "'" !== $this->sql[ $this->bytes_already_read ]
 				) {
 					return null; // Invalid input.
@@ -2708,7 +2758,7 @@ private function read_number(): ?int {
 			$this->bytes_already_read += strspn( $this->sql, '01', $this->bytes_already_read );
 			if ( $is_quoted ) {
 				if (
-					$this->bytes_already_read >= strlen( $this->sql )
+					$this->bytes_already_read >= $this->sql_length
 					|| "'" !== $this->sql[ $this->bytes_already_read ]
 				) {
 					return null; // Invalid input.
@@ -2737,11 +2787,12 @@ private function read_number(): ?int {
 				( 'e' === $byte || 'E' === $byte )
 				&& null !== $next_byte
 				&& (
-					strspn( $next_byte, self::DIGIT_MASK ) > 0
+					( $next_byte >= '0' && $next_byte <= '9' )
 					|| (
 						( '+' === $next_byte || '-' === $next_byte )
-						&& $this->bytes_already_read + 2 < strlen( $this->sql )
-						&& strspn( $this->sql[ $this->bytes_already_read + 2 ], self::DIGIT_MASK ) > 0
+						&& $this->bytes_already_read + 2 < $this->sql_length
+						&& $this->sql[ $this->bytes_already_read + 2 ] >= '0'
+						&& $this->sql[ $this->bytes_already_read + 2 ] <= '9'
 					)
 				);
 			if ( $has_exponent ) {
@@ -2838,12 +2889,11 @@ private function read_quoted_text(): ?int {
 		// in which case the escape sequence is consumed and the loop continues.
 		$at = $this->bytes_already_read;
 		while ( true ) {
-			$at += strcspn( $this->sql, $quote, $at );
-
-			// Unclosed string - unexpected EOF.
-			if ( ( $this->sql[ $at ] ?? null ) !== $quote ) {
+			$quote_at = strpos( $this->sql, $quote, $at );
+			if ( false === $quote_at ) {
 				return null; // Invalid input.
 			}
+			$at = $quote_at;
 
 			/*
 			 * By default, quotes can be escaped with a "\".
@@ -2853,9 +2903,17 @@ private function read_quoted_text(): ?int {
 			 * The quote is escaped only when the number of preceding backslashes
 			 * is odd - "\" is an escape sequence, "\\" is an escaped backslash,
 			 * "\\\" is an escaped backslash and an escape sequence, and so on.
+			 *
+			 * The `($at - $i - 1) >= 0` guard prevents PHP's negative-string-
+			 * offset wraparound (PHP 7.1+) when the closing-quote candidate
+			 * sits at the very start of the input. The `?? null` covers
+			 * positive out-of-range indexes belt-and-suspenders.
 			 */
 			if ( ! $no_backslash_escapes ) {
-				for ( $i = 0; ( $at - $i - 1 ) >= 0 && '\\' === $this->sql[ $at - $i - 1 ]; $i += 1 );
+				$i = 0;
+				while ( ( $at - $i - 1 ) >= 0 && '\\' === ( $this->sql[ $at - $i - 1 ] ?? null ) ) {
+					$i += 1;
+				}
 				if ( 1 === $i % 2 ) {
 					$at += 1;
 					continue;
@@ -2920,17 +2978,11 @@ private function read_mysql_comment(): int {
 	}
 
 	private function read_comment_content(): void {
-		while ( true ) {
-			$this->bytes_already_read += strcspn( $this->sql, '*', $this->bytes_already_read );
-			$this->bytes_already_read += 1; // Consume the '*'.
-			$byte                      = $this->sql[ $this->bytes_already_read ] ?? null;
-			if ( null === $byte ) {
-				break;
-			}
-			if ( '/' === $byte ) {
-				$this->bytes_already_read += 1; // Consume the '/'.
-				break;
-			}
+		$comment_end = strpos( $this->sql, '*/', $this->bytes_already_read );
+		if ( false === $comment_end ) {
+			$this->bytes_already_read = $this->sql_length;
+		} else {
+			$this->bytes_already_read = $comment_end + 2;
 		}
 	}