Store and expose byte start and length in tokens and nodes

Author: JanJakes

wp-includes/mysql/class-wp-mysql-lexer.php

@@ -2247,7 +2247,12 @@ public function get_token(): ?WP_MySQL_Token {
 		if ( null === $this->token_type ) {
 			return null;
 		}
-		return new WP_MySQL_Token( $this->token_type, $this->get_current_token_bytes() );
+		return new WP_MySQL_Token(
+			$this->token_type,
+			$this->get_current_token_bytes(),
+			$this->token_starts_at,
+			$this->bytes_already_read - $this->token_starts_at
+		);
 	}
 
 	/**

wp-includes/parser/class-wp-parser-node.php

@@ -263,6 +263,27 @@ public function get_descendant_tokens( ?int $token_id = null ): array {
 		return $all_descendants;
 	}
 
+	/**
+	 * Get the byte offset in the input where the node begins.
+	 *
+	 * @return int
+	 */
+	public function get_start(): int {
+		return $this->get_first_descendant_token()->start;
+	}
+
+	/**
+	 * Get the byte length of the node in the input.
+	 *
+	 * @return int
+	 */
+	public function get_length(): int {
+		$tokens     = $this->get_descendant_tokens();
+		$last_token = end( $tokens );
+		$start      = $this->get_start();
+		return $last_token->start + $last_token->length - $start;
+	}
+
 	/*
 	 * @TODO: Let's implement a more powerful AST-querying API.
 	 *        See: https://github.com/WordPress/sqlite-database-integration/pull/164#discussion_r1855230501

wp-includes/parser/class-wp-parser-token.php

@@ -23,14 +23,37 @@ class WP_Parser_Token {
 	 */
 	public $value;
 
+	/**
+	 * Byte offset in the input where the token begins.
+	 *
+	 * @var int
+	 */
+	public $start;
+
+	/**
+	 * Byte length of the token in the input.
+	 *
+	 * @var int
+	 */
+	public $length;
+
 	/**
 	 * Constructor.
 	 *
-	 * @param int $id Token type.
-	 * @param string $value Token value.
+	 * @param int    $id      Token type.
+	 * @param string $value   Token value.
+	 * @param int    $start   Byte offset in the input where the token begins.
+	 * @param int    $length  Byte length of the token in the input.
 	 */
-	public function __construct( int $id, string $value ) {
-		$this->id    = $id;
-		$this->value = $value;
+	public function __construct(
+		int $id,
+		string $value,
+		int $start,
+		int $length
+	) {
+		$this->id     = $id;
+		$this->value  = $value;
+		$this->start  = $start;
+		$this->length = $length;
 	}
 }