feat: sync parser with server SQL (0.1.21 → current, OSS + Enterprise)

src/autocomplete/budgeted-content-assist.ts

@@ -29,7 +29,43 @@ import {
   Rule,
   Terminal,
 } from "chevrotain"
-import { parser } from "../parser/parser"
+import {
+  ALIAS_STOP_STATEMENT_TOKENS,
+  IMPLICIT_ALIAS_STOP_TOKENS,
+  parser,
+} from "../parser/parser"
+import { As, Case, IdentifierKeyword, Semicolon, When } from "../parser/lexer"
+
+// The parser's implicit-alias GATEs (a join / clause / statement word is not
+// an alias without AS), which content assist would otherwise ignore: the
+// alias path would consume e.g. `ASOF` in `FROM t ASOF JOIN u` and prune the
+// join path.
+function isStoppedIdentifier(
+  terminalType: TokenType,
+  ruleStack: string[],
+  tokenVector: IToken[],
+  idx: number,
+): boolean {
+  if (terminalType !== IdentifierKeyword) return false
+  // `CASE WHEN`: never a simple-CASE operand named `when`
+  const tokenType = tokenVector[idx].tokenType
+  if (tokenType === When && idx > 0 && tokenVector[idx - 1].tokenType === Case)
+    return true
+  // `SHOW `, `COPY `, `GRANT ` ...: a statement keyword, not the table of an
+  // implicit SELECT (`show WHERE ...`) at statement start
+  if (
+    ALIAS_STOP_STATEMENT_TOKENS.has(tokenType) &&
+    (idx === 0 || tokenVector[idx - 1].tokenType === Semicolon) &&
+    (ruleStack.includes("implicitSelectStatement") ||
+      ruleStack.includes("pivotStatement"))
+  )
+    return true
+  const n = ruleStack.length
+  if (n < 2 || ruleStack[n - 1] !== "identifier") return false
+  const stops = IMPLICIT_ALIAS_STOP_TOKENS.get(ruleStack[n - 2])
+  if (!stops || !stops.has(tokenVector[idx].tokenType)) return false
+  return idx === 0 || tokenVector[idx - 1].tokenType !== As
+}
 
 export interface ContentAssistSuggestion {
   nextTokenType: TokenType
@@ -147,7 +183,15 @@ function nextPossibleTokensAfterBudgeted(
       if (currIdx < tokenVectorLength - 1) {
         const nextIdx = currIdx + 1
         const actualToken = tokenVector[nextIdx]
-        if (tokMatcher(actualToken, prod.terminalType)) {
+        if (
+          tokMatcher(actualToken, prod.terminalType) &&
+          !isStoppedIdentifier(
+            prod.terminalType,
+            currRuleStack,
+            tokenVector,
+            nextIdx,
+          )
+        ) {
           possiblePaths.push({
             idx: nextIdx,
             def: currDef.slice(1),

src/autocomplete/content-assist.ts

@@ -2,6 +2,7 @@ import { type ILexingError, IToken, TokenType } from "chevrotain"
 import { parse as parseRaw } from "../parser/parser"
 import { visitor } from "../parser/visitor"
 import { QuestDBLexer } from "../parser/lexer"
+import { CLAUSE_IDENTIFIER_KEYWORD_NAMES } from "../parser/tokens"
 import type { Statement } from "../parser/ast"
 import {
   IDENTIFIER_KEYWORD_TOKENS,
@@ -36,6 +37,7 @@ const WORD_BOUNDARY_CHARS = new Set([
   "%",
   "'",
   '"',
+  "`",
   "|",
   "&",
   "^",
@@ -580,10 +582,18 @@ function extractTables(fullSql: string, tokens: IToken[]): ExtractResult {
 
       let alias: string | undefined
       let aliasStart = tableNameResult.nextIndex
-      if (tokens[aliasStart]?.tokenType.name === "As") {
+      const explicitAs = tokens[aliasStart]?.tokenType.name === "As"
+      if (explicitAs) {
         aliasStart++
       }
-      if (isIdentifierLike(tokens[aliasStart])) {
+      if (
+        isIdentifierLike(tokens[aliasStart]) &&
+        // `FROM t GROUP BY`, `FROM t LEFT JOIN`: clause words are no alias
+        (explicitAs ||
+          !CLAUSE_IDENTIFIER_KEYWORD_NAMES.has(
+            tokens[aliasStart].tokenType.name,
+          ))
+      ) {
         alias = tokenToNamePart(tokens[aliasStart])
       }
 
@@ -872,6 +882,90 @@ function contextFunctionSuggestions(
   return inSubsample ? [...SUBSAMPLE_METHODS] : []
 }
 
+// Enterprise permission names (Permission.java namePermissionMap)
+export const PERMISSION_NAMES: readonly string[] = [
+  "ALL",
+  "SELECT",
+  "INSERT",
+  "UPDATE",
+  "CREATE TABLE",
+  "DROP TABLE",
+  "RENAME TABLE",
+  "TRUNCATE TABLE",
+  "ADD COLUMN",
+  "DROP COLUMN",
+  "RENAME COLUMN",
+  "ALTER COLUMN TYPE",
+  "ALTER COLUMN CACHE",
+  "ALTER SYMBOL CAPACITY",
+  "ADD INDEX",
+  "DROP INDEX",
+  "REINDEX",
+  "ATTACH PARTITION",
+  "DETACH PARTITION",
+  "DROP PARTITION",
+  "SET TABLE PARAM",
+  "SET TABLE TYPE",
+  "SET TABLE FORMAT",
+  "RESUME WAL",
+  "CREATE GROUP",
+  "DROP GROUP",
+  "CREATE USER",
+  "DROP USER",
+  "CREATE SERVICE ACCOUNT",
+  "DROP SERVICE ACCOUNT",
+  "ADD USER",
+  "REMOVE USER",
+  "CANCEL ANY COPY",
+  "BACKUP TABLE",
+  "BACKUP DATABASE",
+  "SNAPSHOT",
+  "VACUUM TABLE",
+  "ENABLE USER",
+  "DISABLE USER",
+  "ADD PASSWORD",
+  "REMOVE PASSWORD",
+  "ADD EXTERNAL ALIAS",
+  "REMOVE EXTERNAL ALIAS",
+  "CREATE JWK",
+  "DROP JWK",
+  "CREATE REST TOKEN",
+  "DROP REST TOKEN",
+  "LIST USERS",
+  "USER DETAILS",
+  "DEDUP ENABLE",
+  "DEDUP DISABLE",
+  "HTTP",
+  "PGWIRE",
+  "ILP",
+  "SQL ENGINE ADMIN",
+  "SYSTEM ADMIN",
+  "CREATE MATERIALIZED VIEW",
+  "DROP MATERIALIZED VIEW",
+  "REFRESH MATERIALIZED VIEW",
+  "SETTINGS",
+  "DATABASE ADMIN",
+  "AUDIT VIEW",
+  "CREATE VIEW",
+  "DROP VIEW",
+  "CREATE LIVE VIEW",
+  "DROP LIVE VIEW",
+  "COMPILE VIEW",
+  "ALTER VIEW",
+  "SET REFRESH LIMIT",
+  "SET REFRESH TYPE",
+  "CONVERT PARTITION TO PARQUET",
+  "CONVERT PARTITION TO NATIVE",
+  "SET PARQUET SETTINGS",
+  "SET STORAGE POLICY",
+  "REMOVE STORAGE POLICY",
+  "ENABLE STORAGE POLICY",
+  "DISABLE STORAGE POLICY",
+  "SWITCH ROLE",
+  "SET MEMORY LIMIT",
+  "RESOURCE GROUP ADMIN",
+]
+
 // Category names valid inside SHOW CREATE DATABASE (INCLUDE|EXCLUDE) ( ... ).
 // The grammar consumes these via `identifier` (they are non-reserved), so only
 // the explicit `ALL` token surfaces on its own; re-inject the rest.
@@ -912,27 +1006,21 @@ function contextKeywordSuggestions(
     idKeywordStacks.some((stack) => stack.includes(r))
 
   const last = tokens[tokens.length - 1]?.tokenType.name
-  const prev = tokens[tokens.length - 2]?.tokenType.name
   const out = new Set<string>()
 
-  // GRANT / REVOKE permission phrases whose words route through `identifier`:
-  //   CONVERT PARTITION [TO PARQUET|NATIVE], SET TABLE FORMAT, SET TABLE TYPE,
-  //   SET STORAGE POLICY, REMOVE STORAGE POLICY.
-  // (SET / TABLE / TO / PARQUET / NATIVE at their spots are explicit CONSUMEs
-  // and already surface; we only add the identifier-path words.)
+  // GRANT / REVOKE permission phrases: the grammar takes any 1-3 words (most
+  // via `identifier`), so offer the next word of every server permission name
+  // that extends the words typed so far in the current list item.
   if (inRule("permissionToken")) {
-    if (last === "Grant" || last === "Revoke") {
-      out.add("CONVERT")
-      out.add("REMOVE")
-    } else if (last === "Convert") {
-      out.add("PARTITION")
-    } else if (last === "Set" || last === "Remove") {
-      out.add("STORAGE")
-    } else if (last === "Table" && prev === "Set") {
-      out.add("FORMAT")
-      out.add("TYPE")
-    } else if (last === "Storage" && (prev === "Set" || prev === "Remove")) {
-      out.add("POLICY")
+    const words: string[] = []
+    for (let i = tokens.length - 1; i >= 0; i--) {
+      const name = tokens[i].tokenType.name
+      if (name === "Grant" || name === "Revoke" || name === "Comma") break
+      words.unshift(tokens[i].image.toUpperCase())
+    }
+    const prefix = words.length > 0 ? words.join(" ") + " " : ""
+    for (const perm of PERMISSION_NAMES) {
+      if (perm.startsWith(prefix)) out.add(perm.split(" ")[words.length])
     }
   }
 
@@ -951,6 +1039,44 @@ function contextKeywordSuggestions(
   return [...out]
 }
 
+const QUIET_EXPRESSION_ANCHORS = [
+  "sampleByClause",
+  "alignToClause",
+  "fillValue",
+]
+// Literal / variable / punctuation tokens stay valid there (no keyword noise)
+const QUIET_EXPRESSION_KEPT_TOKENS = new Set([
+  "DurationLiteral",
+  "NumberLiteral",
+  "LongLiteral",
+  "DecimalLiteral",
+  "FloatLiteral",
+  "StringLiteral",
+  "EscapeStringLiteral",
+  "VariableReference",
+  "BindVariable",
+  "Minus",
+  "LParen",
+])
+
+/**
+ * If the ruleStack is at the top level of the expression directly under
+ * SAMPLE BY, ALIGN TO (TIME ZONE / WITH OFFSET) or a FILL value, return that
+ * anchor rule. Nested expressions (function arguments, parentheses) are not
+ * top level.
+ */
+function quietExpressionAnchor(ruleStack: string[]): string | undefined {
+  for (let i = ruleStack.length - 2; i >= 0; i--) {
+    if (!QUIET_EXPRESSION_ANCHORS.includes(ruleStack[i])) continue
+    if (ruleStack[i + 1] !== "expression") return undefined
+    const nested = ruleStack
+      .slice(i + 2)
+      .some((r) => r === "expression" || r === "selectStatement")
+    return nested ? undefined : ruleStack[i]
+  }
+  return undefined
+}
+
 /**
  * Compute content assist suggestions, handling CTE context specially.
  *
@@ -973,7 +1099,59 @@ function computeSuggestions(tokens: IToken[]): ComputeResult {
     : "statement"
   // Budgeted: aborts after a fixed number of path-exploration steps.
   // The budget keeps autocomplete responsive and falls back to empty suggestions when hit.
-  const { suggestions } = computeContentAssistBudgeted(ruleName, tokens)
+  const { suggestions: rawSuggestions } = computeContentAssistBudgeted(
+    ruleName,
+    tokens,
+  )
+  // timestampWithTimeZoneLiteral opens with explicit TIMESTAMP / TIMESTAMP_NS
+  // tokens. Content assist ignores its GATE, so without this filter those two
+  // keywords would be offered wherever an expression can start. Only the
+  // rule's first token is dropped; WITH TIME ZONE after `timestamp` stays.
+  // columnDefinition only takes DOUBLE PRECISION in CREATE TABLE (a rule
+  // argument gates it); ALTER TABLE ADD COLUMN shares the rule, so drop
+  // PRECISION there as well.
+  // SAMPLE BY / FILL / ALIGN TO ... TIME ZONE / WITH OFFSET take any
+  // expression on the server, but at the top level of those expressions we
+  // keep the short lists offered before (nothing for SAMPLE BY and ALIGN TO,
+  // NULL / PREV / LINEAR / NONE for FILL) instead of every column, function
+  // and operator.
+  let fillOperandStart = false
+  const suggestions = rawSuggestions.filter((s) => {
+    const quiet = quietExpressionAnchor(s.ruleStack)
+    if (quiet === "fillValue" && s.nextTokenType.name === "Null") {
+      fillOperandStart = true
+      return true
+    }
+    return !(
+      (quiet && !QUIET_EXPRESSION_KEPT_TOKENS.has(s.nextTokenType.name)) ||
+      (s.ruleStack[s.ruleStack.length - 1] === "timestampWithTimeZoneLiteral" &&
+        (s.nextTokenType.name === "Timestamp" ||
+          s.nextTokenType.name === "TimestampNs")) ||
+      (s.nextTokenType.name === "Precision" &&
+        (s.ruleStack.includes("alterTableAction") ||
+          s.ruleStack[s.ruleStack.length - 1] === "columnNullModifier")) ||
+      // SET <name> ... takes free-form tokens; only offer TO after the name
+      (s.ruleStack[s.ruleStack.length - 1] === "setStatement" &&
+        (s.nextTokenType.name !== "To" ||
+          tokens[tokens.length - 1]?.tokenType.name === "Set")) ||
+      // ARRAY[...] literals: not offered at every expression start (it was an
+      // identifier before); ARRAY stays suggested as a type name
+      (s.nextTokenType.name === "Array" &&
+        s.ruleStack[s.ruleStack.length - 1] === "arrayLiteral") ||
+      // IS / ILIKE / DECLARE are accepted as column names, not suggested
+      (s.ruleStack[s.ruleStack.length - 1] === "columnDefinition" &&
+        (s.nextTokenType.name === "Is" ||
+          s.nextTokenType.name === "Ilike" ||
+          s.nextTokenType.name === "Declare")) ||
+      // ANY only continues CANCEL in permission names (CANCEL ANY COPY)
+      (s.nextTokenType.name === "Any" &&
+        s.ruleStack[s.ruleStack.length - 1] === "permissionToken" &&
+        tokens[tokens.length - 1]?.tokenType.name !== "Cancel") ||
+      // INCREMENTAL is an obsolete synonym of IMMEDIATE in REFRESH modes
+      (s.nextTokenType.name === "Incremental" &&
+        s.ruleStack[s.ruleStack.length - 1] === "materializedViewRefresh")
+    )
+  })
   const result = suggestions.map((s) => s.nextTokenType)
 
   // Walk every IdentifierKeyword path and union the category flags valid at
@@ -1028,6 +1206,7 @@ function computeSuggestions(tokens: IToken[]): ComputeResult {
   )
 
   const contextKeywords = contextKeywordSuggestions(tokens, suggestions)
+  if (fillOperandStart) contextKeywords.push("PREV", "LINEAR", "NONE")
   const contextFunctions = contextFunctionSuggestions(tokens, suggestions)
 
   return {
@@ -1157,7 +1336,11 @@ export function getContentAssist(
   // problems than it solves (prefix filtering breaks, ambiguity with string values).
   const fullTokens = QuestDBLexer.tokenize(fullSql).tokens
   for (const token of fullTokens) {
-    if (token.tokenType.name !== "StringLiteral") continue
+    if (
+      token.tokenType.name !== "StringLiteral" &&
+      token.tokenType.name !== "EscapeStringLiteral"
+    )
+      continue
     const start = token.startOffset
     const end = token.startOffset + token.image.length
     if (cursorOffset > start && cursorOffset < end) {

src/autocomplete/token-classification.ts

@@ -107,14 +107,20 @@ export const SKIP_TOKENS = new Set([
   "BitOr",
   // Variable references (user-defined, can't suggest)
   "VariableReference",
+  // Bind variables ($1, :name)
+  "BindVariable",
   // Literals (don't suggest literal tokens)
   "StringLiteral",
   "NumberLiteral",
   "LongLiteral",
   "DecimalLiteral",
   "DurationLiteral",
+  "MalformedDurationLiteral",
   "GeohashLiteral",
   "GeohashBinaryLiteral",
+  "HexLiteral",
+  "FloatLiteral",
+  "EscapeStringLiteral",
   "Nan",
   // Whitespace
   "WhiteSpace",