From 62367cf1fd3c6c37dac7658d22f2b7ba0f09f077 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 13:46:32 +0300 Subject: [PATCH 01/36] test: add server-query parity harness Queries copied from the QuestDB server test suites live in tests/fixtures/server-queries.json, each with its source file and line. tests/server-queries.test.ts checks every entry four ways: it parses as the expected number of statements, toSql matches the reviewed output and round-trips, autocomplete suggests each next token, and the formatter preserves it. The token-by-token autocomplete walkthrough moves from docs-autocomplete.test.ts to tests/helpers so both suites share it. --- tests/docs-autocomplete.test.ts | 358 +-------------------- tests/fixtures/server-queries.json | 26 ++ tests/helpers/autocomplete-walkthrough.ts | 365 ++++++++++++++++++++++ tests/server-queries.test.ts | 107 +++++++ 4 files changed, 503 insertions(+), 353 deletions(-) create mode 100644 tests/fixtures/server-queries.json create mode 100644 tests/helpers/autocomplete-walkthrough.ts create mode 100644 tests/server-queries.test.ts diff --git a/tests/docs-autocomplete.test.ts b/tests/docs-autocomplete.test.ts index eeaeacc..e86743d 100644 --- a/tests/docs-autocomplete.test.ts +++ b/tests/docs-autocomplete.test.ts @@ -21,18 +21,12 @@ */ import { describe, it } from "vitest" +import { parseToAst } from "../src/index" import { - tokenize, - createAutocompleteProvider, - parseToAst, - getNextValidTokens, -} from "../src/index" -import type { SchemaInfo } from "../src/autocomplete/types" -import type * as AST from "../src/parser/ast" -import { - tokenNameToKeyword, - IDENTIFIER_KEYWORD_TOKENS, -} from "../src/autocomplete/token-classification" + autocompleteWalkthrough, + buildSchema, + extractSchemaFromAst, +} from "./helpers/autocomplete-walkthrough" import * as fs from "fs" import * as path from "path" @@ -46,348 +40,6 @@ const queries: DocsQuery[] = JSON.parse( fs.readFileSync(fixtureP, "utf-8"), ) as DocsQuery[] -// ============================================================================= -// Schema extraction from AST -// ============================================================================= - -interface ExtractedSchema { - tables: Set - columns: Map> // tableName → column names -} - -/** - * Extract table and column names from a parsed AST. - * Walks all nodes recursively to find table references and column definitions. - */ -function extractSchemaFromAst(statements: AST.Statement[]): ExtractedSchema { - const tables = new Set() - const columns = new Map>() - - function addColumn(table: string, col: string) { - const lower = table.toLowerCase() - if (!columns.has(lower)) columns.set(lower, new Set()) - columns.get(lower)!.add(col) - } - - function getTableName(name: AST.QualifiedName): string { - return name.parts[name.parts.length - 1] - } - - function walkNode(node: unknown): void { - if (!node || typeof node !== "object") return - const n = node as Record - - // Handle arrays - if (Array.isArray(node)) { - for (const item of node) walkNode(item) - return - } - - const type = n.type as string | undefined - - // TableRef: extract table name from FROM/JOIN - if (type === "tableRef") { - const ref = node as AST.TableRef - if ( - ref.table && - (ref.table as AST.QualifiedName).type === "qualifiedName" - ) { - const name = getTableName(ref.table as AST.QualifiedName) - tables.add(name) - } - // Recurse into joins - if (ref.joins) walkNode(ref.joins) - // Recurse into subqueries - if (ref.table && (ref.table as AST.SelectStatement).type === "select") { - walkNode(ref.table) - } - return - } - - // JoinClause: extract table from joined table - if (type === "join") { - const join = node as AST.JoinClause - walkNode(join.table) - if (join.on) walkNode(join.on) - return - } - - // CreateTable: extract table name and column definitions - if (type === "createTable") { - const ct = node as AST.CreateTableStatement - const tableName = getTableName(ct.table) - tables.add(tableName) - if (ct.columns) { - for (const col of ct.columns) { - addColumn(tableName, col.name) - } - } - if (ct.asSelect) walkNode(ct.asSelect) - if (ct.like) tables.add(getTableName(ct.like)) - return - } - - // InsertStatement: extract table name and column names - if (type === "insert") { - const ins = node as AST.InsertStatement - const tableName = getTableName(ins.table) - tables.add(tableName) - if (ins.columns) { - for (const col of ins.columns) { - addColumn(tableName, col) - } - } - if (ins.select) walkNode(ins.select) - return - } - - // SelectStatement: recurse into all parts - if (type === "select") { - const sel = node as AST.SelectStatement - if (sel.from) walkNode(sel.from) - if (sel.columns) walkNode(sel.columns) - if (sel.where) walkNode(sel.where) - if (sel.with) { - for (const cte of sel.with) { - if (cte.name) tables.add(cte.name) - if (cte.query) walkNode(cte.query) - } - } - if (sel.groupBy) walkNode(sel.groupBy) - if (sel.orderBy) walkNode(sel.orderBy) - if (sel.sampleBy) walkNode(sel.sampleBy) - if (sel.latestOn) walkNode(sel.latestOn) - if (sel.declare) walkNode(sel.declare) - return - } - - // AlterTable: extract table name - if (type === "alterTable") { - const alt = node as AST.AlterTableStatement - tables.add(getTableName(alt.table)) - return - } - - // UpdateStatement: extract table name - if (type === "update") { - const upd = node as AST.UpdateStatement - tables.add(getTableName(upd.table)) - if (upd.from) walkNode(upd.from) - if (upd.where) walkNode(upd.where) - return - } - - // For all other node types, recurse into all object/array properties - for (const value of Object.values(n)) { - if (value && typeof value === "object") { - walkNode(value) - } - } - } - - for (const stmt of statements) { - walkNode(stmt) - } - - return { tables, columns } -} - -/** - * Build a SchemaInfo from extracted table/column data - */ -function buildSchema(extracted: ExtractedSchema): SchemaInfo { - const tableList = Array.from(extracted.tables).map((name) => ({ name })) - const columnsMap: Record = {} - - for (const [tableName, cols] of extracted.columns) { - columnsMap[tableName] = Array.from(cols).map((name) => ({ - name, - type: "STRING", - })) - } - - return { tables: tableList, columns: columnsMap } -} - -// ============================================================================= -// Token classification -// ============================================================================= - -/** - * Token types that we cannot verify via suggestion labels: - * - Literals: user-provided values, not suggested - * - Punctuation and operators: not shown as keyword suggestions - * - Variable references: user-defined - */ -const SKIP_TOKEN_TYPES = new Set([ - // Literals - "StringLiteral", - "NumberLiteral", - "DurationLiteral", - "BooleanLiteral", - "NullLiteral", - "GeohashLiteral", - "GeohashBinaryLiteral", - "DecimalLiteral", - "LongLiteral", - // Punctuation - "LParen", - "RParen", - "Comma", - "Dot", - "Semicolon", - "LBracket", - "RBracket", - "AtSign", - "ColonEquals", - // Operators - "Equals", - "NotEquals", - "LessThan", - "LessThanOrEqual", - "GreaterThan", - "GreaterThanOrEqual", - "Plus", - "Minus", - "Star", - "Divide", - "Modulo", - "Concat", - "DoubleColon", - "RegexMatch", - "RegexNotMatch", - "RegexNotEquals", - "IPv4ContainedBy", - "IPv4Contains", - "BitXor", - "BitOr", - "BitAnd", - // Variable references - "VariableReference", - // Interval literals - "IntervalLiteral", -]) - -// ============================================================================= -// Walkthrough logic -// ============================================================================= - -interface WalkthroughStep { - position: number - tokenImage: string - tokenType: string - expectedLabel: string - suggestions: string[] - found: boolean -} - -function autocompleteWalkthrough( - sql: string, - schema: SchemaInfo, -): { - success: boolean - steps: WalkthroughStep[] - failedSteps: WalkthroughStep[] -} { - const { tokens, errors } = tokenize(sql) - if (errors.length > 0) { - return { success: false, steps: [], failedSteps: [] } - } - - const provider = createAutocompleteProvider(schema) - - // Build a set of known table and column names for identifier matching - const knownNames = new Set() - for (const t of schema.tables) knownNames.add(t.name.toLowerCase()) - for (const cols of Object.values(schema.columns)) { - for (const c of cols) knownNames.add(c.name.toLowerCase()) - } - - const steps: WalkthroughStep[] = [] - - for (let i = 0; i < tokens.length; i++) { - const token = tokens[i] - const tokenType = token.tokenType.name - - // Skip literals, punctuation, operators - if (SKIP_TOKEN_TYPES.has(tokenType)) continue - - const prefix = sql.substring(0, token.startOffset) - const labels = provider - .getSuggestions(prefix, prefix.length) - .map((s) => s.label) - - let found: boolean - let expectedLabel: string - - if (tokenType === "Identifier" || tokenType === "QuotedIdentifier") { - // Identifier token: check if it's a known table/column name (reference) - // or an unknown name (new table, alias, new column definition) → skip - const word = - tokenType === "QuotedIdentifier" - ? token.image.slice(1, -1).toLowerCase() - : token.image.toLowerCase() - if (!knownNames.has(word)) continue - - // If the provider suggests ANY known name at this position, it's a - // reference position (FROM, JOIN, WHERE, etc.) → verify our name is there. - // If it suggests none, it's a definition position (CREATE TABLE name, - // column definition, alias) → skip. - const suggestsAnyKnownName = labels.some((l) => - knownNames.has(l.toLowerCase()), - ) - if (!suggestsAnyKnownName) continue - - expectedLabel = word - found = labels.some((l) => l.toLowerCase() === word) - } else { - // Keyword token: check if keyword label appears in suggestions. - // Join prefix tokens (LEFT, ASOF, etc.) are now suggested as compound - // keywords ("LEFT JOIN", "ASOF JOIN"), so also accept those forms. - expectedLabel = tokenNameToKeyword(tokenType) - const expectedUpper = expectedLabel.toUpperCase() - found = labels.some((l) => { - const upper = l.toUpperCase() - return upper === expectedUpper || upper === expectedUpper + " JOIN" - }) - - // Fallback: keyword tokens used as identifiers (e.g., `timestamp` - // as a column name). Accept if the word is a known column/table in - // the schema OR IdentifierKeyword is expected at this position. - if (!found && IDENTIFIER_KEYWORD_TOKENS.has(tokenType)) { - const word = token.image.toLowerCase() - if (knownNames.has(word)) { - found = labels.some((l) => l.toLowerCase() === word) - } - if (!found) { - const rawTokens = getNextValidTokens(prefix) - if ( - rawTokens.includes("IdentifierKeyword") || - rawTokens.includes(tokenType) - ) { - found = true - } - } - } - } - - steps.push({ - position: token.startOffset, - tokenImage: token.image, - tokenType, - expectedLabel, - suggestions: labels.slice(0, 15), - found, - }) - } - - return { - success: true, - steps, - failedSteps: steps.filter((s) => !s.found), - } -} - // ============================================================================= // Test setup // ============================================================================= diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json new file mode 100644 index 0000000..2e2f169 --- /dev/null +++ b/tests/fixtures/server-queries.json @@ -0,0 +1,26 @@ +[ + { + "query": "alter table t force reindex parquet column b partition list '2024-01-02'", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/parquet/StaleParquetColumnTopTest.java:325", + "feature": "baseline", + "toSql": "ALTER TABLE t FORCE REINDEX PARQUET COLUMN b PARTITION LIST '2024-01-02'" + }, + { + "query": "select first(i) from a sample by 1h fill(linear) align to first observation", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/ExplainPlanTest.java:8658", + "feature": "baseline", + "toSql": "SELECT first(i) FROM a SAMPLE BY 1h FILL(LINEAR) ALIGN TO FIRST OBSERVATION" + }, + { + "query": "INSERT INTO dim1 VALUES (1, 'one')", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/lv/LiveViewHashJoinGroupByTest.java:82", + "feature": "baseline", + "toSql": "INSERT INTO dim1 VALUES (1, 'one')" + }, + { + "query": "GRANT SELECT ON pending_grants TO alice", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/acl/AclAuditLogTest.java:1357", + "feature": "baseline", + "toSql": "GRANT SELECT ON pending_grants TO alice" + } +] diff --git a/tests/helpers/autocomplete-walkthrough.ts b/tests/helpers/autocomplete-walkthrough.ts new file mode 100644 index 0000000..6605f02 --- /dev/null +++ b/tests/helpers/autocomplete-walkthrough.ts @@ -0,0 +1,365 @@ +/** + * Shared autocomplete walkthrough used by the docs and server-query suites. + * + * For a SQL string, tokenizes it and walks token-by-token, checking that at + * each position the provider's getSuggestions() includes the actual next word + * (keywords, and known table/column names in reference positions). Literals, + * punctuation and operators are skipped (SKIP_TOKEN_TYPES). + * + * Moved verbatim from tests/docs-autocomplete.test.ts. + */ +import { + tokenize, + createAutocompleteProvider, + getNextValidTokens, +} from "../../src/index" +import type { SchemaInfo } from "../../src/autocomplete/types" +import type * as AST from "../../src/parser/ast" +import { + tokenNameToKeyword, + IDENTIFIER_KEYWORD_TOKENS, +} from "../../src/autocomplete/token-classification" + +// ============================================================================= +// Schema extraction from AST +// ============================================================================= + +export interface ExtractedSchema { + tables: Set + columns: Map> // tableName → column names +} + +/** + * Extract table and column names from a parsed AST. + * Walks all nodes recursively to find table references and column definitions. + */ +export function extractSchemaFromAst( + statements: AST.Statement[], +): ExtractedSchema { + const tables = new Set() + const columns = new Map>() + + function addColumn(table: string, col: string) { + const lower = table.toLowerCase() + if (!columns.has(lower)) columns.set(lower, new Set()) + columns.get(lower)!.add(col) + } + + function getTableName(name: AST.QualifiedName): string { + return name.parts[name.parts.length - 1] + } + + function walkNode(node: unknown): void { + if (!node || typeof node !== "object") return + const n = node as Record + + // Handle arrays + if (Array.isArray(node)) { + for (const item of node) walkNode(item) + return + } + + const type = n.type as string | undefined + + // TableRef: extract table name from FROM/JOIN + if (type === "tableRef") { + const ref = node as AST.TableRef + if ( + ref.table && + (ref.table as AST.QualifiedName).type === "qualifiedName" + ) { + const name = getTableName(ref.table as AST.QualifiedName) + tables.add(name) + } + // Recurse into joins + if (ref.joins) walkNode(ref.joins) + // Recurse into subqueries + if (ref.table && (ref.table as AST.SelectStatement).type === "select") { + walkNode(ref.table) + } + return + } + + // JoinClause: extract table from joined table + if (type === "join") { + const join = node as AST.JoinClause + walkNode(join.table) + if (join.on) walkNode(join.on) + return + } + + // CreateTable: extract table name and column definitions + if (type === "createTable") { + const ct = node as AST.CreateTableStatement + const tableName = getTableName(ct.table) + tables.add(tableName) + if (ct.columns) { + for (const col of ct.columns) { + addColumn(tableName, col.name) + } + } + if (ct.asSelect) walkNode(ct.asSelect) + if (ct.like) tables.add(getTableName(ct.like)) + return + } + + // InsertStatement: extract table name and column names + if (type === "insert") { + const ins = node as AST.InsertStatement + const tableName = getTableName(ins.table) + tables.add(tableName) + if (ins.columns) { + for (const col of ins.columns) { + addColumn(tableName, col) + } + } + if (ins.select) walkNode(ins.select) + return + } + + // SelectStatement: recurse into all parts + if (type === "select") { + const sel = node as AST.SelectStatement + if (sel.from) walkNode(sel.from) + if (sel.columns) walkNode(sel.columns) + if (sel.where) walkNode(sel.where) + if (sel.with) { + for (const cte of sel.with) { + if (cte.name) tables.add(cte.name) + if (cte.query) walkNode(cte.query) + } + } + if (sel.groupBy) walkNode(sel.groupBy) + if (sel.orderBy) walkNode(sel.orderBy) + if (sel.sampleBy) walkNode(sel.sampleBy) + if (sel.latestOn) walkNode(sel.latestOn) + if (sel.declare) walkNode(sel.declare) + return + } + + // AlterTable: extract table name + if (type === "alterTable") { + const alt = node as AST.AlterTableStatement + tables.add(getTableName(alt.table)) + return + } + + // UpdateStatement: extract table name + if (type === "update") { + const upd = node as AST.UpdateStatement + tables.add(getTableName(upd.table)) + if (upd.from) walkNode(upd.from) + if (upd.where) walkNode(upd.where) + return + } + + // For all other node types, recurse into all object/array properties + for (const value of Object.values(n)) { + if (value && typeof value === "object") { + walkNode(value) + } + } + } + + for (const stmt of statements) { + walkNode(stmt) + } + + return { tables, columns } +} + +/** + * Build a SchemaInfo from extracted table/column data + */ +export function buildSchema(extracted: ExtractedSchema): SchemaInfo { + const tableList = Array.from(extracted.tables).map((name) => ({ name })) + const columnsMap: Record = {} + + for (const [tableName, cols] of extracted.columns) { + columnsMap[tableName] = Array.from(cols).map((name) => ({ + name, + type: "STRING", + })) + } + + return { tables: tableList, columns: columnsMap } +} + +// ============================================================================= +// Token classification +// ============================================================================= + +/** + * Token types that we cannot verify via suggestion labels: + * - Literals: user-provided values, not suggested + * - Punctuation and operators: not shown as keyword suggestions + * - Variable references: user-defined + */ +export const SKIP_TOKEN_TYPES = new Set([ + // Literals + "StringLiteral", + "NumberLiteral", + "DurationLiteral", + "BooleanLiteral", + "NullLiteral", + "GeohashLiteral", + "GeohashBinaryLiteral", + "DecimalLiteral", + "LongLiteral", + // Punctuation + "LParen", + "RParen", + "Comma", + "Dot", + "Semicolon", + "LBracket", + "RBracket", + "AtSign", + "ColonEquals", + // Operators + "Equals", + "NotEquals", + "LessThan", + "LessThanOrEqual", + "GreaterThan", + "GreaterThanOrEqual", + "Plus", + "Minus", + "Star", + "Divide", + "Modulo", + "Concat", + "DoubleColon", + "RegexMatch", + "RegexNotMatch", + "RegexNotEquals", + "IPv4ContainedBy", + "IPv4Contains", + "BitXor", + "BitOr", + "BitAnd", + // Variable references + "VariableReference", + // Interval literals + "IntervalLiteral", +]) + +// ============================================================================= +// Walkthrough logic +// ============================================================================= + +export interface WalkthroughStep { + position: number + tokenImage: string + tokenType: string + expectedLabel: string + suggestions: string[] + found: boolean +} + +export function autocompleteWalkthrough( + sql: string, + schema: SchemaInfo, +): { + success: boolean + steps: WalkthroughStep[] + failedSteps: WalkthroughStep[] +} { + const { tokens, errors } = tokenize(sql) + if (errors.length > 0) { + return { success: false, steps: [], failedSteps: [] } + } + + const provider = createAutocompleteProvider(schema) + + // Build a set of known table and column names for identifier matching + const knownNames = new Set() + for (const t of schema.tables) knownNames.add(t.name.toLowerCase()) + for (const cols of Object.values(schema.columns)) { + for (const c of cols) knownNames.add(c.name.toLowerCase()) + } + + const steps: WalkthroughStep[] = [] + + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i] + const tokenType = token.tokenType.name + + // Skip literals, punctuation, operators + if (SKIP_TOKEN_TYPES.has(tokenType)) continue + + const prefix = sql.substring(0, token.startOffset) + const labels = provider + .getSuggestions(prefix, prefix.length) + .map((s) => s.label) + + let found: boolean + let expectedLabel: string + + if (tokenType === "Identifier" || tokenType === "QuotedIdentifier") { + // Identifier token: check if it's a known table/column name (reference) + // or an unknown name (new table, alias, new column definition) → skip + const word = + tokenType === "QuotedIdentifier" + ? token.image.slice(1, -1).toLowerCase() + : token.image.toLowerCase() + if (!knownNames.has(word)) continue + + // If the provider suggests ANY known name at this position, it's a + // reference position (FROM, JOIN, WHERE, etc.) → verify our name is there. + // If it suggests none, it's a definition position (CREATE TABLE name, + // column definition, alias) → skip. + const suggestsAnyKnownName = labels.some((l) => + knownNames.has(l.toLowerCase()), + ) + if (!suggestsAnyKnownName) continue + + expectedLabel = word + found = labels.some((l) => l.toLowerCase() === word) + } else { + // Keyword token: check if keyword label appears in suggestions. + // Join prefix tokens (LEFT, ASOF, etc.) are now suggested as compound + // keywords ("LEFT JOIN", "ASOF JOIN"), so also accept those forms. + expectedLabel = tokenNameToKeyword(tokenType) + const expectedUpper = expectedLabel.toUpperCase() + found = labels.some((l) => { + const upper = l.toUpperCase() + return upper === expectedUpper || upper === expectedUpper + " JOIN" + }) + + // Fallback: keyword tokens used as identifiers (e.g., `timestamp` + // as a column name). Accept if the word is a known column/table in + // the schema OR IdentifierKeyword is expected at this position. + if (!found && IDENTIFIER_KEYWORD_TOKENS.has(tokenType)) { + const word = token.image.toLowerCase() + if (knownNames.has(word)) { + found = labels.some((l) => l.toLowerCase() === word) + } + if (!found) { + const rawTokens = getNextValidTokens(prefix) + if ( + rawTokens.includes("IdentifierKeyword") || + rawTokens.includes(tokenType) + ) { + found = true + } + } + } + } + + steps.push({ + position: token.startOffset, + tokenImage: token.image, + tokenType, + expectedLabel, + suggestions: labels.slice(0, 15), + found, + }) + } + + return { + success: true, + steps, + failedSteps: steps.filter((s) => !s.found), + } +} diff --git a/tests/server-queries.test.ts b/tests/server-queries.test.ts new file mode 100644 index 0000000..809af07 --- /dev/null +++ b/tests/server-queries.test.ts @@ -0,0 +1,107 @@ +/** + * Server SQL parity tests + * + * Every entry in tests/fixtures/server-queries.json is a query copied from the + * QuestDB server test suites (questdb/questdb or questdb/questdb-enterprise), + * with the source file and line it came from. The server accepts each of them, + * so sql-parser must too. For every entry we check: + * + * 1. parse — no errors, and exactly `statements` statements (default 1), + * so a query that is silently split in two fails + * 2. roundtrip — toSql(ast) equals the reviewed `toSql` string, re-parses + * cleanly, and is a fixpoint (toSql of the re-parse is equal) + * 3. autocomplete — token-by-token walkthrough: at each keyword / known + * identifier, the provider suggests the word that comes next + * 4. formatter — format() changes no tokens or meaning and is idempotent + * (assertPreserved); optionally equals the reviewed `format` + */ + +import { describe, it, expect } from "vitest" +import * as fs from "fs" +import * as path from "path" +import { parseToAst } from "../src/index" +import { toSql } from "../src/parser/toSql" +import { format } from "../src/formatter/index" +import { + autocompleteWalkthrough, + buildSchema, + extractSchemaFromAst, +} from "./helpers/autocomplete-walkthrough" +import { assertPreserved } from "./formatter/oracles" + +export interface ServerQuery { + /** SQL exactly as written in the server test */ + query: string + /** "::" of the server test the query comes from */ + source: string + /** Feature / report item the query belongs to */ + feature: string + /** Expected number of statements (default 1) */ + statements?: number + /** Reviewed expected toSql output */ + toSql: string + /** Reviewed expected formatter output (optional) */ + format?: string + /** Reason to skip the autocomplete walkthrough (optional) */ + skipAutocomplete?: string +} + +const fixturePath = path.join(__dirname, "fixtures", "server-queries.json") +const entries: ServerQuery[] = JSON.parse( + fs.readFileSync(fixturePath, "utf-8"), +) as ServerQuery[] + +const label = (q: ServerQuery) => + `[${q.feature}] ${q.query.substring(0, 70).replace(/\n/g, " ")}` + +const cases = entries.map((q) => [label(q), q] as const) + +describe("server SQL parity", () => { + it("fixture entries are well formed", () => { + expect(entries.length).toBeGreaterThan(0) + for (const q of entries) { + expect(q.source).toMatch(/^questdb(-enterprise)?:.+:\d+$/) + expect(q.feature).not.toBe("") + expect(typeof q.toSql).toBe("string") + } + }) + + describe("1. parses without errors, as the expected number of statements", () => { + it.each(cases)("%s", (_label, q) => { + const result = parseToAst(q.query) + expect(result.errors.map((e) => e.message)).toEqual([]) + expect(result.ast).toHaveLength(q.statements ?? 1) + }) + }) + + describe("2. parse + toSql round trip", () => { + it.each(cases)("%s", (_label, q) => { + const sql = toSql(parseToAst(q.query).ast) + expect(sql).toBe(q.toSql) + const again = parseToAst(sql) + expect(again.errors.map((e) => e.message)).toEqual([]) + expect(again.ast).toHaveLength(q.statements ?? 1) + expect(toSql(again.ast)).toBe(sql) + }) + }) + + describe("3. autocomplete suggests each next token", () => { + it.each(cases.filter(([, q]) => !q.skipAutocomplete))("%s", (_label, q) => { + const schema = buildSchema(extractSchemaFromAst(parseToAst(q.query).ast)) + const result = autocompleteWalkthrough(q.query, schema) + expect(result.success).toBe(true) + const failures = result.failedSteps.map( + (s) => + `offset ${s.position}: "${s.expectedLabel}" (${s.tokenType}) not in [${s.suggestions.join(", ")}]`, + ) + expect(failures).toEqual([]) + }) + }) + + describe("4. formatter preserves the query", () => { + it.each(cases)("%s", (_label, q) => { + assertPreserved(q.query) + if (q.format !== undefined) expect(format(q.query)).toBe(q.format) + }) + }) +}) From f46bcadbfbf87d9ca2e8813d506831f22cad1297 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 13:52:30 +0300 Subject: [PATCH 02/36] feat: parse hex, float-suffix, E'...' and typed literals The server accepts four literal forms that sql-parser misread. Most of them were silent: the query parsed without errors but with a different meaning. 0x7ee6... long256 constant was `0 AS x7ee6...` or split in two 1.5f, 1e36d float/double suffix was `1.5 AS f` or split in two E'abc' PostgreSQL string was `E AS abc` or split in two timestamp '..' typed literal (cast) was a column with an alias, or split Lexer: HexLiteral, FloatLiteral and EscapeStringLiteral tokens. `d` is only a double suffix after an exponent, so `1d` stays a day duration. Parser: the typed literal hangs off identifierExpression as an optional string after a castable type name (the server's processPgTimestampCast list plus nameTypeMap aliases). A separate primaryExpression alternative would be simpler, but autocomplete ignores GATEs, so it broke ARRAY[ suggestions and offered every type keyword wherever a value can start. `timestamp [ns] with time zone '...'` is its own small rule; content assist drops its leading TIMESTAMP tokens so only WITH after `timestamp` is new. Tests: 26 server queries (questdb + questdb-enterprise test suites) in the parity fixture, checked for parse, round trip, autocomplete and formatter, plus tests/literal-forms.test.ts for lexer, AST and autocomplete edge cases. Against 53,975 SQL strings from the server tests and docs: 218 now parse, 25 round trips are corrected, and the only newly rejected input is `cancel query 12.01f`, which the server rejects too. --- src/autocomplete/content-assist.ts | 24 +- src/autocomplete/token-classification.ts | 3 + src/formatter/classification.ts | 3 + src/parser/ast.ts | 2 + src/parser/cst-types.d.ts | 22 ++ src/parser/lexer.ts | 32 ++- src/parser/parser.ts | 108 +++++++++ src/parser/toSql.ts | 7 +- src/parser/visitor.ts | 67 ++++++ tests/fixtures/server-queries.json | 156 +++++++++++++ tests/helpers/autocomplete-walkthrough.ts | 3 + tests/literal-forms.test.ts | 269 ++++++++++++++++++++++ 12 files changed, 691 insertions(+), 5 deletions(-) create mode 100644 tests/literal-forms.test.ts diff --git a/src/autocomplete/content-assist.ts b/src/autocomplete/content-assist.ts index 1de5662..2b9b97e 100644 --- a/src/autocomplete/content-assist.ts +++ b/src/autocomplete/content-assist.ts @@ -973,7 +973,23 @@ function computeSuggestions(tokens: IToken[]): ComputeResult { : "statement" // Budgeted: aborts after a fixed number of path-exploration steps. // The budget keeps autocomplete responsive and falls back to empty suggestions when hit. - const { suggestions } = computeContentAssistBudgeted(ruleName, tokens) + const { suggestions: rawSuggestions } = computeContentAssistBudgeted( + ruleName, + tokens, + ) + // timestampWithTimeZoneLiteral opens with explicit TIMESTAMP / TIMESTAMP_NS + // tokens. Content assist ignores its GATE, so without this filter those two + // keywords would be offered wherever an expression can start. Only the + // rule's first token is dropped; WITH TIME ZONE after `timestamp` stays. + const suggestions = rawSuggestions.filter( + (s) => + !( + s.ruleStack[s.ruleStack.length - 1] === + "timestampWithTimeZoneLiteral" && + (s.nextTokenType.name === "Timestamp" || + s.nextTokenType.name === "TimestampNs") + ), + ) const result = suggestions.map((s) => s.nextTokenType) // Walk every IdentifierKeyword path and union the category flags valid at @@ -1157,7 +1173,11 @@ export function getContentAssist( // problems than it solves (prefix filtering breaks, ambiguity with string values). const fullTokens = QuestDBLexer.tokenize(fullSql).tokens for (const token of fullTokens) { - if (token.tokenType.name !== "StringLiteral") continue + if ( + token.tokenType.name !== "StringLiteral" && + token.tokenType.name !== "EscapeStringLiteral" + ) + continue const start = token.startOffset const end = token.startOffset + token.image.length if (cursorOffset > start && cursorOffset < end) { diff --git a/src/autocomplete/token-classification.ts b/src/autocomplete/token-classification.ts index b5d9558..b14fdd1 100644 --- a/src/autocomplete/token-classification.ts +++ b/src/autocomplete/token-classification.ts @@ -115,6 +115,9 @@ export const SKIP_TOKENS = new Set([ "DurationLiteral", "GeohashLiteral", "GeohashBinaryLiteral", + "HexLiteral", + "FloatLiteral", + "EscapeStringLiteral", "Nan", // Whitespace "WhiteSpace", diff --git a/src/formatter/classification.ts b/src/formatter/classification.ts index 379d242..64c1c4f 100644 --- a/src/formatter/classification.ts +++ b/src/formatter/classification.ts @@ -49,6 +49,9 @@ const literalTokenNames = [ "DurationLiteral", "NumberLiteral", "StringLiteral", + "HexLiteral", + "FloatLiteral", + "EscapeStringLiteral", "QuotedIdentifier", "Identifier", "VariableReference", diff --git a/src/parser/ast.ts b/src/parser/ast.ts index e131baf..5d247dd 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -1270,6 +1270,8 @@ export interface Literal extends AstNode { value: string | number | boolean | null literalType: "string" | "number" | "boolean" | "null" | "geohash" | "duration" raw?: string + /** Typed literal prefix, e.g. "TIMESTAMP" for `timestamp '2020-01-01'` */ + typeName?: string } export interface FunctionCall extends AstNode { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 7f18956..4b2efed 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -2589,6 +2589,7 @@ export interface PrimaryExpressionCstNode extends CstNode { export type PrimaryExpressionCstChildren = { castExpression?: CastExpressionCstNode[]; caseExpression?: CaseExpressionCstNode[]; + timestampWithTimeZoneLiteral?: TimestampWithTimeZoneLiteralCstNode[]; arrayLiteral?: ArrayLiteralCstNode[]; functionCall?: FunctionCallCstNode[]; literal?: LiteralCstNode[]; @@ -2731,6 +2732,21 @@ export type FunctionCallCstChildren = { overClause?: OverClauseCstNode[]; }; +export interface TimestampWithTimeZoneLiteralCstNode extends CstNode { + name: "timestampWithTimeZoneLiteral"; + children: TimestampWithTimeZoneLiteralCstChildren; +} + +export type TimestampWithTimeZoneLiteralCstChildren = { + Timestamp?: IToken[]; + TimestampNs?: IToken[]; + With: IToken[]; + Time: IToken[]; + Zone: IToken[]; + StringLiteral?: IToken[]; + EscapeStringLiteral?: IToken[]; +}; + export interface IdentifierExpressionCstNode extends CstNode { name: "identifierExpression"; children: IdentifierExpressionCstChildren; @@ -2738,6 +2754,8 @@ export interface IdentifierExpressionCstNode extends CstNode { export type IdentifierExpressionCstChildren = { qualifiedName: QualifiedNameCstNode[]; + StringLiteral?: IToken[]; + EscapeStringLiteral?: IToken[]; LParen?: IToken[]; selectStatement?: SelectStatementCstNode[]; Star?: IToken[]; @@ -2868,6 +2886,9 @@ export type LiteralCstChildren = { GeohashLiteral?: IToken[]; GeohashBinaryLiteral?: IToken[]; Nan?: IToken[]; + HexLiteral?: IToken[]; + FloatLiteral?: IToken[]; + EscapeStringLiteral?: IToken[]; }; export interface BooleanLiteralCstNode extends CstNode { @@ -3146,6 +3167,7 @@ export interface ICstNodeVisitor extends ICstVisitor { caseExpression(children: CaseExpressionCstChildren, param?: IN): OUT; functionName(children: FunctionNameCstChildren, param?: IN): OUT; functionCall(children: FunctionCallCstChildren, param?: IN): OUT; + timestampWithTimeZoneLiteral(children: TimestampWithTimeZoneLiteralCstChildren, param?: IN): OUT; identifierExpression(children: IdentifierExpressionCstChildren, param?: IN): OUT; overClause(children: OverClauseCstChildren, param?: IN): OUT; windowClause(children: WindowClauseCstChildren, param?: IN): OUT; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index c7788e3..5392d9f 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -785,6 +785,24 @@ export const GeohashLiteral = createToken({ pattern: /#[a-z0-9/]+/i, }) +// Long256 hex literal: 0x1F, 0x7ee65ec7... (FunctionParser.createConstant -> +// Numbers.extractLong256). Must be before NumberLiteral, which would otherwise +// match the leading "0" and leave "x1F" as an identifier. +export const HexLiteral = createToken({ + name: "HexLiteral", + pattern: /0[xX][0-9a-fA-F]+\b/, +}) + +// Float / double suffix: 1.5f, 10f, 1e36f, 1e36d (FunctionParser.createConstant -> +// Numbers.parseFloat / parseDouble). `f` is not a duration unit, so it never +// collides with DurationLiteral. `d` is only treated as a double suffix after an +// exponent, because `1d` / `1.5d` are durations (days) in SAMPLE BY / TTL. +export const FloatLiteral = createToken({ + name: "FloatLiteral", + pattern: + /(\d[\d_]*\.?[\d_]*|\.\d[\d_]*)(([eE][+-]?\d[\d_]*)?[fF]|[eE][+-]?\d[\d_]*[dD])\b/, +}) + // Long literal suffix: 100000L (must be before DurationLiteral and NumberLiteral) export const LongLiteral = createToken({ name: "LongLiteral", @@ -829,7 +847,7 @@ export const NumberLiteral = createToken({ export const Dot = createToken({ name: "Dot", pattern: /\./, - longer_alt: NumberLiteral, + longer_alt: [FloatLiteral, NumberLiteral], }) // Strings — QuestDB follows PostgreSQL convention: backslash is literal, @@ -839,6 +857,15 @@ export const StringLiteral = createToken({ pattern: /'([^']|'')*'/, }) +// PostgreSQL escape-string constant: E'abc' (ExpressionParser case 'E', +// FunctionParser.createConstant strips the E and the quotes). Uppercase E only, +// glued to the quote, exactly as on the server. Must be before Identifier, +// which would otherwise match the "E" and leave the string as an alias. +export const EscapeStringLiteral = createToken({ + name: "EscapeStringLiteral", + pattern: /E'([^']|'')*'/, +}) + // Quoted Identifiers — same convention: "" escapes a double quote, // backslash is literal. export const QuotedIdentifier = createToken({ @@ -937,11 +964,14 @@ export const allTokens: TokenType[] = [ // Literals (order matters - longer/more specific patterns first) GeohashBinaryLiteral, GeohashLiteral, + HexLiteral, + FloatLiteral, LongLiteral, DecimalLiteral, DurationLiteral, NumberLiteral, StringLiteral, + EscapeStringLiteral, QuotedIdentifier, Identifier, ] diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 65d9557..0aa2160 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -1,6 +1,7 @@ import { CstParser, type TokenType, + type IToken, NoViableAltException, EarlyExitException, tokenMatcher, @@ -336,6 +337,9 @@ import { Include, GeohashBinaryLiteral, GeohashLiteral, + HexLiteral, + FloatLiteral, + EscapeStringLiteral, LongLiteral, DecimalLiteral, Window, @@ -377,6 +381,38 @@ import { IdentifierKeyword, } from "./lexer" +// Types the server accepts in a typed literal ` ''` +// (ExpressionParser.processPgTimestampCast + cannotCastTo): BOOLEAN..LONG256, +// UUID, IPv4, VARCHAR, DECIMAL, plus nameTypeMap aliases. +const TYPED_LITERAL_TYPE_TOKENS: ReadonlySet = new globalThis.Set([ + Boolean, + Byte, + Short, + Char, + Int, + Integer, + Long, + Date, + Timestamp, + TimestampNs, + Float, + Double, + String, + Symbol, + Long256, + Uuid, + Ipv4, + Varchar, + Decimal, +]) +const TYPED_LITERAL_TYPE_ALIASES: ReadonlySet = new globalThis.Set([ + "text", + "smallint", + "bigint", + "real", + "bytea", +]) + class QuestDBParser extends CstParser { constructor(config?: { skipValidations?: boolean }) { super(allTokens, { @@ -4649,10 +4685,29 @@ class QuestDBParser extends CstParser { ]) }) + private isTimestampWithTimeZoneLiteral(): boolean { + const t1 = this.LA(1).tokenType + const t5 = this.LA(5).tokenType + return ( + (t1 === Timestamp || t1 === TimestampNs) && + this.LA(2).tokenType === With && + this.LA(3).tokenType === Time && + this.LA(4).tokenType === Zone && + (t5 === StringLiteral || t5 === EscapeStringLiteral) + ) + } + private primaryExpression = this.RULE("primaryExpression", () => { this.OR([ { ALT: () => this.SUBRULE(this.castExpression) }, { ALT: () => this.SUBRULE(this.caseExpression) }, + { + // timestamp [with time zone] '...' — the WITH TIME ZONE form cannot + // live in identifierExpression's typed-literal option, because then + // autocomplete would offer WITH after every column reference. + GATE: () => this.isTimestampWithTimeZoneLiteral(), + ALT: () => this.SUBRULE(this.timestampWithTimeZoneLiteral), + }, { // ARRAY[...] literal — must come before functionCall/columnRef // since ARRAY is tokenized as Identifier @@ -4886,8 +4941,58 @@ class QuestDBParser extends CstParser { // Parses: qualifiedName [( [args] ) [IGNORE NULLS] [OVER (...)]] // This handles both `schema.func(args)` and `schema.column` in one rule, // deciding based on whether `(` follows the name. + // timestamp with time zone '2020-12-31 15:15:51.663+00:00' + // (ExpressionParser: the literal branch accepts WITH TIME ZONE after a + // timestamp type name, then processPgTimestampCast casts the string) + private timestampWithTimeZoneLiteral = this.RULE( + "timestampWithTimeZoneLiteral", + () => { + this.OR([ + { ALT: () => this.CONSUME(Timestamp) }, + { ALT: () => this.CONSUME(TimestampNs) }, + ]) + this.CONSUME(With) + this.CONSUME(Time) + this.CONSUME(Zone) + this.OR1([ + { ALT: () => this.CONSUME(StringLiteral) }, + { ALT: () => this.CONSUME(EscapeStringLiteral) }, + ]) + }, + ) + + // True when `first` is the only token of the qualifiedName just consumed, is + // a castable type name, and the next token is a string constant. + private isTypedLiteralType(first: IToken): boolean { + const internals = this as unknown as CstParserInternals & { + input: IToken[] + } + if (internals.input[internals.currIdx] !== first) return false + const isType = + TYPED_LITERAL_TYPE_TOKENS.has(first.tokenType) || + (first.tokenType === Identifier && + TYPED_LITERAL_TYPE_ALIASES.has(first.image.toLowerCase())) + const next = this.LA(1).tokenType + return isType && (next === StringLiteral || next === EscapeStringLiteral) + } + private identifierExpression = this.RULE("identifierExpression", () => { + const first = this.LA(1) this.SUBRULE(this.qualifiedName) + // Typed literal: timestamp '2020-01-01', ipv4 '1.2.3.4', long '42' ... + // The server reads a type name directly followed by a string constant as a + // cast (ExpressionParser.processPgTimestampCast). Every castable type is a + // non-reserved keyword, so it already arrives here as a one-part + // qualifiedName; the trailing string turns it into a typed literal instead + // of a column reference followed by an alias. + this.OPTION5({ + GATE: () => this.isTypedLiteralType(first), + DEF: () => + this.OR4([ + { ALT: () => this.CONSUME(StringLiteral) }, + { ALT: () => this.CONSUME(EscapeStringLiteral) }, + ]), + }) this.OPTION(() => { this.CONSUME(LParen) this.OR({ @@ -5126,6 +5231,9 @@ class QuestDBParser extends CstParser { { ALT: () => this.CONSUME(GeohashLiteral) }, { ALT: () => this.CONSUME(GeohashBinaryLiteral) }, { ALT: () => this.CONSUME(NaN) }, + { ALT: () => this.CONSUME(HexLiteral) }, + { ALT: () => this.CONSUME(FloatLiteral) }, + { ALT: () => this.CONSUME(EscapeStringLiteral) }, ]) }) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 83cc8ed..00e6d94 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -1946,8 +1946,11 @@ function unaryExprToSql(expr: AST.UnaryExpression): string { function literalToSql(lit: AST.Literal): string { switch (lit.literalType) { - case "string": - return escapeString(String(lit.value)) + case "string": { + // raw is set for E'...' strings, which must keep their prefix + const str = lit.raw ?? escapeString(String(lit.value)) + return lit.typeName ? `${lit.typeName} ${str}` : str + } case "number": if (lit.raw != null) return lit.raw return String(lit.value) diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 2fa69e4..1ffb0ad 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -110,6 +110,7 @@ import type { LatestOnClauseCstChildren, LimitClauseCstChildren, LiteralCstChildren, + TimestampWithTimeZoneLiteralCstChildren, MaterializedViewPartitionCstChildren, MaterializedViewPeriodCstChildren, MaterializedViewRefreshCstChildren, @@ -4142,6 +4143,9 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.functionCall) { return this.visit(ctx.functionCall) as AST.FunctionCall } + if (ctx.timestampWithTimeZoneLiteral) { + return this.visit(ctx.timestampWithTimeZoneLiteral) as AST.Literal + } if (ctx.literal) { return this.visit(ctx.literal) as AST.Literal } @@ -4359,6 +4363,14 @@ class QuestDBVisitor extends BaseVisitor { ctx.qualifiedName, ) as AST.QualifiedName + // Typed literal: timestamp '2020-01-01' + if (ctx.StringLiteral || ctx.EscapeStringLiteral) { + return { + ...this.typedLiteralString(ctx.StringLiteral, ctx.EscapeStringLiteral), + typeName: qualName.parts[0].toUpperCase(), + } + } + if (ctx.LParen) { // Function call (possibly schema-qualified) const result: AST.FunctionCall = { @@ -4630,9 +4642,64 @@ class QuestDBVisitor extends BaseVisitor { literalType: "number", } } + if (ctx.HexLiteral) { + // Long256 hex constant; keep the text (it does not fit a JS number) + const image = this.tokenImage(ctx.HexLiteral[0]) + return { + type: "literal", + value: image, + literalType: "number", + raw: image, + } + } + if (ctx.FloatLiteral) { + const image = this.tokenImage(ctx.FloatLiteral[0]) + return { + type: "literal", + value: parseFloat(image.replace(/[_fFdD]/g, "")), + literalType: "number", + raw: image, + } + } + if (ctx.EscapeStringLiteral) { + return this.escapeStringLiteral(ctx.EscapeStringLiteral[0]) + } throw new Error("Unknown literal type") } + timestampWithTimeZoneLiteral( + ctx: TimestampWithTimeZoneLiteralCstChildren, + ): AST.Literal { + const type = ctx.TimestampNs ? "TIMESTAMP_NS" : "TIMESTAMP" + return { + ...this.typedLiteralString(ctx.StringLiteral, ctx.EscapeStringLiteral), + typeName: `${type} WITH TIME ZONE`, + } + } + + private typedLiteralString( + stringToken: IToken[] | undefined, + escapeToken: IToken[] | undefined, + ): AST.Literal { + if (escapeToken) return this.escapeStringLiteral(escapeToken[0]) + return { + type: "literal", + value: this.tokenImage(stringToken![0]).slice(1, -1).replace(/''/g, "'"), + literalType: "string", + } + } + + private escapeStringLiteral(token: IToken): AST.Literal { + // E'abc' -> value abc; the server does not process backslashes, so only '' + const raw = this.tokenImage(token) + return { + type: "literal", + value: raw.slice(2, -1).replace(/''/g, "'"), + literalType: "string", + raw, + } + } + booleanLiteral(ctx: BooleanLiteralCstChildren): boolean { return !!ctx.True } diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 2e2f169..5f1efd3 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -22,5 +22,161 @@ "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/acl/AclAuditLogTest.java:1357", "feature": "baseline", "toSql": "GRANT SELECT ON pending_grants TO alice" + }, + { + "query": "insert into tab values (false, cast(24814 as short), 24814, 8260188555232587029, 0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7, 'J', 'ORANGE', '123', '1970-01-01T00:00:01.000000Z')", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/SqlCodeGeneratorTest.java:10357", + "feature": "literal-hex", + "toSql": "INSERT INTO tab VALUES (FALSE, CAST(24814 AS SHORT), 24814, 8260188555232587029, 0x7ee65ec7b6e3bc3a422a8855e9d7bfd29199af5c2aa91ba39c022fa261bdede7, 'J', 'ORANGE', '123', '1970-01-01T00:00:01.000000Z')" + }, + { + "query": "select cast(0x00 as boolean)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/cast/CastTest.java:604", + "feature": "literal-hex", + "toSql": "SELECT CAST(0x00 AS BOOLEAN)" + }, + { + "query": "create table tab as (select 0x11111111 as a, 0x11111111 as b from long_sequence(1))", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/eq/EqLong256FunctionFactoryTest.java:66", + "feature": "literal-hex", + "toSql": "CREATE TABLE tab AS (SELECT 0x11111111 AS a, 0x11111111 AS b FROM long_sequence(1))" + }, + { + "query": "insert into tab values (0xb00ee5505bd95e51dd18889bae1dee3404d446e61d5293f55ff29ba4a01ab073)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/groupby/SumLong256VecGroupByFunctionFactoryTest.java:55", + "feature": "literal-hex", + "toSql": "INSERT INTO tab VALUES (0xb00ee5505bd95e51dd18889bae1dee3404d446e61d5293f55ff29ba4a01ab073)" + }, + { + "query": "INSERT INTO t VALUES (3.14f, 1::TIMESTAMP), (1.0f, 2::TIMESTAMP)", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/qwp/QwpEgressBindRoundTripTest.java:266", + "feature": "literal-float", + "toSql": "INSERT INTO t VALUES (3.14f, 1::TIMESTAMP), (1.0f, 2::TIMESTAMP)" + }, + { + "query": "INSERT INTO t VALUES (1.5f), (-1.5f), (0.0f), (NULL), (CAST(0.0/0.0 AS FLOAT))", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/qwp/QwpEgressTypesExhaustiveTest.java:919", + "feature": "literal-float", + "toSql": "INSERT INTO t VALUES (1.5f), (-1.5f), (0.0f), (NULL), (CAST(0.0 / 0.0 AS FLOAT))" + }, + { + "query": "select rn from z where i > 16777216.0f and rn <= 3", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CompiledFilterRegressionTest.java:5614", + "feature": "literal-float", + "toSql": "SELECT rn FROM z WHERE i > 16777216.0f AND rn <= 3" + }, + { + "query": "SELECT 1e10f * 1e30f * 1e-30f AS v", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/IntArithmeticOverflowFoldingTest.java:111", + "feature": "literal-float", + "toSql": "SELECT 1e10f * 1e30f * 1e-30f AS v" + }, + { + "query": "SELECT CASE WHEN True THEN 1.0f ELSE 1 END", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:13852", + "feature": "literal-float", + "toSql": "SELECT CASE WHEN TRUE THEN 1.0f ELSE 1 END" + }, + { + "query": "select * from x where c in (1.0f, null, 1::byte, 1::short, 1::int, 1::long)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/SqlCompilerImplTest.java:8006", + "feature": "literal-float", + "toSql": "SELECT * FROM x WHERE c IN (1.0f, NULL, 1::BYTE, 1::SHORT, 1::INT, 1::LONG)" + }, + { + "query": "update t1 set f = 5.2f", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/TouchTableFunctionTest.java:71", + "feature": "literal-float", + "toSql": "UPDATE t1 SET f = 5.2f" + }, + { + "query": "with data as (select -1e30d v) select cast(v as decimal(24,2)) from data", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/cast/CastDoubleToDecimalFunctionFactoryTest.java:264", + "feature": "literal-float", + "toSql": "WITH data AS (SELECT -1e30d AS v) SELECT CAST(v AS DECIMAL(24, 2)) FROM data" + }, + { + "query": "select 10f * 0f x", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/math/MulFloatFunctionFactoryTest.java:49", + "feature": "literal-float", + "toSql": "SELECT 10f * 0f AS x" + }, + { + "query": "insert into x values ((2+3*3600L*1000000)::timestamp,2, 8,8,8f,8d,cast(8 as date),8::timestamp,rnd_long256(),'8','8',rnd_geohash(5) ,rnd_geohash(10),rnd_geohash(20),rnd_geohash(40))", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/groupby/CountColumnTest.java:74", + "feature": "literal-float", + "toSql": "INSERT INTO x VALUES ((2 + 3 * 3600L * 1000000)::TIMESTAMP, 2, 8, 8, 8f, 8d, CAST(8 AS DATE), 8::TIMESTAMP, rnd_long256(), '8', '8', rnd_geohash(5), rnd_geohash(10), rnd_geohash(20), rnd_geohash(40))" + }, + { + "query": "SELECT x, count() FROM (SELECT dateadd(E'h', 1, ts) x FROM jan) SAMPLE BY 1d", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/TimestampOffsetPushdownTest.java:1048", + "feature": "literal-escape-string", + "toSql": "SELECT x, count() FROM (SELECT dateadd(E'h', 1, ts) AS x FROM jan) SAMPLE BY 1d" + }, + { + "query": "select ipv4 '2.1.1.1' & '2.2.2.2'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/IPv4Test.java:92", + "feature": "literal-typed", + "toSql": "SELECT IPV4 '2.1.1.1' & '2.2.2.2'" + }, + { + "query": "select null >= ipv4 '34.11.45.3'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/IPv4Test.java:1046", + "feature": "literal-typed", + "toSql": "SELECT NULL >= IPV4 '34.11.45.3'" + }, + { + "query": "select ip & ipv4 '255.255.255.255' from test", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/IPv4Test.java:1255", + "feature": "literal-typed", + "toSql": "SELECT ip & IPV4 '255.255.255.255' FROM test" + }, + { + "query": "select ipv4 '0.0.0.0' - ipv4 '1.1.1.1'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/IPv4Test.java:2705", + "feature": "literal-typed", + "toSql": "SELECT IPV4 '0.0.0.0' - IPV4 '1.1.1.1'" + }, + { + "query": "select (uuid '11111111-1111-1111-1111-111111111111') = cast('11111111-1111-1111-1111-111111111111' as uuid) from long_sequence(1)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/UuidTest.java:1067", + "feature": "literal-typed", + "toSql": "SELECT (UUID '11111111-1111-1111-1111-111111111111') = CAST('11111111-1111-1111-1111-111111111111' AS UUID) FROM long_sequence(1)" + }, + { + "query": "select typeOf(uuid '11111111-1111-1111-1111-111111111111') from long_sequence(1)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/UuidTest.java:1210", + "feature": "literal-typed", + "toSql": "SELECT typeOf(UUID '11111111-1111-1111-1111-111111111111') FROM long_sequence(1)" + }, + { + "query": "SELECT DATE_TRUNC('hour', TIMESTAMP '2017-03-17T02:09:30.111111Z') as truncated", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/date/DateTruncFunctionFactoryTest.java:63", + "feature": "literal-typed", + "toSql": "SELECT DATE_TRUNC('hour', TIMESTAMP '2017-03-17T02:09:30.111111Z') AS truncated" + }, + { + "query": "SELECT DATE_TRUNC('day', TIMESTAMP_NS '2017-03-17T02:09:30.111111111Z') as truncated", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/date/DateTruncFunctionFactoryTest.java:120", + "feature": "literal-typed", + "toSql": "SELECT DATE_TRUNC('day', TIMESTAMP_NS '2017-03-17T02:09:30.111111111Z') AS truncated" + }, + { + "query": "insert into t values (timestamp with time zone '2020-12-31 15:15:51.663+00:00')", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/InsertTest.java:1789", + "feature": "literal-typed", + "toSql": "INSERT INTO t VALUES (TIMESTAMP WITH TIME ZONE '2020-12-31 15:15:51.663+00:00')" + }, + { + "query": "insert into test (test_timestamp, test_value) values (timestamp with time zone '2020-12-31 15:15:51.663+00:00', '256')", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/AbstractSqlParserTest.java:160", + "feature": "literal-typed", + "toSql": "INSERT INTO test (test_timestamp, test_value) VALUES (TIMESTAMP WITH TIME ZONE '2020-12-31 15:15:51.663+00:00', '256')" + }, + { + "query": "SELECT * FROM (\nSELECT \n n.nspname\n ,c.relname\n ,a.attname\n ,a.atttypid\n ,a.attnotnull OR (t.typtype = 'd' AND t.typnotnull) AS attnotnull\n ,a.atttypmod\n ,a.attlen\n ,t.typtypmod\n ,row_number() OVER (PARTITION BY a.attrelid ORDER BY a.attnum) AS attnum\n , nullif(a.attidentity, '') as attidentity\n ,null as attgenerated\n ,pg_catalog.pg_get_expr(def.adbin, def.adrelid) AS adsrc\n ,dsc.description\n ,t.typbasetype\n ,t.typtype \nFROM pg_catalog.pg_namespace n\nJOIN pg_catalog.pg_class c ON (c.relnamespace = n.oid)\nJOIN pg_catalog.pg_attribute a ON (a.attrelid=c.oid)\nJOIN pg_catalog.pg_type t ON (a.atttypid = t.oid)\nLEFT JOIN pg_catalog.pg_attrdef def ON (a.attrelid=def.adrelid AND a.attnum = def.adnum)\nLEFT JOIN pg_catalog.pg_description dsc ON (c.oid=dsc.objoid AND a.attnum = dsc.objsubid)\nLEFT JOIN pg_catalog.pg_class dc ON (dc.oid=dsc.classoid AND dc.relname='pg_class')\nLEFT JOIN pg_catalog.pg_namespace dn ON (dc.relnamespace=dn.oid AND dn.nspname='pg_catalog')\nWHERE \n c.relkind in ('r','p','v','f','m')\n and a.attnum > 0 \n AND NOT a.attisdropped\n AND c.relname LIKE E'test'\n) c WHERE true\nORDER BY nspname,c.relname,attnum;", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGJobContextTest.java:8217", + "feature": "literal-escape-string", + "toSql": "SELECT * FROM (SELECT n.nspname, c.relname, a.attname, a.atttypid, a.attnotnull OR (t.typtype = 'd' AND t.typnotnull) AS attnotnull, a.atttypmod, a.attlen, t.typtypmod, row_number() OVER (PARTITION BY a.attrelid ORDER BY a.attnum) AS attnum, nullif(a.attidentity, '') AS attidentity, NULL AS attgenerated, pg_catalog.pg_get_expr(def.adbin, def.adrelid) AS adsrc, dsc.description, t.typbasetype, t.typtype FROM pg_catalog.pg_namespace AS n JOIN pg_catalog.pg_class AS c ON (c.relnamespace = n.oid) JOIN pg_catalog.pg_attribute AS a ON (a.attrelid = c.oid) JOIN pg_catalog.pg_type AS t ON (a.atttypid = t.oid) LEFT JOIN pg_catalog.pg_attrdef AS def ON (a.attrelid = def.adrelid AND a.attnum = def.adnum) LEFT JOIN pg_catalog.pg_description AS dsc ON (c.oid = dsc.objoid AND a.attnum = dsc.objsubid) LEFT JOIN pg_catalog.pg_class AS dc ON (dc.oid = dsc.classoid AND dc.relname = 'pg_class') LEFT JOIN pg_catalog.pg_namespace AS dn ON (dc.relnamespace = dn.oid AND dn.nspname = 'pg_catalog') WHERE c.relkind IN ('r', 'p', 'v', 'f', 'm') AND a.attnum > 0 AND NOT a.attisdropped AND c.relname LIKE E'test') AS c WHERE TRUE ORDER BY nspname, c.relname, attnum" } ] diff --git a/tests/helpers/autocomplete-walkthrough.ts b/tests/helpers/autocomplete-walkthrough.ts index 6605f02..29b7ddd 100644 --- a/tests/helpers/autocomplete-walkthrough.ts +++ b/tests/helpers/autocomplete-walkthrough.ts @@ -206,6 +206,9 @@ export const SKIP_TOKEN_TYPES = new Set([ "GeohashBinaryLiteral", "DecimalLiteral", "LongLiteral", + "HexLiteral", + "FloatLiteral", + "EscapeStringLiteral", // Punctuation "LParen", "RParen", diff --git a/tests/literal-forms.test.ts b/tests/literal-forms.test.ts new file mode 100644 index 0000000..c594c57 --- /dev/null +++ b/tests/literal-forms.test.ts @@ -0,0 +1,269 @@ +/** + * Literal forms the QuestDB server accepts (see reports §7): + * - long256 hex constants 0x1F FunctionParser.createConstant + * - float / double suffixes 1.5f, 10f, 1e36d FunctionParser.createConstant + * - PostgreSQL escape strings E'abc' ExpressionParser case 'E' + * - typed literals timestamp '...' ExpressionParser.processPgTimestampCast + * + * Server queries for these live in tests/fixtures/server-queries.json and are + * checked by tests/server-queries.test.ts (parse, round trip, autocomplete, + * formatter). This file pins the lexer, AST and edge-case behaviour. + */ + +import { describe, it, expect } from "vitest" +import { + parseToAst, + tokenize, + getNextValidTokens, + createAutocompleteProvider, +} from "../src/index" +import { toSql } from "../src/parser/toSql" +import { format } from "../src/formatter/index" +import type * as AST from "../src/parser/ast" + +const tokenTypes = (sql: string) => + tokenize(sql).tokens.map((t) => `${t.tokenType.name}:${t.image}`) + +/** First select item of a single SELECT statement */ +const firstItem = (sql: string): AST.Expression => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + const select = result.ast[0] as AST.SelectStatement + const item = select.columns[0] + return (item as { expression: AST.Expression }).expression +} + +const roundTrip = (sql: string) => toSql(parseToAst(sql).ast) + +describe("lexer", () => { + it("reads a hex constant as one token", () => { + expect(tokenTypes("0x1F")).toEqual(["HexLiteral:0x1F"]) + expect(tokenTypes("0x7ee65ec7b6e3bc3a")).toEqual([ + "HexLiteral:0x7ee65ec7b6e3bc3a", + ]) + }) + + it("reads float and double suffixes as one token", () => { + expect(tokenTypes("1.5f")).toEqual(["FloatLiteral:1.5f"]) + expect(tokenTypes("10f")).toEqual(["FloatLiteral:10f"]) + expect(tokenTypes("1e-30f")).toEqual(["FloatLiteral:1e-30f"]) + expect(tokenTypes("-1e30d")).toEqual(["Minus:-", "FloatLiteral:1e30d"]) + expect(tokenTypes(".5f")).toEqual(["FloatLiteral:.5f"]) + }) + + it("keeps day durations: `d` is only a double suffix after an exponent", () => { + expect(tokenTypes("1d")).toEqual(["DurationLiteral:1d"]) + expect(tokenTypes("1.5d")).toEqual(["DurationLiteral:1.5d"]) + expect(tokenTypes("SAMPLE BY 1d")).toContain("DurationLiteral:1d") + }) + + it("reads E'...' as one escape-string token, uppercase E only", () => { + expect(tokenTypes("E'abc'")).toEqual(["EscapeStringLiteral:E'abc'"]) + expect(tokenTypes("E'it''s'")).toEqual(["EscapeStringLiteral:E'it''s'"]) + // the server only treats an uppercase E as the escape prefix + expect(tokenTypes("e'abc'")).toEqual([ + "Identifier:e", + "StringLiteral:'abc'", + ]) + // an identifier ending in E is not an escape string + expect(tokenTypes("nameE'x'")).toEqual([ + "Identifier:nameE", + "StringLiteral:'x'", + ]) + }) +}) + +describe("AST", () => { + it("hex constant is a number literal that keeps its text", () => { + expect(firstItem("SELECT 0x1F")).toEqual({ + type: "literal", + value: "0x1F", + literalType: "number", + raw: "0x1F", + }) + }) + + it("float suffix is a number literal that keeps its text", () => { + expect(firstItem("SELECT 1.5f")).toEqual({ + type: "literal", + value: 1.5, + literalType: "number", + raw: "1.5f", + }) + }) + + it("E'...' is a string literal; only '' is unescaped", () => { + expect(firstItem("SELECT E'it''s \\n'")).toEqual({ + type: "literal", + value: "it's \\n", + literalType: "string", + raw: "E'it''s \\n'", + }) + }) + + it("typed literal is a string literal with its type name", () => { + expect(firstItem("SELECT timestamp '2020-01-01'")).toEqual({ + type: "literal", + value: "2020-01-01", + literalType: "string", + typeName: "TIMESTAMP", + }) + expect(firstItem("SELECT ipv4 '1.2.3.4'")).toMatchObject({ + typeName: "IPV4", + }) + expect(firstItem("SELECT bigint '42'")).toMatchObject({ + typeName: "BIGINT", + }) + expect( + firstItem("SELECT timestamp with time zone '2020-12-31 15:15:51+00:00'"), + ).toMatchObject({ + value: "2020-12-31 15:15:51+00:00", + typeName: "TIMESTAMP WITH TIME ZONE", + }) + }) +}) + +describe("typed literals only apply to castable type names", () => { + it.each([ + "boolean", + "byte", + "short", + "char", + "int", + "integer", + "long", + "date", + "timestamp", + "timestamp_ns", + "float", + "double", + "string", + "symbol", + "long256", + "uuid", + "ipv4", + "varchar", + "decimal", + "text", + "smallint", + "bigint", + "real", + "bytea", + ])("%s 'x' is a typed literal", (type) => { + const sql = `SELECT ${type} 'x' FROM t` + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + expect(roundTrip(sql)).toBe(`SELECT ${type.toUpperCase()} 'x' FROM t`) + }) + + it("a non-type name followed by a string stays a column with an alias", () => { + expect(roundTrip("SELECT price 'p' FROM t")).toBe( + "SELECT price AS p FROM t", + ) + }) + + it("type names are still plain columns when no string follows", () => { + expect(roundTrip("SELECT timestamp, date, symbol FROM t")).toBe( + "SELECT timestamp, date, symbol FROM t", + ) + expect(roundTrip("SELECT * FROM t WHERE date = '2020-01-01'")).toBe( + "SELECT * FROM t WHERE date = '2020-01-01'", + ) + expect(roundTrip("SELECT ts timestamp FROM t")).toBe( + "SELECT ts AS timestamp FROM t", + ) + }) + + it("a type name used as a function is still a function call", () => { + expect(roundTrip("SELECT timestamp(ts) FROM t")).toBe( + "SELECT timestamp(ts) FROM t", + ) + }) +}) + +describe("no silent splits", () => { + it.each([ + ["SELECT * FROM t WHERE l = 0x01", "SELECT * FROM t WHERE l = 0x01"], + ["SELECT 1.5f FROM t", "SELECT 1.5f FROM t"], + ["UPDATE t1 SET f = 5.2f", "UPDATE t1 SET f = 5.2f"], + [ + "SELECT * FROM t WHERE relname LIKE E'x'", + "SELECT * FROM t WHERE relname LIKE E'x'", + ], + [ + "SELECT * FROM t WHERE ts > timestamp '2020-01-01'", + "SELECT * FROM t WHERE ts > TIMESTAMP '2020-01-01'", + ], + ])("%s", (sql, expected) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + expect(toSql(result.ast)).toBe(expected) + }) +}) + +describe("autocomplete", () => { + const provider = createAutocompleteProvider({ + tables: [{ name: "t" }], + columns: { t: [{ name: "ts", type: "TIMESTAMP" }] }, + }) + + it("does not suggest anything inside an E'...' string", () => { + const sql = "SELECT * FROM t WHERE ts = E'ab" + expect(provider.getSuggestions(sql + "c'", sql.length)).toEqual([]) + }) + + it("offers WITH after `timestamp` (timestamp with time zone '...')", () => { + expect(getNextValidTokens("INSERT INTO t VALUES (timestamp ")).toContain( + "With", + ) + }) + + it("does not offer WITH after an ordinary column in an expression", () => { + expect(getNextValidTokens("SELECT * FROM t WHERE ts > x ")).not.toContain( + "With", + ) + }) + + it.each([ + "SELECT ", + "SELECT * FROM t WHERE ", + "SELECT * FROM t WHERE ts > ", + "INSERT INTO t VALUES (", + "UPDATE t SET ts = ", + ])("does not add type keywords where a value can start: %s", (sql) => { + const labels = provider.getSuggestions(sql, sql.length).map((s) => s.label) + expect(labels).not.toContain("TIMESTAMP") + expect(labels).not.toContain("TIMESTAMP_NS") + }) + + it("continues after a typed literal", () => { + const sql = "SELECT * FROM t WHERE ts > timestamp '2020-01-01' " + const labels = provider.getSuggestions(sql, sql.length).map((s) => s.label) + expect(labels).toContain("AND") + }) +}) + +describe("formatter keeps literals intact", () => { + it("formats a query with typed, hex and float literals", () => { + expect( + format( + "select * from test where ip = ipv4 '0.0.0.1' and l = 0x01 and f > 1.5f", + ), + ).toBe( + "select *\nfrom test\nwhere\n ip = ipv4 '0.0.0.1'\n and l = 0x01\n and f > 1.5f", + ) + }) + + it("formats a query with an escape string", () => { + expect( + format( + "SELECT x, count() FROM (SELECT dateadd(E'h', 1, ts) x FROM jan) SAMPLE BY 1d", + ), + ).toBe( + "SELECT x, count()\nFROM (\n SELECT dateadd(E'h', 1, ts) x\n FROM jan\n)\nSAMPLE BY 1d", + ) + }) +}) From a964c1cd283209199673e3249aa870cebab05f22 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 14:11:27 +0300 Subject: [PATCH 03/36] feat: parse GROUP BY / SAMPLE BY ROLLUP, CUBE and GROUPING SETS Fixes a regression on master and adds the grouping-sets syntax of questdb/questdb#6853. The server (SqlParser.parseGroupByColumns, parseExplicitGroupingSets, hasGroupingSyntaxLookahead and the SAMPLE BY branch) accepts: GROUP BY ROLLUP(a, ...) | CUBE(a, ...) GROUP BY GROUPING SETS ((a, b), (a), ()) every set parenthesised GROUP BY expr, ..., ROLLUP(...) | CUBE(...) ROLLUP/CUBE last SAMPLE BY ROLLUP(...) | CUBE(...) | GROUPING SETS (...) [FROM ... TO ...] [FILL(...)] [ALIGN TO ...] Nothing may follow ROLLUP/CUBE/GROUPING SETS, GROUPING SETS must be the only GROUP BY item, and keys must be column references (ensureColumnReferences). cube, rollup, grouping and sets are not reserved: they are grouping syntax only when `(` (ROLLUP/CUBE) or SETS (GROUPING) follows. Before: #11 added the four words as reserved keywords with no grammar, so `SELECT rollup, cube, sets, grouping FROM t`, `CREATE TABLE t (sets INT, cube INT)` and `SELECT grouping(a) FROM t GROUP BY a` failed (they worked in 0.1.21), and every ROLLUP/CUBE/GROUPING SETS query failed. Now: - tokens.ts: Cube, Grouping, Rollup, Sets are non-reserved (IDENTIFIER_KEYWORD_NAMES), which alone fixes the regression. - parser: groupByClause takes groupingSetsSpec when LA(1..2) is ROLLUP( / CUBE( / GROUPING SETS, otherwise plain expressions with an optional trailing `, ROLLUP(...)|CUBE(...)`. The LA gates keep `GROUP BY rollup`, `GROUP BY a, rollup` and `GROUP BY grouping(x)` as plain keys, and stop `GROUP BY a, grouping sets (...)` (rejected by the server) from being read as `grouping` plus a second statement. sampleByClause takes an optional groupingSetsSpec right after the interval. - AST: new GroupingSets node {kind: rollup|cube|groupingSets, columns | sets}; SelectStatement.groupingSets holds the GROUP BY part (plain keys stay in groupBy), SampleByClause.groupingSets the SAMPLE BY part. Existing fields are unchanged. - toSql prints ROLLUP(...), CUBE(...), GROUPING SETS ((...), ()). - Autocomplete offers ROLLUP/CUBE/GROUPING after GROUP BY and after the SAMPLE BY interval, ROLLUP/CUBE after a plain GROUP BY key, SETS after GROUPING, and columns inside the parentheses. Not done: the server also runs parseGroupByColumns for a GROUP BY after `FROM t PIVOT (...)`; sql-parser does not accept any GROUP BY after a PIVOT clause (clause order), so that is left for a PIVOT change. The GROUP BY inside PIVOT (...) only takes plain keys on the server too. Tests: 28 server queries (GroupingSetsTest, LateralJoinTest, SampleByFillTest) in the parity fixture, plus tests/grouping-sets.test.ts (identifiers, AST, round trip, rejected forms, autocomplete). Red before: 128 failing (84 fixture checks + 44 focused); after: all 303 pass, full suite 15491 passed. Corpus (53,975 server/docs SQL): 226 fixed, 0 round trips changed, 0 regressed, 0 throws, formatter meaning unchanged. --- src/parser/ast.ts | 18 ++ src/parser/cst-types.d.ts | 59 +++++- src/parser/lexer.ts | 8 + src/parser/parser.ts | 104 +++++++++- src/parser/toSql.ts | 23 ++- src/parser/tokens.ts | 13 ++ src/parser/visitor.ts | 72 ++++++- tests/fixtures/server-queries.json | 168 ++++++++++++++++ tests/grouping-sets.test.ts | 307 +++++++++++++++++++++++++++++ 9 files changed, 762 insertions(+), 10 deletions(-) create mode 100644 tests/grouping-sets.test.ts diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 5d247dd..ca0cbd3 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -74,7 +74,10 @@ export interface SelectStatement extends AstNode { where?: Expression sampleBy?: SampleByClause latestOn?: LatestOnClause + /** Plain GROUP BY keys (the prefix of a composite `GROUP BY a, ROLLUP(b)`) */ groupBy?: Expression[] + /** GROUP BY ROLLUP(...) / CUBE(...) / GROUPING SETS (...), after any plain keys */ + groupingSets?: GroupingSets pivot?: PivotClause /** Named window definitions: SELECT ... WINDOW w AS (...) [, w2 AS (...)] */ namedWindows?: NamedWindow[] @@ -1172,6 +1175,21 @@ export interface SampleByClause extends AstNode { alignTo?: AlignToClause from?: Expression to?: Expression + /** SAMPLE BY 1h ROLLUP(...) / CUBE(...) / GROUPING SETS (...) */ + groupingSets?: GroupingSets +} + +/** + * ROLLUP(a, b), CUBE(a, b) or GROUPING SETS ((a, b), (a), ()) in GROUP BY or + * after the SAMPLE BY interval. Keys are column references. + */ +export interface GroupingSets extends AstNode { + type: "groupingSets" + kind: "rollup" | "cube" | "groupingSets" + /** ROLLUP / CUBE keys */ + columns?: ColumnRef[] + /** GROUPING SETS: one entry per set, `[]` is the empty set `()` */ + sets?: ColumnRef[][] } export interface AlignToClause extends AstNode { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 4b2efed..dba0549 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -493,6 +493,7 @@ export type SampleByClauseCstChildren = { DurationLiteral?: IToken[]; VariableReference?: IToken[]; Identifier?: IToken[]; + groupingSetsSpec?: GroupingSetsSpecCstNode[]; fromToClause?: FromToClauseCstNode[]; fillClause?: FillClauseCstNode[]; alignToClause?: AlignToClauseCstNode[]; @@ -590,8 +591,60 @@ export interface GroupByClauseCstNode extends CstNode { export type GroupByClauseCstChildren = { Group: IToken[]; By: IToken[]; - expression: (ExpressionCstNode)[]; + groupingSetsSpec?: GroupingSetsSpecCstNode[]; + expression?: (ExpressionCstNode)[]; + Comma?: (IToken)[]; + rollupOrCubeSpec?: RollupOrCubeSpecCstNode[]; +}; + +export interface GroupingSetsSpecCstNode extends CstNode { + name: "groupingSetsSpec"; + children: GroupingSetsSpecCstChildren; +} + +export type GroupingSetsSpecCstChildren = { + rollupOrCubeSpec?: RollupOrCubeSpecCstNode[]; + explicitGroupingSets?: ExplicitGroupingSetsCstNode[]; +}; + +export interface RollupOrCubeSpecCstNode extends CstNode { + name: "rollupOrCubeSpec"; + children: RollupOrCubeSpecCstChildren; +} + +export type RollupOrCubeSpecCstChildren = { + Rollup?: IToken[]; + Cube?: IToken[]; + LParen: IToken[]; + columnRef: (ColumnRefCstNode)[]; + Comma?: IToken[]; + RParen: IToken[]; +}; + +export interface ExplicitGroupingSetsCstNode extends CstNode { + name: "explicitGroupingSets"; + children: ExplicitGroupingSetsCstChildren; +} + +export type ExplicitGroupingSetsCstChildren = { + Grouping: IToken[]; + Sets: IToken[]; + LParen: IToken[]; + groupingSet: (GroupingSetCstNode)[]; + Comma?: IToken[]; + RParen: IToken[]; +}; + +export interface GroupingSetCstNode extends CstNode { + name: "groupingSet"; + children: GroupingSetCstChildren; +} + +export type GroupingSetCstChildren = { + LParen: IToken[]; + columnRef?: (ColumnRefCstNode)[]; Comma?: IToken[]; + RParen: IToken[]; }; export interface OrderByClauseCstNode extends CstNode { @@ -3036,6 +3089,10 @@ export interface ICstNodeVisitor extends ICstVisitor { alignToClause(children: AlignToClauseCstChildren, param?: IN): OUT; fromToClause(children: FromToClauseCstChildren, param?: IN): OUT; groupByClause(children: GroupByClauseCstChildren, param?: IN): OUT; + groupingSetsSpec(children: GroupingSetsSpecCstChildren, param?: IN): OUT; + rollupOrCubeSpec(children: RollupOrCubeSpecCstChildren, param?: IN): OUT; + explicitGroupingSets(children: ExplicitGroupingSetsCstChildren, param?: IN): OUT; + groupingSet(children: GroupingSetCstChildren, param?: IN): OUT; orderByClause(children: OrderByClauseCstChildren, param?: IN): OUT; orderByItem(children: OrderByItemCstChildren, param?: IN): OUT; limitClause(children: LimitClauseCstChildren, param?: IN): OUT; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index 5392d9f..8364869 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -304,6 +304,10 @@ import { Lzo, Storage, Subsample, + Cube, + Grouping, + Rollup, + Sets, Policy, Local, Remote, @@ -638,6 +642,10 @@ export { Lzo, Storage, Subsample, + Cube, + Grouping, + Rollup, + Sets, Policy, Local, Remote, diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 0aa2160..8edd75b 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -375,6 +375,10 @@ import { // Storage policy tokens Storage, Subsample, + Cube, + Grouping, + Rollup, + Sets, Policy, Local, Remote, @@ -512,6 +516,23 @@ class QuestDBParser extends CstParser { ) } + // ROLLUP( / CUBE( at LA(offset): SqlParser.hasGroupingSyntaxLookahead. + private isRollupOrCubeStart(offset: number): boolean { + const t = this.LA(offset).tokenType + return ( + (t === Rollup || t === Cube) && this.LA(offset + 1).tokenType === LParen + ) + } + + // ROLLUP( / CUBE( / GROUPING SETS at LA(offset). + private isGroupingSetsStart(offset: number): boolean { + return ( + this.isRollupOrCubeStart(offset) || + (this.LA(offset).tokenType === Grouping && + this.LA(offset + 1).tokenType === Sets) + ) + } + // QuestDB accepts these case-sensitively. See SqlParser.isValidSampleByPeriodLetter. private isSampleByPeriodLetter(image: string): boolean { return image.length === 1 && "nUTsmhdwMy".includes(image) @@ -1291,6 +1312,9 @@ class QuestDBParser extends CstParser { ALT: () => this.CONSUME(Identifier), }, ]) + // SAMPLE BY 1h ROLLUP(a) | CUBE(a) | GROUPING SETS ((a), ()): right after + // the interval, without a plain-key prefix (questdb#6853). + this.OPTION3(() => this.SUBRULE(this.groupingSetsSpec)) // Java order: FROM/TO → FILL → ALIGN TO this.OPTION(() => this.SUBRULE(this.fromToClause)) this.OPTION1(() => this.SUBRULE(this.fillClause)) @@ -1433,14 +1457,90 @@ class QuestDBParser extends CstParser { // GROUP BY, HAVING, ORDER BY, LIMIT // ========================================================================== + // GROUP BY keys (SqlParser.parseGroupByColumns, questdb#6853): + // GROUP BY ROLLUP(a, ...) | CUBE(a, ...) | GROUPING SETS ((a, b), (a), ()) + // GROUP BY expr, ... [, ROLLUP(...) | CUBE(...)] + // ROLLUP/CUBE must be the last item and GROUPING SETS the only one. The + // words are not reserved: they are grouping syntax only when `(` (ROLLUP, + // CUBE) or SETS (GROUPING) follows, otherwise plain keys (`GROUP BY rollup`, + // `GROUP BY grouping(x)`), hence the LA(2) gates. private groupByClause = this.RULE("groupByClause", () => { this.CONSUME(Group) this.CONSUME(By) - this.SUBRULE(this.expression) + this.OR([ + { + GATE: () => this.isGroupingSetsStart(1), + ALT: () => this.SUBRULE(this.groupingSetsSpec), + }, + { + ALT: () => { + this.SUBRULE(this.expression) + // Stop before `, ROLLUP(` / `, CUBE(` (taken below) and before + // `, GROUPING SETS`, which the server rejects after plain keys. + this.MANY({ + GATE: () => !this.isGroupingSetsStart(2), + DEF: () => { + this.CONSUME(Comma) + this.SUBRULE1(this.expression) + }, + }) + this.OPTION(() => { + this.CONSUME1(Comma) + this.SUBRULE(this.rollupOrCubeSpec) + }) + }, + }, + ]) + }) + + // ROLLUP(...) | CUBE(...) | GROUPING SETS (...) + private groupingSetsSpec = this.RULE("groupingSetsSpec", () => { + this.OR([ + { ALT: () => this.SUBRULE(this.rollupOrCubeSpec) }, + { ALT: () => this.SUBRULE(this.explicitGroupingSets) }, + ]) + }) + + // ROLLUP(col, ...) | CUBE(col, ...): at least one key, column references + // only (SqlParser.parseParenthesizedColumnList + ensureColumnReferences). + private rollupOrCubeSpec = this.RULE("rollupOrCubeSpec", () => { + this.OR([ + { ALT: () => this.CONSUME(Rollup) }, + { ALT: () => this.CONSUME(Cube) }, + ]) + this.CONSUME(LParen) + this.SUBRULE(this.columnRef) this.MANY(() => { this.CONSUME(Comma) - this.SUBRULE1(this.expression) + this.SUBRULE1(this.columnRef) }) + this.CONSUME(RParen) + }) + + // GROUPING SETS ((a, b), (a), ()): every set is parenthesised, `()` is the + // empty set (SqlParser.parseExplicitGroupingSets). + private explicitGroupingSets = this.RULE("explicitGroupingSets", () => { + this.CONSUME(Grouping) + this.CONSUME(Sets) + this.CONSUME(LParen) + this.SUBRULE(this.groupingSet) + this.MANY(() => { + this.CONSUME(Comma) + this.SUBRULE1(this.groupingSet) + }) + this.CONSUME(RParen) + }) + + private groupingSet = this.RULE("groupingSet", () => { + this.CONSUME(LParen) + this.OPTION(() => { + this.SUBRULE(this.columnRef) + this.MANY(() => { + this.CONSUME(Comma) + this.SUBRULE1(this.columnRef) + }) + }) + this.CONSUME(RParen) }) private orderByClause = this.RULE("orderByClause", () => { diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 00e6d94..82cae89 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -201,6 +201,9 @@ function selectToSql(stmt: AST.SelectStatement): string { if (stmt.sampleBy) { parts.push("SAMPLE BY") parts.push(stmt.sampleBy.duration) + if (stmt.sampleBy.groupingSets) { + parts.push(groupingSetsToSql(stmt.sampleBy.groupingSets)) + } // FROM/TO (must come before FILL and ALIGN TO per Java order) if (stmt.sampleBy.from) { parts.push("FROM") @@ -246,10 +249,14 @@ function selectToSql(stmt: AST.SelectStatement): string { } } - // GROUP BY - if (stmt.groupBy && stmt.groupBy.length > 0) { + // GROUP BY [plain keys][, ROLLUP(...) | CUBE(...) | GROUPING SETS (...)] + const groupByItems = (stmt.groupBy ?? []).map(expressionToSql) + if (stmt.groupingSets) { + groupByItems.push(groupingSetsToSql(stmt.groupingSets)) + } + if (groupByItems.length > 0) { parts.push("GROUP BY") - parts.push(stmt.groupBy.map(expressionToSql).join(", ")) + parts.push(groupByItems.join(", ")) } // PIVOT (inline pivot clause in SELECT) @@ -2193,6 +2200,16 @@ function qualifiedNameToSql(name: AST.QualifiedName): string { return name.parts.map(escapeIdentifier).join(".") } +function groupingSetsToSql(spec: AST.GroupingSets): string { + const keys = (cols: AST.ColumnRef[] | undefined) => + (cols ?? []).map(expressionToSql).join(", ") + if (spec.kind === "groupingSets") { + const sets = (spec.sets ?? []).map((set) => `(${keys(set)})`) + return `GROUPING SETS (${sets.join(", ")})` + } + return `${spec.kind === "cube" ? "CUBE" : "ROLLUP"}(${keys(spec.columns)})` +} + function escapeString(value: string): string { return `'${value.replace(/'/g, "''")}'` } diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index 604ff21..6e4ed8b 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -416,6 +416,13 @@ export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ "ServiceAccounts", // CREATE LIVE VIEW ... START FROM NOW (same word as the now() function) "Now", + // GROUP BY / SAMPLE BY grouping sets (questdb#6853). Not reserved on the + // server: grouping syntax only when `(` (ROLLUP/CUBE) or SETS (GROUPING) + // follows, so they stay usable as column/table names and grouping(x). + "Cube", + "Grouping", + "Rollup", + "Sets", ]) for (const name of IDENTIFIER_KEYWORD_NAMES) { @@ -715,6 +722,12 @@ export const Policy = getToken("Policy") export const Local = getToken("Local") export const Remote = getToken("Remote") +// GROUP BY / SAMPLE BY grouping sets +export const Cube = getToken("Cube") +export const Grouping = getToken("Grouping") +export const Rollup = getToken("Rollup") +export const Sets = getToken("Sets") + // Data types (from grammar/dataTypes.ts) export const Binary = getToken("Binary") export const Boolean = getToken("Boolean") diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 1ffb0ad..b9e4e0b 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -97,6 +97,10 @@ import type { GrantStatementCstChildren, GrantTableTargetCstChildren, GroupByClauseCstChildren, + GroupingSetCstChildren, + GroupingSetsSpecCstChildren, + ExplicitGroupingSetsCstChildren, + RollupOrCubeSpecCstChildren, IdentifierCstChildren, IdentifierCstNode, IdentifierExpressionCstChildren, @@ -213,6 +217,10 @@ type ConvertPartitionTargetResult = { where?: AST.Expression withParams?: AST.TableParam[] } +type GroupByClauseResult = { + groupBy?: AST.Expression[] + groupingSets?: AST.GroupingSets +} type PivotBodyResult = { aggregations: AST.PivotAggregation[] pivots: AST.PivotForClause[] @@ -477,7 +485,9 @@ class QuestDBVisitor extends BaseVisitor { } if (ctx.groupByClause) { - result.groupBy = this.visit(ctx.groupByClause) as AST.Expression[] + const groupBy = this.visit(ctx.groupByClause) as GroupByClauseResult + if (groupBy.groupBy) result.groupBy = groupBy.groupBy + if (groupBy.groupingSets) result.groupingSets = groupBy.groupingSets } if (ctx.pivotBody) { @@ -702,7 +712,11 @@ class QuestDBVisitor extends BaseVisitor { result.latestOn = this.visitSafe(ctx.latestOnClause) as AST.LatestOnClause } if (ctx.groupByClause) { - result.groupBy = this.visitSafe(ctx.groupByClause) as AST.Expression[] + const groupBy = this.visitSafe(ctx.groupByClause) as + | GroupByClauseResult + | undefined + if (groupBy?.groupBy) result.groupBy = groupBy.groupBy + if (groupBy?.groupingSets) result.groupingSets = groupBy.groupingSets } if (ctx.subsampleClause) { result.subsample = this.visitSafe( @@ -1000,6 +1014,9 @@ class QuestDBVisitor extends BaseVisitor { ? this.tokenImage(ctx.VariableReference[0]) : this.tokenImage(ctx.Identifier![0]), } + if (ctx.groupingSetsSpec) { + result.groupingSets = this.visit(ctx.groupingSetsSpec) as AST.GroupingSets + } if (ctx.fillClause) { result.fill = this.visit(ctx.fillClause) as string[] } @@ -1092,8 +1109,55 @@ class QuestDBVisitor extends BaseVisitor { // GROUP BY, HAVING, ORDER BY, LIMIT // ========================================================================== - groupByClause(ctx: GroupByClauseCstChildren): AST.Expression[] { - return ctx.expression.map((e: CstNode) => this.visit(e) as AST.Expression) + groupByClause(ctx: GroupByClauseCstChildren): GroupByClauseResult { + const result: GroupByClauseResult = {} + if (!ctx.expression && !ctx.groupingSetsSpec) { + // Only reachable from error recovery (`GROUP BY;`): no key at all. + throw new Error("GROUP BY key expected") + } + if (ctx.expression) { + result.groupBy = ctx.expression.map( + (e: CstNode) => this.visit(e) as AST.Expression, + ) + } + if (ctx.groupingSetsSpec) { + result.groupingSets = this.visit(ctx.groupingSetsSpec) as AST.GroupingSets + } else if (ctx.rollupOrCubeSpec) { + result.groupingSets = this.visit(ctx.rollupOrCubeSpec) as AST.GroupingSets + } + return result + } + + groupingSetsSpec(ctx: GroupingSetsSpecCstChildren): AST.GroupingSets { + return this.visit( + (ctx.rollupOrCubeSpec ?? ctx.explicitGroupingSets)!, + ) as AST.GroupingSets + } + + rollupOrCubeSpec(ctx: RollupOrCubeSpecCstChildren): AST.GroupingSets { + return { + type: "groupingSets", + kind: ctx.Cube ? "cube" : "rollup", + columns: ctx.columnRef.map( + (c: CstNode) => this.visit(c) as AST.ColumnRef, + ), + } + } + + explicitGroupingSets(ctx: ExplicitGroupingSetsCstChildren): AST.GroupingSets { + return { + type: "groupingSets", + kind: "groupingSets", + sets: ctx.groupingSet.map( + (g: CstNode) => this.visit(g) as AST.ColumnRef[], + ), + } + } + + groupingSet(ctx: GroupingSetCstChildren): AST.ColumnRef[] { + return (ctx.columnRef ?? []).map( + (c: CstNode) => this.visit(c) as AST.ColumnRef, + ) } orderByClause(ctx: OrderByClauseCstChildren): AST.OrderByItem[] { diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 5f1efd3..211d9ff 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -178,5 +178,173 @@ "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGJobContextTest.java:8217", "feature": "literal-escape-string", "toSql": "SELECT * FROM (SELECT n.nspname, c.relname, a.attname, a.atttypid, a.attnotnull OR (t.typtype = 'd' AND t.typnotnull) AS attnotnull, a.atttypmod, a.attlen, t.typtypmod, row_number() OVER (PARTITION BY a.attrelid ORDER BY a.attnum) AS attnum, nullif(a.attidentity, '') AS attidentity, NULL AS attgenerated, pg_catalog.pg_get_expr(def.adbin, def.adrelid) AS adsrc, dsc.description, t.typbasetype, t.typtype FROM pg_catalog.pg_namespace AS n JOIN pg_catalog.pg_class AS c ON (c.relnamespace = n.oid) JOIN pg_catalog.pg_attribute AS a ON (a.attrelid = c.oid) JOIN pg_catalog.pg_type AS t ON (a.atttypid = t.oid) LEFT JOIN pg_catalog.pg_attrdef AS def ON (a.attrelid = def.adrelid AND a.attnum = def.adnum) LEFT JOIN pg_catalog.pg_description AS dsc ON (c.oid = dsc.objoid AND a.attnum = dsc.objsubid) LEFT JOIN pg_catalog.pg_class AS dc ON (dc.oid = dsc.classoid AND dc.relname = 'pg_class') LEFT JOIN pg_catalog.pg_namespace AS dn ON (dc.relnamespace = dn.oid AND dn.nspname = 'pg_catalog') WHERE c.relkind IN ('r', 'p', 'v', 'f', 'm') AND a.attnum > 0 AND NOT a.attisdropped AND c.relname LIKE E'test') AS c WHERE TRUE ORDER BY nspname, c.relname, attnum" + }, + { + "query": "SELECT a, b, grouping_id(a, b) g, sum(v) FROM t GROUP BY CUBE(a, b)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:72", + "feature": "grouping-sets", + "toSql": "SELECT a, b, grouping_id(a, b) AS g, sum(v) FROM t GROUP BY CUBE(a, b)" + }, + { + "query": "SELECT trim(s) k, a, sum(v) sv FROM t WHERE v > 10 GROUP BY k, ROLLUP(a)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:73", + "feature": "grouping-sets", + "toSql": "SELECT trim(s) AS k, a, sum(v) AS sv FROM t WHERE v > 10 GROUP BY k, ROLLUP(a)" + }, + { + "query": "SELECT a, b, GROUPING_ID(a, b) AS gid, SUM(v) FROM t GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY gid, a, b", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:128", + "feature": "grouping-sets", + "toSql": "SELECT a, b, GROUPING_ID(a, b) AS gid, SUM(v) FROM t GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY gid, a, b" + }, + { + "query": "SELECT trim(s) k, b, GROUPING(b) g, SUM(v) s FROM t GROUP BY trim(s), ROLLUP(b) ORDER BY k, g", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:166", + "feature": "grouping-sets", + "toSql": "SELECT trim(s) AS k, b, GROUPING(b) AS g, SUM(v) AS s FROM t GROUP BY trim(s), ROLLUP(b) ORDER BY k, g" + }, + { + "query": "SELECT x, SUM(y) FROM t GROUP BY cube(x) ORDER BY x", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:427", + "feature": "grouping-sets", + "toSql": "SELECT x, SUM(y) FROM t GROUP BY CUBE(x) ORDER BY x" + }, + { + "query": "SELECT * FROM (SELECT a,b,SUM(v) s FROM t GROUP BY CUBE(a,b)) WHERE a=1 ORDER BY a,b", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:439", + "feature": "grouping-sets", + "toSql": "SELECT * FROM (SELECT a, b, SUM(v) AS s FROM t GROUP BY CUBE(a, b)) WHERE a = 1 ORDER BY a, b" + }, + { + "query": "SELECT * FROM t2 JOIN t3 ON t2.a = t3.b GROUP BY ROLLUP(a, b) ORDER BY a, b", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:649", + "feature": "grouping-sets", + "toSql": "SELECT * FROM t2 JOIN t3 ON t2.a = t3.b GROUP BY ROLLUP(a, b) ORDER BY a, b" + }, + { + "query": "SELECT 1 AS x FROM t GROUP BY GROUPING SETS (())", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:683", + "feature": "grouping-sets", + "toSql": "SELECT 1 AS x FROM t GROUP BY GROUPING SETS (())" + }, + { + "query": "SELECT * FROM (t GROUP BY GROUPING SETS (()))", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:829", + "feature": "grouping-sets", + "toSql": "SELECT * FROM (t GROUP BY GROUPING SETS (()))" + }, + { + "query": "SELECT t.b AS w, GROUPING(w) g, count() n FROM t JOIN u ON t.a = u.a GROUP BY ROLLUP(t.b, w) ORDER BY w, g", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:989", + "feature": "grouping-sets", + "toSql": "SELECT t.b AS w, GROUPING(w) AS g, count() AS n FROM t JOIN u ON t.a = u.a GROUP BY ROLLUP(t.b, w) ORDER BY w, g" + }, + { + "query": "SELECT a + 1 AS k, b, GROUPING(k) gk, GROUPING(b) gb, sum(v) s FROM t GROUP BY a + 1, ROLLUP(b) ORDER BY k, b", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:1012", + "feature": "grouping-sets", + "toSql": "SELECT a + 1 AS k, b, GROUPING(k) AS gk, GROUPING(b) AS gb, sum(v) AS s FROM t GROUP BY a + 1, ROLLUP(b) ORDER BY k, b" + }, + { + "query": "SELECT a, CASE WHEN GROUPING(a) = 1 THEN 'total' ELSE 'row' END k, sum(v) s FROM t GROUP BY ROLLUP(a) ORDER BY a", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:1610", + "feature": "grouping-sets", + "toSql": "SELECT a, CASE WHEN GROUPING(a) = 1 THEN 'total' ELSE 'row' END AS k, sum(v) AS s FROM t GROUP BY ROLLUP(a) ORDER BY a" + }, + { + "query": "SELECT a,b,SUM(v) FROM t GROUP BY a, CUBE(b) LIMIT 0", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:1813", + "feature": "grouping-sets", + "toSql": "SELECT a, b, SUM(v) FROM t GROUP BY a, CUBE(b) LIMIT 0" + }, + { + "query": "CREATE TABLE t (cube INT, rollup INT, grouping INT, v INT)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:2055", + "feature": "grouping-sets", + "toSql": "CREATE TABLE t (cube INT, rollup INT, grouping INT, v INT)" + }, + { + "query": "SELECT cube, rollup, grouping, SUM(v) FROM t GROUP BY cube, rollup, grouping ORDER BY cube, rollup, grouping", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:2058", + "feature": "grouping-sets", + "toSql": "SELECT cube, rollup, grouping, SUM(v) FROM t GROUP BY cube, rollup, grouping ORDER BY cube, rollup, grouping" + }, + { + "query": "SELECT rollup, cube, SUM(v) FROM t GROUP BY rollup, cube ORDER BY rollup, cube", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:2063", + "feature": "grouping-sets", + "toSql": "SELECT rollup, cube, SUM(v) FROM t GROUP BY rollup, cube ORDER BY rollup, cube" + }, + { + "query": "SELECT cube::string AS k, SUM(v) AS s FROM t GROUP BY cube::string ORDER BY k", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:2081", + "feature": "grouping-sets", + "toSql": "SELECT cube::STRING AS k, SUM(v) AS s FROM t GROUP BY cube::STRING ORDER BY k" + }, + { + "query": "SELECT grouping + 1 AS k, SUM(v) AS s FROM t GROUP BY grouping + 1 ORDER BY k", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:2084", + "feature": "grouping-sets", + "toSql": "SELECT grouping + 1 AS k, SUM(v) AS s FROM t GROUP BY grouping + 1 ORDER BY k" + }, + { + "query": "SELECT cube, SUM(v) AS s FROM t GROUP BY grouping sets ((cube), ()) ORDER BY cube", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:2088", + "feature": "grouping-sets", + "toSql": "SELECT cube, SUM(v) AS s FROM t GROUP BY GROUPING SETS ((cube), ()) ORDER BY cube" + }, + { + "query": "SELECT o.k, l.a, l.cnt\nFROM o LEFT JOIN LATERAL (\n SELECT a, count(*) AS cnt\n FROM t\n WHERE t.k = o.k\n GROUP BY GROUPING SETS ((a), ())\n) l ON true\nORDER BY o.k, l.a\n", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/join/LateralJoinTest.java:54", + "feature": "grouping-sets", + "toSql": "SELECT o.k, l.a, l.cnt FROM o LEFT JOIN LATERAL (SELECT a, count(*) AS cnt FROM t WHERE t.k = o.k GROUP BY GROUPING SETS ((a), ())) AS l ON TRUE ORDER BY o.k, l.a" + }, + { + "query": "SELECT ts, x, a, count() c FROM v SAMPLE BY 1d ROLLUP(x, a) FILL(NULL) ORDER BY ts, x, a", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:3522", + "feature": "grouping-sets", + "toSql": "SELECT ts, x, a, count() AS c FROM v SAMPLE BY 1d ROLLUP(x, a) FILL(NULL) ORDER BY ts, x, a" + }, + { + "query": "SELECT ts, a, COUNT(*) FROM t SAMPLE BY 1d CUBE(a, a) FILL(NULL)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:3965", + "feature": "grouping-sets", + "toSql": "SELECT ts, a, COUNT(*) FROM t SAMPLE BY 1d CUBE(a, a) FILL(NULL)" + }, + { + "query": "SELECT ts, a, b, COUNT(*) FROM t SAMPLE BY 1d GROUPING SETS ((a, b), (b, a)) FILL(0)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:4009", + "feature": "grouping-sets", + "toSql": "SELECT ts, a, b, COUNT(*) FROM t SAMPLE BY 1d GROUPING SETS ((a, b), (b, a)) FILL(0)" + }, + { + "query": "SELECT ts, COUNT(*) FROM t SAMPLE BY 1d GROUPING SETS ((), ()) FROM '2024-01-01' TO '2024-01-03' FILL(NULL)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:3998", + "feature": "grouping-sets", + "toSql": "SELECT ts, COUNT(*) FROM t SAMPLE BY 1d GROUPING SETS ((), ()) FROM '2024-01-01' TO '2024-01-03' FILL(NULL)" + }, + { + "query": "SELECT ts, a, count() c FROM t WHERE v < 0 SAMPLE BY 1d ROLLUP(a) FROM '2024-01-01' TO '2024-01-04' FILL(NULL)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:4340", + "feature": "grouping-sets", + "toSql": "SELECT ts, a, count() AS c FROM t WHERE v < 0 SAMPLE BY 1d ROLLUP(a) FROM '2024-01-01' TO '2024-01-04' FILL(NULL)" + }, + { + "query": "SELECT * FROM t o JOIN LATERAL (SELECT ts, count() n FROM u WHERE u.a = o.a SAMPLE BY 1d GROUPING SETS (())) l", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:2194", + "feature": "grouping-sets", + "toSql": "SELECT * FROM t AS o JOIN LATERAL (SELECT ts, count() AS n FROM u WHERE u.a = o.a SAMPLE BY 1d GROUPING SETS (())) AS l" + }, + { + "query": "WITH s AS (SELECT ts, a, v FROM t) SELECT ts, a, sum(v) FROM s SAMPLE BY 1d ROLLUP(a) ORDER BY ts, a", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupingSetsTest.java:4709", + "feature": "grouping-sets", + "toSql": "WITH s AS (SELECT ts, a, v FROM t) SELECT ts, a, sum(v) FROM s SAMPLE BY 1d ROLLUP(a) ORDER BY ts, a" + }, + { + "query": "SELECT ts, sum(val) FROM x SAMPLE BY 1d GROUPING SETS (()) FILL(NULL)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/groupby/SampleByFillTest.java:2452", + "feature": "grouping-sets", + "toSql": "SELECT ts, sum(val) FROM x SAMPLE BY 1d GROUPING SETS (()) FILL(NULL)" } ] diff --git a/tests/grouping-sets.test.ts b/tests/grouping-sets.test.ts new file mode 100644 index 0000000..5742049 --- /dev/null +++ b/tests/grouping-sets.test.ts @@ -0,0 +1,307 @@ +/** + * GROUP BY ROLLUP / CUBE / GROUPING SETS and the SAMPLE BY grouping form + * (questdb/questdb#6853, reports §1.1). + * + * Server grammar (SqlParser.parseGroupByColumns, parseExplicitGroupingSets, + * hasGroupingSyntaxLookahead and the SAMPLE BY branch of parseSelectClause): + * GROUP BY ROLLUP(col, ...) | CUBE(col, ...) + * GROUP BY GROUPING SETS ((a, b), (a), ()) -- only GROUP BY item + * GROUP BY expr, ..., ROLLUP(...) | CUBE(...) -- ROLLUP/CUBE last + * SAMPLE BY [ROLLUP(...) | CUBE(...) | GROUPING SETS (...)] + * [FROM ... TO ...] [FILL(...)] [ALIGN TO ...] + * `rollup`, `cube`, `grouping` and `sets` are not reserved words: they are + * grouping syntax only when `(` (ROLLUP/CUBE) or SETS (GROUPING) follows. + * + * Server queries for this feature live in tests/fixtures/server-queries.json. + */ + +import { describe, it, expect } from "vitest" +import { + parseToAst, + getNextValidTokens, + createAutocompleteProvider, +} from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const parseOne = (sql: string): AST.SelectStatement => { + const result = parseToAst(sql) + expect(result.errors.map((e) => e.message)).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] as AST.SelectStatement +} + +const col = (...parts: string[]) => ({ + type: "column", + name: { type: "qualifiedName", parts }, +}) + +/** true when sql is rejected or does not read as a single statement */ +const rejected = (sql: string) => { + const result = parseToAst(sql) + return result.errors.length > 0 || result.ast.length !== 1 +} + +describe("the grouping words stay usable as identifiers", () => { + it.each([ + "SELECT rollup, cube, sets, grouping FROM t", + "CREATE TABLE t (sets INT, cube INT)", + "CREATE TABLE cube (sets INT, rollup INT, grouping INT)", + "SELECT grouping(a) FROM t GROUP BY a", + "SELECT grouping_id(a, b) FROM t GROUP BY a, b", + "SELECT a FROM t GROUP BY rollup", + "SELECT a FROM t GROUP BY cube, rollup, grouping, sets", + "SELECT a FROM t GROUP BY a, rollup", + "SELECT a FROM t GROUP BY grouping + 1", + "SELECT a FROM t GROUP BY a, grouping(b)", + "SELECT * FROM rollup", + "SELECT sets.a FROM t AS sets", + "SELECT ts, sum(v) FROM t SAMPLE BY 1h FILL(NULL)", + ])("%s", (sql) => { + const stmt = parseToAst(sql) + expect(stmt.errors.map((e) => e.message)).toEqual([]) + expect(stmt.ast).toHaveLength(1) + expect(toSql(stmt.ast)).toBe(sql) + }) + + it("reads GROUP BY grouping(b) as a plain function key", () => { + const stmt = parseOne("SELECT a FROM t GROUP BY grouping(b)") + expect(stmt.groupBy).toHaveLength(1) + expect(stmt.groupBy![0].type).toBe("function") + expect(stmt.groupingSets).toBeUndefined() + }) + + it("reads GROUP BY rollup as a plain column key", () => { + const stmt = parseOne("SELECT a FROM t GROUP BY rollup") + expect(stmt.groupBy).toEqual([col("rollup")]) + expect(stmt.groupingSets).toBeUndefined() + }) +}) + +describe("AST", () => { + it("GROUP BY ROLLUP(a, b)", () => { + const stmt = parseOne("SELECT a, b, sum(v) FROM t GROUP BY ROLLUP(a, b)") + expect(stmt.groupBy).toBeUndefined() + expect(stmt.groupingSets).toEqual({ + type: "groupingSets", + kind: "rollup", + columns: [col("a"), col("b")], + }) + }) + + it("GROUP BY cube(t.a) (any case, qualified columns)", () => { + const stmt = parseOne("SELECT 1 FROM t GROUP BY cube(t.a)") + expect(stmt.groupingSets).toEqual({ + type: "groupingSets", + kind: "cube", + columns: [col("t", "a")], + }) + }) + + it("GROUP BY GROUPING SETS ((a, b), (a), ())", () => { + const stmt = parseOne( + "SELECT a, b, sum(v) FROM t GROUP BY GROUPING SETS ((a, b), (a), ())", + ) + expect(stmt.groupBy).toBeUndefined() + expect(stmt.groupingSets).toEqual({ + type: "groupingSets", + kind: "groupingSets", + sets: [[col("a"), col("b")], [col("a")], []], + }) + }) + + it("composite GROUP BY expr, ROLLUP(...): plain keys stay in groupBy", () => { + const stmt = parseOne( + "SELECT trim(s) k, b, sum(v) FROM t GROUP BY trim(s), k, CUBE(b)", + ) + expect(stmt.groupBy).toHaveLength(2) + expect(stmt.groupBy![0].type).toBe("function") + expect(stmt.groupBy![1]).toEqual(col("k")) + expect(stmt.groupingSets).toEqual({ + type: "groupingSets", + kind: "cube", + columns: [col("b")], + }) + }) + + it("SAMPLE BY interval ROLLUP(...)", () => { + const stmt = parseOne( + "SELECT ts, sym, sum(q) FROM t SAMPLE BY 1h ROLLUP(sym) FILL(NULL)", + ) + expect(stmt.groupBy).toBeUndefined() + expect(stmt.groupingSets).toBeUndefined() + expect(stmt.sampleBy).toMatchObject({ + type: "sampleBy", + duration: "1h", + fill: ["NULL"], + groupingSets: { + type: "groupingSets", + kind: "rollup", + columns: [col("sym")], + }, + }) + }) + + it("SAMPLE BY interval GROUPING SETS (...)", () => { + const stmt = parseOne( + "SELECT ts, sym, sum(x) FROM t SAMPLE BY 1d GROUPING SETS ((sym), ())", + ) + expect(stmt.sampleBy!.groupingSets).toEqual({ + type: "groupingSets", + kind: "groupingSets", + sets: [[col("sym")], []], + }) + }) +}) + +describe("round trip", () => { + it.each([ + "SELECT a, sum(b) FROM t GROUP BY ROLLUP(a)", + "SELECT a, b, sum(c) FROM t GROUP BY CUBE(a, b)", + "SELECT a, b, grouping_id(a, b) AS gid, sum(v) FROM t GROUP BY GROUPING SETS ((a, b), (a), ())", + "SELECT 1 FROM t GROUP BY GROUPING SETS (())", + "SELECT trim(s) AS k, b, grouping(b) AS g, sum(v) FROM t GROUP BY trim(s), ROLLUP(b)", + "SELECT a, b FROM t GROUP BY a, rollup, CUBE(b)", + "SELECT grouping(sym), ts, sym, sum(q) FROM t SAMPLE BY 1h ROLLUP(sym) FILL(NULL)", + "SELECT ts, sym, sum(x) FROM t SAMPLE BY 1d GROUPING SETS ((sym), ()) FILL(NONE) ALIGN TO CALENDAR", + "SELECT ts, sym, sum(x) FROM t SAMPLE BY 1d CUBE(sym, side) FROM '2024-01-01' TO '2024-01-02' FILL(0) ALIGN TO CALENDAR TIME ZONE 'Europe/Riga' WITH OFFSET '00:30'", + "SELECT ts, a, sum(v) FROM t SAMPLE BY 1d ROLLUP(a) ORDER BY ts, a LIMIT 10", + "t GROUP BY ROLLUP(a)", + "t SAMPLE BY 1h GROUPING SETS ((a))", + ])("%s", (sql) => { + expect(toSql(parseToAst(sql).ast)).toBe(sql) + expect(parseToAst(sql).errors).toEqual([]) + }) + + it("normalises keyword case", () => { + expect(toSql(parseToAst("select a from t group by rollup(a)").ast)).toBe( + "SELECT a FROM t GROUP BY ROLLUP(a)", + ) + expect( + toSql( + parseToAst("select a from t sample by 1h grouping sets ((a),())").ast, + ), + ).toBe("SELECT a FROM t SAMPLE BY 1h GROUPING SETS ((a), ())") + }) +}) + +describe("what the server rejects stays rejected", () => { + it.each([ + // nothing may follow ROLLUP / CUBE / GROUPING SETS + "SELECT a, b FROM t GROUP BY ROLLUP(a), b", + "SELECT a, b FROM t GROUP BY CUBE(a), CUBE(b)", + "SELECT a FROM t GROUP BY GROUPING SETS ((a)), a", + // GROUPING SETS must be the only GROUP BY item + "SELECT a, c1 FROM t GROUP BY a, GROUPING SETS ((c1), ())", + "SELECT a, c1, count() FROM t GROUP BY a, c1, grouping sets ((c2))", + // every set needs parentheses + "SELECT a FROM t GROUP BY GROUPING SETS (a)", + "SELECT a FROM t GROUP BY GROUPING SETS ()", + "SELECT a FROM t GROUP BY GROUPING SETS", + // ROLLUP / CUBE need at least one column + "SELECT a FROM t GROUP BY ROLLUP()", + "SELECT a FROM t GROUP BY a, CUBE()", + // keys must be column references + "SELECT sum(v) FROM t GROUP BY ROLLUP(a + b)", + "SELECT sum(v) FROM t GROUP BY CUBE(1)", + "SELECT sum(v) FROM t GROUP BY GROUPING SETS ((a + b))", + "SELECT sum(v) FROM t GROUP BY GROUPING SETS ((a, ))", + "SELECT sum(v) FROM t SAMPLE BY 1h ROLLUP(a + b)", + // SAMPLE BY: grouping right after the interval, only one + "SELECT ts, a FROM t SAMPLE BY 1h FILL(NULL) ROLLUP(a)", + "SELECT ts, a FROM t SAMPLE BY 1h ROLLUP(a), CUBE(a)", + "SELECT ts, a FROM t SAMPLE BY 1h ROLLUP(a) CUBE(a)", + "SELECT ts, a FROM t SAMPLE BY 1h GROUPING a", + ])("%s", (sql) => { + expect(rejected(sql)).toBe(true) + }) + + it("GROUPING SETS after plain keys is an error, not a second statement", () => { + const result = parseToAst( + "SELECT a, c1, count() FROM t GROUP BY a, c1, grouping sets ((c2))", + ) + expect(result.errors.length).toBeGreaterThan(0) + }) +}) + +describe("autocomplete", () => { + it.each([ + "SELECT a FROM t GROUP BY ", + "SELECT ts, a FROM t SAMPLE BY 1h ", + "t SAMPLE BY 1h ", + ])("offers ROLLUP, CUBE and GROUPING after: %s", (sql) => { + const tokens = getNextValidTokens(sql) + expect(tokens).toContain("Rollup") + expect(tokens).toContain("Cube") + expect(tokens).toContain("Grouping") + }) + + it("offers ROLLUP and CUBE (not GROUPING) after a plain GROUP BY key", () => { + const tokens = getNextValidTokens("SELECT a FROM t GROUP BY a, ") + expect(tokens).toContain("Rollup") + expect(tokens).toContain("Cube") + expect(tokens).not.toContain("Grouping") + }) + + it("offers SETS after GROUPING", () => { + expect(getNextValidTokens("SELECT a FROM t GROUP BY GROUPING ")).toContain( + "Sets", + ) + expect( + getNextValidTokens("SELECT a FROM t SAMPLE BY 1h GROUPING "), + ).toEqual(["Sets"]) + }) + + it("keeps FROM / FILL / ALIGN after the SAMPLE BY grouping", () => { + const tokens = getNextValidTokens( + "SELECT ts, a FROM t SAMPLE BY 1h ROLLUP(a) ", + ) + expect(tokens).toContain("Fill") + expect(tokens).toContain("Align") + expect(tokens).toContain("From") + expect(tokens).not.toContain("Rollup") + }) + + it.each([ + "SELECT ", + "SELECT * FROM t WHERE ", + "SELECT * FROM t WHERE a > ", + "SELECT * FROM t ORDER BY ", + ])("does not offer grouping keywords where they are invalid: %s", (sql) => { + const tokens = getNextValidTokens(sql) + expect(tokens).not.toContain("Rollup") + expect(tokens).not.toContain("Cube") + expect(tokens).not.toContain("Sets") + }) + + const provider = createAutocompleteProvider({ + tables: [{ name: "trades" }], + columns: { + trades: [ + { name: "symbol", type: "SYMBOL" }, + { name: "side", type: "SYMBOL" }, + { name: "ts", type: "TIMESTAMP" }, + ], + }, + }) + const labels = (sql: string) => + provider.getSuggestions(sql, sql.length).map((s) => s.label) + + it("suggests the keywords and columns after GROUP BY", () => { + const l = labels("SELECT symbol, count() FROM trades GROUP BY ") + expect(l).toContain("ROLLUP") + expect(l).toContain("CUBE") + expect(l).toContain("GROUPING") + expect(l).toContain("symbol") + }) + + it.each([ + "SELECT symbol, count() FROM trades GROUP BY ROLLUP(", + "SELECT symbol, count() FROM trades GROUP BY symbol, CUBE(side, ", + "SELECT symbol, count() FROM trades GROUP BY GROUPING SETS ((", + "SELECT symbol, count() FROM trades GROUP BY GROUPING SETS ((symbol), (", + "SELECT ts, symbol, count() FROM trades SAMPLE BY 1h ROLLUP(", + ])("suggests columns inside the parentheses: %s", (sql) => { + expect(labels(sql)).toContain("side") + }) +}) From ac9b1864202f3269a13e6591898ac113fa890a5a Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 14:18:12 +0300 Subject: [PATCH 04/36] feat: parse DEDUP SKIP, the DEDUPLICATE synonym and ALTER DEDUP without ENABLE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the dedup syntax of questdb/questdb#7885 and fixes older dedup gaps (reports §1.2). The server accepts (SqlParser CREATE TABLE dedup block, SqlCompilerImpl.alterTableDedupEnable and the ALTER TABLE dedup branch): CREATE TABLE / CTAS ... (DEDUP | DEDUPLICATE) (UPSERT | SKIP) KEYS(c, ...) [FORMAT PARQUET | NATIVE] [OWNED BY ...] ALTER TABLE t (DEDUP | DEDUPLICATE) DISABLE ALTER TABLE t (DEDUP | DEDUPLICATE) [ENABLE] (UPSERT | SKIP) KEYS(c, ...) ENABLE has been optional since v7.3. SHOW CREATE TABLE prints `DEDUP SKIP KEYS(...)`. `skip` and `deduplicate` are not reserved (neither is in SqlKeywords.KEYWORDS; upsert is, and stays reserved). Before: every SKIP form failed ("Expecting Upsert"), ALTER ... DEDUP UPSERT KEYS without ENABLE failed, ALTER ... DEDUPLICATE failed, and `... WAL DEDUPLICATE UPSERT KEYS(ts)` was silently split into three statements. `skip` was a reserved token with no grammar, so `SELECT skip FROM t` failed. Now: - keywords.ts: new `deduplicate`; tokens.ts: Deduplicate and Skip are non-reserved (IDENTIFIER_KEYWORD_NAMES). - parser: new rules dedupKeyword (DEDUP | DEDUPLICATE) and dedupModeKeys ((UPSERT | SKIP) KEYS(...)), shared by the CREATE TABLE dedupClause and the ALTER TABLE dedup action, where ENABLE is now optional. - AST: new optional `dedupMode: "upsert" | "skip"` on CreateTableStatement (set whenever dedupKeys is) and on DedupEnableAction. Existing fields are unchanged. - toSql prints the mode instead of the hard-coded UPSERT (absent mode still prints UPSERT). DEDUPLICATE is printed as DEDUP and ALTER always prints ENABLE (canonical form; same meaning on the server). The existing output spacing (`KEYS (..)` for ALTER) is kept. - Formatter: DEDUPLICATE is a clause/action phrase like DEDUP. - Autocomplete: UPSERT and SKIP after DEDUP / DEDUPLICATE, plus ENABLE and DISABLE after ALTER TABLE t DEDUP; DEDUPLICATE is offered wherever DEDUP is. Tests: 16 server queries (DedupSkipTest, CreateTableDedupTest, ShowCreateTableTest text block, WalWriterTest, ImportIODispatcherTest, O3ParquetStaleReaderTest, AlterTableSetTypeRestartTest, Enterprise EntShowCreateTableTest) in the parity fixture, plus tests/dedup-skip.test.ts (AST mode, round trips incl. FORMAT PARQUET / OWNED BY / CTAS, rejected forms, skip/deduplicate as identifiers, autocomplete). Red before: 83 failing (48 fixture checks + 35 focused); after: all 346 pass, full suite 15604 passed. Corpus (53,975 server/docs SQL): 27 fixed, 0 round trips changed, 0 regressed, 0 throws, formatter meaning unchanged. --- src/formatter/phrases.ts | 2 + src/grammar/keywords.ts | 1 + src/parser/ast.ts | 7 + src/parser/cst-types.d.ts | 35 +++-- src/parser/lexer.ts | 2 + src/parser/parser.ts | 45 +++--- src/parser/toSql.ts | 6 +- src/parser/tokens.ts | 3 + src/parser/visitor.ts | 37 +++-- tests/dedup-skip.test.ts | 222 +++++++++++++++++++++++++++++ tests/fixtures/server-queries.json | 96 +++++++++++++ 11 files changed, 420 insertions(+), 36 deletions(-) create mode 100644 tests/dedup-skip.test.ts diff --git a/src/formatter/phrases.ts b/src/formatter/phrases.ts index 56db73a..6ee86c7 100644 --- a/src/formatter/phrases.ts +++ b/src/formatter/phrases.ts @@ -81,6 +81,7 @@ const updatePhrases: Phrase[] = [ const createOptionPhrases: Phrase[] = clause( ["Dedup"], + ["Deduplicate"], ["Ttl"], ["Expire", "Rows"], ["Storage", "Policy"], @@ -106,6 +107,7 @@ const alterActionPhrases: Phrase[] = action( ["Force", "Drop", "Partition", "List"], ["Force", "Reindex", "Parquet"], ["Dedup"], + ["Deduplicate"], ["Resume", "Wal"], ["Suspend", "Wal"], ["Rebase", "Wal"], diff --git a/src/grammar/keywords.ts b/src/grammar/keywords.ts index ef345e2..2a6884c 100644 --- a/src/grammar/keywords.ts +++ b/src/grammar/keywords.ts @@ -51,6 +51,7 @@ export const keywords: string[] = [ "database", "declare", "dedup", + "deduplicate", "default", "deferred", "delay", diff --git a/src/parser/ast.ts b/src/parser/ast.ts index ca0cbd3..9e74158 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -182,9 +182,14 @@ export interface CreateTableStatement extends AstNode { volume?: string ownedBy?: string dedupKeys?: string[] + /** DEDUP UPSERT | SKIP KEYS(...) mode; set whenever dedupKeys is */ + dedupMode?: DedupMode tableFormat?: "parquet" | "native" } +/** Deduplication mode: UPSERT replaces the existing row, SKIP keeps it */ +export type DedupMode = "upsert" | "skip" + export interface TableParam extends AstNode { type: "tableParam" name: string @@ -686,6 +691,8 @@ export interface DedupDisableAction { export interface DedupEnableAction { actionType: "dedupEnable" keys: string[] + /** UPSERT | SKIP; printed as UPSERT when absent */ + dedupMode?: DedupMode } export interface SetTypeWalAction { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index dba0549..eb6cd1d 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -909,8 +909,28 @@ export interface DedupClauseCstNode extends CstNode { } export type DedupClauseCstChildren = { - Dedup: IToken[]; - Upsert: IToken[]; + dedupKeyword: DedupKeywordCstNode[]; + dedupModeKeys: DedupModeKeysCstNode[]; +}; + +export interface DedupKeywordCstNode extends CstNode { + name: "dedupKeyword"; + children: DedupKeywordCstChildren; +} + +export type DedupKeywordCstChildren = { + Dedup?: IToken[]; + Deduplicate?: IToken[]; +}; + +export interface DedupModeKeysCstNode extends CstNode { + name: "dedupModeKeys"; + children: DedupModeKeysCstChildren; +} + +export type DedupModeKeysCstChildren = { + Upsert?: IToken[]; + Skip?: IToken[]; Keys: IToken[]; LParen: IToken[]; identifier: (IdentifierCstNode)[]; @@ -1574,7 +1594,7 @@ export type AlterTableActionCstChildren = { Policy?: (IToken)[]; Rename?: IToken[]; To?: IToken[]; - identifier?: (IdentifierCstNode)[]; + identifier?: IdentifierCstNode[]; Alter?: IToken[]; Type?: (IToken)[]; dataType?: DataTypeCstNode[]; @@ -1601,13 +1621,10 @@ export type AlterTableActionCstChildren = { Wal?: (IToken)[]; storagePolicy?: StoragePolicyCstNode[]; tableFormatKind?: TableFormatKindCstNode[]; - Dedup?: IToken[]; + dedupKeyword?: DedupKeywordCstNode[]; Disable?: (IToken)[]; Enable?: (IToken)[]; - Upsert?: IToken[]; - Keys?: IToken[]; - LParen?: IToken[]; - RParen?: IToken[]; + dedupModeKeys?: DedupModeKeysCstNode[]; Suspend?: IToken[]; With?: IToken[]; Resume?: IToken[]; @@ -3110,6 +3127,8 @@ export interface ICstNodeVisitor extends ICstVisitor { optionalTableFormat(children: OptionalTableFormatCstChildren, param?: IN): OUT; batchClause(children: BatchClauseCstChildren, param?: IN): OUT; dedupClause(children: DedupClauseCstChildren, param?: IN): OUT; + dedupKeyword(children: DedupKeywordCstChildren, param?: IN): OUT; + dedupModeKeys(children: DedupModeKeysCstChildren, param?: IN): OUT; createUserStatement(children: CreateUserStatementCstChildren, param?: IN): OUT; createGroupStatement(children: CreateGroupStatementCstChildren, param?: IN): OUT; createServiceAccountStatement(children: CreateServiceAccountStatementCstChildren, param?: IN): OUT; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index 8364869..3da28ff 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -64,6 +64,7 @@ import { Decimal, Declare, Dedup, + Deduplicate, Default, DefaultTransactionReadOnly, DeltaBinaryPacked, @@ -402,6 +403,7 @@ export { Decimal, Declare, Dedup, + Deduplicate, Default, DefaultTransactionReadOnly, DeltaBinaryPacked, diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 8edd75b..da3dff9 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -125,7 +125,9 @@ import { Nanosecond, Nanoseconds, Dedup, + Deduplicate, Upsert, + Skip, Keys, Add, Attach, @@ -1903,12 +1905,12 @@ class QuestDBParser extends CstParser { ]) }) - // Optional DEDUP UPSERT KEYS + // Optional DEDUP | DEDUPLICATE UPSERT | SKIP KEYS(...) this.OPTION8(() => { this.SUBRULE(this.dedupClause) }) - // Optional FORMAT { PARQUET | NATIVE } (after DEDUP UPSERT KEYS) + // Optional FORMAT { PARQUET | NATIVE } (after DEDUP UPSERT|SKIP KEYS) this.SUBRULE2(this.optionalTableFormat) // Optional OWNED BY @@ -1948,9 +1950,26 @@ class QuestDBParser extends CstParser { }) }) + // CREATE TABLE ... (DEDUP | DEDUPLICATE) (UPSERT | SKIP) KEYS(col, ...) private dedupClause = this.RULE("dedupClause", () => { - this.CONSUME(Dedup) - this.CONSUME(Upsert) + this.SUBRULE(this.dedupKeyword) + this.SUBRULE(this.dedupModeKeys) + }) + + // DEDUPLICATE is a server synonym of DEDUP (SqlKeywords.isDeduplicateKeyword) + private dedupKeyword = this.RULE("dedupKeyword", () => { + this.OR([ + { ALT: () => this.CONSUME(Dedup) }, + { ALT: () => this.CONSUME(Deduplicate) }, + ]) + }) + + // (UPSERT | SKIP) KEYS(col, ...) — shared by CREATE TABLE and ALTER TABLE + private dedupModeKeys = this.RULE("dedupModeKeys", () => { + this.OR([ + { ALT: () => this.CONSUME(Upsert) }, + { ALT: () => this.CONSUME(Skip) }, + ]) this.CONSUME(Keys) this.CONSUME(LParen) this.SUBRULE(this.identifier) @@ -3112,24 +3131,18 @@ class QuestDBParser extends CstParser { ]) }, }, - // DEDUP (DISABLE | ENABLE UPSERT KEYS (...)) + // (DEDUP | DEDUPLICATE) (DISABLE | [ENABLE] (UPSERT | SKIP) KEYS (...)) + // ENABLE is optional on the server since v7.3 + // (SqlCompilerImpl.alterTableDedupEnable). { ALT: () => { - this.CONSUME(Dedup) + this.SUBRULE(this.dedupKeyword) this.OR6([ { ALT: () => this.CONSUME(Disable) }, { ALT: () => { - this.CONSUME(Enable) - this.CONSUME(Upsert) - this.CONSUME(Keys) - this.CONSUME(LParen) - this.SUBRULE5(this.identifier) - this.MANY6(() => { - this.CONSUME6(Comma) - this.SUBRULE6(this.identifier) - }) - this.CONSUME(RParen) + this.OPTION2(() => this.CONSUME(Enable)) + this.SUBRULE(this.dedupModeKeys) }, }, ]) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 82cae89..68162d2 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -767,7 +767,7 @@ function createTableToSql(stmt: AST.CreateTableStatement): string { if (stmt.dedupKeys && stmt.dedupKeys.length > 0) { parts.push( - `DEDUP UPSERT KEYS(${stmt.dedupKeys.map(escapeIdentifier).join(", ")})`, + `DEDUP ${(stmt.dedupMode ?? "upsert").toUpperCase()} KEYS(${stmt.dedupKeys.map(escapeIdentifier).join(", ")})`, ) } @@ -1009,7 +1009,9 @@ function alterTableToSql(stmt: AST.AlterTableStatement): string { parts.push("DEDUP DISABLE") break case "dedupEnable": - parts.push("DEDUP ENABLE UPSERT KEYS") + parts.push( + `DEDUP ENABLE ${(action.dedupMode ?? "upsert").toUpperCase()} KEYS`, + ) parts.push(`(${action.keys.map(escapeIdentifier).join(", ")})`) break case "setTypeWal": diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index 6e4ed8b..a2d7ccd 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -214,6 +214,8 @@ export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ "Partitions", "PartitionBy", "Dedup", + "Deduplicate", + "Skip", "Wal", "Bypass", "Batch", @@ -520,6 +522,7 @@ export const Database = getToken("Database") export const Datestyle = getToken("Datestyle") export const Declare = getToken("Declare") export const Dedup = getToken("Dedup") +export const Deduplicate = getToken("Deduplicate") export const Default = getToken("Default") export const DefaultTransactionReadOnly = getToken("DefaultTransactionReadOnly") export const Deferred = getToken("Deferred") diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index b9e4e0b..9b3a7a1 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -70,6 +70,8 @@ import type { DeclareAssignmentCstChildren, DeclareClauseCstChildren, DedupClauseCstChildren, + DedupKeywordCstChildren, + DedupModeKeysCstChildren, DropGroupStatementCstChildren, DropMaterializedViewStatementCstChildren, DropServiceAccountStatementCstChildren, @@ -242,6 +244,8 @@ function tokenFloat(image: string): number { type StoragePolicyKind = "toParquet" | "toRemote" | "dropLocal" | "dropRemote" +type DedupModeKeys = { mode: AST.DedupMode; keys: string[] } + // Get the base visitor class from the parser const BaseVisitor = parser.getBaseCstVisitorConstructor() @@ -1461,7 +1465,9 @@ class QuestDBVisitor extends BaseVisitor { } if (ctx.dedupClause) { - result.dedupKeys = this.visit(ctx.dedupClause) as string[] + const dedup = this.visit(ctx.dedupClause) as DedupModeKeys + result.dedupKeys = dedup.keys + result.dedupMode = dedup.mode } // FORMAT { PARQUET | NATIVE } — accepted in three positions; take the @@ -2143,10 +2149,21 @@ class QuestDBVisitor extends BaseVisitor { return "DAYS" } - dedupClause(ctx: DedupClauseCstChildren): string[] { - return ctx.identifier.map((id: IdentifierCstNode) => - this.extractIdentifierName(id.children), - ) + dedupClause(ctx: DedupClauseCstChildren): DedupModeKeys { + return this.visit(ctx.dedupModeKeys) as DedupModeKeys + } + + dedupKeyword(_ctx: DedupKeywordCstChildren): void { + // DEDUP | DEDUPLICATE carry no value; printed as DEDUP + } + + dedupModeKeys(ctx: DedupModeKeysCstChildren): DedupModeKeys { + return { + mode: ctx.Skip ? "skip" : "upsert", + keys: ctx.identifier.map((id: IdentifierCstNode) => + this.extractIdentifierName(id.children), + ), + } } // ========================================================================== @@ -2668,18 +2685,18 @@ class QuestDBVisitor extends BaseVisitor { } } - if (ctx.Dedup && ctx.Disable) { + if (ctx.dedupKeyword && ctx.Disable) { return { actionType: "dedupDisable", } } - if (ctx.Dedup && ctx.Enable) { + if (ctx.dedupKeyword && ctx.dedupModeKeys) { + const dedup = this.visit(ctx.dedupModeKeys) as DedupModeKeys return { actionType: "dedupEnable", - keys: ctx.identifier!.map((id: IdentifierCstNode) => - this.extractIdentifierName(id.children), - ), + keys: dedup.keys, + dedupMode: dedup.mode, } } diff --git a/tests/dedup-skip.test.ts b/tests/dedup-skip.test.ts new file mode 100644 index 0000000..bf81db4 --- /dev/null +++ b/tests/dedup-skip.test.ts @@ -0,0 +1,222 @@ +/** + * DEDUP UPSERT | SKIP KEYS, the DEDUPLICATE synonym, ALTER TABLE DEDUP + * without ENABLE (questdb/questdb#7885, reports §1.2). + * + * Server grammar (SqlParser CREATE TABLE dedup block, + * SqlCompilerImpl.alterTableDedupEnable and the ALTER TABLE dedup branch): + * CREATE TABLE ... WAL (DEDUP | DEDUPLICATE) (UPSERT | SKIP) KEYS(col, ...) + * [FORMAT PARQUET | NATIVE] [OWNED BY ...] + * ALTER TABLE t (DEDUP | DEDUPLICATE) DISABLE + * ALTER TABLE t (DEDUP | DEDUPLICATE) [ENABLE] (UPSERT | SKIP) KEYS(col, ...) + * `skip` and `deduplicate` are not reserved words on the server. + * + * Server queries for this feature live in tests/fixtures/server-queries.json. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const parseOne = (sql: string): AST.Statement => { + const result = parseToAst(sql) + expect(result.errors.map((e) => e.message)).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +/** true when sql is rejected or does not read as a single statement */ +const rejected = (sql: string) => { + const result = parseToAst(sql) + return result.errors.length > 0 || result.ast.length !== 1 +} + +const CREATE = + "CREATE TABLE t (ts TIMESTAMP, s SYMBOL) TIMESTAMP(ts) PARTITION BY DAY WAL" + +describe("CREATE TABLE dedup mode", () => { + it.each([ + [`${CREATE} DEDUP UPSERT KEYS(ts, s)`, "upsert"], + [`${CREATE} DEDUP SKIP KEYS(ts, s)`, "skip"], + [`${CREATE} DEDUPLICATE UPSERT KEYS(ts, s)`, "upsert"], + [`${CREATE} DEDUPLICATE SKIP KEYS(ts, s)`, "skip"], + ])("%s", (sql, mode) => { + const stmt = parseOne(sql) as AST.CreateTableStatement + expect(stmt.type).toBe("createTable") + expect(stmt.dedupKeys).toEqual(["ts", "s"]) + expect(stmt.dedupMode).toBe(mode) + }) + + it("has no dedupMode without a DEDUP clause", () => { + const stmt = parseOne(CREATE) as AST.CreateTableStatement + expect(stmt.dedupKeys).toBeUndefined() + expect(stmt.dedupMode).toBeUndefined() + }) + + it.each([ + [`${CREATE} DEDUP SKIP KEYS(ts, s)`, `${CREATE} DEDUP SKIP KEYS(ts, s)`], + [`${CREATE} DEDUP UPSERT KEYS(ts)`, `${CREATE} DEDUP UPSERT KEYS(ts)`], + [`${CREATE} DEDUPLICATE SKIP KEYS(ts)`, `${CREATE} DEDUP SKIP KEYS(ts)`], + [`${CREATE} dedup skip keys(ts)`, `${CREATE} DEDUP SKIP KEYS(ts)`], + [ + `${CREATE} DEDUP SKIP KEYS(ts) FORMAT PARQUET`, + `${CREATE} FORMAT PARQUET DEDUP SKIP KEYS(ts)`, + ], + [ + `${CREATE} DEDUPLICATE SKIP KEYS(ts) FORMAT PARQUET OWNED BY bob`, + `${CREATE} FORMAT PARQUET DEDUP SKIP KEYS(ts) OWNED BY bob`, + ], + [ + "CREATE TABLE t AS (SELECT * FROM x) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP SKIP KEYS(ts) FORMAT PARQUET", + // FORMAT position is not tracked; toSql prints it before DEDUP + "CREATE TABLE t AS (SELECT * FROM x) TIMESTAMP(ts) PARTITION BY DAY WAL FORMAT PARQUET DEDUP SKIP KEYS(ts)", + ], + ])("round trips %s", (sql, expected) => { + const result = parseToAst(sql) + expect(result.errors.map((e) => e.message)).toEqual([]) + expect(result.ast).toHaveLength(1) + expect(toSql(result.ast)).toBe(expected) + const again = parseToAst(expected) + expect(again.errors).toEqual([]) + expect(toSql(again.ast)).toBe(expected) + }) + + it("keeps the CTAS dedup mode", () => { + const stmt = parseOne( + "CREATE TABLE t AS (SELECT * FROM x) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP SKIP KEYS(ts) FORMAT PARQUET", + ) as AST.CreateTableStatement + expect(stmt.dedupMode).toBe("skip") + expect(stmt.dedupKeys).toEqual(["ts"]) + expect(stmt.tableFormat).toBe("parquet") + }) +}) + +describe("ALTER TABLE dedup", () => { + const action = (sql: string) => + (parseOne(sql) as AST.AlterTableStatement).action + + it.each([ + ["ALTER TABLE t DEDUP ENABLE UPSERT KEYS (ts, s)", "upsert"], + ["ALTER TABLE t DEDUP ENABLE SKIP KEYS (ts, s)", "skip"], + ["ALTER TABLE t DEDUP UPSERT KEYS(ts, s)", "upsert"], + ["ALTER TABLE t DEDUP SKIP KEYS(ts, s)", "skip"], + ["ALTER TABLE t DEDUPLICATE ENABLE SKIP KEYS(ts, s)", "skip"], + ["ALTER TABLE t DEDUPLICATE UPSERT KEYS(ts, s)", "upsert"], + ])("%s", (sql, mode) => { + expect(action(sql)).toEqual({ + actionType: "dedupEnable", + keys: ["ts", "s"], + dedupMode: mode, + }) + }) + + it.each(["ALTER TABLE t DEDUP DISABLE", "ALTER TABLE t DEDUPLICATE DISABLE"])( + "%s", + (sql) => { + expect(action(sql)).toEqual({ actionType: "dedupDisable" }) + expect(toSql(parseToAst(sql).ast)).toBe("ALTER TABLE t DEDUP DISABLE") + }, + ) + + it.each([ + [ + "ALTER TABLE t DEDUP SKIP KEYS(ts, s)", + "ALTER TABLE t DEDUP ENABLE SKIP KEYS (ts, s)", + ], + [ + "ALTER TABLE t DEDUP UPSERT KEYS(ts)", + "ALTER TABLE t DEDUP ENABLE UPSERT KEYS (ts)", + ], + [ + "ALTER TABLE t DEDUPLICATE ENABLE SKIP KEYS(ts)", + "ALTER TABLE t DEDUP ENABLE SKIP KEYS (ts)", + ], + [ + "ALTER TABLE t DEDUP ENABLE SKIP KEYS (ts)", + "ALTER TABLE t DEDUP ENABLE SKIP KEYS (ts)", + ], + ])("round trips %s", (sql, expected) => { + expect(toSql(parseToAst(sql).ast)).toBe(expected) + expect(toSql(parseToAst(expected).ast)).toBe(expected) + }) +}) + +describe("rejected dedup forms", () => { + it.each([ + // CreateTableDedupTest: server errors + "alter table a dedup enable skipped keys(ts)", + "alter table a dedup skip (ts)", + "ALTER TABLE t DEDUP ENABLE KEYS(ts)", + "ALTER TABLE t DEDUP KEYS(ts)", + `${CREATE} DEDUP KEYS(ts)`, + `${CREATE} DEDUP SKIP (ts)`, + `${CREATE} DEDUPLICATE KEYS(ts)`, + `${CREATE} DEDUP ENABLE SKIP KEYS(ts)`, + ])("%s", (sql) => { + expect(rejected(sql)).toBe(true) + }) +}) + +describe("skip and deduplicate stay usable as identifiers", () => { + it.each([ + "SELECT skip FROM t", + "SELECT deduplicate FROM t", + "SELECT skip, deduplicate FROM skip", + "CREATE TABLE skip (skip INT, deduplicate INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP SKIP KEYS(ts, skip)", + "CREATE TABLE deduplicate (ts TIMESTAMP)", + "SELECT * FROM t WHERE skip > 0", + "ALTER TABLE skip DEDUP DISABLE", + "ALTER TABLE t DEDUP ENABLE SKIP KEYS (ts, skip)", + ])("%s", (sql) => { + const stmt = parseToAst(sql) + expect(stmt.errors.map((e) => e.message)).toEqual([]) + expect(stmt.ast).toHaveLength(1) + expect(toSql(stmt.ast)).toBe(sql) + }) +}) + +describe("autocomplete", () => { + it.each([`${CREATE} DEDUP `, `${CREATE} DEDUPLICATE `])( + "offers UPSERT and SKIP after: %s", + (sql) => { + expect([...getNextValidTokens(sql)].sort()).toEqual(["Skip", "Upsert"]) + }, + ) + + it.each(["ALTER TABLE t DEDUP ", "ALTER TABLE t DEDUPLICATE "])( + "offers ENABLE, DISABLE, UPSERT and SKIP after: %s", + (sql) => { + expect([...getNextValidTokens(sql)].sort()).toEqual([ + "Disable", + "Enable", + "Skip", + "Upsert", + ]) + }, + ) + + it("offers UPSERT and SKIP after ALTER TABLE t DEDUP ENABLE", () => { + expect( + [...getNextValidTokens("ALTER TABLE t DEDUP ENABLE ")].sort(), + ).toEqual(["Skip", "Upsert"]) + }) + + it("offers KEYS after SKIP", () => { + expect(getNextValidTokens(`${CREATE} DEDUP SKIP `)).toEqual(["Keys"]) + expect(getNextValidTokens("ALTER TABLE t DEDUP SKIP ")).toEqual(["Keys"]) + }) + + it("offers DEDUP and DEDUPLICATE after WAL in CREATE TABLE", () => { + const tokens = getNextValidTokens(`${CREATE} `) + expect(tokens).toContain("Dedup") + expect(tokens).toContain("Deduplicate") + }) + + it("does not offer SKIP or DEDUPLICATE at statement start or after SELECT", () => { + for (const sql of ["", "SELECT ", "SELECT a FROM t "]) { + const tokens = getNextValidTokens(sql) + expect(tokens).not.toContain("Deduplicate") + expect(tokens).not.toContain("Skip") + } + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 211d9ff..4a27471 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -346,5 +346,101 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/groupby/SampleByFillTest.java:2452", "feature": "grouping-sets", "toSql": "SELECT ts, sum(val) FROM x SAMPLE BY 1d GROUPING SETS (()) FILL(NULL)" + }, + { + "query": "CREATE TABLE pt (s SYMBOL INDEX, v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP SKIP KEYS(ts, s)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/parquet/O3ParquetStaleReaderTest.java:2806", + "feature": "dedup-skip", + "toSql": "CREATE TABLE pt (s SYMBOL INDEX, v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP SKIP KEYS(ts, s)" + }, + { + "query": "CREATE TABLE t (ts TIMESTAMP, k1 VARCHAR, k2 STRING, v LONG) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP SKIP KEYS(ts, k1, k2)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/wal/DedupSkipTest.java:1390", + "feature": "dedup-skip", + "toSql": "CREATE TABLE t (ts TIMESTAMP, k1 VARCHAR, k2 STRING, v LONG) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP SKIP KEYS(ts, k1, k2)" + }, + { + "query": "CREATE TABLE t (ts TIMESTAMP, sym SYMBOL, v LONG) TIMESTAMP(ts) PARTITION BY DAY FORMAT PARQUET WAL DEDUP SKIP KEYS(ts, sym)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/wal/DedupSkipTest.java:405", + "feature": "dedup-skip", + "toSql": "CREATE TABLE t (ts TIMESTAMP, sym SYMBOL, v LONG) TIMESTAMP(ts) PARTITION BY DAY WAL FORMAT PARQUET DEDUP SKIP KEYS(ts, sym)" + }, + { + "query": "CREATE TABLE c (ts TIMESTAMP, k INT, v LONG) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP SKIP KEYS(ts, k)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/AlterTableSetTypeRestartTest.java:711", + "feature": "dedup-skip", + "toSql": "CREATE TABLE c (ts TIMESTAMP, k INT, v LONG) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP SKIP KEYS(ts, k)" + }, + { + "query": "CREATE TABLE x AS (\n SELECT timestamp_sequence('2024-01-01', 1_000_000) ts, 'a'::SYMBOL sym, x v FROM long_sequence(3)\n) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP SKIP KEYS(ts, sym)\n", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CreateTableDedupTest.java:190", + "feature": "dedup-skip", + "toSql": "CREATE TABLE x AS (SELECT timestamp_sequence('2024-01-01', 1_000_000) AS ts, 'a'::SYMBOL AS sym, x AS v FROM long_sequence(3)) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP SKIP KEYS(ts, sym)" + }, + { + "query": "create table testAddIndexOnDeduplicatedColumn (ts TIMESTAMP, x long, s symbol, i int) timestamp(ts) PARTITION BY DAY WAL DEDUPLICATE UPSERT KEYS (ts, i, s ) ", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CreateTableDedupTest.java:61", + "feature": "dedup-skip", + "toSql": "CREATE TABLE testAddIndexOnDeduplicatedColumn (ts TIMESTAMP, x LONG, s SYMBOL, i INT) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP UPSERT KEYS(ts, i, s)" + }, + { + "query": "create table testDeduplicationEnabledTimestampAndSymbol (ts TIMESTAMP, x long, s symbol) timestamp(ts) PARTITION BY DAY WAL DEDUPLICATE UPSERT KEYS(ts, s)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CreateTableDedupTest.java:715", + "feature": "dedup-skip", + "toSql": "CREATE TABLE testDeduplicationEnabledTimestampAndSymbol (ts TIMESTAMP, x LONG, s SYMBOL) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP UPSERT KEYS(ts, s)" + }, + { + "query": "CREATE TABLE 'foo' ( \n\tts TIMESTAMP,\n\ts SYMBOL,\n\ti INT\n) timestamp(ts) PARTITION BY DAY\nDEDUP SKIP KEYS(ts,s);\n", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/ShowCreateTableTest.java:110", + "feature": "dedup-skip", + "toSql": "CREATE TABLE foo (ts TIMESTAMP, s SYMBOL, i INT) TIMESTAMP(ts) PARTITION BY DAY DEDUP SKIP KEYS(ts, s)" + }, + { + "query": "create table foo ( ts timestamp, s symbol, i int ) timestamp(ts) partition by day wal dedup skip keys(ts, s)", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/griffin/EntShowCreateTableTest.java:67", + "feature": "dedup-skip", + "toSql": "CREATE TABLE foo (ts TIMESTAMP, s SYMBOL, i INT) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP SKIP KEYS(ts, s)" + }, + { + "query": "ALTER TABLE t DEDUP ENABLE SKIP KEYS(ts, k)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/wal/DedupSkipTest.java:190", + "feature": "dedup-skip", + "toSql": "ALTER TABLE t DEDUP ENABLE SKIP KEYS (ts, k)" + }, + { + "query": "ALTER TABLE t DEDUP ENABLE SKIP KEYS(ts)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/wal/WalWriterTest.java:5983", + "feature": "dedup-skip", + "toSql": "ALTER TABLE t DEDUP ENABLE SKIP KEYS (ts)" + }, + { + "query": "ALTER TABLE t DEDUP SKIP KEYS(ts, sym)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/wal/DedupSkipTest.java:1484", + "feature": "dedup-skip", + "toSql": "ALTER TABLE t DEDUP ENABLE SKIP KEYS (ts, sym)" + }, + { + "query": "ALTER TABLE t DEDUP UPSERT KEYS(ts, sym)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/wal/DedupSkipTest.java:1477", + "feature": "dedup-skip", + "toSql": "ALTER TABLE t DEDUP ENABLE UPSERT KEYS (ts, sym)" + }, + { + "query": "alter table trips dedup upsert keys(Pickup_DateTime)", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/http/ImportIODispatcherTest.java:1063", + "feature": "dedup-skip", + "toSql": "ALTER TABLE trips DEDUP ENABLE UPSERT KEYS (Pickup_DateTime)" + }, + { + "query": "Alter table dups dedup upsert keys(ts)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CreateTableDedupTest.java:125", + "feature": "dedup-skip", + "toSql": "ALTER TABLE dups DEDUP ENABLE UPSERT KEYS (ts)" + }, + { + "query": "ALTER TABLE x DEDUPLICATE SKIP KEYS(ts)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CreateTableDedupTest.java:149", + "feature": "dedup-skip", + "toSql": "ALTER TABLE x DEDUP ENABLE SKIP KEYS (ts)" } ] From e59df460e92e9c0aba4555a37c6a3ade64a510d6 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 14:26:03 +0300 Subject: [PATCH 05/36] fix: require ON in EXPIRE ROWS KEEP HIGHEST|LOWEST and end WHEN at IN VOLUME Server (OSS #7263, SqlParser.parseExpireRowsClause) parses the relative retention form as `KEEP [N] HIGHEST|LOWEST ON [PARTITION BY ]` (expectTok "on"), and SHOW CREATE MATERIALIZED VIEW prints it with ON (RowExpiryUtil.appendKeepByClause). RowExpiryWindowTest:966-1001 asserts "'on' expected" for the form without ON. In CREATE MATERIALIZED VIEW the WHEN predicate ends at the clause boundary (expireRowsClauseBoundary): CLEANUP EVERY, IN VOLUME (and WITH / DEDUP, which mat views then reject; Ent OWNED BY via isExpireRowsClauseBoundary). Before: `... EXPIRE ROWS KEEP 2 LOWEST ON v PARTITION BY k` failed with "Unexpected token 'ON'" in CREATE and ALTER MATERIALIZED VIEW ... SET EXPIRE ROWS, while `KEEP HIGHEST v ...` (server error) was accepted. `EXPIRE ROWS WHEN v < 2.0 IN VOLUME 'v1'` was silently misparsed: IN was read as the IN operator, round trip `... IN VOLUME; v1` (two statements). Change: - parser expireRowsClause: CONSUME(On) between HIGHEST|LOWEST and the column. The WHEN predicate is parsed with a parser flag (inExpireRowsPredicate, restored in finally); setExpression's IN/BETWEEN option is gated off when the flag is set and the next tokens are IN VOLUME, so the predicate ends there and the mat view IN VOLUME clause takes it. Outside EXPIRE ROWS `x IN volume` is unchanged. CLEANUP, WITH, DEDUP and OWNED are not operators, so the predicate already stopped there. - toSql prints `HIGHEST|LOWEST ON `. AST unchanged (keepExtremum `column` already holds the ON column; visitor reads ctx.On only in the KEEP LATEST branch). - tests/fixtures/docs-queries.json: the docs example used the old form without ON; now `KEEP HIGHEST ON px ...` (upstream docs need the same fix). Tests: 17 server queries (MatViewExpireRowsTest, MatViewExpireRowsHardeningTest text block, RowExpiryKeepLatestTest, RowExpiryWindowTest, SqlCompilerImplTest incl. IN VOLUME + OWNED BY, Ent EntRowExpiryAclTest), tests/expire-rows-keep-on.test.ts (AST, toSql, negatives without ON, IN operator kept inside the predicate, `x IN volume` outside, autocomplete), parser.test.ts EXPIRE ROWS describes, a formatter fixture. Red 74 failing -> green; full suite 15702 passed. Corpus: fixed 26 / rtChanged 0 / regressed 6 / throws 0 / formatterChangedMeaning 0. All 6 regressions are the form without ON that the server rejects: 5 from RowExpiryWindowTest:973-993 (expects "'on' expected") and the old docs example. Suggestions: +ON after KEEP [N] HIGHEST|LOWEST (CREATE and ALTER); +PARTITION, CLEANUP, IN, OWNED after KEEP ... ON . Left out: `CLEANUP EVERY h` without a number (server defaults to 1, low priority). --- src/parser/ast.ts | 2 +- src/parser/cst-types.d.ts | 2 +- src/parser/parser.ts | 135 ++++++++++-------- src/parser/toSql.ts | 2 +- tests/expire-rows-keep-on.test.ts | 220 +++++++++++++++++++++++++++++ tests/fixtures/docs-queries.json | 2 +- tests/fixtures/server-queries.json | 102 +++++++++++++ tests/formatter/fixtures.ts | 12 ++ tests/parser.test.ts | 24 ++++ 9 files changed, 441 insertions(+), 60 deletions(-) create mode 100644 tests/expire-rows-keep-on.test.ts diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 9e74158..cc7db06 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -241,7 +241,7 @@ export interface ExpireRowsClause { predicate?: Expression // keepLatest: KEEP LATEST [ON ] PARTITION BY on?: string - // keepExtremum: KEEP [] HIGHEST|LOWEST [PARTITION BY ] + // keepExtremum: KEEP [] HIGHEST|LOWEST ON [PARTITION BY ] keepCount?: number extremum?: "highest" | "lowest" column?: string diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index eb6cd1d..d3626fa 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -1044,7 +1044,7 @@ export type ExpireRowsClauseCstChildren = { expression?: ExpressionCstNode[]; Keep?: IToken[]; Latest?: IToken[]; - On?: IToken[]; + On?: (IToken)[]; identifier?: (IdentifierCstNode)[]; Partition?: (IToken)[]; By?: (IToken)[]; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index da3dff9..99e15d6 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -438,6 +438,9 @@ class QuestDBParser extends CstParser { private _semicolonBoundedRecovery = false private _statementStartIdx = -1 + // True while parsing an EXPIRE ROWS WHEN predicate (see expireRowsClause). + private inExpireRowsPredicate = false + findReSyncTokenType(): TokenType { if (this._semicolonBoundedRecovery) { const currIdx = (this as unknown as CstParserInternals).currIdx @@ -2134,8 +2137,9 @@ class QuestDBParser extends CstParser { // EXPIRE ROWS row-retention clause (materialized views): // EXPIRE ROWS WHEN // EXPIRE ROWS KEEP LATEST [ON ] PARTITION BY - // EXPIRE ROWS KEEP [] (HIGHEST|LOWEST) [PARTITION BY ] + // EXPIRE ROWS KEEP [] (HIGHEST|LOWEST) ON [PARTITION BY ] // [ CLEANUP EVERY ] + // (server: SqlParser.parseExpireRowsClause; ON is required after HIGHEST|LOWEST) private expireRowsClause = this.RULE("expireRowsClause", () => { this.CONSUME(Expire) this.CONSUME(Rows) @@ -2144,7 +2148,15 @@ class QuestDBParser extends CstParser { { ALT: () => { this.CONSUME(When) - this.SUBRULE(this.expression) + // The predicate ends before `IN VOLUME` (server + // expireRowsClauseBoundary); setExpression reads this flag. + const outer = this.inExpireRowsPredicate + this.inExpireRowsPredicate = true + try { + this.SUBRULE(this.expression) + } finally { + this.inExpireRowsPredicate = outer + } }, }, // KEEP ... @@ -2169,7 +2181,7 @@ class QuestDBParser extends CstParser { }) }, }, - // KEEP [] HIGHEST|LOWEST [PARTITION BY ] + // KEEP [] HIGHEST|LOWEST ON [PARTITION BY ] { ALT: () => { this.OPTION1(() => this.CONSUME(NumberLiteral)) @@ -2177,6 +2189,7 @@ class QuestDBParser extends CstParser { { ALT: () => this.CONSUME(Highest) }, { ALT: () => this.CONSUME(Lowest) }, ]) + this.CONSUME1(On) this.SUBRULE3(this.identifier) this.OPTION2(() => { this.CONSUME1(Partition) @@ -4601,65 +4614,75 @@ class QuestDBParser extends CstParser { // Precedence 11: [NOT] IN, [NOT] BETWEEN, [NOT] LIKE, [NOT] ILIKE, WITHIN private setExpression = this.RULE("setExpression", () => { this.SUBRULE(this.bitOrExpression) - this.OPTION(() => { - this.OR([ - { - ALT: () => { - this.OPTION1(() => this.CONSUME(Not)) - this.CONSUME(In) - this.OR1([ - { - GATE: () => this.LA(1).tokenType === LParen, - ALT: () => { - this.CONSUME(LParen) - this.SUBRULE2(this.expression) - this.MANY(() => { - this.CONSUME(Comma) - this.SUBRULE3(this.expression) - }) - this.CONSUME(RParen) + this.OPTION({ + // An EXPIRE ROWS WHEN predicate ends before `IN VOLUME` (the mat view + // volume clause), so `IN` there is not the IN operator. + GATE: () => + !( + this.inExpireRowsPredicate && + this.LA(1).tokenType === In && + this.LA(2).tokenType === Volume + ), + DEF: () => { + this.OR([ + { + ALT: () => { + this.OPTION1(() => this.CONSUME(Not)) + this.CONSUME(In) + this.OR1([ + { + GATE: () => this.LA(1).tokenType === LParen, + ALT: () => { + this.CONSUME(LParen) + this.SUBRULE2(this.expression) + this.MANY(() => { + this.CONSUME(Comma) + this.SUBRULE3(this.expression) + }) + this.CONSUME(RParen) + }, }, - }, - { - ALT: () => { - this.SUBRULE3(this.bitOrExpression, { LABEL: "inValue" }) + { + ALT: () => { + this.SUBRULE3(this.bitOrExpression, { LABEL: "inValue" }) + }, }, - }, - ]) + ]) + }, }, - }, - { - ALT: () => { - this.OPTION2(() => this.CONSUME1(Not)) - this.CONSUME(Between) - this.SUBRULE1(this.bitOrExpression, { LABEL: "betweenLow" }) - this.CONSUME(And) - this.SUBRULE2(this.bitOrExpression, { LABEL: "betweenHigh" }) + { + ALT: () => { + this.OPTION2(() => this.CONSUME1(Not)) + this.CONSUME(Between) + this.SUBRULE1(this.bitOrExpression, { LABEL: "betweenLow" }) + this.CONSUME(And) + this.SUBRULE2(this.bitOrExpression, { LABEL: "betweenHigh" }) + }, }, - }, - { - ALT: () => { - this.CONSUME(Within) - this.CONSUME1(LParen) - this.SUBRULE5(this.expression) - this.MANY1(() => { - this.CONSUME1(Comma) - this.SUBRULE6(this.expression) - }) - this.CONSUME1(RParen) + { + ALT: () => { + this.CONSUME(Within) + this.CONSUME1(LParen) + this.SUBRULE5(this.expression) + this.MANY1(() => { + this.CONSUME1(Comma) + this.SUBRULE6(this.expression) + }) + this.CONSUME1(RParen) + }, }, - }, - { - ALT: () => { - this.CONSUME2(Not) - this.OR2([ - { ALT: () => this.CONSUME(Like) }, - { ALT: () => this.CONSUME(Ilike) }, - ]) - this.SUBRULE4(this.bitOrExpression, { LABEL: "notLikeRight" }) + { + ALT: () => { + this.CONSUME2(Not) + this.OR2([ + { ALT: () => this.CONSUME(Like) }, + { ALT: () => this.CONSUME(Ilike) }, + ]) + this.SUBRULE4(this.bitOrExpression, { LABEL: "notLikeRight" }) + }, }, - }, - ]) + ]) + }, }) }) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 68162d2..cea8bff 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -1281,7 +1281,7 @@ function expireRowsClauseToSql(clause: AST.ExpireRowsClause): string { parts.push("KEEP") if (clause.keepCount !== undefined) parts.push(String(clause.keepCount)) parts.push(clause.extremum === "lowest" ? "LOWEST" : "HIGHEST") - parts.push(escapeIdentifier(clause.column!)) + parts.push("ON", escapeIdentifier(clause.column!)) if (clause.partitionBy && clause.partitionBy.length > 0) { parts.push( `PARTITION BY ${clause.partitionBy.map(escapeIdentifier).join(", ")}`, diff --git a/tests/expire-rows-keep-on.test.ts b/tests/expire-rows-keep-on.test.ts new file mode 100644 index 0000000..d526efe --- /dev/null +++ b/tests/expire-rows-keep-on.test.ts @@ -0,0 +1,220 @@ +/** + * EXPIRE ROWS KEEP [N] HIGHEST|LOWEST ON and the WHEN ... IN VOLUME + * boundary (questdb/questdb#7263, reports §1.3). + * + * Server grammar (SqlParser.parseExpireRowsClause, expireRowsClauseBoundary): + * EXPIRE ROWS WHEN [CLEANUP EVERY ] + * EXPIRE ROWS KEEP LATEST [ON ] PARTITION BY [CLEANUP EVERY ...] + * EXPIRE ROWS KEEP [] HIGHEST|LOWEST ON [PARTITION BY ] + * [CLEANUP EVERY ...] + * `ON` is required in the HIGHEST/LOWEST form (expectTok "on"). In CREATE + * MATERIALIZED VIEW the WHEN predicate ends at `IN VOLUME` (the tail clause). + * + * Server queries for this feature live in tests/fixtures/server-queries.json. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const parseOne = (sql: string): AST.Statement => { + const result = parseToAst(sql) + expect(result.errors.map((e) => e.message)).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +/** true when sql is rejected or does not read as a single statement */ +const rejected = (sql: string) => { + const result = parseToAst(sql) + return result.errors.length > 0 || result.ast.length !== 1 +} + +const MV = "CREATE MATERIALIZED VIEW mv AS (SELECT * FROM base)" +const ALTER = "ALTER MATERIALIZED VIEW mv SET" + +const createExpire = (sql: string): AST.ExpireRowsClause => { + const stmt = parseOne(sql) as AST.CreateMaterializedViewStatement + expect(stmt.type).toBe("createMaterializedView") + return stmt.expireRows! +} + +describe("KEEP HIGHEST|LOWEST ON ", () => { + it("parses KEEP HIGHEST ON col PARTITION BY keys", () => { + expect( + createExpire(`${MV} EXPIRE ROWS KEEP HIGHEST ON v PARTITION BY k, j`), + ).toEqual({ + mode: "keepExtremum", + extremum: "highest", + column: "v", + partitionBy: ["k", "j"], + }) + }) + + it("parses KEEP N LOWEST ON col without PARTITION BY", () => { + expect(createExpire(`${MV} EXPIRE ROWS KEEP 3 LOWEST ON v`)).toEqual({ + mode: "keepExtremum", + extremum: "lowest", + column: "v", + keepCount: 3, + }) + }) + + it("parses a quoted column and CLEANUP EVERY", () => { + expect( + createExpire( + `${MV} EXPIRE ROWS KEEP HIGHEST ON "my val" PARTITION BY k CLEANUP EVERY 30m`, + ), + ).toEqual({ + mode: "keepExtremum", + extremum: "highest", + column: "my val", + partitionBy: ["k"], + cleanupEvery: "30m", + }) + }) + + it("parses ALTER MATERIALIZED VIEW ... SET EXPIRE ROWS KEEP ... ON", () => { + const stmt = parseOne( + `${ALTER} EXPIRE ROWS KEEP 2 LOWEST ON v PARTITION BY k`, + ) as AST.AlterMaterializedViewStatement + expect(stmt.action).toEqual({ + actionType: "setExpireRows", + expireRows: { + mode: "keepExtremum", + extremum: "lowest", + column: "v", + keepCount: 2, + partitionBy: ["k"], + }, + }) + }) + + it.each([ + [ + `${MV} EXPIRE ROWS KEEP HIGHEST ON v PARTITION BY k`, + `${MV} EXPIRE ROWS KEEP HIGHEST ON v PARTITION BY k`, + ], + [ + `${MV} expire rows keep 1 lowest on v cleanup every 1h`, + `${MV} EXPIRE ROWS KEEP 1 LOWEST ON v CLEANUP EVERY 1h`, + ], + [ + `${ALTER} EXPIRE ROWS KEEP HIGHEST ON "a b" PARTITION BY k`, + `${ALTER} EXPIRE ROWS KEEP HIGHEST ON "a b" PARTITION BY k`, + ], + ])("toSql prints ON: %s", (sql, expected) => { + const out = toSql(parseOne(sql)) + expect(out).toBe(expected) + expect(toSql(parseOne(out))).toBe(out) + }) + + it.each([ + `${MV} EXPIRE ROWS KEEP HIGHEST v PARTITION BY k`, + `${MV} EXPIRE ROWS KEEP 2 LOWEST v`, + `${MV} EXPIRE ROWS KEEP HIGHEST ON PARTITION BY k`, + `${ALTER} EXPIRE ROWS KEEP HIGHEST v PARTITION BY k`, + `${ALTER} EXPIRE ROWS KEEP LOWEST v`, + ])("rejects the form without a column after ON: %s", (sql) => { + expect(rejected(sql)).toBe(true) + }) + + it("keeps KEEP LATEST [ON ts] working", () => { + expect( + createExpire(`${MV} EXPIRE ROWS KEEP LATEST ON ts PARTITION BY k`), + ).toEqual({ mode: "keepLatest", on: "ts", partitionBy: ["k"] }) + expect( + createExpire(`${MV} EXPIRE ROWS KEEP LATEST PARTITION BY k, j`), + ).toEqual({ mode: "keepLatest", partitionBy: ["k", "j"] }) + }) +}) + +describe("WHEN predicate ends at IN VOLUME", () => { + it("reads IN VOLUME as the mat view volume clause", () => { + const stmt = parseOne( + `${MV} EXPIRE ROWS WHEN v < 2.0 IN VOLUME 'v1'`, + ) as AST.CreateMaterializedViewStatement + expect(stmt.volume).toBe("v1") + expect(stmt.expireRows?.mode).toBe("when") + expect(toSql(stmt)).toBe(`${MV} EXPIRE ROWS WHEN v < 2.0 IN VOLUME v1`) + }) + + it("stops before IN VOLUME with OWNED BY after it", () => { + const stmt = parseOne( + `${MV} EXPIRE ROWS WHEN v < 2 IN VOLUME vol1 OWNED BY 'g'`, + ) as AST.CreateMaterializedViewStatement + expect(stmt.volume).toBe("vol1") + expect(stmt.ownedBy).toBe("g") + expect(toSql(stmt)).toBe( + `${MV} EXPIRE ROWS WHEN v < 2 IN VOLUME vol1 OWNED BY g`, + ) + }) + + it("stops before IN VOLUME after CLEANUP EVERY and after KEEP", () => { + expect( + toSql( + parseOne( + `${MV} EXPIRE ROWS WHEN v < 2 CLEANUP EVERY 1h IN VOLUME vol1`, + ), + ), + ).toBe(`${MV} EXPIRE ROWS WHEN v < 2 CLEANUP EVERY 1h IN VOLUME vol1`) + expect( + toSql(parseOne(`${MV} EXPIRE ROWS KEEP LOWEST ON v IN VOLUME 'vol1'`)), + ).toBe(`${MV} EXPIRE ROWS KEEP LOWEST ON v IN VOLUME vol1`) + }) + + it("keeps the IN operator inside the predicate", () => { + expect( + toSql( + parseOne( + `${MV} EXPIRE ROWS WHEN k IN ('a', 'b') AND v IN (1, 2) IN VOLUME vol1`, + ), + ), + ).toBe( + `${MV} EXPIRE ROWS WHEN k IN ('a', 'b') AND v IN (1, 2) IN VOLUME vol1`, + ) + expect(toSql(parseOne(`${MV} EXPIRE ROWS WHEN ts IN '2024-01-01'`))).toBe( + `${MV} EXPIRE ROWS WHEN ts IN '2024-01-01'`, + ) + }) + + it("leaves `x IN volume` alone outside EXPIRE ROWS", () => { + expect(toSql(parseOne("SELECT * FROM t WHERE x IN volume"))).toBe( + "SELECT * FROM t WHERE x IN volume", + ) + }) + + it("does not read IN VOLUME as part of an ALTER predicate", () => { + // ALTER ... SET EXPIRE ROWS has no IN VOLUME tail; the server keeps + // IN VOLUME in the predicate text and fails, so this must not parse + // as one clean statement. + expect(rejected(`${ALTER} EXPIRE ROWS WHEN v < 2 IN VOLUME vol1`)).toBe( + true, + ) + }) +}) + +describe("autocomplete", () => { + it("suggests ON after KEEP HIGHEST|LOWEST", () => { + expect(getNextValidTokens(`${MV} EXPIRE ROWS KEEP HIGHEST `)).toEqual([ + "On", + ]) + expect(getNextValidTokens(`${ALTER} EXPIRE ROWS KEEP 2 LOWEST `)).toEqual([ + "On", + ]) + }) + + it("suggests HIGHEST, LOWEST after KEEP ", () => { + expect([...getNextValidTokens(`${MV} EXPIRE ROWS KEEP 2 `)].sort()).toEqual( + ["Highest", "Lowest"], + ) + }) + + it("suggests the clause tail after KEEP HIGHEST ON col", () => { + const tokens = getNextValidTokens(`${MV} EXPIRE ROWS KEEP HIGHEST ON v `) + expect(tokens).toEqual( + expect.arrayContaining(["Partition", "Cleanup", "In", "Owned"]), + ) + }) +}) diff --git a/tests/fixtures/docs-queries.json b/tests/fixtures/docs-queries.json index d44ce80..28bf013 100644 --- a/tests/fixtures/docs-queries.json +++ b/tests/fixtures/docs-queries.json @@ -5410,7 +5410,7 @@ "query": "CREATE MATERIALIZED VIEW price_1h AS (SELECT ts, avg(px) FROM trades SAMPLE BY 1h) PARTITION BY DAY EXPIRE ROWS KEEP LATEST PARTITION BY sym" }, { - "query": "ALTER MATERIALIZED VIEW price_1h SET EXPIRE ROWS KEEP HIGHEST px PARTITION BY sym CLEANUP EVERY 1h" + "query": "ALTER MATERIALIZED VIEW price_1h SET EXPIRE ROWS KEEP HIGHEST ON px PARTITION BY sym CLEANUP EVERY 1h" }, { "query": "ALTER MATERIALIZED VIEW price_1h DROP EXPIRE ROWS" diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 4a27471..e08badd 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -442,5 +442,107 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/CreateTableDedupTest.java:149", "feature": "dedup-skip", "toSql": "ALTER TABLE x DEDUP ENABLE SKIP KEYS (ts)" + }, + { + "query": "create materialized view mv as (select * from base) expire rows keep highest on v partition by k", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewExpireRowsTest.java:1407", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW mv AS (SELECT * FROM base) EXPIRE ROWS KEEP HIGHEST ON v PARTITION BY k" + }, + { + "query": "alter materialized view mv set expire rows keep highest on v partition by k", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewExpireRowsTest.java:1423", + "feature": "expire-rows-keep-on", + "toSql": "ALTER MATERIALIZED VIEW mv SET EXPIRE ROWS KEEP HIGHEST ON v PARTITION BY k" + }, + { + "query": "create materialized view mv as (select * from base) partition by day ttl 3 days expire rows keep 1 highest on v partition by k", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewExpireRowsTest.java:2766", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW mv AS (SELECT * FROM base) PARTITION BY DAY TTL 3 DAYS EXPIRE ROWS KEEP 1 HIGHEST ON v PARTITION BY k" + }, + { + "query": "create materialized view mv as (select * from base) partition by day expire rows keep 1 highest on v partition by k cleanup every 30m", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewExpireRowsTest.java:2976", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW mv AS (SELECT * FROM base) PARTITION BY DAY EXPIRE ROWS KEEP 1 HIGHEST ON v PARTITION BY k CLEANUP EVERY 30m" + }, + { + "query": "create materialized view mv2 as (select * from base2) EXPIRE ROWS KEEP HIGHEST ON v partition by k", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewExpireRowsTest.java:3725", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW mv2 AS (SELECT * FROM base2) EXPIRE ROWS KEEP HIGHEST ON v PARTITION BY k" + }, + { + "query": "ALTER MATERIALIZED VIEW mv SET EXPIRE ROWS KEEP HIGHEST ON v PARTITION BY \"k limit 1\", k CLEANUP EVERY 1h;", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/RowExpiryKeepLatestTest.java:229", + "feature": "expire-rows-keep-on", + "toSql": "ALTER MATERIALIZED VIEW mv SET EXPIRE ROWS KEEP HIGHEST ON v PARTITION BY \"k limit 1\", k CLEANUP EVERY 1h" + }, + { + "query": "create materialized view mv as (select * from base) expire rows keep highest on \"my val\" partition by k", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/RowExpiryWindowTest.java:668", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW mv AS (SELECT * FROM base) EXPIRE ROWS KEEP HIGHEST ON \"my val\" PARTITION BY k" + }, + { + "query": "create materialized view mvbad as (select * from base) expire rows keep 2 lowest on v partition by k, nope", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/RowExpiryWindowTest.java:762", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW mvbad AS (SELECT * FROM base) EXPIRE ROWS KEEP 2 LOWEST ON v PARTITION BY k, nope" + }, + { + "query": "create materialized view mv as (select * from base) expire rows keep lowest on v partition by k", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/RowExpiryWindowTest.java:1093", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW mv AS (SELECT * FROM base) EXPIRE ROWS KEEP LOWEST ON v PARTITION BY k" + }, + { + "query": "ALTER MATERIALIZED VIEW mvalt SET EXPIRE ROWS KEEP 2 LOWEST ON nope PARTITION BY k", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/RowExpiryWindowTest.java:1028", + "feature": "expire-rows-keep-on", + "toSql": "ALTER MATERIALIZED VIEW mvalt SET EXPIRE ROWS KEEP 2 LOWEST ON nope PARTITION BY k" + }, + { + "query": "CREATE MATERIALIZED VIEW mv AS (SELECT * FROM base)\nEXPIRE ROWS KEEP HIGHEST ON v PARTITION BY k CLEANUP EVERY 1s", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewExpireRowsHardeningTest.java:764", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW mv AS (SELECT * FROM base) EXPIRE ROWS KEEP HIGHEST ON v PARTITION BY k CLEANUP EVERY 1s" + }, + { + "query": "create materialized view no_keys as (select * from erb_base) expire rows KEEP HIGHEST ON v OWNED BY 'group'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/SqlCompilerImplTest.java:8590", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW no_keys AS (SELECT * FROM erb_base) EXPIRE ROWS KEEP HIGHEST ON v OWNED BY \"group\"" + }, + { + "query": "create materialized view top_n as (select * from erb_base) expire rows KEEP 2 HIGHEST ON v PARTITION BY k OWNED BY 'group'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/SqlCompilerImplTest.java:8589", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW top_n AS (SELECT * FROM erb_base) EXPIRE ROWS KEEP 2 HIGHEST ON v PARTITION BY k OWNED BY \"group\"" + }, + { + "query": "create materialized view in_volume as (select * from erb_base) expire rows when v < 2 in volume vol1 owned by 'group'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/SqlCompilerImplTest.java:8620", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW in_volume AS (SELECT * FROM erb_base) EXPIRE ROWS WHEN v < 2 IN VOLUME vol1 OWNED BY \"group\"" + }, + { + "query": "CREATE MATERIALIZED VIEW mvv AS (SELECT * FROM base) EXPIRE ROWS WHEN v < 2.0 IN VOLUME 'vol1'", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewExpireRowsTest.java:2426", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW mvv AS (SELECT * FROM base) EXPIRE ROWS WHEN v < 2.0 IN VOLUME vol1" + }, + { + "query": "CREATE MATERIALIZED VIEW mvv AS (SELECT * FROM base) EXPIRE ROWS KEEP HIGHEST ON v PARTITION BY k IN VOLUME 'vol1'", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewExpireRowsTest.java:2426", + "feature": "expire-rows-keep-on", + "toSql": "CREATE MATERIALIZED VIEW mvv AS (SELECT * FROM base) EXPIRE ROWS KEEP HIGHEST ON v PARTITION BY k IN VOLUME vol1" + }, + { + "query": "ALTER MATERIALIZED VIEW mv SET EXPIRE ROWS KEEP 1 HIGHEST ON v PARTITION BY k", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/acl/EntRowExpiryAclTest.java:586", + "feature": "expire-rows-keep-on", + "toSql": "ALTER MATERIALIZED VIEW mv SET EXPIRE ROWS KEEP 1 HIGHEST ON v PARTITION BY k" } ] diff --git a/tests/formatter/fixtures.ts b/tests/formatter/fixtures.ts index 7892615..4c18b20 100644 --- a/tests/formatter/fixtures.ts +++ b/tests/formatter/fixtures.ts @@ -115,6 +115,18 @@ export const fixtures: Fixture[] = [ "EXPIRE ROWS KEEP LATEST PARTITION BY sym", ].join("\n"), }, + { + name: "EXPIRE ROWS KEEP HIGHEST ON stays on one line", + input: + "CREATE MATERIALIZED VIEW price_1h AS (SELECT ts, px, sym FROM trades) EXPIRE ROWS KEEP 2 HIGHEST ON px PARTITION BY sym CLEANUP EVERY 1h", + expected: [ + "CREATE MATERIALIZED VIEW price_1h AS (", + " SELECT ts, px, sym", + " FROM trades", + ")", + "EXPIRE ROWS KEEP 2 HIGHEST ON px PARTITION BY sym CLEANUP EVERY 1h", + ].join("\n"), + }, { name: "UPDATE joins break like SELECT joins", input: "UPDATE t SET a = 1 FROM u JOIN v ON u.x = v.x WHERE t.id = u.id", diff --git a/tests/parser.test.ts b/tests/parser.test.ts index 300c076..3e8ab7f 100644 --- a/tests/parser.test.ts +++ b/tests/parser.test.ts @@ -8197,6 +8197,25 @@ orders PIVOT (sum(amount) FOR status IN ('open'))` ).ast[0] as AST.CreateMaterializedViewStatement expect(s.expireRows?.mode).toBe("keepLatest") }) + it("KEEP N HIGHEST ON col PARTITION BY", () => { + const s = parseToAst( + "CREATE MATERIALIZED VIEW mv AS (SELECT * FROM base) EXPIRE ROWS KEEP 2 HIGHEST ON v PARTITION BY k", + ).ast[0] as AST.CreateMaterializedViewStatement + expect(s.expireRows).toEqual({ + mode: "keepExtremum", + extremum: "highest", + column: "v", + keepCount: 2, + partitionBy: ["k"], + }) + }) + it("KEEP HIGHEST without ON is rejected", () => { + expect( + parseToAst( + "CREATE MATERIALIZED VIEW mv AS (SELECT * FROM base) EXPIRE ROWS KEEP HIGHEST v PARTITION BY k", + ).errors.length, + ).toBeGreaterThan(0) + }) it("ALTER ... DROP EXPIRE", () => { expect( parseToAst("ALTER MATERIALIZED VIEW mv DROP EXPIRE").errors, @@ -8529,6 +8548,11 @@ orders PIVOT (sum(amount) FOR status IN ('open'))` "create materialized view mv as (select * from base) expire rows when ts < now()", "create materialized view mv as (select * from base) expire rows when ts > now()", "create materialized view mv as (select * from base) expire rows when ts < cast(null as timestamp)", + "create materialized view mv as (select * from base) expire rows keep highest on v partition by k", + "create materialized view mv as (select * from base) expire rows keep 2 lowest on v partition by k, j cleanup every 1h", + "create materialized view mv as (select * from base) expire rows keep lowest on v", + "create materialized view mv as (select * from base) expire rows when v < 2.0 in volume 'v1'", + "alter materialized view mv set expire rows keep 1 highest on v partition by k", ] for (const query of queries) { it(`round-trips: ${query.slice(0, 64)}`, () => { From ac28eef9b981e4f2f5699463b74c7188eba1cad5 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 14:36:43 +0300 Subject: [PATCH 06/36] feat: parse implicit select bodies in mat views, CTAS, views and COPY The server parses the query body of CREATE MATERIALIZED VIEW (SqlParser.parseCreateMatView), CREATE TABLE AS (parseCreateTableAsSelect, '(' required), CREATE VIEW (parseCreateView), ALTER VIEW (parseViewSql) and COPY (...) TO (parseCopy) with parseDml, which reads a bare table, a table with clauses, a UNION of tables, a table function or `t PIVOT (...)` as an implicit SELECT * FROM .... OSS #7263 made passthrough mat views valid (MatViewTest: `create materialized view v5 as (base_price where price > 0)`, `... v4 as base_price`). Before: all of these failed with "Expecting Select" / "Unexpected token", except the parenthesised CREATE VIEW form (no set operations there). `CREATE TABLE y AS (x UNION ALL append)`, `COPY (t WHERE x > 0) TO ...`, `copy (generate_series(...)) TO ...`, `ALTER VIEW v AS t WHERE ...` and `CREATE VIEW v AS t WHERE x > 0` all errored. Change: - createMaterializedViewBody, createViewBody, alterViewStatement: body is `(SELECT...)` | `(implicitSelectStatement)` | `SELECT...` | unparenthesised implicitSelectStatement (gated on LA(1) != '('), as createViewBody already did for the parenthesised implicit form. The mat view body previously had independent optional '(' and ')', which also accepted unbalanced parentheses; they are now paired. - createTableBody (CTAS) and copyTo: inside the required parentheses, selectStatement (LA(1) SELECT/WITH/DECLARE) or implicitSelectStatement. - implicitSelectBody gains the optional PIVOT (...) clause (same position as in simpleSelect), so `(t PIVOT (...) ORDER BY ...)` works in COPY and in FROM subqueries/CTEs, as on the server. The top-level `t PIVOT (...)` still goes to pivotStatement (unchanged AST). - createViewBody now uses implicitSelectStatement (with UNION/EXCEPT/ INTERSECT) instead of implicitSelectBody. - Visitor: implicit bodies become SelectStatement { implicit: true } in the existing fields (CreateMaterializedViewStatement.query, CreateTable asSelect, CreateView/AlterView query, CopyTo source). No AST type changes. Mat view asParens is now taken from the body parenthesis only: before, `AS SELECT ... TIMESTAMP(ts)` set asParens because TIMESTAMP's '(' is in the same CST node. toSql unchanged (implicit selects already print without SELECT * FROM; ALTER VIEW always prints parentheses). - grammar-drift test: Select/Declare/With listed as the inline mat view query (as for live views), since the unparenthesised SELECT is now an OR alternative. Tests: 20 server queries (MatViewTest x5, ArrayTest, O3FailureTest x2, O3Test x4, QueryRegistryTest, IPv4Test, SqlParserTest x2 incl. ALTER VIEW, Ent EntSqlParserTest, CopyExportTest x4) and tests/implicit-select-bodies.test.ts (AST shape, round trips, unbalanced parentheses / clauses after a parenthesised body / CTAS without '(' rejected, bare `COPY t TO` unchanged, top-level PIVOT unchanged, autocomplete). Red 95 failing (55 fixture + 40 focused) -> green; full suite 15833 passed. Corpus: fixed 31 / rtChanged 0 / regressed 8 / throws 0 / formatterChangedMeaning 0. Fixed: the queries above plus more CTAS bare-table forms and `select ... from (cities PIVOT (...))`. All 8 regressions are truncated concatenation fragments with an unclosed parenthesis (`create materialized view test as (select ts,`, ...), which the server rejects with "')' expected"; the old optional ')' accepted them. Suggestions: after `CREATE TABLE y AS (`, `COPY (`, `CREATE [MATERIALIZED] VIEW v AS [(]`, `ALTER VIEW v AS [(]` table names and UNNEST are offered; after the table inside those bodies the usual table-source follow-ups (alias, joins, WHERE, ...); PIVOT after `SELECT * FROM (t `. The 41 default positions are unchanged. --- src/parser/cst-types.d.ts | 16 +- src/parser/parser.ts | 87 +++++++- src/parser/visitor.ts | 33 ++- tests/fixtures/server-queries.json | 120 ++++++++++ tests/formatter/grammar-drift.test.ts | 4 + tests/implicit-select-bodies.test.ts | 306 ++++++++++++++++++++++++++ 6 files changed, 550 insertions(+), 16 deletions(-) create mode 100644 tests/implicit-select-bodies.test.ts diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index d3626fa..fb375db 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -257,6 +257,10 @@ export type ImplicitSelectBodyCstChildren = { sampleByClause?: SampleByClauseCstNode[]; latestOnClause?: LatestOnClauseCstNode[]; groupByClause?: GroupByClauseCstNode[]; + Pivot?: IToken[]; + LParen?: IToken[]; + pivotBody?: PivotBodyCstNode[]; + RParen?: IToken[]; subsampleClause?: SubsampleClauseCstNode[]; orderByClause?: OrderByClauseCstNode[]; limitClause?: LimitClauseCstNode[]; @@ -808,7 +812,7 @@ export type CreateViewBodyCstChildren = { LParen?: (IToken)[]; selectStatement?: (SelectStatementCstNode)[]; RParen?: (IToken)[]; - implicitSelectBody?: ImplicitSelectBodyCstNode[]; + implicitSelectStatement?: (ImplicitSelectStatementCstNode)[]; Owned?: IToken[]; By?: IToken[]; stringOrIdentifier?: StringOrIdentifierCstNode[]; @@ -830,6 +834,7 @@ export type CreateTableBodyCstChildren = { As?: IToken[]; LParen?: (IToken)[]; selectStatement?: SelectStatementCstNode[]; + implicitSelectStatement?: ImplicitSelectStatementCstNode[]; RParen?: (IToken)[]; Comma?: (IToken)[]; castDefinition?: CastDefinitionCstNode[]; @@ -1014,8 +1019,9 @@ export type CreateMaterializedViewBodyCstChildren = { materializedViewRefresh?: MaterializedViewRefreshCstNode[]; As: IToken[]; LParen?: (IToken)[]; - selectStatement: SelectStatementCstNode[]; + selectStatement?: (SelectStatementCstNode)[]; RParen?: (IToken)[]; + implicitSelectStatement?: (ImplicitSelectStatementCstNode)[]; Comma?: IToken[]; indexDefinition?: IndexDefinitionCstNode[]; Timestamp?: IToken[]; @@ -1488,9 +1494,10 @@ export type AlterViewStatementCstChildren = { View: IToken[]; stringOrQualifiedName: StringOrQualifiedNameCstNode[]; As: IToken[]; - LParen?: IToken[]; + LParen?: (IToken)[]; selectStatement?: (SelectStatementCstNode)[]; - RParen?: IToken[]; + RParen?: (IToken)[]; + implicitSelectStatement?: (ImplicitSelectStatementCstNode)[]; }; export interface AlterUserStatementCstNode extends CstNode { @@ -2045,6 +2052,7 @@ export interface CopyToCstNode extends CstNode { export type CopyToCstChildren = { LParen?: IToken[]; selectStatement?: SelectStatementCstNode[]; + implicitSelectStatement?: ImplicitSelectStatementCstNode[]; RParen?: IToken[]; tableName?: TableNameCstNode[]; To: IToken[]; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 99e15d6..9135410 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -925,6 +925,13 @@ class QuestDBParser extends CstParser { this.OPTION1(() => this.SUBRULE(this.sampleByClause)) this.OPTION2(() => this.SUBRULE(this.latestOnClause)) this.OPTION3(() => this.SUBRULE(this.groupByClause)) + // PIVOT clause, as in simpleSelect: (t PIVOT (...) ORDER BY ...) + this.OPTION7(() => { + this.CONSUME(Pivot) + this.CONSUME(LParen) + this.SUBRULE(this.pivotBody) + this.CONSUME(RParen) + }) this.OPTION6(() => this.SUBRULE(this.subsampleClause)) this.OPTION4(() => this.SUBRULE(this.orderByClause)) this.OPTION5(() => this.SUBRULE(this.limitClause)) @@ -1723,13 +1730,19 @@ class QuestDBParser extends CstParser { }, }, { + // (t [WHERE ...] [UNION ...]) — server parseDml implicit select ALT: () => { this.CONSUME1(LParen) - this.SUBRULE(this.implicitSelectBody) + this.SUBRULE(this.implicitSelectStatement) this.CONSUME1(RParen) }, }, { ALT: () => this.SUBRULE1(this.selectStatement) }, + { + // Unparenthesised implicit select: CREATE VIEW v AS t WHERE ... + GATE: () => this.LA(1).tokenType !== LParen, + ALT: () => this.SUBRULE1(this.implicitSelectStatement), + }, ]) // Optional OWNED BY this.OPTION2(() => { @@ -1757,12 +1770,21 @@ class QuestDBParser extends CstParser { this.SUBRULE(this.stringOrQualifiedName) this.OR1([ - // CREATE TABLE AS (SELECT ...) + // CREATE TABLE AS (SELECT ...) | (t [WHERE ...] [UNION ...]) { ALT: () => { this.CONSUME(As) this.CONSUME(LParen) - this.SUBRULE(this.selectStatement) + this.OR7([ + { + GATE: () => { + const la1 = this.LA(1).tokenType + return la1 === Select || la1 === With || la1 === Declare + }, + ALT: () => this.SUBRULE(this.selectStatement), + }, + { ALT: () => this.SUBRULE(this.implicitSelectStatement) }, + ]) this.CONSUME(RParen) this.MANY2(() => { this.CONSUME(Comma) @@ -2099,9 +2121,33 @@ class QuestDBParser extends CstParser { this.SUBRULE1(this.materializedViewPeriod) }) this.CONSUME(As) - this.OPTION4(() => this.CONSUME(LParen)) - this.SUBRULE(this.selectStatement) - this.OPTION5(() => this.CONSUME(RParen)) + // Body: (SELECT ...) | SELECT ... | (t [WHERE ...]) | t — server + // parseDml reads a bare table as an implicit SELECT (OSS #7263). + this.OR3([ + { + GATE: () => { + const la2 = this.LA(2).tokenType + return la2 === Select || la2 === With || la2 === Declare + }, + ALT: () => { + this.CONSUME(LParen) + this.SUBRULE(this.selectStatement) + this.CONSUME(RParen) + }, + }, + { + ALT: () => { + this.CONSUME2(LParen) + this.SUBRULE(this.implicitSelectStatement) + this.CONSUME2(RParen) + }, + }, + { ALT: () => this.SUBRULE1(this.selectStatement) }, + { + GATE: () => this.LA(1).tokenType !== LParen, + ALT: () => this.SUBRULE1(this.implicitSelectStatement), + }, + ]) // Optional `, INDEX(col [CAPACITY n])` clauses (mat-view symbol indexes) this.MANY(() => { this.CONSUME(Comma) @@ -2776,15 +2822,31 @@ class QuestDBParser extends CstParser { this.CONSUME(View) this.SUBRULE(this.stringOrQualifiedName) this.CONSUME(As) + // Same body forms as CREATE VIEW (server parseViewSql → parseDml) this.OR([ { + GATE: () => { + const la2 = this.LA(2).tokenType + return la2 === Select || la2 === With || la2 === Declare + }, ALT: () => { this.CONSUME(LParen) this.SUBRULE(this.selectStatement) this.CONSUME(RParen) }, }, + { + ALT: () => { + this.CONSUME1(LParen) + this.SUBRULE(this.implicitSelectStatement) + this.CONSUME1(RParen) + }, + }, { ALT: () => this.SUBRULE1(this.selectStatement) }, + { + GATE: () => this.LA(1).tokenType !== LParen, + ALT: () => this.SUBRULE1(this.implicitSelectStatement), + }, ]) }) @@ -3894,9 +3956,20 @@ class QuestDBParser extends CstParser { private copyTo = this.RULE("copyTo", () => { this.OR([ { + // COPY (SELECT ...) | (t [WHERE ...]) | (fn(...)) | (t PIVOT (...)) + // — server parseCopy → parseDml ALT: () => { this.CONSUME(LParen) - this.SUBRULE(this.selectStatement) + this.OR1([ + { + GATE: () => { + const la1 = this.LA(1).tokenType + return la1 === Select || la1 === With || la1 === Declare + }, + ALT: () => this.SUBRULE(this.selectStatement), + }, + { ALT: () => this.SUBRULE(this.implicitSelectStatement) }, + ]) this.CONSUME(RParen) }, }, diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 9b3a7a1..a7585df 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -722,6 +722,17 @@ class QuestDBVisitor extends BaseVisitor { if (groupBy?.groupBy) result.groupBy = groupBy.groupBy if (groupBy?.groupingSets) result.groupingSets = groupBy.groupingSets } + if (ctx.pivotBody) { + const body = this.visitSafe(ctx.pivotBody) as PivotBodyResult | undefined + if (body) { + result.pivot = { + type: "pivotClause", + aggregations: body.aggregations, + pivots: body.pivots, + groupBy: body.groupBy, + } + } + } if (ctx.subsampleClause) { result.subsample = this.visitSafe( ctx.subsampleClause, @@ -1391,6 +1402,10 @@ class QuestDBVisitor extends BaseVisitor { result.asSelect = this.visit( ctx.selectStatement[0], ) as AST.SelectStatement + } else if (ctx.implicitSelectStatement) { + result.asSelect = this.visit( + ctx.implicitSelectStatement[0], + ) as AST.SelectStatement } if (ctx.castDefinition) { @@ -1745,7 +1760,7 @@ class QuestDBVisitor extends BaseVisitor { view: this.visit(ctx.stringOrQualifiedName) as AST.QualifiedName, query: ctx.selectStatement ? (this.visit(ctx.selectStatement[0]) as AST.SelectStatement) - : (this.visit(ctx.implicitSelectBody!) as AST.SelectStatement), + : (this.visit(ctx.implicitSelectStatement!) as AST.SelectStatement), } if (ctx.LParen) { result.asParens = true @@ -1831,9 +1846,13 @@ class QuestDBVisitor extends BaseVisitor { const result: AST.CreateMaterializedViewStatement = { type: "createMaterializedView", view: this.visit(ctx.stringOrQualifiedName) as AST.QualifiedName, - query: this.visit(ctx.selectStatement[0]) as AST.SelectStatement, + query: ctx.selectStatement + ? (this.visit(ctx.selectStatement[0]) as AST.SelectStatement) + : (this.visit(ctx.implicitSelectStatement!) as AST.SelectStatement), } - if (ctx.LParen) { + // The body's "(" follows AS; TIMESTAMP(col) also consumes an LParen here. + const timestampOffset = ctx.Timestamp?.[0].startOffset ?? Infinity + if (ctx.LParen?.some((t) => t.startOffset < timestampOffset)) { result.asParens = true } if (ctx.If) { @@ -2207,7 +2226,9 @@ class QuestDBVisitor extends BaseVisitor { return { type: "alterView", view: this.visit(ctx.stringOrQualifiedName) as AST.QualifiedName, - query: this.visit(ctx.selectStatement![0]) as AST.SelectStatement, + query: ctx.selectStatement + ? (this.visit(ctx.selectStatement[0]) as AST.SelectStatement) + : (this.visit(ctx.implicitSelectStatement!) as AST.SelectStatement), } } @@ -3220,7 +3241,9 @@ class QuestDBVisitor extends BaseVisitor { copyTo(ctx: CopyToCstChildren): AST.CopyToStatement { const source = ctx.selectStatement ? (this.visit(ctx.selectStatement[0]) as AST.SelectStatement) - : (this.visit(ctx.tableName!) as AST.QualifiedName) + : ctx.implicitSelectStatement + ? (this.visit(ctx.implicitSelectStatement[0]) as AST.SelectStatement) + : (this.visit(ctx.tableName!) as AST.QualifiedName) const result: AST.CopyToStatement = { type: "copyTo", source, diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index e08badd..4fcad27 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -544,5 +544,125 @@ "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/acl/EntRowExpiryAclTest.java:586", "feature": "expire-rows-keep-on", "toSql": "ALTER MATERIALIZED VIEW mv SET EXPIRE ROWS KEEP 1 HIGHEST ON v PARTITION BY k" + }, + { + "query": "create materialized view price_copy as (base_price where \"declare\" > 0)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewTest.java:10596", + "feature": "implicit-select-bodies", + "toSql": "CREATE MATERIALIZED VIEW price_copy AS (base_price WHERE \"declare\" > 0)" + }, + { + "query": "create materialized view v3 as (base_price)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewTest.java:10746", + "feature": "implicit-select-bodies", + "toSql": "CREATE MATERIALIZED VIEW v3 AS (base_price)" + }, + { + "query": "create materialized view v4 as base_price", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewTest.java:10747", + "feature": "implicit-select-bodies", + "toSql": "CREATE MATERIALIZED VIEW v4 AS base_price" + }, + { + "query": "create materialized view v5 as (base_price where price > 0)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewTest.java:10748", + "feature": "implicit-select-bodies", + "toSql": "CREATE MATERIALIZED VIEW v5 AS (base_price WHERE price > 0)" + }, + { + "query": "create materialized view v6 as (base_price where price * 2 > 0)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewTest.java:10749", + "feature": "implicit-select-bodies", + "toSql": "CREATE MATERIALIZED VIEW v6 AS (base_price WHERE price * 2 > 0)" + }, + { + "query": "create table z as (x union all y)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/ArrayTest.java:4765", + "feature": "implicit-select-bodies", + "toSql": "CREATE TABLE z AS (x UNION ALL y)" + }, + { + "query": "CREATE ATOMIC TABLE y AS (x UNION ALL append)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/o3/O3FailureTest.java:1415", + "feature": "implicit-select-bodies", + "toSql": "CREATE ATOMIC TABLE y AS (x UNION ALL append)" + }, + { + "query": "create atomic table x1 as (x) timestamp(ts) partition by DAY", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/o3/O3FailureTest.java:4558", + "feature": "implicit-select-bodies", + "toSql": "CREATE ATOMIC TABLE x1 AS (x) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "create table y as (w union all append1 union all append2)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/o3/O3Test.java:3664", + "feature": "implicit-select-bodies", + "toSql": "CREATE TABLE y AS (w UNION ALL append1 UNION ALL append2)" + }, + { + "query": "create table y as (x order by 1)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/o3/O3Test.java:4523", + "feature": "implicit-select-bodies", + "toSql": "CREATE TABLE y AS (x ORDER BY 1)" + }, + { + "query": "create table y1 as (y)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/o3/O3Test.java:8548", + "feature": "implicit-select-bodies", + "toSql": "CREATE TABLE y1 AS (y)" + }, + { + "query": "CREATE TABLE tab2 AS (tab WHERE x>8)", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/http/QueryRegistryTest.java:79", + "feature": "implicit-select-bodies", + "toSql": "CREATE TABLE tab2 AS (tab WHERE x > 8)" + }, + { + "query": "create table y as (x), cast(col as ipv4)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/IPv4Test.java:727", + "feature": "implicit-select-bodies", + "toSql": "CREATE TABLE y AS (x), CAST(col AS IPV4)" + }, + { + "query": "create table x as (tab), cast(b as double), cast(b as long)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:2989", + "feature": "implicit-select-bodies", + "toSql": "CREATE TABLE x AS (tab), CAST(b AS DOUBLE), CAST(b AS LONG)" + }, + { + "query": "alter view v1 as bla", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:1009", + "feature": "implicit-select-bodies", + "toSql": "ALTER VIEW v1 AS (bla)" + }, + { + "query": "alter view v1 as bla", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/griffin/EntSqlParserTest.java:586", + "feature": "implicit-select-bodies", + "toSql": "ALTER VIEW v1 AS (bla)" + }, + { + "query": "copy (generate_series(0, '9999-01-01', '1U')) TO 'very_large_table' WITH FORMAT PARQUET;", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CopyExportTest.java:226", + "feature": "implicit-select-bodies", + "toSql": "COPY (generate_series(0, '9999-01-01', '1U')) TO 'very_large_table' WITH FORMAT PARQUET" + }, + { + "query": "COPY (test_table where x > 0) TO 'mmap_fail_output' WITH FORMAT parquet", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CopyExportTest.java:716", + "feature": "implicit-select-bodies", + "toSql": "COPY (test_table WHERE x > 0) TO 'mmap_fail_output' WITH FORMAT parquet" + }, + { + "query": "copy (monthly_sales PIVOT (SUM(amount) FOR month IN ('JAN', 'FEB', 'MAR') GROUP BY empid) ORDER BY empid) to 'pivot_output' with format parquet", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CopyExportTest.java:2736", + "feature": "implicit-select-bodies", + "toSql": "COPY (monthly_sales PIVOT (SUM(amount) FOR month IN ('JAN', 'FEB', 'MAR') GROUP BY empid) ORDER BY empid) TO 'pivot_output' WITH format parquet" + }, + { + "query": "copy (test_table ORDER BY x DESC) to 'output1' with format parquet", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CopyExportTest.java:4251", + "feature": "implicit-select-bodies", + "toSql": "COPY (test_table ORDER BY x DESC) TO 'output1' WITH format parquet" } ] diff --git a/tests/formatter/grammar-drift.test.ts b/tests/formatter/grammar-drift.test.ts index 401ae15..fbc9944 100644 --- a/tests/formatter/grammar-drift.test.ts +++ b/tests/formatter/grammar-drift.test.ts @@ -102,6 +102,10 @@ const INLINE_BY_DESIGN: Record = { "Start", "Limit", "Time Zone", + // The query after AS, with or without parentheses. + "Select", + "Declare", + "With", ], createLiveView: [ "Live View", diff --git a/tests/implicit-select-bodies.test.ts b/tests/implicit-select-bodies.test.ts new file mode 100644 index 0000000..2e9ebbe --- /dev/null +++ b/tests/implicit-select-bodies.test.ts @@ -0,0 +1,306 @@ +/** + * Implicit select bodies (bare table / table + clauses / table function / + * PIVOT) in CREATE MATERIALIZED VIEW, CTAS, CREATE/ALTER VIEW and + * COPY (...) TO (reports §1.4, §3.4, §3.5, §3.6). + * + * Server grammar: all of these bodies go through SqlParser.parseDml, which + * reads `t [WHERE ...] [... clauses]`, `t UNION ...`, `fn(...)` and + * `t PIVOT (...)` as an implicit `SELECT * FROM ...`: + * CREATE MATERIALIZED VIEW v AS ( body ) | body (parseCreateMatView) + * CREATE [ATOMIC] TABLE t AS ( body ) [, CAST/INDEX] (parseCreateTableAsSelect, '(' required) + * CREATE VIEW v AS ( body ) | body (parseCreateView) + * ALTER VIEW v AS ( body ) | body (parseViewSql) + * COPY ( body ) TO '...' (parseCopy) + * + * Server queries for this feature live in tests/fixtures/server-queries.json. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const parseOne = (sql: string): AST.Statement => { + const result = parseToAst(sql) + expect(result.errors.map((e) => e.message)).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +/** true when sql is rejected or does not read as a single statement */ +const rejected = (sql: string) => { + const result = parseToAst(sql) + return result.errors.length > 0 || result.ast.length !== 1 +} + +const roundTrip = (sql: string, expected: string) => { + const out = toSql(parseOne(sql)) + expect(out).toBe(expected) + expect(toSql(parseOne(out))).toBe(expected) +} + +const tableName = (q: AST.SelectStatement, i = 0) => + (q.from?.[i]?.table as AST.QualifiedName).parts.join(".") + +describe("CREATE MATERIALIZED VIEW with an implicit body", () => { + it("parenthesised table + WHERE", () => { + const stmt = parseOne( + "CREATE MATERIALIZED VIEW v5 AS (base_price WHERE price > 0)", + ) as AST.CreateMaterializedViewStatement + expect(stmt.type).toBe("createMaterializedView") + expect(stmt.asParens).toBe(true) + expect(stmt.query.implicit).toBe(true) + expect(tableName(stmt.query)).toBe("base_price") + expect(stmt.query.where).toBeDefined() + }) + + it("bare table without parentheses", () => { + const stmt = parseOne( + "CREATE MATERIALIZED VIEW v4 AS base_price", + ) as AST.CreateMaterializedViewStatement + expect(stmt.asParens).toBeUndefined() + expect(stmt.query.implicit).toBe(true) + expect(tableName(stmt.query)).toBe("base_price") + }) + + it.each([ + [ + "CREATE MATERIALIZED VIEW v5 AS (base_price WHERE price > 0) EXPIRE ROWS WHEN price < 1", + "CREATE MATERIALIZED VIEW v5 AS (base_price WHERE price > 0) EXPIRE ROWS WHEN price < 1", + ], + [ + "CREATE MATERIALIZED VIEW v AS (t) PARTITION BY DAY TTL 2 DAYS OWNED BY bob", + "CREATE MATERIALIZED VIEW v AS (t) PARTITION BY DAY TTL 2 DAYS OWNED BY bob", + ], + [ + "CREATE MATERIALIZED VIEW v AS (t SAMPLE BY 1h), INDEX(s) TIMESTAMP(ts) PARTITION BY DAY", + "CREATE MATERIALIZED VIEW v AS (t SAMPLE BY 1h), INDEX(s) TIMESTAMP(ts) PARTITION BY DAY", + ], + [ + "CREATE MATERIALIZED VIEW v AS (a UNION ALL b)", + "CREATE MATERIALIZED VIEW v AS (a UNION ALL b)", + ], + ])("round trips %s", (sql, expected) => roundTrip(sql, expected)) + + it("keeps the SELECT forms", () => { + roundTrip( + "CREATE MATERIALIZED VIEW v AS (SELECT * FROM t SAMPLE BY 1h)", + "CREATE MATERIALIZED VIEW v AS (SELECT * FROM t SAMPLE BY 1h)", + ) + roundTrip( + "CREATE MATERIALIZED VIEW v AS SELECT * FROM t SAMPLE BY 1h", + "CREATE MATERIALIZED VIEW v AS SELECT * FROM t SAMPLE BY 1h", + ) + roundTrip( + "CREATE MATERIALIZED VIEW v AS (WITH c AS (SELECT * FROM t) SELECT * FROM c)", + "CREATE MATERIALIZED VIEW v AS (WITH c AS (SELECT * FROM t) SELECT * FROM c)", + ) + }) + + it("does not read the TIMESTAMP(...) parenthesis as a parenthesised body", () => { + const stmt = parseOne( + "CREATE MATERIALIZED VIEW v AS SELECT ts, avg(x) FROM t SAMPLE BY 1h TIMESTAMP(ts)", + ) as AST.CreateMaterializedViewStatement + expect(stmt.asParens).toBeUndefined() + expect(toSql(stmt)).toBe( + "CREATE MATERIALIZED VIEW v AS SELECT ts, avg(x) FROM t SAMPLE BY 1h TIMESTAMP(ts)", + ) + }) + + it("rejects unbalanced parentheses", () => { + expect(rejected("CREATE MATERIALIZED VIEW v AS (SELECT * FROM t")).toBe( + true, + ) + expect(rejected("CREATE MATERIALIZED VIEW v AS (base_price")).toBe(true) + expect(rejected("CREATE MATERIALIZED VIEW v AS")).toBe(true) + }) +}) + +describe("CREATE TABLE AS with an implicit body", () => { + it("bare table", () => { + const stmt = parseOne("CREATE TABLE y AS (x)") as AST.CreateTableStatement + expect(stmt.asSelect?.implicit).toBe(true) + expect(tableName(stmt.asSelect!)).toBe("x") + }) + + it("UNION ALL of bare tables", () => { + const stmt = parseOne( + "CREATE ATOMIC TABLE y AS (x UNION ALL append)", + ) as AST.CreateTableStatement + expect(stmt.atomic).toBe(true) + expect(stmt.asSelect?.implicit).toBe(true) + expect(stmt.asSelect?.setOperations).toHaveLength(1) + expect(stmt.asSelect?.setOperations?.[0].operator).toBe("UNION") + expect(stmt.asSelect?.setOperations?.[0].all).toBe(true) + }) + + it.each([ + [ + "create table y as (x where a > 1 limit 10), index(s) timestamp(ts) partition by day wal", + "CREATE TABLE y AS (x WHERE a > 1 LIMIT 10), INDEX(s) TIMESTAMP(ts) PARTITION BY DAY WAL", + ], + [ + "create table y as (long_sequence(10))", + "CREATE TABLE y AS (long_sequence(10))", + ], + [ + "create table y as (select * from x)", + "CREATE TABLE y AS (SELECT * FROM x)", + ], + ])("round trips %s", (sql, expected) => roundTrip(sql, expected)) + + it("still requires the parentheses (server expects '(')", () => { + expect(rejected("CREATE TABLE y AS x")).toBe(true) + expect(rejected("CREATE TABLE y AS (x")).toBe(true) + }) +}) + +describe("CREATE VIEW with an implicit body", () => { + it("unparenthesised table + WHERE", () => { + const stmt = parseOne( + "CREATE VIEW v AS t WHERE x > 0", + ) as AST.CreateViewStatement + expect(stmt.asParens).toBeUndefined() + expect(stmt.query.implicit).toBe(true) + expect(tableName(stmt.query)).toBe("t") + expect(stmt.query.where).toBeDefined() + }) + + it.each([ + ["CREATE VIEW v AS t WHERE x > 0", "CREATE VIEW v AS t WHERE x > 0"], + [ + "create view v as t where x > 0 owned by bob", + "CREATE VIEW v AS t WHERE x > 0 OWNED BY bob", + ], + ["CREATE VIEW v AS (t WHERE x > 0)", "CREATE VIEW v AS (t WHERE x > 0)"], + ["CREATE VIEW v AS (a UNION b)", "CREATE VIEW v AS (a UNION b)"], + ["CREATE VIEW v AS a UNION b", "CREATE VIEW v AS a UNION b"], + [ + "CREATE OR REPLACE VIEW v AS SELECT * FROM t", + "CREATE OR REPLACE VIEW v AS SELECT * FROM t", + ], + ])("round trips %s", (sql, expected) => roundTrip(sql, expected)) + + it("rejects clauses after a parenthesised body", () => { + expect(rejected("CREATE VIEW v AS (t) WHERE x > 0")).toBe(true) + }) +}) + +describe("ALTER VIEW with an implicit body", () => { + it.each([ + ["ALTER VIEW v AS (t WHERE x > 0)", "ALTER VIEW v AS (t WHERE x > 0)"], + ["ALTER VIEW v AS t WHERE x > 0", "ALTER VIEW v AS (t WHERE x > 0)"], + ["alter view v1 as bla", "ALTER VIEW v1 AS (bla)"], + ["ALTER VIEW v AS (a UNION ALL b)", "ALTER VIEW v AS (a UNION ALL b)"], + ["ALTER VIEW v AS SELECT * FROM t", "ALTER VIEW v AS (SELECT * FROM t)"], + ])("round trips %s", (sql, expected) => roundTrip(sql, expected)) + + it("builds an implicit select", () => { + const stmt = parseOne( + "ALTER VIEW v AS t WHERE x > 0", + ) as AST.AlterViewStatement + expect(stmt.type).toBe("alterView") + expect(stmt.query.implicit).toBe(true) + expect(tableName(stmt.query)).toBe("t") + }) +}) + +describe("COPY (...) TO with an implicit body", () => { + it("table + WHERE", () => { + const stmt = parseOne( + "COPY (test_table WHERE x > 0) TO 'o' WITH FORMAT parquet", + ) as AST.CopyToStatement + const source = stmt.source as AST.SelectStatement + expect(source.type).toBe("select") + expect(source.implicit).toBe(true) + expect(tableName(source)).toBe("test_table") + expect(source.where).toBeDefined() + }) + + it("table function", () => { + const stmt = parseOne( + "copy (generate_series(0, '9999-01-01', '1U')) TO 'big' WITH FORMAT PARQUET", + ) as AST.CopyToStatement + const source = stmt.source as AST.SelectStatement + expect(source.implicit).toBe(true) + expect(source.from?.[0].table).toMatchObject({ type: "tableFunctionCall" }) + }) + + it("PIVOT body", () => { + const stmt = parseOne( + "copy (monthly_sales PIVOT (SUM(amount) FOR month IN ('JAN') GROUP BY empid) ORDER BY empid) to 'p' with format parquet", + ) as AST.CopyToStatement + const source = stmt.source as AST.SelectStatement + expect(source.implicit).toBe(true) + expect(source.pivot?.type).toBe("pivotClause") + expect(source.orderBy).toHaveLength(1) + }) + + it("keeps a bare table name source", () => { + const stmt = parseOne("COPY t TO 'o'") as AST.CopyToStatement + expect(stmt.source).toMatchObject({ type: "qualifiedName", parts: ["t"] }) + }) + + it.each([ + ["COPY (a UNION ALL b) TO 'o'", "COPY (a UNION ALL b) TO 'o'"], + ["COPY (SELECT * FROM t) TO 'o'", "COPY (SELECT * FROM t) TO 'o'"], + ])("round trips %s", (sql, expected) => roundTrip(sql, expected)) + + it("rejects an unclosed body", () => { + expect(rejected("COPY (t WHERE x > 0 TO 'o'")).toBe(true) + }) +}) + +describe("implicit PIVOT body in a subquery", () => { + it("parses (t PIVOT (...) ORDER BY ...) as a table source", () => { + roundTrip( + "SELECT * FROM (monthly_sales PIVOT (sum(amount) FOR month IN ('JAN')) ORDER BY empid)", + "SELECT * FROM (monthly_sales PIVOT (sum(amount) FOR month IN ('JAN')) ORDER BY empid)", + ) + }) + + it("keeps the top-level PIVOT statement", () => { + const stmt = parseOne( + "monthly_sales PIVOT (sum(amount) FOR month IN ('JAN')) ORDER BY empid", + ) + expect(stmt.type).toBe("pivot") + }) +}) + +describe("autocomplete", () => { + const tableSource = ["Identifier", "QuotedIdentifier", "LParen"] + it.each([ + "CREATE TABLE y AS (", + "COPY (", + "CREATE MATERIALIZED VIEW v AS (", + "CREATE MATERIALIZED VIEW v AS ", + "ALTER VIEW v AS ", + "ALTER VIEW v AS (", + "CREATE VIEW v AS ", + ])("offers SELECT and table sources after %s", (sql) => { + const tokens = getNextValidTokens(sql) + expect(tokens).toEqual(expect.arrayContaining(["Select", ...tableSource])) + }) + + it.each([ + "CREATE TABLE y AS (x ", + "COPY (t ", + "CREATE MATERIALIZED VIEW v AS (t ", + "ALTER VIEW v AS (t ", + ])("offers clauses after the table in %s", (sql) => { + const tokens = getNextValidTokens(sql) + expect(tokens).toEqual( + expect.arrayContaining(["Where", "Union", "Order", "Limit", "RParen"]), + ) + }) + + it("offers clauses after an unparenthesised CREATE VIEW table", () => { + expect(getNextValidTokens("CREATE VIEW v AS t ")).toEqual( + expect.arrayContaining(["Where", "Union", "Owned"]), + ) + }) + + it("offers PIVOT after the table inside COPY (", () => { + expect(getNextValidTokens("COPY (t ")).toContain("Pivot") + }) +}) From 55c851853e9cbf53ab995f1e76bb4f8ff7ca968a Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 14:44:27 +0300 Subject: [PATCH 07/36] fix: unescape '' in single-quoted statement values so they round-trip The visitor unquoted single-quoted strings in statement positions with image.slice(1, -1) and never turned '' into ', while toSql re-escapes them with escapeString. Every doubled quote got doubled again: CREATE USER bob WITH PASSWORD 'it''s' -> PASSWORD 'it''''s' COPY t FROM 'f.csv' WITH DELIMITER '''' -> DELIMITER '''''' The server only strips the outer quotes in these positions (GenericLexer.unquote, std/GenericLexer.java:111: password EntSqlCompilerImpl:2202, DELIMITER SqlParser:3165). So the SQL text has to survive unchanged, and before this fix it did not. Change: a shared unquoteSingle() helper (strip quotes, '' -> ') is now used in every position whose value toSql prints back single-quoted: passwords (CREATE/ALTER USER, SERVICE ACCOUNT), EXTERNAL ALIAS (CREATE/ALTER GROUP), TTL, JWK public keys, DROP TOKEN, IN VOLUME, partition lists (DROP/DETACH/ ATTACH/FORCE DROP/CONVERT, REINDEX PARTITION), SUSPEND WAL code/message (table + live view), REBASE WAL INTO, COPY file/destination/options/CANCEL id, CANCEL QUERY, live view START FROM, ANCHOR DAILY time/zone, SAMPLE BY TIME ZONE/OFFSET, mat view REFRESH START/TIME ZONE/RANGE (extractMaybeString), and INSERT BATCH o3MaxLag. The places that already unescaped (expression literals, typed literals, identifier StringLiteral) now use the same helper. Deliberately unchanged: single-quoted NAMES that toSql prints as "..." identifiers (FROM 'x', stringOrQualifiedName, tableNameOrString, OWNED BY via stringOrIdentifier). The server keeps their inner text verbatim, so FROM 'it''s' is table it''s (SqlParser:3047/7305/8076 unquote; questdb-ent EntShowCreateSupportTest: CREATE USER 'o''brien' stores principal o''brien). The current output "it''s" names that same table. Unescaping would print "it's", which is a different table, so the report's "wrong table" claim for this case was incorrect. extractIdentifierName is left as it is. Tests: tests/single-quote-unescape.test.ts (30 round trips, AST values, pinned verbatim names, expression literals). 3 server queries with '' escapes (EscapeTest:52/62, LatestByTest:3797) were added to server-queries.json as guards. They already passed: no server test uses '' in a statement value position. Red -> green: 34 failing -> all pass; full suite 15884 passed. Corpus: fixed 0 / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0 (the corpus has no '' in these positions). Suggestions: unchanged (0/43 positions). --- src/parser/visitor.ts | 103 ++++++++++++------- tests/fixtures/server-queries.json | 18 ++++ tests/single-quote-unescape.test.ts | 148 ++++++++++++++++++++++++++++ 3 files changed, 231 insertions(+), 38 deletions(-) create mode 100644 tests/single-quote-unescape.test.ts diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index a7585df..6145b40 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -242,6 +242,18 @@ function tokenFloat(image: string): number { return parseFloat(image.replace(/_/g, "")) } +// Value of a single-quoted SQL string token: strip the outer quotes and turn +// the '' escape into '. toSql re-escapes with escapeString, so the SQL text +// round-trips unchanged. Single-quoted NAMES printed back as "..." identifiers +// (FROM 'x', stringOrQualifiedName, tableNameOrString, OWNED BY) deliberately +// do not use this: the server's GenericLexer.unquote only strips the outer +// quotes, so FROM 'it''s' names table it''s (questdb-ent +// EntShowCreateSupportTest: CREATE USER 'o''brien' stores o''brien), which is +// what "it''s" means too. +function unquoteSingle(image: string): string { + return image.slice(1, -1).replace(/''/g, "'") +} + type StoragePolicyKind = "toParquet" | "toRemote" | "dropLocal" | "dropRemote" type DedupModeKeys = { mode: AST.DedupMode; keys: string[] } @@ -774,7 +786,8 @@ class QuestDBVisitor extends BaseVisitor { const varImage = ctx.VariableReference[0].image // e.g. "@subquery" table = { type: "qualifiedName", parts: [varImage] } as AST.QualifiedName } else if (ctx.StringLiteral) { - // Single-quoted table name: FROM 'sys.copy_export_log' + // Single-quoted table name: FROM 'sys.copy_export_log'. Kept verbatim + // (no '' unescape), like the server's GenericLexer.unquote. table = { type: "qualifiedName", parts: [ctx.StringLiteral[0].image.slice(1, -1)], @@ -1260,7 +1273,7 @@ class QuestDBVisitor extends BaseVisitor { const o3MaxLag = ctx.DurationLiteral?.[0]?.image ?? (ctx.StringLiteral?.[0]?.image - ? ctx.StringLiteral[0].image.slice(1, -1) + ? unquoteSingle(ctx.StringLiteral[0].image) : undefined) return { size: tokenInt(ctx.NumberLiteral[0].image), @@ -1468,7 +1481,7 @@ class QuestDBVisitor extends BaseVisitor { (t: IToken) => t.startOffset > volumeOffset, ) result.volume = volumeStr - ? volumeStr.image.slice(1, -1) + ? unquoteSingle(volumeStr.image) : ctx.identifier?.length ? this.extractIdentifierName( ctx.identifier[ctx.identifier.length - 1].children, @@ -1792,7 +1805,7 @@ class QuestDBVisitor extends BaseVisitor { } if (ctx.Password) { if (ctx.StringLiteral) { - result.password = ctx.StringLiteral[0].image.slice(1, -1) + result.password = unquoteSingle(ctx.StringLiteral[0].image) } else if (ctx.identifier) { result.password = this.extractIdentifierName(ctx.identifier[0].children) } @@ -1809,7 +1822,7 @@ class QuestDBVisitor extends BaseVisitor { ifNotExists: !!ctx.If, } if (ctx.StringLiteral) { - result.externalAlias = ctx.StringLiteral[0].image.slice(1, -1) + result.externalAlias = unquoteSingle(ctx.StringLiteral[0].image) } return result } @@ -1829,7 +1842,7 @@ class QuestDBVisitor extends BaseVisitor { } if (ctx.Password && !ctx.No) { if (ctx.StringLiteral) { - result.password = ctx.StringLiteral[0].image.slice(1, -1) + result.password = unquoteSingle(ctx.StringLiteral[0].image) } else if (ctx.identifier) { result.password = this.extractIdentifierName(ctx.identifier[0].children) } @@ -1907,7 +1920,7 @@ class QuestDBVisitor extends BaseVisitor { (t: IToken) => t.startOffset > volumeOffset, ) result.volume = volumeStr - ? volumeStr.image.slice(1, -1) + ? unquoteSingle(volumeStr.image) : ctx.identifier?.length ? this.extractIdentifierName( ctx.identifier[ctx.identifier.length - 1].children, @@ -2054,7 +2067,7 @@ class QuestDBVisitor extends BaseVisitor { } else if (ctx.StringLiteral) { result.startFrom = { kind: "timestamp", - value: ctx.StringLiteral[0].image.slice(1, -1), + value: unquoteSingle(ctx.StringLiteral[0].image), } } else { result.startFrom = { kind: "now" } @@ -2094,11 +2107,11 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.NumberLiteral) { result.code = tokenInt(ctx.NumberLiteral[0].image) } else if (ctx.StringLiteral && ctx.StringLiteral.length > 0) { - result.code = ctx.StringLiteral[0].image.slice(1, -1) + result.code = unquoteSingle(ctx.StringLiteral[0].image) } const strings = ctx.StringLiteral || [] if (strings.length > 0) { - result.message = strings[strings.length - 1].image.slice(1, -1) + result.message = unquoteSingle(strings[strings.length - 1].image) } } return result @@ -2113,10 +2126,10 @@ class QuestDBVisitor extends BaseVisitor { } const result: AST.AnchorClause = { kind: "daily", - time: ctx.StringLiteral![0].image.slice(1, -1), + time: unquoteSingle(ctx.StringLiteral![0].image), } if (ctx.StringLiteral && ctx.StringLiteral.length > 1) { - result.timezone = ctx.StringLiteral[1].image.slice(1, -1) + result.timezone = unquoteSingle(ctx.StringLiteral[1].image) } return result } @@ -2244,7 +2257,7 @@ class QuestDBVisitor extends BaseVisitor { memoryLimit: this.visit(ctx.memoryLimit) as string, } } - const alias = ctx.StringLiteral![0].image.slice(1, -1) + const alias = unquoteSingle(ctx.StringLiteral![0].image) return { type: "alterGroup", group, @@ -2381,7 +2394,7 @@ class QuestDBVisitor extends BaseVisitor { actionType: "rebaseWal", } if (ctx.Into && ctx.StringLiteral && ctx.StringLiteral.length > 0) { - result.targetDir = ctx.StringLiteral[0].image.slice(1, -1) + result.targetDir = unquoteSingle(ctx.StringLiteral[0].image) } return result } @@ -2436,7 +2449,9 @@ class QuestDBVisitor extends BaseVisitor { return { actionType: "password", noPassword: !!ctx.No, - password: ctx.StringLiteral?.[0]?.image?.slice(1, -1), + password: ctx.StringLiteral + ? unquoteSingle(ctx.StringLiteral[0].image) + : undefined, } } if (ctx.Create && ctx.Token) { @@ -2444,7 +2459,7 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.DurationLiteral) { ttl = ctx.DurationLiteral[0].image } else if (ctx.Ttl && ctx.StringLiteral) { - ttl = ctx.StringLiteral[0].image.slice(1, -1) + ttl = unquoteSingle(ctx.StringLiteral[0].image) } const result: AST.AlterUserCreateTokenAction = { actionType: "createToken", @@ -2458,8 +2473,8 @@ class QuestDBVisitor extends BaseVisitor { // JWK public key: WITH PUBLIC KEY X '...' Y '...' if (ctx.Public && ctx.Key && ctx.Identifier) { // In JWK path, StringLiteral[0] and [1] are the public key values - result.publicKeyX = ctx.StringLiteral![0].image.slice(1, -1) - result.publicKeyY = ctx.StringLiteral![1].image.slice(1, -1) + result.publicKeyX = unquoteSingle(ctx.StringLiteral![0].image) + result.publicKeyY = unquoteSingle(ctx.StringLiteral![1].image) } return result } @@ -2468,7 +2483,9 @@ class QuestDBVisitor extends BaseVisitor { tokenType: ctx.Jwk ? "JWK" : "REST", token: ctx.Identifier?.[0]?.image ?? - ctx.StringLiteral?.[0]?.image?.slice(1, -1), + (ctx.StringLiteral + ? unquoteSingle(ctx.StringLiteral[0].image) + : undefined), } } @@ -2482,7 +2499,7 @@ class QuestDBVisitor extends BaseVisitor { ctx: AlterTableForceActionCstChildren, ): AST.ForceDropPartitionAction | AST.ForceReindexParquetAction { const partitions = (ctx.StringLiteral ?? []).map((s: IToken) => - s.image.slice(1, -1), + unquoteSingle(s.image), ) if (ctx.Drop) { return { actionType: "forceDropPartition", partitions } @@ -2643,7 +2660,7 @@ class QuestDBVisitor extends BaseVisitor { } if (ctx.StringLiteral) { result.partitions = ctx.StringLiteral.map((s: IToken) => - s.image.slice(1, -1), + unquoteSingle(s.image), ) } if (ctx.Where && ctx.expression) { @@ -2656,7 +2673,9 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.Attach) { return { actionType: "attachPartition", - partitions: ctx.StringLiteral!.map((s: IToken) => s.image.slice(1, -1)), + partitions: ctx.StringLiteral!.map((s: IToken) => + unquoteSingle(s.image), + ), } } @@ -2667,7 +2686,7 @@ class QuestDBVisitor extends BaseVisitor { } if (ctx.StringLiteral) { result.partitions = ctx.StringLiteral.map((s: IToken) => - s.image.slice(1, -1), + unquoteSingle(s.image), ) } if (ctx.Where && ctx.expression) { @@ -2729,12 +2748,12 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.NumberLiteral) { result.code = tokenInt(ctx.NumberLiteral[0].image) } else if (ctx.StringLiteral && ctx.StringLiteral.length > 0) { - result.code = ctx.StringLiteral[0].image.slice(1, -1) + result.code = unquoteSingle(ctx.StringLiteral[0].image) } // Message is the last StringLiteral const strings = ctx.StringLiteral || [] if (strings.length > 0) { - result.message = strings[strings.length - 1].image.slice(1, -1) + result.message = unquoteSingle(strings[strings.length - 1].image) } } return result @@ -2758,7 +2777,7 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.Rebase) { const result: AST.RebaseWalAction = { actionType: "rebaseWal" } if (ctx.Into && ctx.StringLiteral && ctx.StringLiteral.length > 0) { - result.targetDir = ctx.StringLiteral[0].image.slice(1, -1) + result.targetDir = unquoteSingle(ctx.StringLiteral[0].image) } return result } @@ -2978,7 +2997,9 @@ class QuestDBVisitor extends BaseVisitor { ctx.NumberLiteral?.[0] ?? ctx.Identifier?.[0] ?? ctx.StringLiteral?.[0] return { type: "cancelQuery", - queryId: token?.image?.replace(/^'|'$/g, "") ?? "", + queryId: ctx.StringLiteral + ? unquoteSingle(ctx.StringLiteral[0].image) + : (token?.image ?? ""), } } @@ -3204,7 +3225,7 @@ class QuestDBVisitor extends BaseVisitor { copyCancel(ctx: CopyCancelCstChildren): AST.CopyCancelStatement { let id = "" if (ctx.StringLiteral) { - id = ctx.StringLiteral[0].image.slice(1, -1) + id = unquoteSingle(ctx.StringLiteral[0].image) } else if (ctx.NumberLiteral) { id = ctx.NumberLiteral[0].image } else if (ctx.Identifier) { @@ -3299,7 +3320,7 @@ class QuestDBVisitor extends BaseVisitor { result.quoted = true } } else if (ctx.StringLiteral) { - result.value = ctx.StringLiteral[0].image.slice(1, -1) + result.value = unquoteSingle(ctx.StringLiteral[0].image) result.quoted = true } else if (ctx.expression) { const expr = this.visit(ctx.expression) as AST.Expression @@ -3435,7 +3456,7 @@ class QuestDBVisitor extends BaseVisitor { // Optional LIST of partition names if (ctx.StringLiteral) { result.partitions = ctx.StringLiteral.map((s: IToken) => - s.image.slice(1, -1), + unquoteSingle(s.image), ) } @@ -3636,8 +3657,12 @@ class QuestDBVisitor extends BaseVisitor { ) } if (ctx.Partition && ctx.stringOrIdentifier) { - result.partitions = ctx.stringOrIdentifier.map( - (s: CstNode) => this.visit(s) as string, + // Partition names are values (printed back single-quoted), unlike the + // OWNED BY name that stringOrIdentifier() returns verbatim. + result.partitions = ctx.stringOrIdentifier.map((s: CstNode) => + s.children.StringLiteral + ? unquoteSingle((s.children.StringLiteral[0] as IToken).image) + : (this.visit(s) as string), ) } if (ctx.Exclusive) { @@ -4682,7 +4707,7 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.StringLiteral) { const raw = this.tokenImage(ctx.StringLiteral[0]) // Remove outer quotes and unescape '' to ' - const value = raw.slice(1, -1).replace(/''/g, "'") + const value = unquoteSingle(raw) return { type: "literal", value, @@ -4788,7 +4813,7 @@ class QuestDBVisitor extends BaseVisitor { if (escapeToken) return this.escapeStringLiteral(escapeToken[0]) return { type: "literal", - value: this.tokenImage(stringToken![0]).slice(1, -1).replace(/''/g, "'"), + value: unquoteSingle(this.tokenImage(stringToken![0])), literalType: "string", } } @@ -4808,6 +4833,8 @@ class QuestDBVisitor extends BaseVisitor { return !!ctx.True } + // Names (OWNED BY): a single-quoted name keeps its inner text verbatim, as + // the server's GenericLexer.unquote does (see unquoteSingle). stringOrIdentifier(ctx: StringOrIdentifierCstChildren): string { if (ctx.StringLiteral) { return ctx.StringLiteral[0].image.slice(1, -1) @@ -4821,6 +4848,7 @@ class QuestDBVisitor extends BaseVisitor { stringOrQualifiedName( ctx: StringOrQualifiedNameCstChildren, ): AST.QualifiedName { + // Name kept verbatim (no '' unescape), see unquoteSingle. if (ctx.StringLiteral) { return { type: "qualifiedName", @@ -4852,6 +4880,7 @@ class QuestDBVisitor extends BaseVisitor { } tableNameOrString(ctx: TableNameOrStringCstChildren): AST.QualifiedName { + // Name kept verbatim (no '' unescape), see unquoteSingle. if (ctx.StringLiteral) { return { type: "qualifiedName", @@ -4878,8 +4907,7 @@ class QuestDBVisitor extends BaseVisitor { const ctx: CstChildrenRecord = "name" in node && "children" in node ? (node as CstNode).children : node if (ctx.StringLiteral) { - const raw = this.tokenImage(ctx.StringLiteral[0] as IToken) - return raw.slice(1, -1) + return unquoteSingle(this.tokenImage(ctx.StringLiteral[0] as IToken)) } if (ctx.NumberLiteral) { return this.tokenImage(ctx.NumberLiteral[0] as IToken) @@ -4981,8 +5009,7 @@ class QuestDBVisitor extends BaseVisitor { } // QuestDB accepts single-quoted strings in identifier positions if (ctx.StringLiteral) { - const raw = this.tokenImage(ctx.StringLiteral[0] as IToken) - return raw.slice(1, -1).replace(/''/g, "'") + return unquoteSingle(this.tokenImage(ctx.StringLiteral[0] as IToken)) } // Handle keyword tokens used as identifiers // Find the first token in the context diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 4fcad27..55a3f29 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -664,5 +664,23 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/CopyExportTest.java:4251", "feature": "implicit-select-bodies", "toSql": "COPY (test_table ORDER BY x DESC) TO 'output1' WITH format parquet" + }, + { + "query": "insert into t values ( '1st ''', '2nd ''''', '3rd ''''''' );", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/EscapeTest.java:52", + "feature": "single-quote-unescape", + "toSql": "INSERT INTO t VALUES ('1st ''', '2nd ''''', '3rd ''''''')" + }, + { + "query": "select strpos('abc''def', '''' ) ", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/EscapeTest.java:62", + "feature": "single-quote-unescape", + "toSql": "SELECT strpos('abc''def', '''')" + }, + { + "query": "SELECT s, v FROM escaped_symbols_0 WHERE (s IN ('O''Brien', 'O''''Brien', 'x') AND s NOT IN ('O''Brien', 'x')) LATEST ON ts PARTITION BY s ORDER BY v", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/LatestByTest.java:3797", + "feature": "single-quote-unescape", + "toSql": "SELECT s, v FROM escaped_symbols_0 WHERE (s IN ('O''Brien', 'O''''Brien', 'x') AND s NOT IN ('O''Brien', 'x')) LATEST ON ts PARTITION BY s ORDER BY v" } ] diff --git a/tests/single-quote-unescape.test.ts b/tests/single-quote-unescape.test.ts new file mode 100644 index 0000000..c780770 --- /dev/null +++ b/tests/single-quote-unescape.test.ts @@ -0,0 +1,148 @@ +/** + * Single-quoted values in statement positions (passwords, aliases, partition + * names, COPY options, time zones, ...) must round-trip unchanged: the visitor + * turns `''` into `'` and toSql re-escapes it (reports §2 X4). + * + * The QuestDB server never unescapes `''` in these positions + * (GenericLexer.unquote only strips the outer quotes), so the SQL text has to + * survive exactly. Before this fix 'it''s' was printed as 'it''''s'. + * + * Single-quoted NAMES that toSql prints as "..." identifiers (FROM 'x', + * INSERT INTO 'x', OWNED BY 'x', ...) keep the inner text verbatim: the server + * reads FROM 'it''s' as table it''s (see questdb-ent EntShowCreateSupportTest: + * CREATE USER 'o''brien' stores principal o''brien), which is exactly what + * "it''s" means, so they must not be unescaped. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst } from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const parseOne = (sql: string): T => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] as T +} + +const roundTrip = (sql: string) => toSql([parseOne(sql)]) + +describe("single-quoted values round-trip with '' escapes", () => { + const sameText = [ + "CREATE USER bob WITH PASSWORD 'it''s'", + "ALTER USER bob WITH PASSWORD 'it''s'", + "CREATE SERVICE ACCOUNT svc WITH PASSWORD 'it''s'", + "ALTER SERVICE ACCOUNT svc WITH PASSWORD 'it''s'", + "CREATE GROUP g WITH EXTERNAL ALIAS 'cn=o''brien'", + "ALTER GROUP g WITH EXTERNAL ALIAS 'cn=o''brien'", + "ALTER GROUP g DROP EXTERNAL ALIAS 'cn=o''brien'", + "ALTER USER bob CREATE TOKEN TYPE REST WITH TTL '1''d'", + "ALTER USER bob CREATE TOKEN TYPE JWK WITH PUBLIC KEY X 'a''b' Y 'c''d'", + "ALTER USER bob DROP TOKEN TYPE REST 'tok''en'", + "COPY t FROM 'it''s.csv' WITH DELIMITER ''''", + "COPY t TO 'it''s' WITH FORMAT parquet", + "COPY 'it''s' CANCEL", + "CANCEL QUERY 'it''s'", + "CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY IN VOLUME 'v''1'", + "ALTER TABLE t DROP PARTITION LIST '2024''01'", + "ALTER TABLE t DETACH PARTITION LIST '2024''01'", + "ALTER TABLE t ATTACH PARTITION LIST '2024''01'", + "ALTER TABLE t FORCE DROP PARTITION LIST '2024''01'", + "ALTER TABLE t CONVERT PARTITION LIST '2024''01' TO PARQUET", + "ALTER TABLE t SUSPEND WAL WITH 'c''1', 'm''sg'", + "ALTER TABLE t REBASE WAL INTO 'd''ir'", + "ALTER MATERIALIZED VIEW v REBASE WAL INTO 'd''ir'", + "ALTER LIVE VIEW lv SUSPEND WAL WITH 'c''1', 'm''sg'", + "REINDEX TABLE t PARTITION '2024''01' LOCK EXCLUSIVE", + "REFRESH MATERIALIZED VIEW v RANGE FROM '2024''01' TO '2025''01'", + "SELECT * FROM t SAMPLE BY 1h ALIGN TO CALENDAR TIME ZONE 'Europe/it''s' WITH OFFSET '00''15'", + "CREATE MATERIALIZED VIEW v REFRESH EVERY 1h START '2024''01' TIME ZONE 'it''s' AS (SELECT ts, count() FROM t SAMPLE BY 1h)", + "CREATE LIVE VIEW lv FLUSH EVERY 1s START FROM 'it''s' AS SELECT ts, x FROM base", + "SELECT avg(x) OVER w FROM t WINDOW w AS (ORDER BY ts ANCHOR DAILY '09''30' 'Europe/it''s')", + ] + it.each(sameText)("%s", (sql) => { + const out = roundTrip(sql) + expect(out).toBe(sql) + expect(roundTrip(out)).toBe(sql) + }) +}) + +describe("AST holds the unescaped value", () => { + it("password", () => { + const stmt = parseOne( + "CREATE USER bob WITH PASSWORD 'it''s'", + ) + expect(stmt.password).toBe("it's") + }) + + it("COPY file and DELIMITER", () => { + const stmt = parseOne( + "COPY t FROM 'it''s.csv' WITH DELIMITER ''''", + ) + expect(stmt.file).toBe("it's.csv") + expect(stmt.options).toEqual([ + { type: "copyOption", key: "DELIMITER", value: "'", quoted: true }, + ]) + }) + + it("partition list", () => { + const stmt = parseOne( + "ALTER TABLE t DROP PARTITION LIST '2024''01', 'b'", + ) + expect((stmt.action as AST.DropPartitionAction).partitions).toEqual([ + "2024'01", + "b", + ]) + }) + + it("external alias", () => { + const stmt = parseOne( + "CREATE GROUP g WITH EXTERNAL ALIAS 'cn=o''brien'", + ) + expect(stmt.externalAlias).toBe("cn=o'brien") + }) + + it("strings without doubled quotes are unchanged", () => { + const stmt = parseOne( + "CREATE USER bob WITH PASSWORD 'secret'", + ) + expect(stmt.password).toBe("secret") + expect( + parseOne("CREATE USER bob WITH PASSWORD ''") + .password, + ).toBe("") + }) +}) + +describe("single-quoted names keep the server's verbatim text", () => { + it("FROM 'it''s' stays table it''s", () => { + const stmt = parseOne("SELECT * FROM 'it''s'") + const from = stmt.from![0] + expect((from.table as AST.QualifiedName).parts).toEqual(["it''s"]) + expect(roundTrip("SELECT * FROM 'it''s'")).toBe("SELECT * FROM \"it''s\"") + }) + + it("INSERT INTO 'it''s' stays table it''s", () => { + expect(roundTrip("INSERT INTO 'it''s' VALUES (1)")).toBe( + "INSERT INTO \"it''s\" VALUES (1)", + ) + }) + + it("OWNED BY 'o''brien' stays principal o''brien", () => { + const stmt = parseOne( + "CREATE TABLE t (a INT) OWNED BY 'o''brien'", + ) + expect(stmt.ownedBy).toBe("o''brien") + expect(roundTrip("CREATE TABLE t (a INT) OWNED BY 'o''brien'")).toBe( + "CREATE TABLE t (a INT) OWNED BY \"o''brien\"", + ) + }) + + it("string literals in expressions are still unescaped", () => { + const stmt = parseOne("SELECT 'it''s'") + const item = stmt.columns[0] as { expression: AST.Literal } + expect(item.expression.value).toBe("it's") + expect(roundTrip("SELECT 'it''s'")).toBe("SELECT 'it''s'") + }) +}) From 52c8d03e60ee6d59fb137e072edca39c938fb621 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 14:54:56 +0300 Subject: [PATCH 08/36] fix: end duration literals at a word boundary and add the ns unit Server: the lexer reads a number glued to letters as one token, so `1hour`, `1days`, `1sec`, `111ml` are rejected (SampleByTest:250 "Invalid unit: 1hour", SampleByTest:262 "1min", AlterTableO3MaxLagTest:363/370 "interval qualifier", SqlParser.parseTtlHoursOrMonths "invalid time unit"). SqlUtil.expectMicros (o3MaxLag) accepts s, m, ms, h, d, us and ns (SqlUtilTest:829 "1000ns"). Before: DurationLiteral had no trailing boundary, so invalid units were silently split into a duration plus a second statement: SAMPLE BY 1hour -> `1h; our`, TTL 1days -> `TTL 1 DAY; ays`, o3MaxLag = 1sec -> `1s; ec`, and the valid o3MaxLag = 100ns -> `100n; s`. Now: - DurationLiteral gets the `ns` unit and a (?![\w\u0080-\uFFFF]) boundary. - New MalformedDurationLiteral token (after DurationLiteral, before NumberLiteral): digits + a duration-unit letter + more word characters (`1hour`, `1days`, `1.5hour`, `100nsx`). No grammar rule accepts it, so it is a parse error, like on the server. A boundary alone would have turned `TTL 1days` into a valid `TTL 1 DAYS` (NumberLiteral + DAYS) and `SELECT 1hour` into `1 AS hour`. A dedicated token was chosen over a NumberLiteral lookahead: NumberLiteral is untouched (GEOHASH(8c), MEMORY LIMIT 1G still split as before) and the formatter keeps a glued run like `10.5mFROM` as one word instead of backtracking into opaque fragments. Classified as a literal in the formatter and never suggested in autocomplete. - All valid units still lex as one token: 1h 30m 1M 1y 100T 1U 1n 1_000T 2W 12H 1.5d 1ms 1us 100ns 2Y 2D. Tests: 11 server queries (guards for n/U/T/m/ms/us units, TTL 1D, INSERT BATCH o3MaxLag, TOLERANCE w, GEOHASH(8c), MEMORY LIMIT 1G) + tests/duration-boundary.test.ts. Red before: 24 failing -> 548/548 pass; full suite 15975 passed. Corpus: fixed 0 / rtChanged 0 / regressed 3 / throws 0 / formatterChangedMeaning 0. All 3 regressions are server-rejected inputs that used to be silently split: `o3MaxLag = 111days`, `o3MaxLag = 111ml` (AlterTableO3MaxLagTest expects "interval qualifier") and `sample by 1hour` (SampleByTest expects "Invalid unit: 1hour"). --- src/autocomplete/token-classification.ts | 1 + src/formatter/classification.ts | 1 + src/parser/lexer.ts | 23 ++- tests/duration-boundary.test.ts | 163 ++++++++++++++++++++++ tests/fixtures/server-queries.json | 66 +++++++++ tests/helpers/autocomplete-walkthrough.ts | 1 + 6 files changed, 252 insertions(+), 3 deletions(-) create mode 100644 tests/duration-boundary.test.ts diff --git a/src/autocomplete/token-classification.ts b/src/autocomplete/token-classification.ts index b14fdd1..681e9e9 100644 --- a/src/autocomplete/token-classification.ts +++ b/src/autocomplete/token-classification.ts @@ -113,6 +113,7 @@ export const SKIP_TOKENS = new Set([ "LongLiteral", "DecimalLiteral", "DurationLiteral", + "MalformedDurationLiteral", "GeohashLiteral", "GeohashBinaryLiteral", "HexLiteral", diff --git a/src/formatter/classification.ts b/src/formatter/classification.ts index 64c1c4f..29f95bb 100644 --- a/src/formatter/classification.ts +++ b/src/formatter/classification.ts @@ -47,6 +47,7 @@ const literalTokenNames = [ "LongLiteral", "DecimalLiteral", "DurationLiteral", + "MalformedDurationLiteral", "NumberLiteral", "StringLiteral", "HexLiteral", diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index 3da28ff..2c37c7c 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -837,11 +837,27 @@ export const DecimalLiteral = createToken({ // disambiguation (e.g. rejecting nanos in storage policy) is enforced by // per-context grammar rules. // Supports decimal values like 1.5d, 0.5h. -// Multi-char units (ms, us) must appear before single-char alternatives to -// avoid prefix match. +// Multi-char units (ms, us, ns) must appear before single-char alternatives to +// avoid prefix match. `ns` is an o3MaxLag unit (SqlUtil.expectMicros). +// A duration ends at a word boundary: the server reads `1hour`, `1days`, `1sec` +// as one token and rejects it, so they must not lex as `1h` + `our`. export const DurationLiteral = createToken({ name: "DurationLiteral", - pattern: /\d[\d_]*(\.\d[\d_]*)?(ms|us|[smhdwyMDWYNnTUuH])/, + pattern: + /\d[\d_]*(\.\d[\d_]*)?(ms|us|ns|[smhdwyMDWYNnTUuH])(?![\w\u0080-\uFFFF])/, +}) + +// A number glued to a word that starts with a duration unit but is not a valid +// duration (`1hour`, `1days`, `1sec`, `100nsx`, `1.5hour`). The server lexer +// reads it as one token and rejects it ("Invalid unit: 1hour", "invalid +// interval qualifier", "invalid time unit"). No grammar rule accepts this +// token, so it is a parse error instead of `1` + `hour` (which would read as +// `TTL 1 HOUR` or `SELECT 1 AS hour`). Numbers glued to other letters keep +// splitting (`GEOHASH(8c)`, `MEMORY LIMIT 1G`). Must come after +// DurationLiteral and before NumberLiteral. +export const MalformedDurationLiteral = createToken({ + name: "MalformedDurationLiteral", + pattern: /\d[\d_]*(\.\d[\d_]*)?[smhdwyMDWYNnTUuH][\w\u0080-\uFFFF]+/, }) // Numbers (negation is handled as unary minus in the parser, not in the lexer) @@ -979,6 +995,7 @@ export const allTokens: TokenType[] = [ LongLiteral, DecimalLiteral, DurationLiteral, + MalformedDurationLiteral, NumberLiteral, StringLiteral, EscapeStringLiteral, diff --git a/tests/duration-boundary.test.ts b/tests/duration-boundary.test.ts new file mode 100644 index 0000000..a6a7588 --- /dev/null +++ b/tests/duration-boundary.test.ts @@ -0,0 +1,163 @@ +/** + * Duration literals (reports §2 X5, §3.1): + * - a duration ends at a word boundary. The server lexer reads `1hour`, + * `1days`, `1sec` as ONE token and rejects it (SampleByTest:250 + * "Invalid unit: 1hour", AlterTableO3MaxLagTest:363 "interval qualifier", + * SqlParser.parseTtlHoursOrMonths "invalid time unit"). sql-parser used to + * split them into a duration plus a second statement (`1h; our`). + * - the `ns` unit (SqlUtil.expectMicros accepts s, m, ms, h, d, us, ns; + * SqlUtilTest:829 `expectMicros("1000ns")`). + * + * Valid server queries are guarded in tests/fixtures/server-queries.json + * (feature "duration-boundary"). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, tokenize } from "../src/index" +import { toSql } from "../src/parser/toSql" + +const tokenTypes = (sql: string) => + tokenize(sql).tokens.map((t) => `${t.tokenType.name}:${t.image}`) + +const roundTrip = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return toSql(result.ast) +} + +const expectError = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors.length).toBeGreaterThan(0) +} + +describe("lexer", () => { + it.each([ + "1h", + "30m", + "1M", + "1y", + "100T", + "1U", + "1n", + "1_000T", + "2W", + "12H", + "1.5d", + "1ms", + "1us", + "100ns", + "5_600ns", + "2Y", + "2D", + "1000000000n", + ])("reads %s as one duration token", (d) => { + expect(tokenTypes(d)).toEqual([`DurationLiteral:${d}`]) + }) + + it("ends a duration before punctuation", () => { + expect(tokenTypes("1h)")).toEqual(["DurationLiteral:1h", "RParen:)"]) + expect(tokenTypes("1h,")).toEqual(["DurationLiteral:1h", "Comma:,"]) + expect(tokenTypes("100ns;")).toEqual([ + "DurationLiteral:100ns", + "Semicolon:;", + ]) + }) + + it.each([ + "1hour", + "1days", + "1sec", + "1min", + "100nsx", + "111ml", + "1.5hour", + "1_000days", + "6h30m", + "1hé", + ])("reads %s as one malformed duration, not a duration and a word", (w) => { + expect(tokenTypes(w)).toEqual([`MalformedDurationLiteral:${w}`]) + }) + + it("keeps numbers glued to non-duration letters (geohash, memory sizes)", () => { + expect(tokenTypes("8c")).toEqual(["NumberLiteral:8", "Identifier:c"]) + expect(tokenTypes("5b")).toEqual(["NumberLiteral:5", "Identifier:b"]) + expect(tokenTypes("1G")).toEqual(["NumberLiteral:1", "Identifier:G"]) + expect(tokenTypes("64K")).toEqual(["NumberLiteral:64", "Identifier:K"]) + }) + + it("keeps other numeric literals", () => { + expect(tokenTypes("1e5")).toEqual(["NumberLiteral:1e5"]) + expect(tokenTypes("1.5")).toEqual(["NumberLiteral:1.5"]) + expect(tokenTypes("1_000_000")).toEqual(["NumberLiteral:1_000_000"]) + expect(tokenTypes("100L")).toEqual(["LongLiteral:100L"]) + expect(tokenTypes("10.5m")).toEqual(["DecimalLiteral:10.5m"]) + expect(tokenTypes("1.5f")).toEqual(["FloatLiteral:1.5f"]) + }) +}) + +describe("ns unit", () => { + it("parses o3MaxLag in nanoseconds", () => { + expect(roundTrip("ALTER TABLE t SET PARAM o3MaxLag = 100ns")).toBe( + "ALTER TABLE t SET PARAM o3MaxLag = 100ns", + ) + expect( + roundTrip( + "CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WITH o3MaxLag=1000ns", + ), + ).toBe( + "CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WITH o3MaxLag = 1000ns", + ) + expect( + roundTrip("INSERT BATCH 100 o3MaxLag 500ns INTO x SELECT * FROM y"), + ).toBe("INSERT BATCH 100 o3MaxLag 500ns INTO x SELECT * FROM y") + }) +}) + +describe("invalid units are errors, not a second statement", () => { + it.each([ + // SampleByTest:250 / :262 ("Invalid unit: 1hour" / "1min") + "select b, sum(a), k from x sample by 1hour", + "select b, sum(a), k from x sample by 1min", + // AlterTableO3MaxLagTest:363 / :370 ("interval qualifier") + "ALTER TABLE X SET PARAM o3MaxLag = 111days", + "ALTER TABLE X SET PARAM o3MaxLag = 111ml", + "ALTER TABLE t SET PARAM o3MaxLag = 1sec", + "CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WITH o3MaxLag=1sec", + // parseTtlHoursOrMonths: "invalid time unit" + "CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY TTL 1days", + "ALTER TABLE t SET TTL 1hours", + "ALTER TABLE t SET TTL 1_000days", + "INSERT BATCH 100 o3MaxLag 3days INTO x SELECT * FROM y", + "SELECT * FROM a ASOF JOIN b TOLERANCE 1hour", + "SELECT 1hour", + "SELECT 1.5hour", + ])("%s", (sql) => { + expectError(sql) + }) + + it("long-form TTL with a space still parses", () => { + expect( + roundTrip( + "CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY TTL 1 DAYS", + ), + ).toBe( + "CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY TTL 1 DAY", + ) + expect(roundTrip("ALTER TABLE t SET TTL 12 HOURS")).toBe( + "ALTER TABLE t SET TTL 12 HOURS", + ) + }) + + it("geohash precision and memory sizes still parse", () => { + expect(roundTrip("CREATE TABLE t (g GEOHASH(8c), h GEOHASH(5b))")).toBe( + "CREATE TABLE t (g GEOHASH(8c), h GEOHASH(5b))", + ) + expect(roundTrip("ALTER USER u SET MEMORY LIMIT 64K")).toBe( + "ALTER USER u SET MEMORY LIMIT 64K", + ) + expect(roundTrip("ALTER USER u SET MEMORY LIMIT 512M")).toBe( + "ALTER USER u SET MEMORY LIMIT 512M", + ) + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 55a3f29..4eb8c5b 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -682,5 +682,71 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/LatestByTest.java:3797", "feature": "single-quote-unescape", "toSql": "SELECT s, v FROM escaped_symbols_0 WHERE (s IN ('O''Brien', 'O''''Brien', 'x') AND s NOT IN ('O''Brien', 'x')) LATEST ON ts PARTITION BY s ORDER BY v" + }, + { + "query": "select sym, last(price) as price, ts from base_price sample by 1000000000n", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewTest.java:9487", + "feature": "duration-boundary", + "toSql": "SELECT sym, last(price) AS price, ts FROM base_price SAMPLE BY 1000000000n" + }, + { + "query": "select x,sum(y) from tab sample by 2U align to first observation", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:11184", + "feature": "duration-boundary", + "toSql": "SELECT x, sum(y) FROM tab SAMPLE BY 2U ALIGN TO FIRST OBSERVATION" + }, + { + "query": "select sum(a), k from x sample by 100T fill(none) align to first observation", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/groupby/SampleByNanoTimestampTest.java:5436", + "feature": "duration-boundary", + "toSql": "SELECT sum(a), k FROM x SAMPLE BY 100T FILL(NONE) ALIGN TO FIRST OBSERVATION" + }, + { + "query": "SELECT ts, s, first(arr) a FROM tango SAMPLE BY 1m FILL(PREV) ORDER BY s, ts", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/ArrayTest.java:852", + "feature": "duration-boundary", + "toSql": "SELECT ts, s, first(arr) AS a FROM tango SAMPLE BY 1m FILL(PREV) ORDER BY s, ts" + }, + { + "query": "alter table x1 set param o3MaxLag = 100ms", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/AlterTableO3MaxLagTest.java:429", + "feature": "duration-boundary", + "toSql": "ALTER TABLE x1 SET PARAM o3MaxLag = 100ms" + }, + { + "query": "alter table x1 set param o3MaxLag = 10us", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/AlterTableO3MaxLagTest.java:434", + "feature": "duration-boundary", + "toSql": "ALTER TABLE x1 SET PARAM o3MaxLag = 10us" + }, + { + "query": "create table x (gh GEOHASH(8c), t TIMESTAMP) timestamp(t) partition by DAY WITH maxUncommittedRows=10000, o3MaxLag=250ms;", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:4572", + "feature": "duration-boundary", + "toSql": "CREATE TABLE x (gh GEOHASH(8c), t TIMESTAMP) TIMESTAMP(t) PARTITION BY DAY WITH maxUncommittedRows = 10000, o3MaxLag = 250ms" + }, + { + "query": "CREATE TABLE tango (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY HOUR TTL 1D", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/TtlTest.java:264", + "feature": "duration-boundary", + "toSql": "CREATE TABLE tango (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY HOUR TTL 1 DAY" + }, + { + "query": "insert batch 20000 o3MaxLag 3d into x select * from top", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/o3/O3Test.java:5151", + "feature": "duration-boundary", + "toSql": "INSERT BATCH 20000 o3MaxLag 3d INTO x SELECT * FROM top" + }, + { + "query": "select * from tab t1 asof join tab t2 tolerance 15250285w", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:1269", + "feature": "duration-boundary", + "toSql": "SELECT * FROM tab AS t1 ASOF JOIN tab AS t2 TOLERANCE 15250285w" + }, + { + "query": "ALTER GROUP test_group SET MEMORY LIMIT 1G", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/cairo/wal/transfer/ReplicationAclTest.java:2310", + "feature": "duration-boundary", + "toSql": "ALTER GROUP test_group SET MEMORY LIMIT 1G" } ] diff --git a/tests/helpers/autocomplete-walkthrough.ts b/tests/helpers/autocomplete-walkthrough.ts index 29b7ddd..710188b 100644 --- a/tests/helpers/autocomplete-walkthrough.ts +++ b/tests/helpers/autocomplete-walkthrough.ts @@ -200,6 +200,7 @@ export const SKIP_TOKEN_TYPES = new Set([ "StringLiteral", "NumberLiteral", "DurationLiteral", + "MalformedDurationLiteral", "BooleanLiteral", "NullLiteral", "GeohashLiteral", From 65d4abd21b24b8044d8d8ff36e76fd03eb2f0f07 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 15:07:55 +0300 Subject: [PATCH 09/36] feat: parse bind variables $n and :name Server: ExpressionNode.of types a literal token as BIND_VARIABLE when it is `$` + digits or starts with `:` (FunctionParser.createBindVariable binds it). `:name` is glued in ExpressionParser.processColon: a lone `:` that does not follow a completed operand (literal, constant, `)`, `]`; isCompletedOperand) and is not followed by whitespace takes the next token as the name. So `arr[1:n]` / `arr[i:j]` stay slices, while `x = :sym`, `IN (:a, :b)`, `LIMIT :lo, :hi`, `-:v` and even `arr[:1]` are bind variables (ArrayTest:4428 "undefined bind variable: :1"). SAMPLE BY takes an expression and, when it is not a full period, a one-letter unit (SqlParser.expectSample): `SAMPLE BY $1 d`. Before: `$` was a lexer error and `:name` "Unexpected token ':'" in every position (WHERE, VALUES, LIMIT, SELECT items, function args, IN lists, DECLARE values, UPDATE SET, SAMPLE BY FROM/TO, ...). Now: - New BindVariable token (custom lexer pattern, before Colon): `$\d+` with a word boundary (`$1abc`, `$a`, `$` stay lexer errors), and `:[A-Za-z0-9_\u0080-\uFFFF]+` only when the previous token does not end an operand (Identifier, QuotedIdentifier, `)`, `]`, literals, bind/@ variables, TRUE/FALSE/NULL/NaN). Keywords are operators/clause words (`LIMIT :n`, `THEN :x`), except a non-reserved keyword glued to the colon, which is a column name (`arr[offset:n]`). `::` and `:=` keep lexing first. - primaryExpression accepts BindVariable -> new AST node `{ type: "bindVariable", name: "$1" | ":sym" }` (name as written); toSql prints it verbatim. Covers LIMIT, FROM/TO and every expression slot. - sampleByClause: `BindVariable ` (gated to the SAMPLE BY unit letters); duration is "$1 d". `SAMPLE BY $1` without a unit is an error, as on the server. - Formatter: BindVariable is a literal; a slice Colon followed by a gap and a word/`:` keeps the gap (`arr[: 5]`), since gluing would create `:5`. Autocomplete never suggests the token; walkthrough skips BindVariable and Colon. - `arr[:5]` (no space) now reads as index by bind variable `:5` (server semantics) instead of an open-start slice; `arr[: 5]` stays a slice. parser.test.ts updated accordingly. Tests: 38 server queries (OSS CoveringIndex*, PGJobContextTest, LimitTest, QwpEgressBind*, MatViewExpireRowsHardeningTest, DeclareTest, SqlCodeGeneratorTest, ArrayCreateFunctionFactoryTest, InLongFunction*, MonotonicTimestampPruningTest, UpdateTest, SampleByNanoTimestampTest, HashJoinGroupByRuntimeConstantTest, PercentileWindowFunctionTest, SubsampleTest, CopyExportTest, CompiledFilterRegressionTest, DeclareOperatorPrecedenceTest, GroupByTest, IntervalFilterTest, OrderByEncodeSortTest, ParquetTest, RuntimeConstFunctionTest, SubsampleWildcardProjectionTest, LatestByTest, ILikeSymbol*, ArrayTest slice guards; Ent ViewAuditTest, TableColdPartitionsTest) + tests/bind-variables.test.ts. Red before: 150 failing (110 fixture + 40 focused) -> 727/727 pass; full suite 16201 passed. Corpus: fixed 528 (305 `$n`, 223 `:name`) / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0. Left out: ALIGN TO CALENDAR TIME ZONE $1 / WITH OFFSET $1 and FILL($1), FILL(PREV($1)) (11 corpus strings) - those positions are not expressions in sql-parser yet (AST stores strings) and belong to sample-by-fill-align. TOLERANCE $1 and SWITCH ROLE ... TIMEOUT $1 stay errors (server rejects). --- src/autocomplete/token-classification.ts | 2 + src/formatter/classification.ts | 1 + src/formatter/spacing.ts | 8 + src/parser/ast.ts | 10 + src/parser/cst-types.d.ts | 3 + src/parser/lexer.ts | 69 ++++- src/parser/parser.ts | 17 ++ src/parser/toSql.ts | 2 + src/parser/visitor.ts | 10 +- tests/bind-variables.test.ts | 330 ++++++++++++++++++++++ tests/fixtures/server-queries.json | 228 +++++++++++++++ tests/helpers/autocomplete-walkthrough.ts | 3 + tests/parser.test.ts | 6 +- 13 files changed, 685 insertions(+), 4 deletions(-) create mode 100644 tests/bind-variables.test.ts diff --git a/src/autocomplete/token-classification.ts b/src/autocomplete/token-classification.ts index 681e9e9..cff8799 100644 --- a/src/autocomplete/token-classification.ts +++ b/src/autocomplete/token-classification.ts @@ -107,6 +107,8 @@ export const SKIP_TOKENS = new Set([ "BitOr", // Variable references (user-defined, can't suggest) "VariableReference", + // Bind variables ($1, :name) + "BindVariable", // Literals (don't suggest literal tokens) "StringLiteral", "NumberLiteral", diff --git a/src/formatter/classification.ts b/src/formatter/classification.ts index 29f95bb..306e4b8 100644 --- a/src/formatter/classification.ts +++ b/src/formatter/classification.ts @@ -56,6 +56,7 @@ const literalTokenNames = [ "QuotedIdentifier", "Identifier", "VariableReference", + "BindVariable", ] export const wordTokenNames: ReadonlySet = new Set([ diff --git a/src/formatter/spacing.ts b/src/formatter/spacing.ts index 19710b1..daf0191 100644 --- a/src/formatter/spacing.ts +++ b/src/formatter/spacing.ts @@ -32,6 +32,9 @@ const isComment = (piece: Piece) => const isNumber = (piece: Piece) => /^\d/.test(piece.token.image) +const startsLikeBindVariableName = (piece: Piece) => + /^[\w\u0080-\uFFFF:]/.test(piece.token.image) + const isUncertain = (piece: Piece) => piece.token.kind === "opaque" || piece.token.kind === "tolerant" @@ -64,6 +67,11 @@ export const gapBetween = (previous: Piece | null, next: Piece): string => { if (nextName === "Dot" && isNumber(previous)) return " " if (previousName === "Dot" && isNumber(next)) return " " if (nextName !== null && noSpaceBefore.has(nextName)) return "" + // `arr[: 5]`, `: name`, `: :x`: gluing the colon to what follows would lex + // as a `:name` bind variable or `::`, so keep the gap. + if (previousName === "Colon" && startsLikeBindVariableName(next)) { + return preserved(next) + } if (previousName !== null && noSpaceAfter.has(previousName)) return "" if (nextName === "LParen" && previous.token.kind === "word") { return preserved(next) diff --git a/src/parser/ast.ts b/src/parser/ast.ts index cc7db06..3f34283 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -1253,6 +1253,7 @@ export type Expression = | UnaryExpression | ColumnRef | VariableRef + | BindVariable | Literal | FunctionCall | CaseExpression @@ -1404,6 +1405,15 @@ export interface VariableRef extends AstNode { name: string } +/** + * Bind variable: positional `$1` or named `:name`. `name` is the token as + * written, including the `$` / `:` sigil. + */ +export interface BindVariable extends AstNode { + type: "bindVariable" + name: string +} + export interface ParenExpression extends AstNode { type: "paren" expression: Expression diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index fb375db..3a924fb 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -496,6 +496,8 @@ export type SampleByClauseCstChildren = { By: IToken[]; DurationLiteral?: IToken[]; VariableReference?: IToken[]; + BindVariable?: IToken[]; + sampleByUnit?: IToken[]; Identifier?: IToken[]; groupingSetsSpec?: GroupingSetsSpecCstNode[]; fromToClause?: FromToClauseCstNode[]; @@ -2672,6 +2674,7 @@ export type PrimaryExpressionCstChildren = { functionCall?: FunctionCallCstNode[]; literal?: LiteralCstNode[]; VariableReference?: IToken[]; + BindVariable?: IToken[]; identifierExpression?: IdentifierExpressionCstNode[]; LParen?: IToken[]; selectStatement?: SelectStatementCstNode[]; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index 2c37c7c..4daa081 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -1,4 +1,4 @@ -import { createToken, Lexer, TokenType } from "chevrotain" +import { createToken, IToken, Lexer, TokenType } from "chevrotain" // Import all keyword tokens from tokens.ts (auto-generated from grammar arrays) // Re-export all named tokens for parser usage @@ -781,6 +781,72 @@ export const VariableReference = createToken({ // Bare @ sign (only matches when not followed by a word character) export const AtSign = createToken({ name: "AtSign", pattern: /@/ }) +// Bind variables: `$1` (positional) and `:name` (named). +// Server: ExpressionNode.of types a token as BIND_VARIABLE when it is `$` + +// digits or starts with `:`. A named variable is glued in +// ExpressionParser.processColon: a lone `:` that does not follow a completed +// operand (literal, constant, `)`, `]`) and is not followed by whitespace takes +// the next token as its name. So `arr[1:n]` / `arr[i:j]` stay slices, while +// `x = :sym`, `IN (:a, :b)`, `LIMIT :n` and `arr[:1]` are bind variables. +// `::` and `:=` are matched earlier by DoubleColon / ColonEquals. +const POSITIONAL_BIND_VARIABLE = /\$\d+(?![\w$\u0080-\uFFFF])/y +const NAMED_BIND_VARIABLE = /:[a-zA-Z0-9_\u0080-\uFFFF]+/y + +const COMPLETED_OPERAND_TOKEN_NAMES: ReadonlySet = new globalThis.Set([ + "Identifier", + "QuotedIdentifier", + "RParen", + "RBracket", + "BindVariable", + "VariableReference", + "True", + "False", + "Null", + "Nan", +]) + +/** + * Whether the token before a `:` ends an operand, which makes the colon an + * array-slice separator rather than the start of a `:name` bind variable. + * Keywords are operators / clause words (`= :x`, `LIMIT :x`, `THEN :x`) + * except the value constants; a non-reserved keyword glued to the colon is a + * column name (`arr[offset:n]`). + */ +function endsOperand(prev: IToken, colonOffset: number): boolean { + const type = prev.tokenType + if (COMPLETED_OPERAND_TOKEN_NAMES.has(type.name)) return true + if (type.name.endsWith("Literal")) return true + return ( + prev.endOffset !== undefined && + prev.endOffset + 1 === colonOffset && + (type.CATEGORIES ?? []).includes(IdentifierKeyword) + ) +} + +function matchBindVariable( + text: string, + offset: number, + tokens: IToken[], +): RegExpExecArray | null { + const ch = text.charAt(offset) + if (ch === "$") { + POSITIONAL_BIND_VARIABLE.lastIndex = offset + return POSITIONAL_BIND_VARIABLE.exec(text) + } + if (ch !== ":") return null + const prev = tokens[tokens.length - 1] + if (prev && endsOperand(prev, offset)) return null + NAMED_BIND_VARIABLE.lastIndex = offset + return NAMED_BIND_VARIABLE.exec(text) +} + +export const BindVariable = createToken({ + name: "BindVariable", + pattern: matchBindVariable, + start_chars_hint: ["$", ":"], + line_breaks: false, +}) + // ============================================================================= // Literals // ============================================================================= @@ -974,6 +1040,7 @@ export const allTokens: TokenType[] = [ BitXor, BitOr, RegexMatch, + BindVariable, Colon, VariableReference, AtSign, diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 9135410..6e0861e 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -260,6 +260,7 @@ import { Lock, Exclusive, VariableReference, + BindVariable, ColonEquals, RegexMatch, RegexNotMatch, @@ -1316,6 +1317,20 @@ class QuestDBParser extends CstParser { this.OR([ { ALT: () => this.CONSUME(DurationLiteral) }, { ALT: () => this.CONSUME(VariableReference) }, + { + // SAMPLE BY $1 d / :n h — a bind variable is not a full period, so + // the server requires a one-letter unit after it + // (SqlParser.expectSample: "one letter sample by period unit expected"). + ALT: () => { + this.CONSUME(BindVariable) + this.OR1([ + { + GATE: () => this.isSampleByPeriodLetter(this.LA(1).image), + ALT: () => this.CONSUME1(Identifier, { LABEL: "sampleByUnit" }), + }, + ]) + }, + }, // Bare single-letter period unit with no leading digit, e.g. `SAMPLE BY w` // (equivalent to `SAMPLE BY 1w`). Lexes as an Identifier; the GATE keeps // this alternative restricted to valid unit letters. @@ -4943,6 +4958,8 @@ class QuestDBParser extends CstParser { { ALT: () => this.CONSUME(VariableReference), }, + // Bind variable: $1 or :name + { ALT: () => this.CONSUME(BindVariable) }, { ALT: () => this.SUBRULE(this.identifierExpression) }, { ALT: () => { diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index cea8bff..0c06048 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -1848,6 +1848,8 @@ function expressionToSql(expr: AST.Expression): string { return qualifiedNameToSql(expr.name) case "variable": return `@${expr.name}` + case "bindVariable": + return expr.name case "literal": return literalToSql(expr) case "function": diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 6145b40..460fe09 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -1040,7 +1040,9 @@ class QuestDBVisitor extends BaseVisitor { ? this.tokenImage(ctx.DurationLiteral[0]) : ctx.VariableReference ? this.tokenImage(ctx.VariableReference[0]) - : this.tokenImage(ctx.Identifier![0]), + : ctx.BindVariable + ? `${this.tokenImage(ctx.BindVariable[0])} ${this.tokenImage(ctx.sampleByUnit![0])}` + : this.tokenImage(ctx.Identifier![0]), } if (ctx.groupingSetsSpec) { result.groupingSets = this.visit(ctx.groupingSetsSpec) as AST.GroupingSets @@ -4285,6 +4287,12 @@ class QuestDBVisitor extends BaseVisitor { name: image.substring(1), // strip leading @ } } + if (ctx.BindVariable) { + return { + type: "bindVariable", + name: ctx.BindVariable[0].image, // "$1" or ":name" + } + } if (ctx.identifierExpression) { return this.visit(ctx.identifierExpression) as AST.Expression } diff --git a/tests/bind-variables.test.ts b/tests/bind-variables.test.ts new file mode 100644 index 0000000..defa197 --- /dev/null +++ b/tests/bind-variables.test.ts @@ -0,0 +1,330 @@ +/** + * Bind variables (reports §3.1): `$1` (positional) and `:name` (named). + * + * Server: ExpressionNode.of turns a literal token into BIND_VARIABLE when it + * is `$` + digits or starts with `:`; FunctionParser.createBindVariable binds + * it. A `:name` token is glued by ExpressionParser.processColon: a single `:` + * that does NOT follow a completed operand (literal, constant, `)` or `]`) and + * is not followed by whitespace takes the next token as the variable name. + * So `arr[1:n]` / `arr[i:j]` stay array slices, while `x = :sym`, + * `IN (:a, :b)`, `LIMIT :lo, :hi` and even `arr[:1]` (ArrayTest:4428 + * "undefined bind variable: :1") are bind variables. + * + * Server queries live in tests/fixtures/server-queries.json + * (feature "bind-variables"). + */ + +import { describe, it, expect } from "vitest" +import { + parseToAst, + tokenize, + getNextValidTokens, + createAutocompleteProvider, +} from "../src/index" +import { toSql } from "../src/parser/toSql" +import { format } from "../src/formatter/index" +import type * as AST from "../src/parser/ast" + +const tokenTypes = (sql: string) => + tokenize(sql).tokens.map((t) => `${t.tokenType.name}:${t.image}`) + +const parseOne = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +const roundTrip = (sql: string) => toSql([parseOne(sql)]) + +const expectError = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors.length).toBeGreaterThan(0) +} + +const whereOf = (sql: string) => + (parseOne(sql) as AST.SelectStatement).where as AST.Expression + +describe("lexer", () => { + it.each(["$1", "$2", "$12", "$100"])("reads %s as one bind variable", (v) => { + expect(tokenTypes(`a = ${v}`)).toEqual([ + "Identifier:a", + "Equals:=", + `BindVariable:${v}`, + ]) + }) + + it.each([ + ["x = :sym", "BindVariable::sym"], + ["(:sym", "BindVariable::sym"], + [", :k2", "BindVariable::k2"], + ["-:sv", "BindVariable::sv"], + ["SELECT :b0", "BindVariable::b0"], + ["LIMIT :lim", "BindVariable::lim"], + ["IN :date", "BindVariable::date"], + ["@a := :value", "BindVariable::value"], + ["TIME ZONE :tz", "BindVariable::tz"], + ["a[:1]", "BindVariable::1"], + ["a[1, :n]", "BindVariable::n"], + ["CASE WHEN x THEN :a", "BindVariable::a"], + ])("reads the :name in `%s` as a bind variable", (sql, expected) => { + expect(tokenTypes(sql)).toContain(expected) + expect(tokenTypes(sql)).not.toContain("Colon::") + }) + + it.each([ + "arr[1:n]", + "arr[i:j]", + "arr[i:j+1]", + "arr[(i):j]", + "arr[x[1]:j]", + "arr[$1:n]", + "arr[:a:b]", + "arr['a':b]", + "arr[1.5:b]", + "arr[null:b]", + "arr[@v:b]", + "arr[offset:b]", + 'arr["i":j]', + ])("keeps the slice colon in %s", (sql) => { + expect(tokenTypes(sql)).toContain("Colon::") + }) + + it("does not glue a colon followed by whitespace", () => { + expect(tokenTypes("x = : sym")).toEqual([ + "Identifier:x", + "Equals:=", + "Colon::", + "Identifier:sym", + ]) + }) + + it("keeps :: and := as operators", () => { + expect(tokenTypes("x::int")).toEqual([ + "Identifier:x", + "DoubleColon:::", + "Int:int", + ]) + expect(tokenTypes(":b0::LONG")).toEqual([ + "BindVariable::b0", + "DoubleColon:::", + "Long:LONG", + ]) + expect(tokenTypes("@x := 1")).toEqual([ + "VariableReference:@x", + "ColonEquals::=", + "NumberLiteral:1", + ]) + expect(tokenTypes("@x:=:a")).toEqual([ + "VariableReference:@x", + "ColonEquals::=", + "BindVariable::a", + ]) + }) + + it("leaves $ and : inside strings and quoted names alone", () => { + expect(tokenTypes("'$1'")).toEqual(["StringLiteral:'$1'"]) + expect(tokenTypes("':keep'")).toEqual(["StringLiteral:':keep'"]) + expect(tokenTypes('"price$1"')).toEqual(['QuotedIdentifier:"price$1"']) + }) + + it.each(["$", "$a", "$1abc", "$1_"])("rejects %s", (v) => { + expect(tokenize(`SELECT ${v}`).errors.length).toBeGreaterThan(0) + }) +}) + +describe("AST", () => { + it("parses $1 as a bindVariable node", () => { + expect(whereOf("SELECT * FROM t WHERE a = $1")).toEqual({ + type: "binary", + operator: "=", + left: { type: "column", name: { type: "qualifiedName", parts: ["a"] } }, + right: { type: "bindVariable", name: "$1" }, + }) + }) + + it("parses :name as a bindVariable node", () => { + const where = whereOf( + "SELECT * FROM t WHERE a = :sym", + ) as AST.BinaryExpression + expect(where.right).toEqual({ type: "bindVariable", name: ":sym" }) + }) + + it("parses bind variables in LIMIT", () => { + const stmt = parseOne( + "SELECT * FROM t LIMIT $1, :hi", + ) as AST.SelectStatement + expect(stmt.limit?.lowerBound).toEqual({ type: "bindVariable", name: "$1" }) + expect(stmt.limit?.upperBound).toEqual({ + type: "bindVariable", + name: ":hi", + }) + }) + + it("parses a bind variable as a select item and function argument", () => { + const stmt = parseOne("SELECT $1 AS a, f(:x) FROM t") as AST.SelectStatement + const items = stmt.columns as AST.ExpressionSelectItem[] + expect(items[0].expression).toEqual({ type: "bindVariable", name: "$1" }) + expect(items[0].alias).toBe("a") + expect((items[1].expression as AST.FunctionCall).args).toEqual([ + { type: "bindVariable", name: ":x" }, + ]) + }) + + it("parses `arr[:1]` as an index by bind variable :1, like the server", () => { + const stmt = parseOne("SELECT arr[:1] FROM t") as AST.SelectStatement + const item = (stmt.columns as AST.ExpressionSelectItem[])[0] + expect(item.expression).toEqual({ + type: "arrayAccess", + array: { + type: "column", + name: { type: "qualifiedName", parts: ["arr"] }, + }, + subscripts: [{ type: "bindVariable", name: ":1" }], + }) + }) + + it("keeps `arr[1:n]` an array slice", () => { + const stmt = parseOne("SELECT arr[1:n] FROM t") as AST.SelectStatement + const item = (stmt.columns as AST.ExpressionSelectItem[])[0] + const access = item.expression as AST.ArrayAccessExpression + expect(access.subscripts[0]).toMatchObject({ + type: "arraySlice", + start: { type: "literal", value: 1 }, + end: { type: "column" }, + }) + }) + + it("keeps the open-start slice when the colon is followed by whitespace", () => { + const stmt = parseOne("SELECT arr[: 5] FROM t") as AST.SelectStatement + const item = (stmt.columns as AST.ExpressionSelectItem[])[0] + const access = item.expression as AST.ArrayAccessExpression + expect(access.subscripts[0]).toMatchObject({ + type: "arraySlice", + end: { type: "literal", value: 5 }, + }) + }) + + it("parses SAMPLE BY $1 with the unit kept in the duration", () => { + const stmt = parseOne( + "SELECT ts, count() FROM t SAMPLE BY $1 d", + ) as AST.SelectStatement + expect(stmt.sampleBy?.duration).toBe("$1 d") + }) +}) + +describe("round trip", () => { + it.each([ + ["SELECT * FROM t WHERE a = $1", "SELECT * FROM t WHERE a = $1"], + ["select * from t where a=:sym", "SELECT * FROM t WHERE a = :sym"], + ["insert into t values ($1, :b, 3)", "INSERT INTO t VALUES ($1, :b, 3)"], + ["select * from t limit $1", "SELECT * FROM t LIMIT $1"], + ["select * from t limit :lo,:hi", "SELECT * FROM t LIMIT :lo, :hi"], + ["select * from t sample by $1 h", "SELECT * FROM t SAMPLE BY $1 h"], + [ + "select * from t sample by :n T fill(null)", + "SELECT * FROM t SAMPLE BY :n T FILL(NULL)", + ], + [ + "select * from t sample by 1h from $1 to $2", + "SELECT * FROM t SAMPLE BY 1h FROM $1 TO $2", + ], + ["select $1::int + :x", "SELECT $1::INT + :x"], + ["select -$1, -:x", "SELECT -$1, -:x"], + ["select arr[:1] from t", "SELECT arr[:1] FROM t"], + ["select arr[1:n] from t", "SELECT arr[1:n] FROM t"], + ["select arr[$1:$2] from t", "SELECT arr[$1:$2] FROM t"], + [ + "select case when a then :x else $2 end from t", + "SELECT CASE WHEN a THEN :x ELSE $2 END FROM t", + ], + ["update t set a = :a where b = $1", "UPDATE t SET a = :a WHERE b = $1"], + ["declare @a := :v select @a", "DECLARE @a := :v SELECT @a"], + ])("%s", (sql, expected) => { + const out = roundTrip(sql) + expect(out).toBe(expected) + expect(roundTrip(out)).toBe(expected) + }) +}) + +describe("errors", () => { + it.each([ + // SqlParser.expectSample: a bind variable interval needs a unit letter + "SELECT * FROM t SAMPLE BY $1", + "SELECT * FROM t SAMPLE BY $1 FILL(NULL)", + // not a sample by period letter + "SELECT * FROM t SAMPLE BY $1 x", + // a bare colon is not a value + "SELECT * FROM t WHERE a = : sym", + // a colon after a completed operand is not a bind variable + "SELECT a :b FROM t", + "SELECT * FROM t WHERE a = $", + ])("rejects %s", (sql) => { + expectError(sql) + }) +}) + +describe("autocomplete", () => { + const provider = createAutocompleteProvider({ + tables: [{ name: "t" }], + columns: { t: [{ name: "a", type: "INT" }] }, + }) + const labels = (sql: string) => + provider.getSuggestions(sql, sql.length).map((s) => s.label) + + it.each(["SELECT * FROM t WHERE a = $1 ", "SELECT * FROM t WHERE a = :sym "])( + "continues the expression after a bind variable: %s", + (sql) => { + expect(labels(sql)).toContain("AND") + expect(labels(sql)).toContain("ORDER") + }, + ) + + it("does not suggest anything for the bind variable token itself", () => { + for (const sql of [ + "SELECT * FROM t WHERE a = ", + "SELECT * FROM t LIMIT ", + ]) { + expect(labels(sql).some((l) => /^[$:]/.test(l))).toBe(false) + } + }) + + it("accepts a bind variable where an expression can start", () => { + expect(getNextValidTokens("SELECT * FROM t WHERE a = ")).toContain( + "BindVariable", + ) + expect(getNextValidTokens("SELECT * FROM t SAMPLE BY ")).toContain( + "BindVariable", + ) + }) +}) + +describe("formatter", () => { + it("keeps bind variables glued", () => { + expect( + format( + "select a, $1 from t where a = :sym and b in (:k1, :k2) limit :lo, :hi", + ), + ).toBe( + "select a, $1\nfrom t\nwhere a = :sym and b in (:k1, :k2)\nlimit :lo, :hi", + ) + }) + + it("keeps unary minus and array subscripts glued", () => { + expect( + format("SELECT id FROM neg_min WHERE -:sv = s and arr[:1] > -$1"), + ).toBe("SELECT id\nFROM neg_min\nWHERE -:sv = s and arr[:1] > -$1") + }) + + it("does not glue a slice colon to the following word", () => { + expect(format("SELECT arr[: 5], arr[1: n], arr[1 :n] FROM t")).toBe( + "SELECT arr[: 5], arr[1: n], arr[1:n]\nFROM t", + ) + }) + + it("keeps a SAMPLE BY bind variable and its unit", () => { + expect(format("select count() from t sample by $1 d")).toBe( + "select count()\nfrom t\nsample by $1 d", + ) + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 4eb8c5b..936f963 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -748,5 +748,233 @@ "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/cairo/wal/transfer/ReplicationAclTest.java:2310", "feature": "duration-boundary", "toSql": "ALTER GROUP test_group SET MEMORY LIMIT 1G" + }, + { + "query": "SELECT sym, val FROM t_bv_latest WHERE sym = $1 LATEST ON ts PARTITION BY sym", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/covering/CoveringIndexBindVariableKeyTest.java:46", + "feature": "bind-variables", + "toSql": "SELECT sym, val FROM t_bv_latest WHERE sym = $1 LATEST ON ts PARTITION BY sym" + }, + { + "query": "SELECT sym, price FROM t_in_dup_bind WHERE sym IN ($1, $2) LATEST ON ts PARTITION BY sym", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/covering/CoveringIndexTest.java:12920", + "feature": "bind-variables", + "toSql": "SELECT sym, price FROM t_in_dup_bind WHERE sym IN ($1, $2) LATEST ON ts PARTITION BY sym" + }, + { + "query": "INSERT INTO test VALUES($1, $2);", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGJobContextTest.java:6412", + "feature": "bind-variables", + "toSql": "INSERT INTO test VALUES ($1, $2)" + }, + { + "query": "select * from y limit $1, $2;", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/LimitTest.java:723", + "feature": "bind-variables", + "toSql": "SELECT * FROM y LIMIT $1, $2" + }, + { + "query": "SELECT $1::UUID AS u FROM long_sequence(1)", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/qwp/QwpEgressBindFuzzTest.java:227", + "feature": "bind-variables", + "toSql": "SELECT $1::UUID AS u FROM long_sequence(1)" + }, + { + "query": "SELECT CAST($1 AS BOOLEAN) AS v FROM long_sequence(1)", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/qwp/QwpEgressBindRoundTripTest.java:384", + "feature": "bind-variables", + "toSql": "SELECT CAST($1 AS BOOLEAN) AS v FROM long_sequence(1)" + }, + { + "query": "alter materialized view mv set expire rows when v > $1", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewExpireRowsHardeningTest.java:1188", + "feature": "bind-variables", + "toSql": "ALTER MATERIALIZED VIEW mv SET EXPIRE ROWS WHEN v > $1" + }, + { + "query": "DECLARE @x := $1, @y := $2 SELECT @x, @y", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:856", + "feature": "bind-variables", + "toSql": "DECLARE @x := $1, @y := $2 SELECT @x, @y" + }, + { + "query": "select $1+x, $2 from long_sequence(10)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/SqlCodeGeneratorTest.java:7919", + "feature": "bind-variables", + "toSql": "SELECT $1 + x, $2 FROM long_sequence(10)" + }, + { + "query": "SELECT ARRAY[1.0, $1]", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/array/ArrayCreateFunctionFactoryTest.java:60", + "feature": "bind-variables", + "toSql": "SELECT ARRAY[1.0, $1]" + }, + { + "query": "SELECT rn FROM x WHERE (a*$1) IS NULL", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/bool/InLongFunctionFactoryTest.java:668", + "feature": "bind-variables", + "toSql": "SELECT rn FROM x WHERE (a * $1) IS NULL" + }, + { + "query": "SELECT * FROM trades WHERE date_trunc('day', timestamp) BETWEEN $1 AND $2", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/MonotonicTimestampPruningTest.java:183", + "feature": "bind-variables", + "toSql": "SELECT * FROM trades WHERE date_trunc('day', timestamp) BETWEEN $1 AND $2" + }, + { + "query": "UPDATE up SET x = $1 WHERE x > 1 and x < 4", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/UpdateTest.java:3011", + "feature": "bind-variables", + "toSql": "UPDATE up SET x = $1 WHERE x > 1 AND x < 4" + }, + { + "query": "select k, s, first(lat) lat, last(lon) lon from x where s in ('a') sample by $1 T align to calendar", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/groupby/SampleByNanoTimestampTest.java:143", + "feature": "bind-variables", + "toSql": "SELECT k, s, first(lat) AS lat, last(lon) AS lon FROM x WHERE s IN ('a') SAMPLE BY $1 T ALIGN TO CALENDAR" + }, + { + "query": "SELECT r.reading_ts, count(*) n, sum(r.energy_kwh) energy FROM r JOIN p ON r.plant_id = p.plant_id SAMPLE BY 1d FROM $1 TO $2 ALIGN TO CALENDAR TIME ZONE 'Europe/Berlin'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/HashJoinGroupByRuntimeConstantTest.java:450", + "feature": "bind-variables", + "toSql": "SELECT r.reading_ts, count(*) AS n, sum(r.energy_kwh) AS energy FROM r JOIN p ON r.plant_id = p.plant_id SAMPLE BY 1d FROM $1 TO $2 ALIGN TO CALENDAR TIME ZONE 'Europe/Berlin'" + }, + { + "query": "SELECT k, percentile_disc($1, 0.5) OVER () p, quantile_disc($2, 0.5) OVER (PARTITION BY k) q FROM t", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/window/PercentileWindowFunctionTest.java:1933", + "feature": "bind-variables", + "toSql": "SELECT k, percentile_disc($1, 0.5) OVER () AS p, quantile_disc($2, 0.5) OVER (PARTITION BY k) AS q FROM t" + }, + { + "query": "EXPLAIN SELECT price, ts FROM t SUBSAMPLE cadence($1)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SubsampleTest.java:5481", + "feature": "bind-variables", + "toSql": "EXPLAIN SELECT price, ts FROM t SUBSAMPLE cadence($1)" + }, + { + "query": "DECLARE @flag := ($1 = 1) SELECT * FROM v_flag", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/view/ViewAuditTest.java:644", + "feature": "bind-variables", + "toSql": "DECLARE @flag := ($1 = 1) SELECT * FROM v_flag" + }, + { + "query": "COPY (SELECT * FROM ts_table2 WHERE x <= $1) TO 'bind_streaming' WITH FORMAT parquet", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CopyExportTest.java:2208", + "feature": "bind-variables", + "toSql": "COPY (SELECT * FROM ts_table2 WHERE x <= $1) TO 'bind_streaming' WITH FORMAT parquet" + }, + { + "query": "select * from alcatel_traffic_tmp where deviceName not in ($1,$2) and time < '1970-01-01T00:00:00.300000Z'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/SqlCodeGeneratorTest.java:9952", + "feature": "bind-variables", + "toSql": "SELECT * FROM alcatel_traffic_tmp WHERE deviceName NOT IN ($1, $2) AND time < '1970-01-01T00:00:00.300000Z'" + }, + { + "query": "SELECT price FROM t_latest_bind WHERE sym = :sym LATEST ON ts PARTITION BY sym", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/covering/CoveringIndexTest.java:6363", + "feature": "bind-variables", + "toSql": "SELECT price FROM t_latest_bind WHERE sym = :sym LATEST ON ts PARTITION BY sym" + }, + { + "query": "SELECT count() cnt FROM t_pf_mk_bind WHERE sym IN (:k1, :k2)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/covering/CoveringIndexTest.java:12786", + "feature": "bind-variables", + "toSql": "SELECT count() AS cnt FROM t_pf_mk_bind WHERE sym IN (:k1, :k2)" + }, + { + "query": "SELECT id FROM neg_min WHERE -:sv = s", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CompiledFilterRegressionTest.java:7460", + "feature": "bind-variables", + "toSql": "SELECT id FROM neg_min WHERE -:sv = s" + }, + { + "query": "select k, f, l from a4b where (:fv + f) * 1.0 > 1.5", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CompiledFilterRegressionTest.java:8560", + "feature": "bind-variables", + "toSql": "SELECT k, f, l FROM a4b WHERE (:fv + f) * 1.0 > 1.5" + }, + { + "query": "DECLARE @a := :value SELECT * FROM (SELECT @a AS value)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareOperatorPrecedenceTest.java:75", + "feature": "bind-variables", + "toSql": "DECLARE @a := :value SELECT * FROM (SELECT @a AS value)" + }, + { + "query": "SELECT :b0::LONG + 1 AS e0, count() AS a0 FROM t", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupByTest.java:4090", + "feature": "bind-variables", + "toSql": "SELECT :b0::LONG + 1 AS e0, count() AS a0 FROM t" + }, + { + "query": "select * from x where ts in :date and ts != :ts1", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/IntervalFilterTest.java:224", + "feature": "bind-variables", + "toSql": "SELECT * FROM x WHERE ts IN :date AND ts != :ts1" + }, + { + "query": "select * from x where ts not in :i", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/IntervalFilterTest.java:641", + "feature": "bind-variables", + "toSql": "SELECT * FROM x WHERE ts NOT IN :i" + }, + { + "query": "select * from y limit :lo,:hi", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/LimitTest.java:1206", + "feature": "bind-variables", + "toSql": "SELECT * FROM y LIMIT :lo, :hi" + }, + { + "query": "SELECT count(*) cnt, min(v) mn, max(v) mx FROM (SELECT * FROM x ORDER BY v LIMIT :n)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/OrderByEncodeSortTest.java:642", + "feature": "bind-variables", + "toSql": "SELECT count(*) AS cnt, min(v) AS mn, max(v) AS mx FROM (SELECT * FROM x ORDER BY v LIMIT :n)" + }, + { + "query": "SELECT c0, ts FROM t WHERE NOT ((:b0::BYTE)::DECIMAL(38, 1) >= c0)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/ParquetTest.java:789", + "feature": "bind-variables", + "toSql": "SELECT c0, ts FROM t WHERE NOT ((:b0::BYTE)::DECIMAL(38, 1) >= c0)" + }, + { + "query": "SELECT k, mode(l > :b0::INT * 17_161::SHORT) m FROM t ORDER BY k", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/RuntimeConstFunctionTest.java:1114", + "feature": "bind-variables", + "toSql": "SELECT k, mode(l > :b0::INT * 17_161::SHORT) AS m FROM t ORDER BY k" + }, + { + "query": "SELECT * FROM ca SUBSAMPLE minmax(x, :target)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SubsampleWildcardProjectionTest.java:974", + "feature": "bind-variables", + "toSql": "SELECT * FROM ca SUBSAMPLE minmax(x, :target)" + }, + { + "query": "SELECT sym FROM x WHERE sym ILIKE $1", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/regex/ILikeSymbolFunctionFactoryTest.java:78", + "feature": "bind-variables", + "toSql": "SELECT sym FROM x WHERE sym ILIKE $1" + }, + { + "query": "SELECT * FROM t WHERE :b0 LATEST ON ts PARTITION BY c3", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/LatestByTest.java:1280", + "feature": "bind-variables", + "toSql": "SELECT * FROM t WHERE :b0 LATEST ON ts PARTITION BY c3" + }, + { + "query": "SELECT count() FROM table_cold_partitions(:table_name)", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/cairo/cold/storage/TableColdPartitionsTest.java:92", + "feature": "bind-variables", + "toSql": "SELECT count() FROM table_cold_partitions(:table_name)" + }, + { + "query": "SELECT arr[1:n] FROM tango", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/ArrayTest.java:488", + "feature": "bind-variables", + "toSql": "SELECT arr[1:n] FROM tango" + }, + { + "query": "SELECT arr[i:j+j-i-i] x FROM tango", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/ArrayTest.java:677", + "feature": "bind-variables", + "toSql": "SELECT arr[i:j + j - i - i] AS x FROM tango" } ] diff --git a/tests/helpers/autocomplete-walkthrough.ts b/tests/helpers/autocomplete-walkthrough.ts index 710188b..0579e5c 100644 --- a/tests/helpers/autocomplete-walkthrough.ts +++ b/tests/helpers/autocomplete-walkthrough.ts @@ -218,6 +218,7 @@ export const SKIP_TOKEN_TYPES = new Set([ "Semicolon", "LBracket", "RBracket", + "Colon", "AtSign", "ColonEquals", // Operators @@ -244,6 +245,8 @@ export const SKIP_TOKEN_TYPES = new Set([ "BitAnd", // Variable references "VariableReference", + // Bind variables ($1, :name) + "BindVariable", // Interval literals "IntervalLiteral", ]) diff --git a/tests/parser.test.ts b/tests/parser.test.ts index 3e8ab7f..53d9159 100644 --- a/tests/parser.test.ts +++ b/tests/parser.test.ts @@ -5606,8 +5606,10 @@ orders PIVOT (sum(amount) FOR status IN ('open'))` expect(result.ast[0].type).toBe("select") }) - it("should parse array subscript with open-ended slice (:end)", () => { - const result = parseToAst("SELECT arr[:5] FROM t") + it("should parse array subscript with open-ended slice (: end)", () => { + // `arr[:5]` (no space) is an index by bind variable `:5`, as on the + // server (ArrayTest:4428), see tests/bind-variables.test.ts + const result = parseToAst("SELECT arr[: 5] FROM t") expect(result.errors).toHaveLength(0) expect(result.ast[0].type).toBe("select") }) From 8624c3c57cba67e51d7ad83d1b5bd5d5931ba75a Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 15:16:33 +0300 Subject: [PATCH 10/36] feat: lex backtick-quoted identifiers and $ inside identifiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server: GenericLexer treats '`' as a quote (Chars.isQuote: ', ", `). A backtick token has no escapes - it ends at the next backtick, so `a``b` is two tokens - and GenericLexer.unquote strips the quotes wherever an identifier is expected (PivotTest `where \`BTC-USD_buy\` > 25780`, CopyExportTest `copy \`test_table❤️\` to ...`, DeclareTest `\`@c\` + 1`). SqlParser.validateIdentifier allows letters, digits, '_' and '$' after the first character of an unquoted identifier (SqlParserTest testGoodNonQuotedAlias: `select 'a' a1$_`, `select 'a' _a1$`); `$`+digits at the start of a token stays a bind variable (ExpressionNode.of). Before: '`' and '$' were lexer errors ("unexpected character"), so `a$b` lexed as `a` + error + `b` and `select$x` as SELECT + ... Now: - QuotedIdentifier also matches `...` (no escapes, may be empty like ""). visitor unquoteIdentifier(): "..." still unescapes "", `...` content is verbatim (`a""b` names a""b). toSql prints the canonical "..." form. - Identifier allows '$' after the first character; the keyword negative lookahead (tokens.ts createKeywordToken) includes '$', so `select$x`, `from$` are identifiers. `$1` stays BindVariable, `$a` stays an error. - Named bind variables allow '$' after the first name char (`:a$b`, processColon glues the next token); `:$1` stays Colon + `$1` (formatter lexeme oracle). - toSql escapeIdentifier leaves names with a non-leading '$' unquoted (`"price$1"` -> price$1, the same identifier on the server); a leading '$' still needs quotes. - Autocomplete: '`' is a word boundary like '"'. Backtick table names already resolve columns (QuotedIdentifier images are sliced). Tests: 7 server queries (PivotTest, CopyExportTest, DeclareTest x2, SqlParserTest x2, RowExpiryWindowTest guard) + tests/lexer-quoting.test.ts. Red before: 62 failing (17 fixture checks + 45 focused) -> all pass; full suite 16282 passed. Corpus: fixed 6 / rtChanged 1 / regressed 0 / throws 0 / formatterChangedMeaning 0. Fixed: the 5 queries above plus `select 'a' ``` which the server rejects (empty alias) - accepted and the empty alias dropped by toSql, exactly as sql-parser already does for `select 'a' ""` (pre-existing looseness, not changed here). rtChanged: RowExpiryWindowTest `"price$1"` now prints unquoted price$1 (same name). --- src/autocomplete/content-assist.ts | 1 + src/parser/lexer.ts | 15 +- src/parser/toSql.ts | 2 +- src/parser/tokens.ts | 5 +- src/parser/visitor.ts | 17 +- tests/fixtures/server-queries.json | 42 +++++ tests/lexer-quoting.test.ts | 278 +++++++++++++++++++++++++++++ 7 files changed, 349 insertions(+), 11 deletions(-) create mode 100644 tests/lexer-quoting.test.ts diff --git a/src/autocomplete/content-assist.ts b/src/autocomplete/content-assist.ts index 2b9b97e..50ee2c1 100644 --- a/src/autocomplete/content-assist.ts +++ b/src/autocomplete/content-assist.ts @@ -36,6 +36,7 @@ const WORD_BOUNDARY_CHARS = new Set([ "%", "'", '"', + "`", "|", "&", "^", diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index 4daa081..1624f86 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -790,7 +790,10 @@ export const AtSign = createToken({ name: "AtSign", pattern: /@/ }) // `x = :sym`, `IN (:a, :b)`, `LIMIT :n` and `arr[:1]` are bind variables. // `::` and `:=` are matched earlier by DoubleColon / ColonEquals. const POSITIONAL_BIND_VARIABLE = /\$\d+(?![\w$\u0080-\uFFFF])/y -const NAMED_BIND_VARIABLE = /:[a-zA-Z0-9_\u0080-\uFFFF]+/y +// The name may contain `$` after its first character, like an identifier +// (`:a$b`); `:$1` stays a slice colon followed by a positional variable. +const NAMED_BIND_VARIABLE = + /:[a-zA-Z0-9_\u0080-\uFFFF][a-zA-Z0-9_$\u0080-\uFFFF]*/y const COMPLETED_OPERAND_TOKEN_NAMES: ReadonlySet = new globalThis.Set([ "Identifier", @@ -959,19 +962,23 @@ export const EscapeStringLiteral = createToken({ }) // Quoted Identifiers — same convention: "" escapes a double quote, -// backslash is literal. +// backslash is literal. Backticks quote too (GenericLexer, Chars.isQuote): +// `BTC-USD_buy`. A backtick token has no escapes, it ends at the next +// backtick (`a``b` is two tokens, as on the server). export const QuotedIdentifier = createToken({ name: "QuotedIdentifier", - pattern: /"([^"]|"")*"/, + pattern: /"([^"]|"")*"|`[^`]*`/, }) // Identifiers (must come after keywords) // Uses Unicode ranges to support non-ASCII identifiers (e.g., café, 日本語) // \u0080-\uFFFF covers all non-ASCII characters; Chevrotain's regex analyzer // cannot handle \p{L} Unicode property escapes, so we use explicit ranges. +// `$` is allowed after the first character (SqlParser.validateIdentifier: +// `a1$_`, `_a1$`); a leading `$` is a bind variable or an error. export const Identifier = createToken({ name: "Identifier", - pattern: /[a-zA-Z_\u0080-\uFFFF][a-zA-Z0-9_\u0080-\uFFFF]*/, + pattern: /[a-zA-Z_\u0080-\uFFFF][a-zA-Z0-9_$\u0080-\uFFFF]*/, }) // ============================================================================= diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 0c06048..3b0d9b9 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -2220,7 +2220,7 @@ function escapeString(value: string): string { function escapeIdentifier(name: string): string { if ( - /^[a-zA-Z_\u0080-\uFFFF][a-zA-Z0-9_\u0080-\uFFFF]*$/.test(name) && + /^[a-zA-Z_\u0080-\uFFFF][a-zA-Z0-9_$\u0080-\uFFFF]*$/.test(name) && !RESERVED_KEYWORDS.has(name.toLowerCase()) ) { return name diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index a2d7ccd..6be738d 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -46,12 +46,13 @@ function toPascalCase(str: string): string { * identifiers. JavaScript's \b only considers ASCII \w characters, so * "in\b" would incorrectly match the "in" prefix of "inção" (since ç * is non-ASCII and not in \w). The lookahead ensures the keyword is not - * followed by any valid identifier character, including Unicode. + * followed by any valid identifier character, including Unicode and `$` + * (`select$x` is one identifier, SqlParser.validateIdentifier). */ function createKeywordToken(name: string, pattern: string): TokenType { return createToken({ name, - pattern: new RegExp(`${pattern}(?![\\w\\u0080-\\uFFFF])`, "i"), + pattern: new RegExp(`${pattern}(?![\\w$\\u0080-\\uFFFF])`, "i"), }) } diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 460fe09..a76e4f9 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -254,6 +254,13 @@ function unquoteSingle(image: string): string { return image.slice(1, -1).replace(/''/g, "'") } +// Strip a QuotedIdentifier's quotes. "..." unescapes "" (as before); a +// `...` identifier has no escapes (GenericLexer), its content is verbatim. +function unquoteIdentifier(image: string): string { + const inner = image.slice(1, -1) + return image.startsWith("`") ? inner : inner.replace(/""/g, '"') +} + type StoragePolicyKind = "toParquet" | "toRemote" | "dropLocal" | "dropRemote" type DedupModeKeys = { mode: AST.DedupMode; keys: string[] } @@ -4927,8 +4934,9 @@ class QuestDBVisitor extends BaseVisitor { return this.tokenImage(ctx.Identifier[0] as IToken) } if (ctx.QuotedIdentifier) { - const raw = this.tokenImage(ctx.QuotedIdentifier[0] as IToken) - return raw.slice(1, -1).replace(/""/g, '"') + return unquoteIdentifier( + this.tokenImage(ctx.QuotedIdentifier[0] as IToken), + ) } if (ctx.IdentifierKeyword) { return this.tokenImage(ctx.IdentifierKeyword[0] as IToken) @@ -5012,8 +5020,9 @@ class QuestDBVisitor extends BaseVisitor { return this.tokenImage(ctx.Identifier[0] as IToken) } if (ctx.QuotedIdentifier) { - const raw = this.tokenImage(ctx.QuotedIdentifier[0] as IToken) - return raw.slice(1, -1).replace(/""/g, '"') + return unquoteIdentifier( + this.tokenImage(ctx.QuotedIdentifier[0] as IToken), + ) } // QuestDB accepts single-quoted strings in identifier positions if (ctx.StringLiteral) { diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 936f963..ac73295 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -976,5 +976,47 @@ "source": "questdb:core/src/test/java/io/questdb/test/cairo/ArrayTest.java:677", "feature": "bind-variables", "toSql": "SELECT arr[i:j + j - i - i] AS x FROM tango" + }, + { + "query": "WITH p AS \n(WITH t AS\n(\n\n SELECT timestamp, symbol, side, AVG(price) price, AVG(amount) amount FROM trades WHERE symbol IN 'BTC-USD'\n SAMPLE BY 1m\n)\nSELECT * FROM t\nPIVOT (\n sum(price)\n FOR symbol IN ('BTC-USD') \n side IN ('buy', 'sell') \n GROUP BY timestamp\n) ) SELECT * from p where `BTC-USD_buy` > 25780 or `BTC-USD_sell` > 25780;", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/PivotTest.java:4104", + "feature": "lexer-quoting", + "toSql": "WITH p AS (WITH t AS (SELECT timestamp, symbol, side, AVG(price) AS price, AVG(amount) AS amount FROM trades WHERE symbol IN 'BTC-USD' SAMPLE BY 1m) SELECT * FROM t PIVOT (sum(price) FOR symbol IN ('BTC-USD') side IN ('buy', 'sell') GROUP BY timestamp)) SELECT * FROM p WHERE \"BTC-USD_buy\" > 25780 OR \"BTC-USD_sell\" > 25780" + }, + { + "query": "copy `test_table❤️` to '❤️🍺' with format parquet", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CopyExportTest.java:1622", + "feature": "lexer-quoting", + "toSql": "COPY test_table❤️ TO '❤️🍺' WITH format parquet" + }, + { + "query": "DECLARE OVERRIDABLE @col := `@c` + 1 SELECT @col v FROM q", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:638", + "feature": "lexer-quoting", + "toSql": "DECLARE OVERRIDABLE @col := \"@c\" + 1 SELECT @col AS v FROM q" + }, + { + "query": "DECLARE OVERRIDABLE @x := (1, ``) SELECT d FROM q", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:390", + "feature": "lexer-quoting", + "toSql": "DECLARE OVERRIDABLE @x := (1, \"\") SELECT d FROM q" + }, + { + "query": "select 'a' a1$_", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:6037", + "feature": "lexer-quoting", + "toSql": "SELECT 'a' AS a1$_" + }, + { + "query": "select 'a' _a1$", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:6038", + "feature": "lexer-quoting", + "toSql": "SELECT 'a' AS _a1$" + }, + { + "query": "CREATE TABLE base (k SYMBOL, \"price$1\" DOUBLE, \"price$2\" DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WAL", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/RowExpiryWindowTest.java:168", + "feature": "lexer-quoting", + "toSql": "CREATE TABLE base (k SYMBOL, price$1 DOUBLE, price$2 DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WAL" } ] diff --git a/tests/lexer-quoting.test.ts b/tests/lexer-quoting.test.ts new file mode 100644 index 0000000..ba4cd57 --- /dev/null +++ b/tests/lexer-quoting.test.ts @@ -0,0 +1,278 @@ +/** + * Backtick identifiers and `$` inside identifiers (reports §3.1). + * + * Server: GenericLexer treats '`' as a quote (Chars.isQuote: `'`, `"`, '`'). + * A backtick-quoted token has no escapes: it ends at the next backtick, so + * `a``b` is two tokens. Quoted tokens are identifiers wherever an identifier + * is expected (GenericLexer.unquote strips the outer quotes), e.g. + * PivotTest `where \`BTC-USD_buy\` > 25780`, CopyExportTest + * `copy \`test_table❤️\` to ...`. + * + * SqlParser.validateIdentifier allows letters, digits, '_' and '$' after the + * first character of an unquoted identifier (`select 'a' a1$_`, + * SqlParserTest.testGoodNonQuotedAlias). `$` + digits at the start of a token + * stays a bind variable (ExpressionNode.of). + * + * Server queries live in tests/fixtures/server-queries.json + * (feature "lexer-quoting"). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, tokenize, createAutocompleteProvider } from "../src/index" +import { toSql } from "../src/parser/toSql" +import { format } from "../src/formatter/index" +import type * as AST from "../src/parser/ast" + +const tokenTypes = (sql: string) => + tokenize(sql).tokens.map((t) => `${t.tokenType.name}:${t.image}`) + +const parseOne = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +const roundTrip = (sql: string) => toSql([parseOne(sql)]) + +const expectError = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors.length).toBeGreaterThan(0) +} + +const columnParts = (expr: AST.Expression | undefined) => + (expr as AST.ColumnRef).name.parts + +describe("backtick identifiers", () => { + describe("lexer", () => { + it.each([ + ["`BTC-USD_buy`", "QuotedIdentifier:`BTC-USD_buy`"], + ["`test_table❤️`", "QuotedIdentifier:`test_table❤️`"], + ["`@c`", "QuotedIdentifier:`@c`"], + ["``", "QuotedIdentifier:``"], + ['`a""b`', 'QuotedIdentifier:`a""b`'], + ["`it's`", "QuotedIdentifier:`it's`"], + ["`a b`", "QuotedIdentifier:`a b`"], + ])("reads %s as one quoted identifier", (sql, token) => { + expect(tokenTypes(sql)).toEqual([token]) + }) + + it("has no escape: a doubled backtick ends the token", () => { + expect(tokenTypes("`a``b`")).toEqual([ + "QuotedIdentifier:`a`", + "QuotedIdentifier:`b`", + ]) + }) + + it("does not affect double-quoted identifiers or strings", () => { + expect(tokenTypes("\"a`b\" 'c`d'")).toEqual([ + 'QuotedIdentifier:"a`b"', + "StringLiteral:'c`d'", + ]) + }) + + it("reports an unterminated backtick", () => { + expect(tokenize("SELECT `abc FROM t").errors.length).toBeGreaterThan(0) + }) + }) + + describe("AST", () => { + it("unquotes a column reference", () => { + const stmt = parseOne( + "SELECT * FROM p WHERE `BTC-USD_buy` > 25780", + ) as AST.SelectStatement + const where = stmt.where as AST.BinaryExpression + expect(columnParts(where.left)).toEqual(["BTC-USD_buy"]) + }) + + it("keeps double quotes inside backticks verbatim", () => { + const stmt = parseOne('SELECT `a""b` FROM t') as AST.SelectStatement + const item = stmt.columns[0] as AST.ExpressionSelectItem + expect(columnParts(item.expression)).toEqual(['a""b']) + }) + + it("unquotes qualified names, table names and aliases", () => { + const stmt = parseOne( + "SELECT `t`.`c` AS `my col` FROM `my-table` `t`", + ) as AST.SelectStatement + const item = stmt.columns[0] as AST.ExpressionSelectItem + expect(columnParts(item.expression)).toEqual(["t", "c"]) + expect(item.alias).toBe("my col") + const from = stmt.from![0] + expect((from.table as AST.QualifiedName).parts).toEqual(["my-table"]) + expect(from.alias).toBe("t") + }) + + it("reads a doubled backtick as a column and its alias", () => { + const stmt = parseOne("SELECT `a``b` FROM t") as AST.SelectStatement + const item = stmt.columns[0] as AST.ExpressionSelectItem + expect(columnParts(item.expression)).toEqual(["a"]) + expect(item.alias).toBe("b") + }) + + it("names a CREATE TABLE column", () => { + const stmt = parseOne( + "CREATE TABLE t (`a b` INT, `c` SYMBOL)", + ) as AST.CreateTableStatement + expect(stmt.columns!.map((c) => c.name)).toEqual(["a b", "c"]) + }) + }) + + describe("toSql", () => { + it.each([ + [ + "SELECT * FROM p WHERE `BTC-USD_buy` > 25780", + 'SELECT * FROM p WHERE "BTC-USD_buy" > 25780', + ], + ["SELECT `a` FROM `t`", "SELECT a FROM t"], + ['SELECT `a"b` FROM t', 'SELECT "a""b" FROM t'], + ["SELECT `select` FROM t", 'SELECT "select" FROM t'], + [ + "SELECT `t`.`c` AS `my col` FROM `my-table` `t`", + 'SELECT t.c AS "my col" FROM "my-table" AS t', + ], + ["SELECT `a``b` FROM t", "SELECT a AS b FROM t"], + [ + "INSERT INTO `my-table` (`a b`) VALUES (1)", + 'INSERT INTO "my-table" ("a b") VALUES (1)', + ], + ])("%s", (sql, expected) => { + const out = roundTrip(sql) + expect(out).toBe(expected) + expect(roundTrip(out)).toBe(expected) + }) + }) +}) + +describe("$ inside identifiers", () => { + describe("lexer", () => { + it.each(["a$b", "a1$_", "_a1$", "a$1", "price$1", "x$$y"])( + "reads %s as one identifier", + (name) => { + expect(tokenTypes(name)).toEqual([`Identifier:${name}`]) + }, + ) + + it.each(["select$x", "from$", "table$1", "timestamp$"])( + "does not split a keyword glued to $: %s", + (name) => { + expect(tokenTypes(name)).toEqual([`Identifier:${name}`]) + }, + ) + + it("keeps $n bind variables", () => { + expect(tokenTypes("a = $1")).toEqual([ + "Identifier:a", + "Equals:=", + "BindVariable:$1", + ]) + expect(tokenTypes("a$1 = $1")).toEqual([ + "Identifier:a$1", + "Equals:=", + "BindVariable:$1", + ]) + }) + + it("includes $ in a :name bind variable (processColon glues the next token)", () => { + expect(tokenTypes("a = :a$b")).toEqual([ + "Identifier:a", + "Equals:=", + "BindVariable::a$b", + ]) + }) + + it("keeps `:$1` a colon followed by a positional variable", () => { + expect(tokenTypes("arr[:$1]")).toEqual([ + "Identifier:arr", + "LBracket:[", + "Colon::", + "BindVariable:$1", + "RBracket:]", + ]) + }) + }) + + describe("parser", () => { + it("reads a$b as a column", () => { + const stmt = parseOne("SELECT a$b FROM t") as AST.SelectStatement + const item = stmt.columns[0] as AST.ExpressionSelectItem + expect(columnParts(item.expression)).toEqual(["a$b"]) + expect(item.alias).toBeUndefined() + }) + + it("names a CREATE TABLE column", () => { + const stmt = parseOne( + "CREATE TABLE t (a$b INT)", + ) as AST.CreateTableStatement + expect(stmt.columns!.map((c) => c.name)).toEqual(["a$b"]) + }) + + it.each([ + // identifiers cannot start with $ (validateIdentifier) + "SELECT $a FROM t", + "SELECT 'a' $", + "SELECT * FROM t WHERE a = $", + ])("rejects %s", (sql) => { + expectError(sql) + }) + }) + + describe("toSql", () => { + it.each([ + ["SELECT a$b FROM t", "SELECT a$b FROM t"], + ["select 'a' a1$_", "SELECT 'a' AS a1$_"], + ["select 'a' _a1$", "SELECT 'a' AS _a1$"], + ["CREATE TABLE t (a$b INT)", "CREATE TABLE t (a$b INT)"], + ['SELECT "a$b" FROM "x$y"', "SELECT a$b FROM x$y"], + // a leading $ still needs quotes + ['SELECT "$a", "$1" FROM t', 'SELECT "$a", "$1" FROM t'], + ["SELECT a$1 FROM t WHERE b = $1", "SELECT a$1 FROM t WHERE b = $1"], + ["SELECT select$x FROM t", "SELECT select$x FROM t"], + ])("%s", (sql, expected) => { + const out = roundTrip(sql) + expect(out).toBe(expected) + expect(roundTrip(out)).toBe(expected) + }) + }) +}) + +describe("formatter", () => { + it("keeps backtick identifiers as written", () => { + expect(format("select `BTC-USD_buy` from p where `a b` > 1")).toBe( + "select `BTC-USD_buy`\nfrom p\nwhere `a b` > 1", + ) + }) + + it("keeps $ identifiers whole", () => { + expect(format("select a$b, x from t where a$1 = $1")).toBe( + "select a$b, x\nfrom t\nwhere a$1 = $1", + ) + }) +}) + +describe("autocomplete", () => { + const provider = createAutocompleteProvider({ + tables: [{ name: "my-table" }, { name: "t" }], + columns: { + "my-table": [{ name: "price", type: "DOUBLE" }], + t: [{ name: "a$b", type: "INT" }], + }, + }) + const labels = (sql: string) => + provider.getSuggestions(sql, sql.length).map((s) => s.label) + + it("resolves columns of a backtick-quoted table", () => { + expect(labels("SELECT * FROM `my-table` WHERE ")).toContain("price") + }) + + it("continues the expression after a backtick identifier", () => { + const sql = "SELECT * FROM t WHERE `a$b` " + expect(labels(sql)).toContain("AND") + }) + + it("continues the expression after a $ identifier", () => { + const sql = "SELECT * FROM t WHERE a$b " + expect(labels(sql)).toContain("AND") + expect(labels(sql)).toContain("ORDER") + }) +}) From 82f521b2a6f5e789637d94638fcf9353ae27f4b5 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 15:24:52 +0300 Subject: [PATCH 11/36] feat: nest block comments, run unterminated ones to the end, skip empty statements Server: SqlUtil.fetchNext counts `/*` / `*/` (blockCount), so block comments nest (`SELECT 1 /* a /* nested */ still */ FROM t`, DeclareTest:5139-5168, SqlParserTest:6397-6427 hints with nested comments) and a comment that is never closed runs to the end of the input (SqlParserTest.testBlockCommentUnclosed). SqlCompilerImpl.compileBatch skips empty statements via getNextValidTokenPosition (PGMultiStatementMessageTest: `select 1;;`, `;;`, ` ; ;`, `; /* comment */`, `select 1;;/*c*/;--c\n;select 2;`). Before: BlockComment was a lazy regex, so the first `*/` closed a nested comment and the rest leaked out as `still * / ...` (error or silent extra statement); an unterminated `/*` lexed as Divide + Star; any `;` beyond one per statement gave "Redundant input". Changes: - lexer: BlockComment is a custom matcher (exported blockCommentEnd) that nests and runs to end of input when unclosed; line_breaks kept. - parser: statements = MANY(;) MANY2(statement MANY3(;)). Visitor/AST unchanged; empty statements produce no AST node. cst-types regenerated. - formatter lexer: its BlockComment uses the same nesting (closed comments only); an unterminated (nested) comment stays a tolerant token as before. Deliberately NOT replicated (server tokenizer artefacts): `--` inside a block comment starting a line comment, and quotes inside a block comment (GenericLexer still reads quoted spans there, so `/* '*/' */` stays open; the pairing quote is searched to end of input, so on the server `SELECT 1 /* don't */ FROM t WHERE s = 'x'` is just `SELECT 1`). Decision approved by the supervisor; DeclareTest:5142/5148/5157/5720 (`/* '*/' */`) stay unsupported. Here quotes and `--` inside /* */ are plain text. Tests: 15 server queries (DeclareTest, SqlParserTest, PGMultiStatementMessageTest) in server-queries.json + tests/comments-empty-statements.test.ts (41 tests: nesting, unterminated, quotes/-- inside comments, stray */, empty statements, formatter scan/format, autocomplete). recovery.test.ts: `;;;` and `SELECT 1;;;;;SELECT 2` no longer produce errors (server accepts both). Red before: 57 failing (26 focused + 31 fixture) -> all pass; full suite 16383 passed. SqlParserTest:6427 kept in the focused test, not the fixture: the formatter wraps that long hint comment differently on a second pass (pre-existing, also without nesting). Corpus: fixed 10 / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0. Fixed are the nested-comment DeclareTest / SqlParserTest queries plus three the server rejects for other reasons: MatViewExpireRowsHardeningTest:1221 (contextual "unterminated block comment in EXPIRE ROWS clause"), DeclareTest:5177 (nested lists, semantic), and DeclareTest:5168 whose `FROM @x` prints as "@x" (pre-existing toSql issue). Suggestions: statement starts after `;; ` / `; ` / `; /* c */ ; ` (were empty); normal follow-ups after a nested comment and inside an unterminated one (as for `--` comments). Default positions unchanged. --- src/formatter/lexer.ts | 16 +- src/parser/cst-types.d.ts | 2 +- src/parser/lexer.ts | 37 +++- src/parser/parser.ts | 7 +- tests/comments-empty-statements.test.ts | 233 ++++++++++++++++++++++++ tests/fixtures/server-queries.json | 96 ++++++++++ tests/recovery.test.ts | 7 +- 7 files changed, 389 insertions(+), 9 deletions(-) create mode 100644 tests/comments-empty-statements.test.ts diff --git a/src/formatter/lexer.ts b/src/formatter/lexer.ts index 33249c9..60db9a4 100644 --- a/src/formatter/lexer.ts +++ b/src/formatter/lexer.ts @@ -2,6 +2,7 @@ import { createToken, Lexer, TokenType } from "chevrotain" import { allTokens, BlockComment, + blockCommentEnd, LineComment, QuotedIdentifier, StringLiteral, @@ -40,11 +41,22 @@ const FormatterLineComment = createToken({ group: TRIVIA_GROUP, }) +// Nested like the parser's BlockComment; an unterminated one is tolerant. +const matchClosedBlockComment = ( + text: string, + offset: number, +): [string] | null => { + if (!text.startsWith("/*", offset)) return null + const end = blockCommentEnd(text, offset) + return end === -1 ? null : [text.slice(offset, end)] +} + const FormatterBlockComment = createToken({ name: "BlockComment", - pattern: /\/\*[\s\S]*?\*\//, + pattern: { exec: matchClosedBlockComment }, group: TRIVIA_GROUP, line_breaks: true, + start_chars_hint: ["/"], }) const matchUnterminatedQuoted = @@ -70,7 +82,7 @@ const matchUnterminatedBlockComment = ( offset: number, ): [string] | null => { if (!text.startsWith("/*", offset)) return null - if (text.indexOf("*/", offset + 2) !== -1) return null + if (blockCommentEnd(text, offset) !== -1) return null return [text.slice(offset)] } diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 3a924fb..c705750 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -6,8 +6,8 @@ export interface StatementsCstNode extends CstNode { } export type StatementsCstChildren = { + Semicolon?: (IToken)[]; statement?: StatementCstNode[]; - Semicolon?: IToken[]; }; export interface StatementCstNode extends CstNode { diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index 1624f86..4e0444c 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -997,9 +997,44 @@ export const LineComment = createToken({ group: Lexer.SKIPPED, }) +/** + * End offset (exclusive) of the block comment that starts with `/*` at + * `offset`, or -1 when it is never closed. Block comments nest: every `/*` + * needs its own `*\/` (SqlUtil.fetchNext blockCount). Quotes and `--` inside + * the comment are plain text. + */ +export function blockCommentEnd(text: string, offset: number): number { + let depth = 0 + let index = offset + while (index < text.length - 1) { + const ch = text[index] + const next = text[index + 1] + if (ch === "/" && next === "*") { + depth++ + index += 2 + } else if (ch === "*" && next === "/") { + index += 2 + if (--depth === 0) return index + } else { + index++ + } + } + return -1 +} + +// A block comment that is never closed runs to the end of the input, as on +// the server (SqlParserTest.testBlockCommentUnclosed). +function matchBlockComment(text: string, offset: number): [string] | null { + if (!text.startsWith("/*", offset)) return null + const end = blockCommentEnd(text, offset) + return [text.slice(offset, end === -1 ? text.length : end)] +} + export const BlockComment = createToken({ name: "BlockComment", - pattern: /\/\*[\s\S]*?\*\//, + pattern: { exec: matchBlockComment }, + start_chars_hint: ["/"], + line_breaks: true, group: Lexer.SKIPPED, }) diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 6e0861e..5857f57 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -548,15 +548,18 @@ class QuestDBParser extends CstParser { // Entry point // ========================================================================== + // Empty statements (`;;`, a leading `;`) are skipped, as in + // SqlCompilerImpl.compileBatch (getNextValidTokenPosition). public statements = this.RULE("statements", () => { - this.MANY(() => { + this.MANY(() => this.CONSUME(Semicolon)) + this.MANY2(() => { this._semicolonBoundedRecovery = true try { this.SUBRULE(this.statement) } finally { this._semicolonBoundedRecovery = false } - this.OPTION(() => this.CONSUME(Semicolon)) + this.MANY3(() => this.CONSUME2(Semicolon)) }) }) diff --git a/tests/comments-empty-statements.test.ts b/tests/comments-empty-statements.test.ts new file mode 100644 index 0000000..091f3a4 --- /dev/null +++ b/tests/comments-empty-statements.test.ts @@ -0,0 +1,233 @@ +/** + * Nested and unterminated block comments; empty statements (reports §3.1). + * + * Server: SqlUtil.fetchNext counts `/*` and `*\/` (blockCount), so block + * comments nest (`SELECT 1 /* a /* b *\/ c *\/ FROM t`) and a comment that is + * never closed runs to the end of the input (SqlParserTest.testBlockCommentUnclosed). + * SqlCompilerImpl.compileBatch skips empty statements + * (getNextValidTokenPosition): `select 1;;`, `;;`, `; /* c *\/`. + * + * Not replicated on purpose (server tokenizer artefacts): + * - `--` inside a block comment starting a line comment; + * - quoted text inside a block comment hiding a `*\/` (GenericLexer still + * reads quotes there, pairing an apostrophe with a later quote). + * + * Server queries live in tests/fixtures/server-queries.json + * (feature "comments-empty-statements"). + */ + +import { describe, it, expect } from "vitest" +import { + parseToAst, + tokenize, + createAutocompleteProvider, + getNextValidTokens, +} from "../src/index" +import { toSql } from "../src/parser/toSql" +import { format } from "../src/formatter/index" +import { reconstruct, scan } from "../src/formatter/lexer" + +const tokenTypes = (sql: string) => + tokenize(sql).tokens.map((t) => `${t.tokenType.name}:${t.image}`) + +const parse = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors.map((e) => e.message)).toEqual([]) + return result.ast +} + +const roundTrip = (sql: string, statements = 1) => { + const ast = parse(sql) + expect(ast).toHaveLength(statements) + return toSql(ast) +} + +const expectError = (sql: string) => { + expect(parseToAst(sql).errors.length).toBeGreaterThan(0) +} + +describe("nested block comments", () => { + it.each([ + ["SELECT 1 /* a /* nested */ still */ FROM t", "SELECT 1 FROM t"], + ["SELECT 1 /* a /* b /* c */ d */ e */ FROM t", "SELECT 1 FROM t"], + ["SELECT /* /**/ */ 1", "SELECT 1"], + ["SELECT /**/ 1", "SELECT 1"], + ["SELECT /***/ 1", "SELECT 1"], + ["SELECT a /* x */ + /* y /* z */ */ b FROM t", "SELECT a + b FROM t"], + ["SELECT a FROM t /* ; SELECT b */", "SELECT a FROM t"], + ])("%s", (sql, expected) => { + expect(roundTrip(sql)).toBe(expected) + }) + + it("reads a comment nested in a hint (SqlParserTest:6427)", () => { + // Not in the fixture: the formatter wraps this long hint differently on + // a second pass (pre-existing, also without nesting). + expect( + roundTrip( + "select /*+ NO_INDEX(a) /* THIS IS JUST A REGULAR COMMENT */ AFTER */ a,b,c from xyz where a = 1", + ), + ).toBe("SELECT a, b, c FROM xyz WHERE a = 1") + }) + + it("skips the whole nested comment in the lexer", () => { + expect(tokenTypes("1 /* a /* b */ c */ 2")).toEqual([ + "NumberLiteral:1", + "NumberLiteral:2", + ]) + }) + + it("keeps line numbers after a multi-line comment", () => { + const tokens = tokenize("/* a\n/* b\n*/\n*/ x").tokens + expect(tokens).toHaveLength(1) + expect(tokens[0].startLine).toBe(4) + }) + + it("does not treat -- inside a block comment as a line comment", () => { + expect(roundTrip("SELECT 1 /* -- not a line comment */ FROM t")).toBe( + "SELECT 1 FROM t", + ) + expect(roundTrip("SELECT 1 /* a -- b \n */ FROM t")).toBe("SELECT 1 FROM t") + }) + + it("does not let quotes inside a block comment hide its end", () => { + // Server quirk not replicated: there the apostrophe pairs with the next + // quote, the comment never closes and only `SELECT 1` remains. + expect(roundTrip("SELECT 1 /* don't */ FROM t WHERE s = 'x'")).toBe( + "SELECT 1 FROM t WHERE s = 'x'", + ) + expect(roundTrip("SELECT 1 /* '*/ FROM t")).toBe("SELECT 1 FROM t") + }) + + it("keeps comment markers inside strings and quoted identifiers", () => { + expect(roundTrip("SELECT '/* x' FROM t")).toBe("SELECT '/* x' FROM t") + expect(roundTrip('SELECT "*/" FROM t')).toBe('SELECT "*/" FROM t') + expect(roundTrip("SELECT 1 -- /* x\nFROM t")).toBe("SELECT 1 FROM t") + }) + + it("leaves a stray */ outside a comment as operators", () => { + expectError("SELECT 1 */ FROM t") + expect(tokenTypes("2 */ 3")).toEqual([ + "NumberLiteral:2", + "Star:*", + "Divide:/", + "NumberLiteral:3", + ]) + }) +}) + +describe("unterminated block comments", () => { + it.each([ + ["SELECT 1 /* trailing", "SELECT 1"], + ["SELECT 1 /* trailing\nFROM t", "SELECT 1"], + ["SELECT 1 FROM t /* a /* b */", "SELECT 1 FROM t"], + ["SELECT 1 FROM t /*", "SELECT 1 FROM t"], + ["SELECT 1 FROM t /*/", "SELECT 1 FROM t"], + ])("%s runs to the end of the input", (sql, expected) => { + expect(roundTrip(sql)).toBe(expected) + }) + + it("leaves no tokens behind", () => { + expect(tokenTypes("1 /* a /* b */ 2")).toEqual(["NumberLiteral:1"]) + expect(tokenize("1 /* a").errors).toEqual([]) + }) + + it("still rejects a statement that the comment cuts short", () => { + // SqlParserTest:6596 `select /* HINT_A` -> "[distinct] column expected" + expectError("select /* HINT_A") + expectError("SELECT * FROM t WHERE /* x") + }) + + it("is only a comment input when nothing precedes it", () => { + expect(roundTrip("/* only a comment", 0)).toBe("") + }) +}) + +describe("empty statements", () => { + it.each([ + ["SELECT 1;; ; SELECT 1 ; SELECT 1; /* c */ ;", 3], + ["select 1;;", 1], + ["; SELECT 1", 1], + [";;SELECT 1;;SELECT 2;;", 2], + ["SELECT 1;\n;\nSELECT 2;", 2], + [";", 0], + [";;", 0], + [" ; ; ", 0], + ["; -- c\n; /* d */ ;", 0], + ["", 0], + ])("%s", (sql, count) => { + expect(parse(sql)).toHaveLength(count) + }) + + it("prints only the real statements", () => { + expect(roundTrip("SELECT 1;; ; SELECT 1 ; SELECT 1; /* c */ ;", 3)).toBe( + "SELECT 1;\nSELECT 1;\nSELECT 1", + ) + expect(roundTrip("; ; INSERT INTO t VALUES (1);;", 1)).toBe( + "INSERT INTO t VALUES (1)", + ) + }) + + it("still reports errors inside a statement between empty ones", () => { + const result = parseToAst(";; SELECT FROM ;; SELECT 2") + expect(result.errors.length).toBeGreaterThan(0) + }) + + it("still splits consecutive statements without a semicolon", () => { + expect(parse("SELECT 1 SELECT 2")).toHaveLength(2) + }) +}) + +describe("formatter", () => { + const signature = (sql: string) => + scan(sql).map((t) => `${t.kind}:${t.image}`) + + it("scans a nested comment as one block comment", () => { + const sql = "/* a /* b */ c */ z" + expect(signature(sql)).toEqual([ + "blockComment:/* a /* b */ c */", + "whitespace: ", + "word:z", + ]) + expect(reconstruct(scan(sql))).toBe(sql) + }) + + it("keeps an unterminated nested comment tolerant", () => { + expect(signature("x /* a /* b */ c")).toEqual([ + "word:x", + "whitespace: ", + "tolerant:/* a /* b */ c", + ]) + }) + + it("formats around a nested comment without touching it", () => { + expect(format("SELECT 1 /* a /* nested */ still */ FROM t")).toBe( + "SELECT 1 /* a /* nested */ still */\nFROM t", + ) + expect(format("SELECT 1 FROM t /* a /* b */ c")).toBe( + "SELECT 1\nFROM t /* a /* b */ c", + ) + }) +}) + +describe("autocomplete", () => { + const provider = createAutocompleteProvider({ + tables: [{ name: "trades" }], + columns: { trades: [{ name: "price", type: "DOUBLE" }] }, + }) + const labels = (sql: string) => + provider.getSuggestions(sql, sql.length).map((s) => s.label) + + it("suggests statement starts after empty statements", () => { + expect(labels("SELECT 1;; ")).toContain("SELECT") + expect(labels("; ")).toContain("SELECT") + expect(getNextValidTokens(";; ")).toContain("Select") + }) + + it("never suggests a semicolon", () => { + expect(labels("SELECT 1;; ")).not.toContain(";") + }) + + it("reads past a nested comment", () => { + expect(labels("SELECT * FROM trades /* a /* b */ c */ ")).toContain("WHERE") + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index ac73295..2074a19 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1018,5 +1018,101 @@ "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/RowExpiryWindowTest.java:168", "feature": "lexer-quoting", "toSql": "CREATE TABLE base (k SYMBOL, price$1 DOUBLE, price$2 DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WAL" + }, + { + "query": "DECLARE @x := /* a /* b */ c */ ('a', 'b') SELECT s FROM k WHERE s IN @x", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:5139", + "feature": "comments-empty-statements", + "toSql": "DECLARE @x := ('a', 'b') SELECT s FROM k WHERE s IN @x" + }, + { + "query": "DECLARE @x := ('a' /* a /* b */ ) */ , 'b') SELECT s FROM k WHERE s IN @x", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:5154", + "feature": "comments-empty-statements", + "toSql": "DECLARE @x := ('a', 'b') SELECT s FROM k WHERE s IN @x" + }, + { + "query": "DECLARE @x := (1 /* a /* b */ , */ + 2) SELECT l FROM k WHERE l = @x", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:5162", + "feature": "comments-empty-statements", + "toSql": "DECLARE @x := (1 + 2) SELECT l FROM k WHERE l = @x" + }, + { + "query": "DECLARE OVERRIDABLE /* a /* b */ c */ @y := 2 SELECT @y AS y", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:5714", + "feature": "comments-empty-statements", + "toSql": "DECLARE OVERRIDABLE @y := 2 SELECT @y AS y" + }, + { + "query": "select /* /*+ NO_INDEX(a) */ */ a,b,c from xyz where a = 1", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:6397", + "feature": "comments-empty-statements", + "toSql": "SELECT a, b, c FROM xyz WHERE a = 1" + }, + { + "query": "select /*+ NO_INDEX(a) /*+ IGNORED */ */ a,b,c from xyz where a = 1", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:6417", + "feature": "comments-empty-statements", + "toSql": "SELECT a, b, c FROM xyz WHERE a = 1" + }, + { + "query": "(x where a > 1) 'b a' where x > 1 /* this block comment", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:1534", + "feature": "comments-empty-statements", + "toSql": "(x WHERE a > 1) AS \"b a\" WHERE x > 1" + }, + { + "query": "select 1;;", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGMultiStatementMessageTest.java:890", + "feature": "comments-empty-statements", + "toSql": "SELECT 1" + }, + { + "query": ";;", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGMultiStatementMessageTest.java:909", + "feature": "comments-empty-statements", + "toSql": "", + "statements": 0 + }, + { + "query": " ; ;", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGMultiStatementMessageTest.java:918", + "feature": "comments-empty-statements", + "toSql": "", + "statements": 0 + }, + { + "query": ";--comment", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGMultiStatementMessageTest.java:982", + "feature": "comments-empty-statements", + "toSql": "", + "statements": 0 + }, + { + "query": "; /* comment */", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGMultiStatementMessageTest.java:985", + "feature": "comments-empty-statements", + "toSql": "", + "statements": 0 + }, + { + "query": "/* comment */", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGMultiStatementMessageTest.java:976", + "feature": "comments-empty-statements", + "toSql": "", + "statements": 0 + }, + { + "query": "select 1;/* comment */", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGMultiStatementMessageTest.java:1108", + "feature": "comments-empty-statements", + "toSql": "SELECT 1" + }, + { + "query": "select 1;;/*multiline comment */;--single line comment\n;select 2;", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGMultiStatementMessageTest.java:333", + "feature": "comments-empty-statements", + "toSql": "SELECT 1;\nSELECT 2", + "statements": 2 } ] diff --git a/tests/recovery.test.ts b/tests/recovery.test.ts index a2a9a3e..d168c39 100644 --- a/tests/recovery.test.ts +++ b/tests/recovery.test.ts @@ -452,9 +452,10 @@ describe("Edge cases", () => { expect(result.ast.length).toBe(0) }) - it("semicolons-only input produces error but no AST", () => { + it("semicolons-only input is empty statements: no error, no AST", () => { + // Server compileBatch skips empty statements. const result = parseToAst(";;;") - expect(result.errors.length).toBe(1) + expect(result.errors.length).toBe(0) expect(result.ast.length).toBe(0) }) @@ -481,7 +482,7 @@ describe("Edge cases", () => { it("many semicolons between statements still parses all valid statements", () => { const result = parseToAst("SELECT 1;;;;;SELECT 2;;;;;SELECT 3") - expect(result.errors.length).toBe(2) + expect(result.errors.length).toBe(0) expect(result.ast.length).toBe(3) expect(result.ast.map((s) => s.type)).toEqual([ "select", From b602c4a6cb3868abb2ad00f32b38faf021fbb6a1 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 15:34:31 +0300 Subject: [PATCH 12/36] feat: type 99m / 1d as decimal / double numbers in expressions, lex 1.5e3m Server (FunctionParser.createConstant): a literal token that Numbers.parseDouble accepts is a DOUBLE, and FastDouble skips an optional d/D suffix, so `1d`, `0d`, `1.5D` are doubles. A token ending in m/M (`(c | 32) == 'm'`) is a DECIMAL parsed by DecimalParser.parse, which also accepts an exponent (`1.5e3m`, `2.5e-3M`; Decimal256Test:1839 `1.5e2M`). Before: `99m` / `1d` lexed as DurationLiteral and the AST typed them as literalType "duration" in expressions (`cast(99m as byte)`, `10d * 0d`, `VALUES (..., 100m)`), and `1.5e3m` silently misparsed as `1.5e3 AS m`. Changes: - lexer: DecimalLiteral gains the exponent form `\d[\d_]*(\.[\d_]*)?[eE][+-]?\d[\d_]*[mM]\b` (an exponent never forms a duration, so either case of m is safe). `99m`, `99M`, `1.5M`, `1d`, `1.5d` stay DurationLiteral tokens, so SAMPLE BY / TTL / REFRESH EVERY / window frames / WINDOW JOIN / HORIZON LIST are unchanged. - visitor literal(): a DurationLiteral ending in m/M/d/D is typed `{ literalType: "number", value, raw }` (value parsed without the suffix and underscores; raw kept, so toSql output is unchanged). Other units (1h, 30s, 1ms, 100T, ...) stay durations: they are not numbers on the server. DecimalLiteral value parsing strips M as well as m. - Duration-valued expression positions keep a bare duration: table params (`o3MaxLag = 10m` / `7d`, SqlUtil.expectMicros) and DECLARE values (`DECLARE @unit := 1m ... SAMPLE BY @unit` substitutes the token into SAMPLE BY). A new helper bareDurationLiteral() detects an expression that is a single DurationLiteral token; `DECLARE @x := 1m + 1` is typed as a number expression. - COPY options with an expression value keep a suffixed number verbatim (`COMPRESSION_LEVEL 3d` still prints `3d`, it used to come from the duration path; a decimal like `1.5m` there now also keeps its suffix). AST: no new fields; numbers carry raw as before. No grammar change (cst-types unchanged). Tests: 19 server queries added to tests/fixtures/server-queries.json (CastDecimalToByte/CastByteToDecimal, InsertTest, DecimalFullScaleTest, LiveViewSmokeTest, Ent ViewAuditTest DECLARE, MulDouble, UnionAllCast, GreatestNumeric, GroupByRewrite + guards TtlTest TTL 1M, AlterTableO3MaxLag, HORIZON LIST, mat view SAMPLE BY 10m); no server SQL test uses the exponent-m form (only Decimal256Test unit tests), so it is covered in the focused tests/numeric-literal-typing.test.ts (51 tests: lexer, AST typing, unchanged duration contexts, COPY, autocomplete, formatter). Red -> green: 29 failing focused tests -> 51/51; the 76 fixture checks are guards (toSql was already identical). Full suite 16510 passed. Corpus: fixed 0 / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0 (toSql keeps raw; the corpus has no exponent-m). Suggestions: after `SELECT 1.5e3m ` and `WHERE price > 1.5e3m ` now the normal expression follow-ups (m used to be read as an alias). --- src/parser/lexer.ts | 9 +- src/parser/visitor.ts | 65 +++++- tests/fixtures/server-queries.json | 114 ++++++++++ tests/numeric-literal-typing.test.ts | 315 +++++++++++++++++++++++++++ 4 files changed, 495 insertions(+), 8 deletions(-) create mode 100644 tests/numeric-literal-typing.test.ts diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index 4e0444c..d09ee2c 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -888,11 +888,14 @@ export const LongLiteral = createToken({ pattern: /\d[\d_]*[Ll]\b/, }) -// QuestDB DECIMAL literal with m suffix: 10.5m, 99.99m (fractional only) -// Integer + m (like 5m) is DurationLiteral (minutes), so we only match when there's a decimal point +// QuestDB DECIMAL literal with m suffix: 10.5m, 99.99m, and the exponent form +// 1.5e3m / 2.5e-3M (FunctionParser.createConstant -> DecimalParser.parse, which +// takes an exponent and `m` in either case). Integer + m (like 5m) and 1.5M +// stay DurationLiteral (minutes / months); the visitor types them as decimals +// in expressions. An exponent never forms a duration, so `[mM]` is safe there. export const DecimalLiteral = createToken({ name: "DecimalLiteral", - pattern: /\d[\d_]*\.\d[\d_]*m\b/, + pattern: /\d[\d_]*(\.\d[\d_]*m|(\.[\d_]*)?[eE][+-]?\d[\d_]*[mM])\b/, }) // QuestDB time units for SAMPLE BY (e.g., 1h, 5m, 30s, 1d, 1M, 1y, 100T, 1U, 1.5d) diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index a76e4f9..58edd95 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -1349,7 +1349,11 @@ class QuestDBVisitor extends BaseVisitor { const result: AST.DeclareAssignment = { type: "declareAssignment", name: image.substring(1), // strip leading @ - value: this.visit(ctx.expression) as AST.Expression, + // A bare duration stays a duration: the variable may be substituted + // into SAMPLE BY (`DECLARE @unit := 1m ... SAMPLE BY @unit`). + value: + this.bareDurationLiteral(ctx.expression[0]) ?? + (this.visit(ctx.expression) as AST.Expression), } if (ctx.Overridable) result.overridable = true return result @@ -1771,11 +1775,42 @@ class QuestDBVisitor extends BaseVisitor { name, } if (ctx.expression) { - result.value = this.visit(ctx.expression) as AST.Expression + // o3MaxLag = 10m / 7d is a duration (SqlUtil.expectMicros), not a number + result.value = + this.bareDurationLiteral(ctx.expression[0]) ?? + (this.visit(ctx.expression) as AST.Expression) } return result } + /** + * If an expression CST is nothing but one DurationLiteral token, returns it + * as a duration literal. Used where a value is a duration (o3MaxLag, DECLARE + * values that may feed SAMPLE BY), since `literal` types `10m` / `7d` as + * numbers. + */ + private bareDurationLiteral(node: CstNode): AST.Literal | undefined { + const tokens: IToken[] = [] + const collect = (n: CstNode) => { + for (const children of Object.values(n.children)) { + for (const c of children) { + if ("image" in c) tokens.push(c) + else collect(c) + if (tokens.length > 1) return + } + } + } + collect(node) + if (tokens.length !== 1 || tokens[0].tokenType.name !== "DurationLiteral") { + return undefined + } + return { + type: "literal", + value: this.tokenImage(tokens[0]), + literalType: "duration", + } + } + createViewBody(ctx: CreateViewBodyCstChildren): AST.CreateViewStatement { const result: AST.CreateViewStatement = { type: "createView", @@ -3333,7 +3368,14 @@ class QuestDBVisitor extends BaseVisitor { result.quoted = true } else if (ctx.expression) { const expr = this.visit(ctx.expression) as AST.Expression - if (expr?.type === "literal" && expr.literalType === "number") { + if ( + expr?.type === "literal" && + expr.literalType === "number" && + /[mMdD]$/.test(expr.raw ?? "") + ) { + // keep a suffixed number (3d, 1m) verbatim; it used to be a duration + result.value = expr.raw! + } else if (expr?.type === "literal" && expr.literalType === "number") { result.value = expr.value as number } else if (expr?.type === "literal" && expr.literalType === "string") { result.value = expr.value as string @@ -4753,7 +4795,7 @@ class QuestDBVisitor extends BaseVisitor { const image = this.tokenImage(ctx.DecimalLiteral[0]) return { type: "literal", - value: parseFloat(image.replace(/[m_]/g, "")), + value: parseFloat(image.replace(/[mM_]/g, "")), literalType: "number", raw: image, } @@ -4773,9 +4815,22 @@ class QuestDBVisitor extends BaseVisitor { } } if (ctx.DurationLiteral) { + const image = this.tokenImage(ctx.DurationLiteral[0]) + // In an expression the server types `99m` / `1.5M` as DECIMAL and + // `1d` / `1.5D` as DOUBLE (FunctionParser.createConstant: parseDouble + // skips a d/D suffix, a trailing m/M means decimal). Other units + // (1h, 30s, 1ms, ...) are not numbers there and stay durations. + if (/[mMdD]$/.test(image)) { + return { + type: "literal", + value: parseFloat(image.slice(0, -1).replace(/_/g, "")), + literalType: "number", + raw: image, + } + } return { type: "literal", - value: this.tokenImage(ctx.DurationLiteral[0]), + value: image, literalType: "duration", } } diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 2074a19..9987141 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1114,5 +1114,119 @@ "feature": "comments-empty-statements", "toSql": "SELECT 1;\nSELECT 2", "statements": 2 + }, + { + "query": "SELECT cast(99m as byte)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/cast/CastDecimalToByteFunctionFactoryTest.java:217", + "feature": "numeric-literal-typing", + "toSql": "SELECT CAST(99m AS BYTE)" + }, + { + "query": "select cast(-127m as byte)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/cast/CastDecimalToByteFunctionFactoryTest.java:470", + "feature": "numeric-literal-typing", + "toSql": "SELECT CAST(-127m AS BYTE)" + }, + { + "query": "select cast(cast(2147483647m as DECIMAL(19)) as byte)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/cast/CastDecimalToByteFunctionFactoryTest.java:526", + "feature": "numeric-literal-typing", + "toSql": "SELECT CAST(CAST(2147483647m AS DECIMAL(19)) AS BYTE)" + }, + { + "query": "select 123::byte + 123m", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/cast/CastByteToDecimalFunctionFactoryTest.java:429", + "feature": "numeric-literal-typing", + "toSql": "SELECT 123::BYTE + 123m" + }, + { + "query": "WITH data AS (SELECT 123m AS value) SELECT cast(value as byte) FROM data", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/cast/CastDecimalToByteFunctionFactoryTest.java:113", + "feature": "numeric-literal-typing", + "toSql": "WITH data AS (SELECT 123m AS value) SELECT CAST(value AS BYTE) FROM data" + }, + { + "query": "insert into target select id, val * 2m as price, val, ts from source where id > 1", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/InsertTest.java:742", + "feature": "numeric-literal-typing", + "toSql": "INSERT INTO target SELECT id, val * 2m AS price, val, ts FROM source WHERE id > 1" + }, + { + "query": "insert into mixed_types values (1, 99.99m, 'Item1', true, 100m, 1.5, 'ABC', 0)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/InsertTest.java:819", + "feature": "numeric-literal-typing", + "toSql": "INSERT INTO mixed_types VALUES (1, 99.99m, 'Item1', TRUE, 100m, 1.5, 'ABC', 0)" + }, + { + "query": "INSERT INTO t VALUES (0m), (0)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DecimalFullScaleTest.java:232", + "feature": "numeric-literal-typing", + "toSql": "INSERT INTO t VALUES (0m), (0)" + }, + { + "query": "INSERT INTO base (ts, sym, d) VALUES ('2026-10-01T02:00:00.000000Z', 1, 20m)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/lv/LiveViewSmokeTest.java:9080", + "feature": "numeric-literal-typing", + "toSql": "INSERT INTO base (ts, sym, d) VALUES ('2026-10-01T02:00:00.000000Z', 1, 20m)" + }, + { + "query": "DECLARE @minp := 150.00m SELECT * FROM v_sec", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/view/ViewAuditTest.java:1427", + "feature": "numeric-literal-typing", + "toSql": "DECLARE @minp := 150.00m SELECT * FROM v_sec" + }, + { + "query": "select 10d * 0d x", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/math/MulDoubleFunctionFactoryTest.java:49", + "feature": "numeric-literal-typing", + "toSql": "SELECT 10d * 0d AS x" + }, + { + "query": "create table y as (select 0d a union select 3d union select 5d union select 9d union select 8d)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/UnionAllCastTest.java:864", + "feature": "numeric-literal-typing", + "toSql": "CREATE TABLE y AS (SELECT 0d AS a UNION SELECT 3d UNION SELECT 5d UNION SELECT 9d UNION SELECT 8d)" + }, + { + "query": "select greatest(4::decimal(2,1), 1d, null::decimal(1,0))", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/math/GreatestNumericFunctionFactoryTest.java:61", + "feature": "numeric-literal-typing", + "toSql": "SELECT greatest(4::DECIMAL(2, 1), 1d, NULL::DECIMAL(1, 0))" + }, + { + "query": "create table y as ( select x + 0.1d as d from long_sequence(10) )", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/GroupByRewriteTest.java:340", + "feature": "numeric-literal-typing", + "toSql": "CREATE TABLE y AS (SELECT x + 0.1d AS d FROM long_sequence(10))" + }, + { + "query": "INSERT INTO TS values(to_timestamp('2019-12-04T13:20:49', 'yyyy-MM-ddTHH:mm:ss'),'X',123.33d)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/InsertTest.java:1221", + "feature": "numeric-literal-typing", + "toSql": "INSERT INTO TS VALUES (to_timestamp('2019-12-04T13:20:49', 'yyyy-MM-ddTHH:mm:ss'), 'X', 123.33d)" + }, + { + "query": "CREATE TABLE tango (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY HOUR TTL 1M", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/TtlTest.java:769", + "feature": "numeric-literal-typing", + "toSql": "CREATE TABLE tango (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY HOUR TTL 1 MONTH" + }, + { + "query": "alter table x1 set param o3MaxLag = 7d", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/AlterTableO3MaxLagTest.java:414", + "feature": "numeric-literal-typing", + "toSql": "ALTER TABLE x1 SET PARAM o3MaxLag = 7d" + }, + { + "query": "SELECT avg(p.price) FROM trades AS t HORIZON JOIN prices AS p ON (t.sym = p.sym) LIST (100T, 1s, 1m, 1h, 1d) AS h", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:6658", + "feature": "numeric-literal-typing", + "toSql": "SELECT avg(p.price) FROM trades AS t HORIZON JOIN prices AS p ON (t.sym = p.sym) LIST (100T, 1s, 1m, 1h, 1d) AS h" + }, + { + "query": "CREATE MATERIALIZED VIEW mv3 AS (SELECT ts, avg(x) AS avg_x FROM tab SAMPLE BY 10m) PARTITION BY DAY", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/DdlListenerTest.java:388", + "feature": "numeric-literal-typing", + "toSql": "CREATE MATERIALIZED VIEW mv3 AS (SELECT ts, avg(x) AS avg_x FROM tab SAMPLE BY 10m) PARTITION BY DAY" } ] diff --git a/tests/numeric-literal-typing.test.ts b/tests/numeric-literal-typing.test.ts new file mode 100644 index 0000000..25bcddc --- /dev/null +++ b/tests/numeric-literal-typing.test.ts @@ -0,0 +1,315 @@ +/** + * Decimal `m` / double `d` literals in expressions (reports §3.1). + * + * Server FunctionParser.createConstant types a literal token as: + * - DOUBLE when Numbers.parseDouble accepts it. FastDouble skips an + * optional `d`/`D` suffix, so `1d`, `0d`, `1.5D`, `1e36d` are doubles. + * - DECIMAL when it ends in `m`/`M` (`(c | 32) == 'm'`), parsed by + * DecimalParser, which also takes an exponent: `99m`, `1.5m`, `1.5e3m`, + * `2.5e-3M` (Decimal256Test:1839 `1.5e2M`). + * + * The lexer keeps `99m` / `1d` as DurationLiteral tokens (SAMPLE BY, TTL, + * o3MaxLag, REFRESH EVERY ... need them), but in an expression the visitor + * now types them as numbers with the raw text preserved. `1.5e3m` used to + * lex as `1.5e3` + `m` and silently became `1.5e3 AS m`. + * + * Valid server queries are in tests/fixtures/server-queries.json + * (feature "numeric-literal-typing"). + */ + +import { describe, it, expect } from "vitest" +import { + parseToAst, + tokenize, + getNextValidTokens, + createAutocompleteProvider, +} from "../src/index" +import { toSql } from "../src/parser/toSql" +import { format } from "../src/formatter/index" +import type * as AST from "../src/parser/ast" + +const tokenTypes = (sql: string) => + tokenize(sql).tokens.map((t) => `${t.tokenType.name}:${t.image}`) + +const parseOne = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +/** The single select item of `SELECT ` */ +const selectItem = (sql: string) => { + const select = parseOne(sql) as AST.SelectStatement + expect(select.columns).toHaveLength(1) + return select.columns[0] as AST.ExpressionSelectItem +} + +const roundTrip = (sql: string) => toSql(parseOne(sql)) + +describe("lexer", () => { + it.each([ + "1.5e3m", + "1e5m", + "1.5E3M", + "2.5e-3m", + "2.5e+3M", + "1_000e2m", + "1.e3m", + "1.2300e2m", + ])("reads %s as one decimal token", (d) => { + expect(tokenTypes(d)).toEqual([`DecimalLiteral:${d}`]) + }) + + it("keeps the existing decimal and duration tokens", () => { + expect(tokenTypes("10.5m")).toEqual(["DecimalLiteral:10.5m"]) + expect(tokenTypes("99m")).toEqual(["DurationLiteral:99m"]) + expect(tokenTypes("99M")).toEqual(["DurationLiteral:99M"]) + expect(tokenTypes("1.5M")).toEqual(["DurationLiteral:1.5M"]) + expect(tokenTypes("1d")).toEqual(["DurationLiteral:1d"]) + expect(tokenTypes("1.5d")).toEqual(["DurationLiteral:1.5d"]) + expect(tokenTypes("1e36d")).toEqual(["FloatLiteral:1e36d"]) + expect(tokenTypes("1.5e3")).toEqual(["NumberLiteral:1.5e3"]) + }) + + it("ends an exponent decimal at punctuation", () => { + expect(tokenTypes("1.5e3m)")).toEqual(["DecimalLiteral:1.5e3m", "RParen:)"]) + expect(tokenTypes("1e5m,")).toEqual(["DecimalLiteral:1e5m", "Comma:,"]) + }) +}) + +describe("expression literals", () => { + it.each([ + ["99m", 99], + ["0m", 0], + ["99M", 99], + ["1_000m", 1000], + ["1.5M", 1.5], + ["10.5m", 10.5], + ["1.5e3m", 1500], + ["1e5m", 100000], + ["2.5e-3m", 0.0025], + ["1.5E3M", 1500], + ])("types %s as a decimal number", (raw, value) => { + const item = selectItem(`SELECT ${raw}`) + expect(item.alias).toBeUndefined() + expect(item.expression).toEqual({ + type: "literal", + value, + literalType: "number", + raw, + }) + }) + + it.each([ + ["1d", 1], + ["0d", 0], + ["1D", 1], + ["1.5d", 1.5], + ["10000d", 10000], + ["1_000d", 1000], + ])("types %s as a double number", (raw, value) => { + const item = selectItem(`SELECT ${raw}`) + expect(item.alias).toBeUndefined() + expect(item.expression).toEqual({ + type: "literal", + value, + literalType: "number", + raw, + }) + }) + + it.each(["1h", "30s", "1ms", "100us", "100ns", "1y", "1w", "100T"])( + "keeps %s a duration (not a number on the server)", + (raw) => { + expect(selectItem(`SELECT ${raw}`).expression).toEqual({ + type: "literal", + value: raw, + literalType: "duration", + }) + }, + ) + + it("no longer reads 1.5e3m as 1.5e3 AS m", () => { + expect(roundTrip("SELECT 1.5e3m")).toBe("SELECT 1.5e3m") + expect(roundTrip("SELECT 1.5e3m m")).toBe("SELECT 1.5e3m AS m") + expect(roundTrip("SELECT * FROM t WHERE price > 1.5e3m")).toBe( + "SELECT * FROM t WHERE price > 1.5e3m", + ) + expect(roundTrip("SELECT cast(2.5e-3M AS DECIMAL(10, 4))")).toBe( + "SELECT CAST(2.5e-3M AS DECIMAL(10, 4))", + ) + }) + + it("types numbers inside larger expressions", () => { + const select = parseOne( + "SELECT x * 2m, -99m, 10d * 0d, generate_series(-3d, 3d) FROM t", + ) as AST.SelectStatement + const [mul, neg, dbl] = select.columns.map( + (c) => (c as AST.ExpressionSelectItem).expression, + ) + expect((mul as AST.BinaryExpression).right).toMatchObject({ + literalType: "number", + value: 2, + raw: "2m", + }) + expect((neg as AST.UnaryExpression).operand).toMatchObject({ + literalType: "number", + value: 99, + raw: "99m", + }) + expect((dbl as AST.BinaryExpression).left).toMatchObject({ + literalType: "number", + raw: "10d", + }) + }) + + it("types VALUES literals", () => { + const insert = parseOne( + "INSERT INTO t VALUES (1, 99.99m, 100m, 123.33d, 1.5e3m)", + ) as AST.InsertStatement + const row = insert.values![0] + expect(row.map((e) => (e as AST.Literal).literalType)).toEqual([ + "number", + "number", + "number", + "number", + "number", + ]) + expect(row.map((e) => (e as AST.Literal).raw)).toEqual([ + "1", + "99.99m", + "100m", + "123.33d", + "1.5e3m", + ]) + }) + + it("rejects an exponent decimal glued to a word like before", () => { + // 1e5mx is not a decimal; it stays a number followed by an alias + expect(tokenTypes("1e5mx")).toEqual(["NumberLiteral:1e5", "Identifier:mx"]) + }) +}) + +describe("duration contexts are unchanged", () => { + it("SAMPLE BY", () => { + for (const d of ["1m", "5m", "1M", "1d", "1D"]) { + const select = parseOne( + `SELECT ts, avg(x) FROM t SAMPLE BY ${d}`, + ) as AST.SelectStatement + expect(select.sampleBy?.duration).toBe(d) + } + }) + + it("TTL", () => { + const create = parseOne( + "CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY HOUR TTL 1M", + ) as AST.CreateTableStatement + expect(create.ttl).toEqual({ value: 1, unit: "MONTHS" }) + expect( + roundTrip( + "CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY TTL 2D", + ), + ).toBe( + "CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY TTL 2 DAYS", + ) + }) + + it.each(["7d", "10m", "2h", "100ms", "1.5d"])( + "o3MaxLag = %s stays a duration", + (d) => { + const alter = parseOne( + `ALTER TABLE x SET PARAM o3MaxLag = ${d}`, + ) as AST.AlterTableStatement + const action = alter.action as { params: AST.TableParam[] } + expect(action.params[0].value).toEqual({ + type: "literal", + value: d, + literalType: "duration", + }) + const create = parseOne( + `CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WITH maxUncommittedRows=7, o3MaxLag=${d}`, + ) as AST.CreateTableStatement + expect(create.withParams?.[1].value).toEqual({ + type: "literal", + value: d, + literalType: "duration", + }) + }, + ) + + it("DECLARE keeps a bare duration value (it may be used in SAMPLE BY)", () => { + const select = parseOne( + "DECLARE @period := 1m SELECT ts, sum(x) FROM t SAMPLE BY @period", + ) as AST.SelectStatement + expect(select.declare?.assignments[0].value).toEqual({ + type: "literal", + value: "1m", + literalType: "duration", + }) + // an expression around it is evaluated by the server: typed as a number + const select2 = parseOne( + "DECLARE @x := 1m + 1 SELECT @x", + ) as AST.SelectStatement + expect( + (select2.declare?.assignments[0].value as AST.BinaryExpression).left, + ).toMatchObject({ literalType: "number", raw: "1m" }) + }) + + it("window frames, WINDOW JOIN and HORIZON LIST", () => { + expect( + roundTrip( + "SELECT sum(x) OVER (ORDER BY ts RANGE BETWEEN 1d PRECEDING AND CURRENT ROW) FROM t", + ), + ).toBe( + "SELECT sum(x) OVER (ORDER BY ts RANGE BETWEEN 1d PRECEDING AND CURRENT ROW) FROM t", + ) + expect( + roundTrip( + "SELECT avg(p.price) FROM trades AS t HORIZON JOIN prices AS p ON (t.sym = p.sym) LIST (100T, 1s, 1m, 1h, 1d) AS h", + ), + ).toBe( + "SELECT avg(p.price) FROM trades AS t HORIZON JOIN prices AS p ON (t.sym = p.sym) LIST (100T, 1s, 1m, 1h, 1d) AS h", + ) + }) + + it("keeps a suffixed number verbatim in a COPY option", () => { + expect(roundTrip("COPY t TO 'x' WITH COMPRESSION_LEVEL 3d")).toBe( + "COPY t TO 'x' WITH COMPRESSION_LEVEL 3d", + ) + expect(roundTrip("COPY t TO 'x' WITH ROW_GROUP_SIZE 1m")).toBe( + "COPY t TO 'x' WITH ROW_GROUP_SIZE 1m", + ) + }) +}) + +describe("autocomplete", () => { + const provider = createAutocompleteProvider({ + tables: [{ name: "t" }], + columns: { t: [{ name: "price", type: "DECIMAL" }] }, + }) + const labels = (sql: string) => + provider.getSuggestions(sql, sql.length).map((s) => s.label) + + it("continues after an exponent decimal like after any number", () => { + expect(labels("SELECT 1.5e3m ")).toEqual(labels("SELECT 1.5m ")) + expect(labels("SELECT * FROM t WHERE price > 1.5e3m ")).toEqual( + labels("SELECT * FROM t WHERE price > 1.5m "), + ) + expect(labels("SELECT * FROM t WHERE price > 1.5e3m ")).toContain("AND") + }) + + it("accepts a decimal where an expression can start", () => { + expect(getNextValidTokens("SELECT * FROM t WHERE price > ")).toContain( + "DecimalLiteral", + ) + }) +}) + +describe("formatter", () => { + it("keeps exponent decimals as one token", () => { + expect(format("select 1.5e3m, 2.5E-3M from t")).toBe( + "select 1.5e3m, 2.5E-3M\nfrom t", + ) + }) +}) From 3765d8c8b3fe3cb861c57b778147fb0d1aa4eb26 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 15:48:41 +0300 Subject: [PATCH 13/36] feat: parse AT TIME ZONE, timestamp WITH TIME ZONE casts and DOUBLE PRECISION Server (ExpressionParser): - ` AT TIME ZONE ''`: AT after a completed operand, then TIME, ZONE and a constant; rewritten to to_timezone(expr, tz). `at time 'x'` and a dangling `at time` fail with "did you mean 'at time zone '?" (TimestampAtTimeZoneTest). at/time/zone stay usable as aliases. - `timestamp[_ns] with time zone` as a cast target: inside CAST(.. AS ..) (Scope.CAST_AS) and after `::` (isColonColon on the op stack). - `double precision`: PRECISION after `double` is ignored in expressions (isPrecisionKeyword), SqlParser.toColumnType skips it for CREATE TABLE columns and CTAS `cast(col as type)` (incl. `double precision[]`). ALTER TABLE ADD COLUMN / ALTER COLUMN TYPE (addColumnWithType) do not. Before: every form failed, and `x::timestamp at time zone 'UTC'` was silently split into `x::TIMESTAMP AS at; time AS zone; UTC`. Changes: - keywords: `at` and `precision`, both non-reserved (server reserves neither). - parser: typeCastExpression takes an optional `AT TIME ZONE (StringLiteral | EscapeStringLiteral)` after the `::` casts (gated on LA(2) = TIME so `SELECT ts at FROM t` keeps the alias; not repeated, the server's to_timezone rewrite does not chain). It binds tighter than arithmetic and inside unary minus, like the server. New castDataType rule (CAST and `::`): timestampWithTimeZoneType | doublePrecisionType | dataType, as gated alternatives so autocomplete offers WITH only after TIMESTAMP[_NS] and PRECISION only after DOUBLE. columnDefinition gets an argument: DOUBLE PRECISION only when called from CREATE TABLE (ALTER ADD COLUMN shares the rule); CTAS castDefinition also takes it. dataType itself is unchanged. - AST: new expression node `atTimeZone { expression, timeZone }`. Cast / column dataType strings are "TIMESTAMP WITH TIME ZONE", "TIMESTAMP_NS WITH TIME ZONE", "DOUBLE PRECISION[]..." (spelling kept, same as the existing typed literal's typeName). - toSql prints `expr AT TIME ZONE 'tz'` and the dataType strings. - autocomplete: PRECISION is dropped under alterTableAction (rule args are invisible to content assist). - formatter: WITH after TIMESTAMP/TIMESTAMP_NS followed by TIME ZONE is not a WITH clause (it used to break the line, also for the existing `timestamp with time zone '...'` literal). - cst-types regenerated. Tests: 19 server queries in tests/fixtures/server-queries.json (TimestampAtTimeZoneTest x8 incl. alias guards, InsertTest x2, SimpleTableTest, SqlParserTest x2, TimestampQueryTest x4, PGArraysTest x2); no server test parses DOUBLE PRECISION in CREATE TABLE (only expected model strings), covered in tests/type-syntax.test.ts (51 tests: AST, precedence, aliases, rejected forms, CREATE/CTAS/ALTER, formatter, autocomplete). Red -> green: 78 failing (52 fixture checks + 26 focused) -> all pass; full suite 16637 passed. Corpus (53,975): fixed 21 / rtChanged 2 (the silent 3-statement splits of `::timestamp[_ns] at time zone 'UTC'`) / regressed 3 / throws 0 / formatterChangedMeaning 0. All 3 regressions are server-rejected TimestampAtTimeZoneTest negatives (testFail2 `at time 'UTC'`, testFailDangling2/3 `at time`) that used to split silently into `AS at; time ...`. Suggestions: AT after a completed operand (SELECT/WHERE/ORDER BY/GROUP BY/ OVER/DECLARE values), TIME after `AT`, WITH after `::timestamp` and `CAST(x AS TIMESTAMP`, PRECISION after DOUBLE in casts, CREATE TABLE columns and CTAS casts (not in ALTER ADD COLUMN). The `[` / `)` fallback punctuation after `CAST(x AS TIMESTAMP|DOUBLE ` is no longer shown because a keyword is now suggested. --- src/autocomplete/content-assist.ts | 11 +- src/formatter/layout.ts | 15 ++ src/grammar/keywords.ts | 2 + src/parser/ast.ts | 8 + src/parser/cst-types.d.ts | 54 +++++- src/parser/lexer.ts | 4 + src/parser/parser.ts | 151 +++++++++++++--- src/parser/toSql.ts | 2 + src/parser/tokens.ts | 8 + src/parser/visitor.ts | 42 ++++- tests/fixtures/server-queries.json | 115 ++++++++++++ tests/type-syntax.test.ts | 274 +++++++++++++++++++++++++++++ 12 files changed, 648 insertions(+), 38 deletions(-) create mode 100644 tests/type-syntax.test.ts diff --git a/src/autocomplete/content-assist.ts b/src/autocomplete/content-assist.ts index 50ee2c1..2ac214c 100644 --- a/src/autocomplete/content-assist.ts +++ b/src/autocomplete/content-assist.ts @@ -982,13 +982,18 @@ function computeSuggestions(tokens: IToken[]): ComputeResult { // tokens. Content assist ignores its GATE, so without this filter those two // keywords would be offered wherever an expression can start. Only the // rule's first token is dropped; WITH TIME ZONE after `timestamp` stays. + // columnDefinition only takes DOUBLE PRECISION in CREATE TABLE (a rule + // argument gates it); ALTER TABLE ADD COLUMN shares the rule, so drop + // PRECISION there as well. const suggestions = rawSuggestions.filter( (s) => !( - s.ruleStack[s.ruleStack.length - 1] === + (s.ruleStack[s.ruleStack.length - 1] === "timestampWithTimeZoneLiteral" && - (s.nextTokenType.name === "Timestamp" || - s.nextTokenType.name === "TimestampNs") + (s.nextTokenType.name === "Timestamp" || + s.nextTokenType.name === "TimestampNs")) || + (s.nextTokenType.name === "Precision" && + s.ruleStack.includes("alterTableAction")) ), ) const result = suggestions.map((s) => s.nextTokenType) diff --git a/src/formatter/layout.ts b/src/formatter/layout.ts index 50179fc..2aefb75 100644 --- a/src/formatter/layout.ts +++ b/src/formatter/layout.ts @@ -82,6 +82,10 @@ const opensBlock = (next: Piece | undefined) => startsQuery(next) type Clause = { doc: Doc; headless: boolean } +const timestampWithTimeZone: Phrase[] = [ + { names: ["With", "Time", "Zone"], role: "clause" }, +] + const alwaysBreaksList = (phrase: Phrase | null) => phrase !== null && phrase.names[0] === "Declare" @@ -217,6 +221,17 @@ class StatementBuilder { ) { return null } + // `timestamp[_ns] WITH TIME ZONE` is a type name, not a WITH clause + if ( + piece !== undefined && + piece.token.tokenName === "With" && + this.previous !== null && + (this.previous.token.tokenName === "Timestamp" || + this.previous.token.tokenName === "TimestampNs") && + matchPhrase(this.tokens, this.index, timestampWithTimeZone) !== null + ) { + return null + } return matchPhrase(this.tokens, this.index, phrases) } diff --git a/src/grammar/keywords.ts b/src/grammar/keywords.ts index 2a6884c..417879f 100644 --- a/src/grammar/keywords.ts +++ b/src/grammar/keywords.ts @@ -13,6 +13,7 @@ export const keywords: string[] = [ "as", "asof", "assume", + "at", "atomic", "attach", "backup", @@ -155,6 +156,7 @@ export const keywords: string[] = [ "pivot", "policy", "preceding", + "precision", "prevailing", "primary", "priority", diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 3f34283..03fef06 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -1259,6 +1259,7 @@ export type Expression = | CaseExpression | CastExpression | TypeCastExpression + | AtTimeZoneExpression | InExpression | BetweenExpression | WithinExpression @@ -1372,6 +1373,13 @@ export interface TypeCastExpression extends AstNode { dataType: string } +/** ` AT TIME ZONE ''` (the server rewrites it to to_timezone(expr, tz)) */ +export interface AtTimeZoneExpression extends AstNode { + type: "atTimeZone" + expression: Expression + timeZone: Expression +} + export interface InExpression extends AstNode { type: "in" expression: Expression diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index c705750..6419381 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -1143,7 +1143,8 @@ export interface ColumnDefinitionCstNode extends CstNode { export type ColumnDefinitionCstChildren = { identifier: IdentifierCstNode[]; - dataType: DataTypeCstNode[]; + doublePrecisionType?: DoublePrecisionTypeCstNode[]; + dataType?: DataTypeCstNode[]; Capacity?: IToken[]; NumberLiteral?: IToken[]; Cache?: IToken[]; @@ -1272,7 +1273,8 @@ export type CastDefinitionCstChildren = { LParen: IToken[]; columnRef: ColumnRefCstNode[]; As: IToken[]; - dataType: DataTypeCstNode[]; + doublePrecisionType?: DoublePrecisionTypeCstNode[]; + dataType?: DataTypeCstNode[]; RParen: IToken[]; }; @@ -2648,7 +2650,12 @@ export type TypeCastExpressionCstChildren = { Comma?: IToken[]; RBracket?: IToken[]; DoubleColon?: IToken[]; - dataType?: DataTypeCstNode[]; + castDataType?: CastDataTypeCstNode[]; + At?: IToken[]; + Time?: IToken[]; + Zone?: IToken[]; + StringLiteral?: IToken[]; + EscapeStringLiteral?: IToken[]; }; export interface ArraySubscriptCstNode extends CstNode { @@ -2725,10 +2732,46 @@ export type CastExpressionCstChildren = { LParen: IToken[]; expression: ExpressionCstNode[]; As: IToken[]; - dataType: DataTypeCstNode[]; + castDataType: CastDataTypeCstNode[]; RParen: IToken[]; }; +export interface CastDataTypeCstNode extends CstNode { + name: "castDataType"; + children: CastDataTypeCstChildren; +} + +export type CastDataTypeCstChildren = { + timestampWithTimeZoneType?: TimestampWithTimeZoneTypeCstNode[]; + doublePrecisionType?: DoublePrecisionTypeCstNode[]; + dataType?: DataTypeCstNode[]; +}; + +export interface TimestampWithTimeZoneTypeCstNode extends CstNode { + name: "timestampWithTimeZoneType"; + children: TimestampWithTimeZoneTypeCstChildren; +} + +export type TimestampWithTimeZoneTypeCstChildren = { + Timestamp?: IToken[]; + TimestampNs?: IToken[]; + With: IToken[]; + Time: IToken[]; + Zone: IToken[]; +}; + +export interface DoublePrecisionTypeCstNode extends CstNode { + name: "doublePrecisionType"; + children: DoublePrecisionTypeCstChildren; +} + +export type DoublePrecisionTypeCstChildren = { + Double: IToken[]; + Precision: IToken[]; + LBracket?: IToken[]; + RBracket?: IToken[]; +}; + export interface DataTypeCstNode extends CstNode { name: "dataType"; children: DataTypeCstChildren; @@ -3250,6 +3293,9 @@ export interface ICstNodeVisitor extends ICstVisitor { arrayBracketBody(children: ArrayBracketBodyCstChildren, param?: IN): OUT; arrayElement(children: ArrayElementCstChildren, param?: IN): OUT; castExpression(children: CastExpressionCstChildren, param?: IN): OUT; + castDataType(children: CastDataTypeCstChildren, param?: IN): OUT; + timestampWithTimeZoneType(children: TimestampWithTimeZoneTypeCstChildren, param?: IN): OUT; + doublePrecisionType(children: DoublePrecisionTypeCstChildren, param?: IN): OUT; dataType(children: DataTypeCstChildren, param?: IN): OUT; caseExpression(children: CaseExpressionCstChildren, param?: IN): OUT; functionName(children: FunctionNameCstChildren, param?: IN): OUT; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index d09ee2c..abffa9a 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -309,6 +309,8 @@ import { Grouping, Rollup, Sets, + At, + Precision, Policy, Local, Remote, @@ -648,6 +650,8 @@ export { Grouping, Rollup, Sets, + At, + Precision, Policy, Local, Remote, diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 5857f57..43825a1 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -245,6 +245,8 @@ import { DataPageSize, ParquetVersion, Zone, + At, + Precision, Period, Length, Delay, @@ -1818,10 +1820,10 @@ class QuestDBParser extends CstParser { { ALT: () => { this.CONSUME1(LParen) - this.SUBRULE(this.columnDefinition) + this.SUBRULE(this.columnDefinition, { ARGS: [true] }) this.MANY(() => { this.CONSUME2(Comma) - this.SUBRULE1(this.columnDefinition) + this.SUBRULE1(this.columnDefinition, { ARGS: [true] }) }) this.CONSUME1(RParen) // Optional INDEX clause(s) after column list @@ -2376,27 +2378,42 @@ class QuestDBParser extends CstParser { }, ) - private columnDefinition = this.RULE("columnDefinition", () => { - this.SUBRULE(this.identifier) - this.SUBRULE(this.dataType) - // SYMBOL column options: CAPACITY n, CACHE|NOCACHE, INDEX [CAPACITY n] - this.OPTION(() => { - this.CONSUME(Capacity) - this.CONSUME(NumberLiteral) - }) - this.OPTION1(() => { - this.OR([ - { ALT: () => this.CONSUME(Cache) }, - { ALT: () => this.CONSUME(Nocache) }, - ]) - }) - this.OPTION2(() => { - this.CONSUME(Index) - this.SUBRULE(this.indexTypeOptions) - }) - // Optional PARQUET config - this.OPTION4(() => this.SUBRULE(this.parquetConfig)) - }) + // allowDoublePrecision: CREATE TABLE columns (SqlParser.toColumnType skips + // PRECISION after DOUBLE). ALTER TABLE ADD COLUMN does not + // (SqlCompilerImpl.addColumnWithType); autocomplete drops PRECISION there. + private columnDefinition = this.RULE( + "columnDefinition", + (allowDoublePrecision?: boolean) => { + this.SUBRULE(this.identifier) + this.OR5({ + IGNORE_AMBIGUITIES: true, + DEF: [ + { + GATE: () => !!allowDoublePrecision && this.isDoublePrecision(), + ALT: () => this.SUBRULE(this.doublePrecisionType), + }, + { ALT: () => this.SUBRULE(this.dataType) }, + ], + }) + // SYMBOL column options: CAPACITY n, CACHE|NOCACHE, INDEX [CAPACITY n] + this.OPTION(() => { + this.CONSUME(Capacity) + this.CONSUME(NumberLiteral) + }) + this.OPTION1(() => { + this.OR([ + { ALT: () => this.CONSUME(Cache) }, + { ALT: () => this.CONSUME(Nocache) }, + ]) + }) + this.OPTION2(() => { + this.CONSUME(Index) + this.SUBRULE(this.indexTypeOptions) + }) + // Optional PARQUET config + this.OPTION4(() => this.SUBRULE(this.parquetConfig)) + }, + ) // PARQUET( BLOOM_FILTER | encoding [, compression[(level)]] [, BLOOM_FILTER] ) private parquetConfig = this.RULE("parquetConfig", () => { @@ -2548,7 +2565,17 @@ class QuestDBParser extends CstParser { this.CONSUME(LParen) this.SUBRULE(this.columnRef) this.CONSUME(As) - this.SUBRULE(this.dataType) + // SqlParser.toColumnType: DOUBLE PRECISION is DOUBLE + this.OR({ + IGNORE_AMBIGUITIES: true, + DEF: [ + { + GATE: () => this.isDoublePrecision(), + ALT: () => this.SUBRULE(this.doublePrecisionType), + }, + { ALT: () => this.SUBRULE(this.dataType) }, + ], + }) this.CONSUME(RParen) }) @@ -4884,7 +4911,23 @@ class QuestDBParser extends CstParser { }) this.MANY2(() => { this.CONSUME(DoubleColon) - this.SUBRULE(this.dataType) + this.SUBRULE(this.castDataType) + }) + // AT TIME ZONE '' (ExpressionParser: AT after a completed + // operand, rewritten to to_timezone(expr, tz); the zone must be a + // string constant). Not repeated: the server does not chain it. The + // gate keeps `at` usable as an alias (`SELECT ts at FROM t`). + this.OPTION({ + GATE: () => this.LA(2).tokenType === Time, + DEF: () => { + this.CONSUME(At) + this.CONSUME(Time) + this.CONSUME(Zone) + this.OR([ + { ALT: () => this.CONSUME(StringLiteral) }, + { ALT: () => this.CONSUME(EscapeStringLiteral) }, + ]) + }, }) }) @@ -5033,10 +5076,66 @@ class QuestDBParser extends CstParser { this.CONSUME(LParen) this.SUBRULE(this.expression) this.CONSUME(As) - this.SUBRULE(this.dataType) + this.SUBRULE(this.castDataType) this.CONSUME(RParen) }) + // Cast target (`::` and CAST(x AS ...)): ExpressionParser also reads the + // multi-word spellings `timestamp[_ns] with time zone` and + // `double precision`. They are separate alternatives (not options inside + // dataType) so autocomplete only offers WITH after TIMESTAMP and PRECISION + // after DOUBLE, and only where the server accepts them. + private castDataType = this.RULE("castDataType", () => { + this.OR({ + IGNORE_AMBIGUITIES: true, + DEF: [ + { + GATE: () => { + const t1 = this.LA(1).tokenType + return ( + (t1 === Timestamp || t1 === TimestampNs) && + this.LA(2).tokenType === With && + this.LA(3).tokenType === Time + ) + }, + ALT: () => this.SUBRULE(this.timestampWithTimeZoneType), + }, + { + GATE: () => this.isDoublePrecision(), + ALT: () => this.SUBRULE(this.doublePrecisionType), + }, + { ALT: () => this.SUBRULE(this.dataType) }, + ], + }) + }) + + private isDoublePrecision(): boolean { + return this.LA(1).tokenType === Double && this.LA(2).tokenType === Precision + } + + private timestampWithTimeZoneType = this.RULE( + "timestampWithTimeZoneType", + () => { + this.OR([ + { ALT: () => this.CONSUME(Timestamp) }, + { ALT: () => this.CONSUME(TimestampNs) }, + ]) + this.CONSUME(With) + this.CONSUME(Time) + this.CONSUME(Zone) + }, + ) + + private doublePrecisionType = this.RULE("doublePrecisionType", () => { + this.CONSUME(Double) + this.CONSUME(Precision) + // Array dimensions: DOUBLE PRECISION[], DOUBLE PRECISION[][] + this.MANY(() => { + this.CONSUME(LBracket) + this.CONSUME(RBracket) + }) + }) + private dataType = this.RULE("dataType", () => { this.OR([ { ALT: () => this.CONSUME(Symbol) }, diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 3b0d9b9..5e49cdb 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -1860,6 +1860,8 @@ function expressionToSql(expr: AST.Expression): string { return castExprToSql(expr) case "typeCast": return typeCastExprToSql(expr) + case "atTimeZone": + return `${expressionToSql(expr.expression)} AT TIME ZONE ${expressionToSql(expr.timeZone)}` case "in": return inExprToSql(expr) case "between": diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index 6be738d..742cd39 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -426,6 +426,10 @@ export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ "Grouping", "Rollup", "Sets", + // AT TIME ZONE '' and DOUBLE PRECISION (ExpressionParser); not + // reserved on the server. + "At", + "Precision", ]) for (const name of IDENTIFIER_KEYWORD_NAMES) { @@ -732,6 +736,10 @@ export const Grouping = getToken("Grouping") export const Rollup = getToken("Rollup") export const Sets = getToken("Sets") +// AT TIME ZONE, DOUBLE PRECISION +export const At = getToken("At") +export const Precision = getToken("Precision") + // Data types (from grammar/dataTypes.ts) export const Binary = getToken("Binary") export const Boolean = getToken("Boolean") diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 58edd95..a1d45ff 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -190,6 +190,9 @@ import type { TimeZoneValueCstChildren, TruncateTableStatementCstChildren, TypeCastExpressionCstChildren, + CastDataTypeCstChildren, + TimestampWithTimeZoneTypeCstChildren, + DoublePrecisionTypeCstChildren, UnaryExpressionCstChildren, UnnestArgCstChildren, UnnestClauseCstChildren, @@ -1542,7 +1545,7 @@ class QuestDBVisitor extends BaseVisitor { const result: AST.ColumnDefinition = { type: "columnDefinition", name: this.extractIdentifierName(ctx.identifier[0].children), - dataType: this.visit(ctx.dataType) as string, + dataType: this.visit(ctx.doublePrecisionType ?? ctx.dataType!) as string, } // Distinguish symbol CAPACITY (CONSUME) from index CAPACITY (CONSUME1) by position @@ -1703,7 +1706,7 @@ class QuestDBVisitor extends BaseVisitor { return { type: "castDefinition", column: colRef.name ?? colRef, - dataType: this.visit(ctx.dataType) as string, + dataType: this.visit(ctx.doublePrecisionType ?? ctx.dataType!) as string, } } @@ -4227,16 +4230,27 @@ class QuestDBVisitor extends BaseVisitor { } } - if (ctx.DoubleColon && ctx.dataType) { + if (ctx.DoubleColon && ctx.castDataType) { for (let i = 0; i < ctx.DoubleColon.length; i++) { inner = { type: "typeCast", expression: inner, - dataType: this.visit(ctx.dataType[i]) as string, + dataType: this.visit(ctx.castDataType[i]) as string, } } } + if (ctx.At) { + inner = { + type: "atTimeZone", + expression: inner, + timeZone: this.typedLiteralString( + ctx.StringLiteral, + ctx.EscapeStringLiteral, + ), + } + } + return inner } @@ -4398,10 +4412,28 @@ class QuestDBVisitor extends BaseVisitor { return { type: "cast", expression: this.visit(ctx.expression) as AST.Expression, - dataType: this.visit(ctx.dataType) as string, + dataType: this.visit(ctx.castDataType) as string, } } + castDataType(ctx: CastDataTypeCstChildren): string { + if (ctx.timestampWithTimeZoneType) { + return this.visit(ctx.timestampWithTimeZoneType) as string + } + if (ctx.doublePrecisionType) { + return this.visit(ctx.doublePrecisionType) as string + } + return this.visit(ctx.dataType!) as string + } + + timestampWithTimeZoneType(ctx: TimestampWithTimeZoneTypeCstChildren): string { + return `${ctx.TimestampNs ? "TIMESTAMP_NS" : "TIMESTAMP"} WITH TIME ZONE` + } + + doublePrecisionType(ctx: DoublePrecisionTypeCstChildren): string { + return "DOUBLE PRECISION" + "[]".repeat(ctx.LBracket?.length ?? 0) + } + dataType(ctx: DataTypeCstChildren): string { // GEOHASH with precision: GEOHASH(8c) if (ctx.Geohash) { diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 9987141..9085f3e 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1228,5 +1228,120 @@ "source": "questdb:core/src/test/java/io/questdb/test/cairo/DdlListenerTest.java:388", "feature": "numeric-literal-typing", "toSql": "CREATE MATERIALIZED VIEW mv3 AS (SELECT ts, avg(x) AS avg_x FROM tab SAMPLE BY 10m) PARTITION BY DAY" + }, + { + "query": "select '2022-03-11T22:00:30.555555Z'::timestamp at time zone 'UTC'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/date/TimestampAtTimeZoneTest.java:180", + "feature": "type-syntax", + "toSql": "SELECT '2022-03-11T22:00:30.555555Z'::TIMESTAMP AT TIME ZONE 'UTC'" + }, + { + "query": "select '2022-03-11T22:00:30.555555555Z'::timestamp_ns at time zone 'UTC'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/date/TimestampAtTimeZoneTest.java:188", + "feature": "type-syntax", + "toSql": "SELECT '2022-03-11T22:00:30.555555555Z'::TIMESTAMP_NS AT TIME ZONE 'UTC'" + }, + { + "query": "select '2022-03-11T22:00:30.555555Z'::timestamp at time zone 'UTC' + 5", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/date/TimestampAtTimeZoneTest.java:34", + "feature": "type-syntax", + "toSql": "SELECT '2022-03-11T22:00:30.555555Z'::TIMESTAMP AT TIME ZONE 'UTC' + 5" + }, + { + "query": "select cast('2022-03-11T22:00:30.555555555Z'::timestamp_ns at time zone 'EST' as string)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/date/TimestampAtTimeZoneTest.java:61", + "feature": "type-syntax", + "toSql": "SELECT CAST('2022-03-11T22:00:30.555555555Z'::TIMESTAMP_NS AT TIME ZONE 'EST' AS STRING)" + }, + { + "query": "select date_trunc('day', '2022-03-11T22:00:30.555555Z'::timestamp at time zone 'UTC')", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/date/TimestampAtTimeZoneTest.java:96", + "feature": "type-syntax", + "toSql": "SELECT date_trunc('day', '2022-03-11T22:00:30.555555Z'::TIMESTAMP AT TIME ZONE 'UTC')" + }, + { + "query": "select case when to_timestamp('2022-03-11T22:00:30.555555Z') at time zone 'EST' > 0 then 'abc' else 'cde'end", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/date/TimestampAtTimeZoneTest.java:115", + "feature": "type-syntax", + "toSql": "SELECT CASE WHEN to_timestamp('2022-03-11T22:00:30.555555Z') AT TIME ZONE 'EST' > 0 THEN 'abc' ELSE 'cde' END" + }, + { + "query": "select '2022-03-11T22:00:30.555555Z'::timestamp time", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/date/TimestampAtTimeZoneTest.java:142", + "feature": "type-syntax", + "toSql": "SELECT '2022-03-11T22:00:30.555555Z'::TIMESTAMP AS time" + }, + { + "query": "select '2022-03-11T22:00:30.555555555Z'::timestamp_ns zone", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/date/TimestampAtTimeZoneTest.java:169", + "feature": "type-syntax", + "toSql": "SELECT '2022-03-11T22:00:30.555555555Z'::TIMESTAMP_NS AS zone" + }, + { + "query": "insert into t values (cast('2021-12-31 15:15:51.663+00:00' as timestamp with time zone))", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/InsertTest.java:1798", + "feature": "type-syntax", + "toSql": "INSERT INTO t VALUES (CAST('2021-12-31 15:15:51.663+00:00' AS TIMESTAMP WITH TIME ZONE))" + }, + { + "query": "insert into t values (cast('2021-12-31 15:15:51.663+00:00'::varchar as timestamp with time zone))", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/InsertTest.java:1822", + "feature": "type-syntax", + "toSql": "INSERT INTO t VALUES (CAST('2021-12-31 15:15:51.663+00:00'::VARCHAR AS TIMESTAMP WITH TIME ZONE))" + }, + { + "query": "select cast('2020-12-31 15:15:51.663+00:00' as timestamp with time zone) time, timestamp from t;", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SimpleTableTest.java:83", + "feature": "type-syntax", + "toSql": "SELECT CAST('2020-12-31 15:15:51.663+00:00' AS TIMESTAMP WITH TIME ZONE) AS time, timestamp FROM t" + }, + { + "query": "select cast(t as timestamp with time zone) from x", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:13672", + "feature": "type-syntax", + "toSql": "SELECT CAST(t AS TIMESTAMP WITH TIME ZONE) FROM x" + }, + { + "query": "select cast('2005-04-02 12:00:00-07' as timestamp with time zone) col from x", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:13681", + "feature": "type-syntax", + "toSql": "SELECT CAST('2005-04-02 12:00:00-07' AS TIMESTAMP WITH TIME ZONE) AS col FROM x" + }, + { + "query": "select '2020-01-01T00:00:00.000000001Z'::timestamp_ns with time zone ts", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/TimestampQueryTest.java:1428", + "feature": "type-syntax", + "toSql": "SELECT '2020-01-01T00:00:00.000000001Z'::TIMESTAMP_NS WITH TIME ZONE AS ts" + }, + { + "query": "select concat('2020-01-01T','01:02:03.123456Z')::timestamp with time zone ts", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/TimestampQueryTest.java:2138", + "feature": "type-syntax", + "toSql": "SELECT concat('2020-01-01T', '01:02:03.123456Z')::TIMESTAMP WITH TIME ZONE AS ts" + }, + { + "query": "insert into tt values ('2020-01-01T00:00:00.000000123Z', '2020-01-01T00:00:00.000000123Z'::timestamp_ns with time zone)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/TimestampQueryTest.java:1448", + "feature": "type-syntax", + "toSql": "INSERT INTO tt VALUES ('2020-01-01T00:00:00.000000123Z', '2020-01-01T00:00:00.000000123Z'::TIMESTAMP_NS WITH TIME ZONE)" + }, + { + "query": "select vch::timestamp with time zone ts from tt", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/TimestampQueryTest.java:2151", + "feature": "type-syntax", + "toSql": "SELECT vch::TIMESTAMP WITH TIME ZONE AS ts FROM tt", + "format": "select vch::timestamp with time zone ts\nfrom tt" + }, + { + "query": "select '{1}'::double precision[] as arr from long_sequence(1)", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGArraysTest.java:1000", + "feature": "type-syntax", + "toSql": "SELECT '{1}'::DOUBLE PRECISION[] AS arr FROM long_sequence(1)" + }, + { + "query": "select '{{1},{2}}'::douBLE pREciSioN[][] as arr from long_sequence(1)", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGArraysTest.java:1014", + "feature": "type-syntax", + "toSql": "SELECT '{{1},{2}}'::DOUBLE PRECISION[][] AS arr FROM long_sequence(1)" } ] diff --git a/tests/type-syntax.test.ts b/tests/type-syntax.test.ts new file mode 100644 index 0000000..0f1c114 --- /dev/null +++ b/tests/type-syntax.test.ts @@ -0,0 +1,274 @@ +/** + * Type syntax the QuestDB server accepts in expressions (reports §3.2): + * - ` AT TIME ZONE ''` ExpressionParser case 'a' (BRANCH_TIMESTAMP_ZONE), + * rewritten to to_timezone(expr, tz) + * - `timestamp[_ns] WITH TIME ZONE` as a cast target (`::` and CAST(.. AS ..)) + * - `DOUBLE PRECISION` ExpressionParser isPrecisionKeyword (casts), + * SqlParser.toColumnType (CREATE TABLE columns, CTAS casts) + * + * Server queries live in tests/fixtures/server-queries.json (feature "type-syntax"). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" +import { format } from "../src/formatter/index" +import type * as AST from "../src/parser/ast" + +const ok = (sql: string): AST.Statement[] => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast +} + +const firstItem = (sql: string): AST.Expression => { + const select = ok(sql)[0] as AST.SelectStatement + return (select.columns[0] as { expression: AST.Expression }).expression +} + +const roundTrip = (sql: string) => toSql(ok(sql)) + +const fails = (sql: string) => { + expect(parseToAst(sql).errors.length).toBeGreaterThan(0) +} + +describe("AT TIME ZONE", () => { + it("builds an atTimeZone node", () => { + expect(firstItem("SELECT ts AT TIME ZONE 'UTC' FROM t")).toEqual({ + type: "atTimeZone", + expression: { + type: "column", + name: { type: "qualifiedName", parts: ["ts"] }, + }, + timeZone: { type: "literal", value: "UTC", literalType: "string" }, + }) + expect(roundTrip("select ts at time zone 'UTC' from t")).toBe( + "SELECT ts AT TIME ZONE 'UTC' FROM t", + ) + }) + + it("applies to the cast operand and binds tighter than arithmetic", () => { + const e = firstItem( + "SELECT x::timestamp AT TIME ZONE 'UTC' + 5", + ) as AST.BinaryExpression + expect(e.type).toBe("binary") + expect(e.operator).toBe("+") + const left = e.left as AST.AtTimeZoneExpression + expect(left.type).toBe("atTimeZone") + expect(left.expression).toMatchObject({ + type: "typeCast", + dataType: "TIMESTAMP", + }) + }) + + it("sits inside unary minus", () => { + const e = firstItem("SELECT -ts AT TIME ZONE 'UTC'") as AST.UnaryExpression + expect(e.type).toBe("unary") + expect(e.operand.type).toBe("atTimeZone") + }) + + it("works in WHERE, function args and with E'' strings", () => { + expect( + roundTrip( + "SELECT date_trunc('day', ts AT TIME ZONE E'Europe/London') FROM t WHERE ts AT TIME ZONE 'EST' > 0", + ), + ).toBe( + "SELECT date_trunc('day', ts AT TIME ZONE E'Europe/London') FROM t WHERE ts AT TIME ZONE 'EST' > 0", + ) + }) + + it("keeps alias names at, time and zone", () => { + expect(roundTrip("SELECT ts at FROM t")).toBe("SELECT ts AS at FROM t") + expect(roundTrip("SELECT at, time, zone FROM t")).toBe( + "SELECT at, time, zone FROM t", + ) + expect(roundTrip("SELECT ts time FROM t")).toBe("SELECT ts AS time FROM t") + expect(roundTrip("CREATE TABLE at (at INT)")).toBe( + "CREATE TABLE at (at INT)", + ) + }) + + it.each([ + "SELECT ts AT 'UTC' FROM t", + "SELECT ts AT TIME 'UTC' FROM t", + "SELECT ts AT TIME ZONE FROM t", + "SELECT ts AT TIME ZONE tz FROM t", + ])("rejects incomplete forms: %s", (sql) => fails(sql)) + + // TimestampAtTimeZoneTest testFail2 / testFailDangling2 / testFailDangling3: + // the server reports "did you mean 'at time zone '?". These used to be + // split silently into `... AS at; time AS UTC`. + it.each([ + "select to_timestamp_ns('2022-03-11T22:00:30.555555555Z') at time 'UTC'", + "select to_timestamp('2022-03-11T22:00:30.555555Z') at time", + "select to_timestamp_ns('2022-03-11T22:00:30.555555555Z') at time", + ])("rejects what the server rejects: %s", (sql) => fails(sql)) +}) + +describe("timestamp WITH TIME ZONE as a cast target", () => { + it("parses CAST(.. AS timestamp with time zone)", () => { + expect( + firstItem("SELECT CAST(x AS timestamp with time zone) FROM t"), + ).toEqual({ + type: "cast", + expression: { + type: "column", + name: { type: "qualifiedName", parts: ["x"] }, + }, + dataType: "TIMESTAMP WITH TIME ZONE", + }) + expect( + roundTrip("SELECT CAST(x AS timestamp_ns with time zone) FROM t"), + ).toBe("SELECT CAST(x AS TIMESTAMP_NS WITH TIME ZONE) FROM t") + }) + + it("parses x::timestamp with time zone, with an alias", () => { + expect(firstItem("SELECT x::timestamp with time zone ts FROM t")).toEqual({ + type: "typeCast", + expression: { + type: "column", + name: { type: "qualifiedName", parts: ["x"] }, + }, + dataType: "TIMESTAMP WITH TIME ZONE", + }) + expect(roundTrip("SELECT x::timestamp with time zone ts FROM t")).toBe( + "SELECT x::TIMESTAMP WITH TIME ZONE AS ts FROM t", + ) + }) + + it("keeps the typed literal form", () => { + expect( + roundTrip("SELECT timestamp with time zone '2020-01-01' FROM t"), + ).toBe("SELECT TIMESTAMP WITH TIME ZONE '2020-01-01' FROM t") + }) + + it.each([ + "SELECT CAST(x AS int with time zone) FROM t", + "SELECT x::date with time zone FROM t", + "SELECT CAST(x AS timestamp with time) FROM t", + "CREATE TABLE t (ts TIMESTAMP WITH TIME ZONE)", + "ALTER TABLE t ADD COLUMN ts TIMESTAMP WITH TIME ZONE", + ])("rejects it elsewhere: %s", (sql) => fails(sql)) +}) + +describe("DOUBLE PRECISION", () => { + it("is a cast target", () => { + expect(firstItem("SELECT CAST(x AS DOUBLE PRECISION) FROM t")).toEqual({ + type: "cast", + expression: { + type: "column", + name: { type: "qualifiedName", parts: ["x"] }, + }, + dataType: "DOUBLE PRECISION", + }) + expect(roundTrip("SELECT x::double precision[] FROM t")).toBe( + "SELECT x::DOUBLE PRECISION[] FROM t", + ) + }) + + it("is a column type in CREATE TABLE", () => { + const stmt = ok( + "CREATE TABLE t (x DOUBLE PRECISION, y double precision[], ts TIMESTAMP) TIMESTAMP(ts)", + )[0] as AST.CreateTableStatement + expect(stmt.columns!.map((c) => c.dataType)).toEqual([ + "DOUBLE PRECISION", + "DOUBLE PRECISION[]", + "TIMESTAMP", + ]) + expect(toSql(stmt)).toBe( + "CREATE TABLE t (x DOUBLE PRECISION, y DOUBLE PRECISION[], ts TIMESTAMP) TIMESTAMP(ts)", + ) + }) + + it("is a CTAS cast target", () => { + expect( + roundTrip( + "CREATE TABLE t AS (SELECT * FROM s), CAST(x AS DOUBLE PRECISION)", + ), + ).toBe("CREATE TABLE t AS (SELECT * FROM s), CAST(x AS DOUBLE PRECISION)") + }) + + it("keeps precision usable as a name", () => { + expect(roundTrip("SELECT x precision FROM t")).toBe( + "SELECT x AS precision FROM t", + ) + expect(roundTrip("SELECT x::int precision FROM t")).toBe( + "SELECT x::INT AS precision FROM t", + ) + expect(roundTrip("CREATE TABLE precision (precision DOUBLE)")).toBe( + "CREATE TABLE precision (precision DOUBLE)", + ) + }) + + it("rejects it after other types", () => { + fails("SELECT CAST(x AS INT PRECISION) FROM t") + }) + + it("is not an ALTER TABLE ADD COLUMN type (addColumnWithType)", () => { + // like any trailing word there, PRECISION is not part of the ALTER + const result = parseToAst("ALTER TABLE t ADD COLUMN x DOUBLE PRECISION") + expect(result.ast.length).not.toBe(1) + const alter = result.ast[0] as AST.AlterTableStatement + expect(toSql(alter)).toBe("ALTER TABLE t ADD COLUMN x DOUBLE") + }) +}) + +describe("formatter", () => { + it("keeps WITH TIME ZONE after timestamp on the same line", () => { + expect( + format( + "select x::timestamp with time zone a, timestamp with time zone '2020-01-01' b from t", + ), + ).toBe( + "select\n x::timestamp with time zone a,\n timestamp with time zone '2020-01-01' b\nfrom t", + ) + expect(format("select ts at time zone 'UTC' from t")).toBe( + "select ts at time zone 'UTC'\nfrom t", + ) + }) + + it("still breaks before a WITH clause", () => { + expect(format("with a as (select 1) select * from a")).toBe( + "with a as (\n select 1\n)\nselect *\nfrom a", + ) + }) +}) + +describe("autocomplete", () => { + it.each([ + ["SELECT x::timestamp ", "With"], + ["SELECT x::timestamp_ns ", "With"], + ["SELECT CAST(x AS TIMESTAMP ", "With"], + ["SELECT CAST(x AS TIMESTAMP WITH ", "Time"], + ["SELECT CAST(x AS TIMESTAMP WITH TIME ", "Zone"], + ["SELECT CAST(x AS DOUBLE ", "Precision"], + ["SELECT x::double ", "Precision"], + ["CREATE TABLE t (x DOUBLE ", "Precision"], + ["SELECT ts ", "At"], + ["SELECT * FROM t WHERE ts ", "At"], + ["SELECT ts AT ", "Time"], + ["SELECT ts AT TIME ", "Zone"], + ])("after `%s` offers %s", (sql, token) => { + expect(getNextValidTokens(sql)).toContain(token) + }) + + it.each([ + ["SELECT x::int ", "With"], + ["SELECT CAST(x AS INT ", "With"], + ["SELECT CAST(x AS INT ", "Precision"], + ["CREATE TABLE t (ts TIMESTAMP ", "With"], + ["CREATE TABLE t (x INT ", "Precision"], + ["ALTER TABLE t ADD COLUMN x DOUBLE ", "Precision"], + ["SELECT * FROM t WHERE ts > x ", "With"], + ["SELECT ", "At"], + ["SELECT ", "Precision"], + ["SELECT * FROM t WHERE ", "At"], + ])("after `%s` does not offer %s", (sql, token) => { + expect(getNextValidTokens(sql)).not.toContain(token) + }) + + it("offers ZONE only after AT TIME", () => { + expect(getNextValidTokens("SELECT ts AT TIME ")).toEqual(["Zone"]) + }) +}) From 9b08b99df453296cb4b8855fb646386ad6814462 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 15:56:26 +0300 Subject: [PATCH 14/36] feat: parse IS [NOT] TRUE/FALSE, <> ALL (...) and NOT WITHIN The server accepts three predicate forms that sql-parser rejected: - `a IS [NOT] TRUE|FALSE`: ExpressionParser.validateIsKeyword accepts IS [NOT] NULL|TRUE|FALSE and rewrites it to `=` / `!=` (ExpressionParserTest.testIsTrue/testIsNotTrue/testIsNotFalse). - `a <> ALL (...)` / `a != ALL (...)`: ExpressionParser isAllKeyword turns the preceding `<>`/`!=` into the `<>all` function (allFunctions); any other operator is "unexpected operator", no operator is "missing operator" (ExpressionParserTest.testAllNotEqual/testAllInvalidOperator/ testAllNoOperator, AllNotEqStrFunctionFactoryTest). - `g NOT WITHIN (...)`: NOT before a SET operator becomes OperatorExpression.UnarySetNegation (in / between / within); ExpressionParserFuzzTest generates `x not within y`. Before: all three were parse errors ("Unexpected token"). Changes: - relationalExpression: `Is Not? (Null | True | False)`. New AST node `isBoolean { expression, value, not }` (isNull is unchanged); toSql `a IS [NOT] TRUE|FALSE`. - equalityExpression: optional `All` after NotEquals. AST stays a binary node with operator "<> ALL" / "!= ALL" (as typed; plain <>/!= still print `!=`); the right operand is the usual relationalExpression, so `all(x)` is a paren node and toSql gives `a <> ALL (x)`. Parens are not required (the server's ALL branch does not require them either). toSql precedence table knows the two operators (13, same as <>). - setExpression: `Not?` before Within. WithinExpression gets an optional `not` (set only when true, so plain WITHIN keeps its AST shape); toSql `g NOT WITHIN (...)`. - cst-types regenerated. Tests: 6 server queries (AllNotEqStrFunctionFactoryTest) in server-queries.json; tests/predicates.test.ts covers the expression-only server tests wrapped in WHERE, AST shapes, negatives (IS 3, IS NOT $1, `|| all`, `= all`, `> all`, bare `all`), formatter and suggestions. Red before: 45 failing (18 fixture + 27 focused) -> all pass; full suite 16708 passed. Corpus (54k): fixed 6 (the AllNotEqStrFunctionFactoryTest queries) / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0. Suggestions: +TRUE/FALSE after `IS` and `IS NOT`, +ALL after `<>`/`!=`, +WITHIN after an operand's `NOT`; nothing else changed. --- src/parser/ast.ts | 11 ++ src/parser/cst-types.d.ts | 3 + src/parser/parser.ts | 20 ++- src/parser/toSql.ts | 6 +- src/parser/visitor.ts | 20 ++- tests/fixtures/server-queries.json | 37 +++++ tests/predicates.test.ts | 251 +++++++++++++++++++++++++++++ 7 files changed, 340 insertions(+), 8 deletions(-) create mode 100644 tests/predicates.test.ts diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 03fef06..a5a70f3 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -1264,6 +1264,7 @@ export type Expression = | BetweenExpression | WithinExpression | IsNullExpression + | IsBooleanExpression | ParenExpression | ArrayLiteral | ArrayAccessExpression @@ -1400,6 +1401,8 @@ export interface WithinExpression extends AstNode { type: "within" expression: Expression values: Expression[] + /** `NOT WITHIN` (set only when true) */ + not?: boolean } export interface IsNullExpression extends AstNode { @@ -1408,6 +1411,14 @@ export interface IsNullExpression extends AstNode { not?: boolean } +/** ` IS [NOT] TRUE|FALSE` (the server rewrites it to `= / != true|false`) */ +export interface IsBooleanExpression extends AstNode { + type: "isBoolean" + expression: Expression + value: boolean + not?: boolean +} + export interface VariableRef extends AstNode { type: "variable" name: string diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 6419381..f875ccb 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -2504,6 +2504,7 @@ export type EqualityExpressionCstChildren = { relationalExpression: (RelationalExpressionCstNode)[]; Equals?: IToken[]; NotEquals?: IToken[]; + All?: IToken[]; Like?: IToken[]; Ilike?: IToken[]; RegexMatch?: IToken[]; @@ -2525,6 +2526,8 @@ export type RelationalExpressionCstChildren = { Is?: IToken[]; Not?: IToken[]; Null?: IToken[]; + True?: IToken[]; + False?: IToken[]; }; export interface SetExpressionCstNode extends CstNode { diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 43825a1..4f36db0 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -4691,7 +4691,14 @@ class QuestDBParser extends CstParser { this.OPTION(() => { this.OR([ { ALT: () => this.CONSUME(Equals) }, - { ALT: () => this.CONSUME(NotEquals) }, + { + // `<> ALL (...)` / `!= ALL (...)`: the server rewrites the operator + // to the <>all function; no other operator takes ALL. + ALT: () => { + this.CONSUME(NotEquals) + this.OPTION1(() => this.CONSUME(All)) + }, + }, { ALT: () => this.CONSUME(Like) }, { ALT: () => this.CONSUME(Ilike) }, { ALT: () => this.CONSUME(RegexMatch) }, @@ -4702,7 +4709,7 @@ class QuestDBParser extends CstParser { }) }) - // Precedence 12: <, <=, >, >=, IS [NOT] NULL + // Precedence 12: <, <=, >, >=, IS [NOT] NULL|TRUE|FALSE private relationalExpression = this.RULE("relationalExpression", () => { this.SUBRULE(this.setExpression) this.OPTION(() => { @@ -4722,14 +4729,18 @@ class QuestDBParser extends CstParser { ALT: () => { this.CONSUME(Is) this.OPTION1(() => this.CONSUME(Not)) - this.CONSUME(Null) + this.OR2([ + { ALT: () => this.CONSUME(Null) }, + { ALT: () => this.CONSUME(True) }, + { ALT: () => this.CONSUME(False) }, + ]) }, }, ]) }) }) - // Precedence 11: [NOT] IN, [NOT] BETWEEN, [NOT] LIKE, [NOT] ILIKE, WITHIN + // Precedence 11: [NOT] IN, [NOT] BETWEEN, [NOT] LIKE, [NOT] ILIKE, [NOT] WITHIN private setExpression = this.RULE("setExpression", () => { this.SUBRULE(this.bitOrExpression) this.OPTION({ @@ -4779,6 +4790,7 @@ class QuestDBParser extends CstParser { }, { ALT: () => { + this.OPTION3(() => this.CONSUME3(Not)) this.CONSUME(Within) this.CONSUME1(LParen) this.SUBRULE5(this.expression) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 5e49cdb..ed65603 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -1870,6 +1870,8 @@ function expressionToSql(expr: AST.Expression): string { return withinExprToSql(expr) case "isNull": return isNullExprToSql(expr) + case "isBoolean": + return `${expressionToSql(expr.expression)} IS ${expr.not ? "NOT " : ""}${expr.value ? "TRUE" : "FALSE"}` case "paren": { const inner = [expressionToSql(expr.expression)] if (expr.additionalExpressions) { @@ -1914,6 +1916,8 @@ const OPERATOR_PRECEDENCE: Record = { "~": 13, "!=": 13, "<>": 13, + "<> ALL": 13, + "!= ALL": 13, "!~": 13, LIKE: 13, ILIKE: 13, @@ -2158,7 +2162,7 @@ function betweenExprToSql(expr: AST.BetweenExpression): string { function withinExprToSql(expr: AST.WithinExpression): string { const left = expressionToSql(expr.expression) const values = expr.values.map(expressionToSql).join(", ") - return `${left} WITHIN (${values})` + return `${left}${expr.not ? " NOT" : ""} WITHIN (${values})` } function isNullExprToSql(expr: AST.IsNullExpression): string { diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index a1d45ff..75f1b18 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -3912,8 +3912,10 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.relationalExpression && ctx.relationalExpression.length > 1) { let operator = "" if (ctx.Equals) operator = "=" - else if (ctx.NotEquals) operator = "!=" - else if (ctx.Like) operator = "LIKE" + else if (ctx.NotEquals) { + // `<> ALL` / `!= ALL` keeps the operator as written + operator = ctx.All ? `${ctx.NotEquals[0].image} ALL` : "!=" + } else if (ctx.Like) operator = "LIKE" else if (ctx.Ilike) operator = "ILIKE" else if (ctx.RegexMatch) operator = "~" else if (ctx.RegexNotMatch) operator = "!~" @@ -3933,6 +3935,16 @@ class QuestDBVisitor extends BaseVisitor { relationalExpression(ctx: RelationalExpressionCstChildren): AST.Expression { const left = this.visit(ctx.setExpression[0]) as AST.Expression + // Check for IS [NOT] TRUE / FALSE + if (ctx.Is && (ctx.True || ctx.False)) { + return { + type: "isBoolean", + expression: left, + value: !!ctx.True, + not: !!ctx.Not, + } + } + // Check for IS [NOT] NULL if (ctx.Is) { return { @@ -3998,13 +4010,15 @@ class QuestDBVisitor extends BaseVisitor { // Check for WITHIN if (ctx.Within) { - return { + const result: AST.WithinExpression = { type: "within", expression: left, values: ctx.expression!.map( (e: CstNode) => this.visit(e) as AST.Expression, ), } + if (ctx.Not) result.not = true + return result } // Check for NOT LIKE / NOT ILIKE diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 9085f3e..7bba665 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1343,5 +1343,42 @@ "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGArraysTest.java:1014", "feature": "type-syntax", "toSql": "SELECT '{{1},{2}}'::DOUBLE PRECISION[][] AS arr FROM long_sequence(1)" + }, + { + "query": "select * from long_sequence(1) where 'aaa' <> all('{abc,xyz}'::text[])", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/bool/AllNotEqStrFunctionFactoryTest.java:34", + "feature": "predicates", + "toSql": "SELECT * FROM long_sequence(1) WHERE 'aaa' <> ALL ('{abc,xyz}'::TEXT[])" + }, + { + "query": "select * from tab where a <> all('{}'::text[])", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/bool/AllNotEqStrFunctionFactoryTest.java:45", + "feature": "predicates", + "toSql": "SELECT * FROM tab WHERE a <> ALL ('{}'::TEXT[])" + }, + { + "query": "select * from tab where a <> all('{abc,xyz}'::text[])", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/bool/AllNotEqStrFunctionFactoryTest.java:60", + "feature": "predicates", + "toSql": "SELECT * FROM tab WHERE a <> ALL ('{abc,xyz}'::TEXT[])" + }, + { + "query": "select * from tab where a <> all('{добрий,вечір}'::text[])", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/bool/AllNotEqStrFunctionFactoryTest.java:74", + "feature": "predicates", + "toSql": "SELECT * FROM tab WHERE a <> ALL ('{добрий,вечір}'::TEXT[])", + "format": "select *\nfrom tab\nwhere a <> all('{добрий,вечір}'::text[])" + }, + { + "query": "select * from tab where a <> all('{aaa,bbb,ccc}'::text[])", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/bool/AllNotEqStrFunctionFactoryTest.java:86", + "feature": "predicates", + "toSql": "SELECT * FROM tab WHERE a <> ALL ('{aaa,bbb,ccc}'::TEXT[])" + }, + { + "query": "select * from long_sequence(1) where null <> all('{abc,xyz}'::text[])", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/bool/AllNotEqStrFunctionFactoryTest.java:100", + "feature": "predicates", + "toSql": "SELECT * FROM long_sequence(1) WHERE NULL <> ALL ('{abc,xyz}'::TEXT[])" } ] diff --git a/tests/predicates.test.ts b/tests/predicates.test.ts new file mode 100644 index 0000000..c47bc02 --- /dev/null +++ b/tests/predicates.test.ts @@ -0,0 +1,251 @@ +/** + * Predicates the QuestDB server accepts (reports §3.2): + * - `a IS [NOT] TRUE|FALSE` ExpressionParser.validateIsKeyword (IS [NOT] NULL|TRUE|FALSE, + * rewritten to `=` / `!=`) + * - `a <> ALL (...)`, + * `a != ALL (...)` ExpressionParser isAllKeyword + allFunctions (`<>`/`!=` -> `<>all`); + * any other operator before ALL is "unexpected operator" + * - `a NOT WITHIN (...)` OperatorExpression.UnarySetNegation (NOT before a SET operator: + * in / between / within) + * + * Statement-level server queries live in tests/fixtures/server-queries.json + * (feature "predicates"). The server tests for IS TRUE/FALSE, `!= all` and + * NOT WITHIN are expression-only (ExpressionParserTest, ExpressionParserFuzzTest), + * so they are wrapped in a WHERE clause here. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" +import { format } from "../src/formatter/index" +import type * as AST from "../src/parser/ast" + +const ok = (sql: string): AST.Statement[] => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast +} + +const where = (cond: string): AST.Expression => { + const select = ok(`SELECT * FROM tab WHERE ${cond}`)[0] as AST.SelectStatement + return select.where! +} + +const roundTrip = (sql: string) => toSql(ok(sql)) + +const fails = (sql: string) => { + expect(parseToAst(sql).errors.length).toBeGreaterThan(0) +} + +const col = (...parts: string[]) => ({ + type: "column", + name: { type: "qualifiedName", parts }, +}) + +describe("IS [NOT] TRUE / FALSE", () => { + it("builds an isBoolean node", () => { + expect(where("a IS TRUE")).toEqual({ + type: "isBoolean", + expression: col("a"), + value: true, + not: false, + }) + expect(where("a IS NOT FALSE")).toEqual({ + type: "isBoolean", + expression: col("a"), + value: false, + not: true, + }) + }) + + // ExpressionParserTest.testIsTrue / testIsNotTrue / testIsNotFalse + it.each([ + ["a IS True", "a IS TRUE"], + ["tab.a IS True", "tab.a IS TRUE"], + ["'true' IS true", "'true' IS TRUE"], + ["a IS not True", "a IS NOT TRUE"], + ["tab.a IS NOT True", "tab.a IS NOT TRUE"], + ["'true' IS not true", "'true' IS NOT TRUE"], + ["a IS not False", "a IS NOT FALSE"], + ["tab.a IS NOT False", "tab.a IS NOT FALSE"], + ["'False' IS not False", "'False' IS NOT FALSE"], + ["a is false", "a IS FALSE"], + ["(a > 1) IS NOT TRUE", "(a > 1) IS NOT TRUE"], + ])("WHERE %s", (cond, expected) => { + expect(roundTrip(`SELECT * FROM tab WHERE ${cond}`)).toBe( + `SELECT * FROM tab WHERE ${expected}`, + ) + }) + + it("combines with AND / OR and works in the select list", () => { + expect( + roundTrip( + "select a is true, b is not false from t where a is not true and b is false", + ), + ).toBe( + "SELECT a IS TRUE, b IS NOT FALSE FROM t WHERE a IS NOT TRUE AND b IS FALSE", + ) + }) + + it("keeps IS [NOT] NULL unchanged", () => { + expect(where("a IS NOT NULL")).toEqual({ + type: "isNull", + expression: col("a"), + not: true, + }) + expect(roundTrip("SELECT * FROM t WHERE a IS NULL")).toBe( + "SELECT * FROM t WHERE a IS NULL", + ) + }) + + // ExpressionParserTest.testIsNull / testIsNotNull assertFail cases + it.each([ + "a IS 3", + "a IS NOT 3", + "a IS $1", + "a IS NOT $1", + "a IS", + "a IS NOT", + "a IS NOT NOT TRUE", + ])("rejects WHERE %s", (cond) => { + fails(`SELECT * FROM tab WHERE ${cond}`) + }) + + it("formats", () => { + expect(format("select * from t where a is not false")).toBe( + "select *\nfrom t\nwhere a is not false", + ) + }) + + it("suggests TRUE / FALSE after IS and IS NOT", () => { + const afterIs = getNextValidTokens("SELECT * FROM t WHERE a IS ") + expect(afterIs).toEqual( + expect.arrayContaining(["Not", "Null", "True", "False"]), + ) + const afterIsNot = getNextValidTokens("SELECT * FROM t WHERE a IS NOT ") + expect(afterIsNot).toEqual( + expect.arrayContaining(["Null", "True", "False"]), + ) + expect(afterIsNot).not.toContain("Not") + }) +}) + +describe("<> ALL / != ALL", () => { + it("builds a binary node with the quantified operator", () => { + expect(where("a <> all(b)")).toEqual({ + type: "binary", + operator: "<> ALL", + left: col("a"), + right: { type: "paren", expression: col("b") }, + }) + expect(where("a != ALL (b)")).toMatchObject({ + type: "binary", + operator: "!= ALL", + }) + }) + + // ExpressionParserTest.testAllNotEqual + it.each([ + ["a <> all(b)", "a <> ALL (b)"], + ["a != all(b)", "a != ALL (b)"], + ["a<>ALL('{x}'::text[])", "a <> ALL ('{x}'::TEXT[])"], + ["a <> all(b) and c = 1", "a <> ALL (b) AND c = 1"], + ["not a <> all(b)", "NOT a <> ALL (b)"], + ])("WHERE %s", (cond, expected) => { + expect(roundTrip(`SELECT * FROM tab WHERE ${cond}`)).toBe( + `SELECT * FROM tab WHERE ${expected}`, + ) + }) + + it("keeps plain <> / != unchanged", () => { + expect(where("a <> b")).toEqual({ + type: "binary", + operator: "!=", + left: col("a"), + right: col("b"), + }) + }) + + // ExpressionParserTest.testAllInvalidOperator / testAllNoOperator: + // only <> and != take ALL + it.each(["a || all(b)", "a all(b)", "a = all(b)", "a > all(b)", "a <> all"])( + "rejects WHERE %s", + (cond) => { + fails(`SELECT * FROM tab WHERE ${cond}`) + }, + ) + + it("formats", () => { + expect(format("select * from t where a <> all(b)")).toBe( + "select *\nfrom t\nwhere a <> all(b)", + ) + }) + + it("suggests ALL only after <> / !=", () => { + expect(getNextValidTokens("SELECT * FROM t WHERE a <> ")).toContain("All") + expect(getNextValidTokens("SELECT * FROM t WHERE a != ")).toContain("All") + expect(getNextValidTokens("SELECT * FROM t WHERE a = ")).not.toContain( + "All", + ) + expect(getNextValidTokens("SELECT * FROM t WHERE a < ")).not.toContain( + "All", + ) + }) +}) + +describe("NOT WITHIN", () => { + it("sets not on the within node", () => { + expect(where("g NOT WITHIN (#u33)")).toEqual({ + type: "within", + expression: col("g"), + values: [{ type: "literal", value: "#u33", literalType: "geohash" }], + not: true, + }) + }) + + it("keeps plain WITHIN without a not field", () => { + expect(where("g WITHIN (#u33)")).not.toHaveProperty("not") + }) + + it.each([ + ["g NOT WITHIN (#u33)", "g NOT WITHIN (#u33)"], + ["geo not within(#gk1gj8, #mbx5c0)", "geo NOT WITHIN (#gk1gj8, #mbx5c0)"], + ["NOT g WITHIN (#u33)", "NOT g WITHIN (#u33)"], + [ + "g not within (#u33) and x not in (1, 2)", + "g NOT WITHIN (#u33) AND x NOT IN (1, 2)", + ], + ])("WHERE %s", (cond, expected) => { + expect(roundTrip(`SELECT * FROM tab WHERE ${cond}`)).toBe( + `SELECT * FROM tab WHERE ${expected}`, + ) + }) + + it("builds NOT WITHIN in a LATEST ON query", () => { + expect( + roundTrip( + "select * from x where geo not within(#gk1gj8, #mbx5c0) latest on ts partition by sym", + ), + ).toBe( + "SELECT * FROM x WHERE geo NOT WITHIN (#gk1gj8, #mbx5c0) LATEST ON ts PARTITION BY sym", + ) + }) + + it("rejects NOT WITHIN without a value list", () => { + fails("SELECT * FROM t WHERE g NOT WITHIN") + }) + + it("formats", () => { + expect(format("select * from t where g not within (#u33)")).toBe( + "select *\nfrom t\nwhere g not within (#u33)", + ) + }) + + it("suggests WITHIN after a NOT that follows an operand", () => { + const afterNot = getNextValidTokens("SELECT * FROM t WHERE g NOT ") + expect(afterNot).toEqual( + expect.arrayContaining(["In", "Between", "Within", "Like", "Ilike"]), + ) + }) +}) From 0065ffdb288e95b8c589bd6cfe4ad6f476aa44c5 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:01:55 +0300 Subject: [PATCH 15/36] feat: parse GROUPS window frames and EXCLUDE GROUP / EXCLUDE TIES The server accepts ROWS | RANGE | GROUPS frame modes and EXCLUDE CURRENT ROW | GROUP | TIES | NO OTHERS (ExpressionParser.parseWindowSpec / parseWindowFrameClause). sql-parser rejected GROUPS, EXCLUDE GROUP and EXCLUDE TIES. Adds the groups / ties keywords (non-reserved) and the grammar options, with server queries in the parity fixture and tests/window-frames.test.ts. --- src/grammar/keywords.ts | 1 + src/parser/ast.ts | 4 +- src/parser/cst-types.d.ts | 3 + src/parser/lexer.ts | 2 + src/parser/parser.ts | 7 +- src/parser/toSql.ts | 6 + src/parser/tokens.ts | 2 + src/parser/visitor.ts | 5 + tests/fixtures/server-queries.json | 72 +++++++++++ tests/window-frames.test.ts | 187 +++++++++++++++++++++++++++++ 10 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 tests/window-frames.test.ts diff --git a/src/grammar/keywords.ts b/src/grammar/keywords.ts index 417879f..e8cdc7b 100644 --- a/src/grammar/keywords.ts +++ b/src/grammar/keywords.ts @@ -207,6 +207,7 @@ export const keywords: string[] = [ "table", "tables", "then", + "ties", "time", "timeout", "to", diff --git a/src/parser/ast.ts b/src/parser/ast.ts index a5a70f3..3ba161c 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -1335,10 +1335,10 @@ export interface WindowSpecification extends AstNode { export interface WindowFrame extends AstNode { type: "windowFrame" - mode: "rows" | "range" | "cumulative" + mode: "rows" | "range" | "groups" | "cumulative" start?: WindowFrameBound end?: WindowFrameBound - exclude?: "currentRow" | "noOthers" + exclude?: "currentRow" | "group" | "ties" | "noOthers" } export interface WindowFrameBound extends AstNode { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index f875ccb..c65a1e7 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -2970,6 +2970,7 @@ export interface WindowFrameClauseCstNode extends CstNode { export type WindowFrameClauseCstChildren = { Rows?: IToken[]; Range?: IToken[]; + Groups?: IToken[]; Cumulative?: IToken[]; Between?: IToken[]; windowFrameBound?: (WindowFrameBoundCstNode)[]; @@ -2977,6 +2978,8 @@ export type WindowFrameClauseCstChildren = { Exclude?: IToken[]; Current?: IToken[]; Row?: IToken[]; + Group?: IToken[]; + Ties?: IToken[]; No?: IToken[]; Others?: IToken[]; }; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index abffa9a..bcaa9ff 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -251,6 +251,7 @@ import { Table, Tables, Then, + Ties, Time, Timestamp, TimestampNs, @@ -592,6 +593,7 @@ export { Table, Tables, Then, + Ties, Time, Timestamp, TimestampNs, diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 4f36db0..da635ac 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -356,6 +356,7 @@ import { Exclude, Cumulative, Others, + Ties, Database, Backup, Prevailing, @@ -5468,6 +5469,7 @@ class QuestDBParser extends CstParser { this.OR([ { ALT: () => this.CONSUME(Rows) }, { ALT: () => this.CONSUME(Range) }, + { ALT: () => this.CONSUME(Groups) }, { ALT: () => this.CONSUME(Cumulative) }, ]) this.OPTION(() => { @@ -5485,7 +5487,8 @@ class QuestDBParser extends CstParser { }, ]) }) - // Optional EXCLUDE clause (QuestDB supports EXCLUDE CURRENT ROW and EXCLUDE NO OTHERS) + // Optional EXCLUDE clause: CURRENT ROW | GROUP | TIES | NO OTHERS + // (ExpressionParser.parseWindowFrameClause) this.OPTION1(() => { this.CONSUME(Exclude) this.OR2([ @@ -5495,6 +5498,8 @@ class QuestDBParser extends CstParser { this.CONSUME(Row) }, }, + { ALT: () => this.CONSUME(Group) }, + { ALT: () => this.CONSUME(Ties) }, { ALT: () => { this.CONSUME(No) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index ed65603..23dc396 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -2085,6 +2085,12 @@ function windowFrameToSql(frame: AST.WindowFrame): string { case "currentRow": sql += " EXCLUDE CURRENT ROW" break + case "group": + sql += " EXCLUDE GROUP" + break + case "ties": + sql += " EXCLUDE TIES" + break case "noOthers": sql += " EXCLUDE NO OTHERS" break diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index 742cd39..db4b6e0 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -366,6 +366,7 @@ export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ "Unbounded", "Exclude", "Others", + "Ties", "Nulls", // SAMPLE BY keywords "Fill", @@ -679,6 +680,7 @@ export const System = getToken("System") export const Table = getToken("Table") export const Tables = getToken("Tables") export const Then = getToken("Then") +export const Ties = getToken("Ties") export const Time = getToken("Time") export const To = getToken("To") export const Token = getToken("Token") diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 75f1b18..89e2601 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -4735,6 +4735,7 @@ class QuestDBVisitor extends BaseVisitor { let mode: string if (ctx.Rows) mode = "rows" else if (ctx.Range) mode = "range" + else if (ctx.Groups) mode = "groups" else mode = "cumulative" const result: AST.WindowFrame = { @@ -4752,6 +4753,10 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.Exclude) { if (ctx.Current && ctx.Row) { result.exclude = "currentRow" + } else if (ctx.Group) { + result.exclude = "group" + } else if (ctx.Ties) { + result.exclude = "ties" } else if (ctx.No && ctx.Others) { result.exclude = "noOthers" } diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 7bba665..d1b427b 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1380,5 +1380,77 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/bool/AllNotEqStrFunctionFactoryTest.java:100", "feature": "predicates", "toSql": "SELECT * FROM long_sequence(1) WHERE NULL <> ALL ('{abc,xyz}'::TEXT[])" + }, + { + "query": "select a,b, f(c) over (partition by b order by ts GROUPS unbounded preceding) from xyz", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:710", + "feature": "window-frames", + "toSql": "SELECT a, b, f(c) OVER (PARTITION BY b ORDER BY ts GROUPS UNBOUNDED PRECEDING) FROM xyz" + }, + { + "query": "select a,b, f(c) over (partition by b order by ts GROUPS between unbounded preceding and current row) from xyz", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:755", + "feature": "window-frames", + "toSql": "SELECT a, b, f(c) OVER (PARTITION BY b ORDER BY ts GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM xyz" + }, + { + "query": "select a,b, f(c) over (partition by b order by ts groups between 1+0 preceding and unbounded following exclude group) from xyz", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:425", + "feature": "window-frames", + "toSql": "SELECT a, b, f(c) OVER (PARTITION BY b ORDER BY ts GROUPS BETWEEN 1 + 0 PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE GROUP) FROM xyz" + }, + { + "query": "select a,b, f(c) over (partition by b order by ts rows between 1/1 preceding and unbounded following exclude ties) from xyz", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:480", + "feature": "window-frames", + "toSql": "SELECT a, b, f(c) OVER (PARTITION BY b ORDER BY ts ROWS BETWEEN 1 / 1 PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE TIES) FROM xyz" + }, + { + "query": "select corr(y, x) over (order by ts groups between 1 preceding and current row) from tab", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/window/WindowFunctionTest.java:1043", + "feature": "window-frames", + "toSql": "SELECT corr(y, x) OVER (ORDER BY ts GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM tab" + }, + { + "query": "select cume_dist() over (order by ts groups between 1 preceding and current row) from tab", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/window/WindowFunctionTest.java:2458", + "feature": "window-frames", + "toSql": "SELECT cume_dist() OVER (ORDER BY ts GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM tab" + }, + { + "query": "select ntile(4) over (order by ts groups between 1 preceding and current row) from tab", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/window/WindowFunctionTest.java:17002", + "feature": "window-frames", + "toSql": "SELECT ntile(4) OVER (ORDER BY ts GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM tab" + }, + { + "query": "select cume_dist() over (order by ts range between unbounded preceding and current row exclude group) from tab", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/window/WindowFunctionTest.java:2432", + "feature": "window-frames", + "toSql": "SELECT cume_dist() OVER (ORDER BY ts RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE GROUP) FROM tab" + }, + { + "query": "select cume_dist() over (order by ts range between unbounded preceding and current row exclude ties) from tab", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/window/WindowFunctionTest.java:2436", + "feature": "window-frames", + "toSql": "SELECT cume_dist() OVER (ORDER BY ts RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE TIES) FROM tab" + }, + { + "query": "select nth_value(val, 1) over (order by ts rows between 3 preceding and current row exclude group) from tab", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/window/WindowFunctionTest.java:13936", + "feature": "window-frames", + "toSql": "SELECT nth_value(val, 1) OVER (ORDER BY ts ROWS BETWEEN 3 PRECEDING AND CURRENT ROW EXCLUDE GROUP) FROM tab" + }, + { + "query": "select ts, sym, avg(x) over (partition by sym order by ts range between 2 preceding and current row exclude group) a from base", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/lv/LiveViewCheckpointFunctionCompilerTest.java:706", + "feature": "window-frames", + "toSql": "SELECT ts, sym, avg(x) OVER (PARTITION BY sym ORDER BY ts RANGE BETWEEN 2 PRECEDING AND CURRENT ROW EXCLUDE GROUP) AS a FROM base" + }, + { + "query": "select ts, sym, sum(x) over (partition by sym order by ts rows between 3 preceding and current row exclude ties) s from base", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/lv/LiveViewCheckpointFunctionCompilerTest.java:712", + "feature": "window-frames", + "toSql": "SELECT ts, sym, sum(x) OVER (PARTITION BY sym ORDER BY ts ROWS BETWEEN 3 PRECEDING AND CURRENT ROW EXCLUDE TIES) AS s FROM base" } ] diff --git a/tests/window-frames.test.ts b/tests/window-frames.test.ts new file mode 100644 index 0000000..e5967a3 --- /dev/null +++ b/tests/window-frames.test.ts @@ -0,0 +1,187 @@ +/** + * Window frame GROUPS mode and EXCLUDE GROUP / EXCLUDE TIES (reports §3.2). + * + * Server: ExpressionParser.parseWindowSpec accepts the frame modes + * ROWS | RANGE | GROUPS | CUMULATIVE, and parseWindowFrameClause accepts + * EXCLUDE CURRENT ROW | GROUP | TIES | NO OTHERS (#6626). The same spec parser + * is used for inline OVER (...) and named WINDOW w AS (...). + * + * Statement-level server queries live in tests/fixtures/server-queries.json + * (feature "window-frames"). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const ok = (sql: string): AST.Statement[] => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast +} + +const roundTrip = (sql: string) => toSql(ok(sql)) + +const fails = (sql: string) => { + expect(parseToAst(sql).errors.length).toBeGreaterThan(0) +} + +const overSpec = (sql: string): AST.WindowSpecification => { + const select = ok(sql)[0] as AST.SelectStatement + const item = select.columns[0] as AST.ExpressionSelectItem + return (item.expression as AST.FunctionCall).over! +} + +describe("GROUPS frame mode", () => { + it("builds a windowFrame with mode groups", () => { + const spec = overSpec( + "SELECT sum(x) OVER (ORDER BY ts GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM t", + ) + expect(spec.frame).toEqual({ + type: "windowFrame", + mode: "groups", + start: { + type: "windowFrameBound", + kind: "preceding", + value: { type: "literal", value: 1, literalType: "number", raw: "1" }, + }, + end: { type: "windowFrameBound", kind: "currentRow" }, + }) + }) + + it("round-trips single-bound and BETWEEN forms", () => { + expect( + roundTrip( + "select sum(x) over (order by ts groups unbounded preceding) from t", + ), + ).toBe("SELECT sum(x) OVER (ORDER BY ts GROUPS UNBOUNDED PRECEDING) FROM t") + expect( + roundTrip( + "select sum(x) over (partition by s order by ts groups between 2 preceding and 1 following) from t", + ), + ).toBe( + "SELECT sum(x) OVER (PARTITION BY s ORDER BY ts GROUPS BETWEEN 2 PRECEDING AND 1 FOLLOWING) FROM t", + ) + }) + + it("works in a named WINDOW definition", () => { + expect( + roundTrip( + "SELECT sum(x) OVER w FROM t WINDOW w AS (ORDER BY ts GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW EXCLUDE TIES)", + ), + ).toBe( + "SELECT sum(x) OVER w FROM t WINDOW w AS (ORDER BY ts GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW EXCLUDE TIES)", + ) + const select = ok( + "SELECT sum(x) OVER w FROM t WINDOW w AS (ORDER BY ts GROUPS 1 PRECEDING)", + )[0] as AST.SelectStatement + expect(select.namedWindows?.[0].frame?.mode).toBe("groups") + }) + + it("works after an inherited base window", () => { + expect( + roundTrip( + "SELECT sum(x) OVER w2 FROM t WINDOW w1 AS (PARTITION BY s), w2 AS (w1 ORDER BY ts GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW EXCLUDE GROUP)", + ), + ).toBe( + "SELECT sum(x) OVER w2 FROM t WINDOW w1 AS (PARTITION BY s), w2 AS (w1 ORDER BY ts GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW EXCLUDE GROUP)", + ) + }) +}) + +describe("EXCLUDE GROUP / EXCLUDE TIES", () => { + it("sets exclude on the frame", () => { + expect( + overSpec( + "SELECT sum(x) OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW EXCLUDE GROUP) FROM t", + ).frame?.exclude, + ).toBe("group") + expect( + overSpec( + "SELECT sum(x) OVER (ORDER BY ts RANGE BETWEEN 1 PRECEDING AND CURRENT ROW EXCLUDE TIES) FROM t", + ).frame?.exclude, + ).toBe("ties") + expect( + overSpec( + "SELECT sum(x) OVER (ORDER BY ts ROWS 1 PRECEDING EXCLUDE CURRENT ROW) FROM t", + ).frame?.exclude, + ).toBe("currentRow") + expect( + overSpec( + "SELECT sum(x) OVER (ORDER BY ts ROWS 1 PRECEDING EXCLUDE NO OTHERS) FROM t", + ).frame?.exclude, + ).toBe("noOthers") + }) + + it("round-trips every EXCLUDE form in every frame mode", () => { + for (const mode of ["ROWS", "RANGE", "GROUPS"]) { + for (const ex of ["CURRENT ROW", "GROUP", "TIES", "NO OTHERS"]) { + const sql = `SELECT sum(x) OVER (ORDER BY ts ${mode} BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE ${ex}) FROM t` + expect(roundTrip(sql.toLowerCase())).toBe(sql) + } + } + }) + + it("rejects unknown exclusion forms", () => { + fails("SELECT sum(x) OVER (ORDER BY ts ROWS 1 PRECEDING EXCLUDE) FROM t") + fails( + "SELECT sum(x) OVER (ORDER BY ts ROWS 1 PRECEDING EXCLUDE GROUPS) FROM t", + ) + fails( + "SELECT sum(x) OVER (ORDER BY ts ROWS 1 PRECEDING EXCLUDE NO TIES) FROM t", + ) + fails( + "SELECT sum(x) OVER (ORDER BY ts ROWS 1 PRECEDING EXCLUDE TIES OTHERS) FROM t", + ) + }) +}) + +describe("groups / ties stay identifiers", () => { + it("as column, table and alias names", () => { + expect(roundTrip("SELECT ties, groups FROM t")).toBe( + "SELECT ties, groups FROM t", + ) + expect(roundTrip("SELECT x AS ties FROM ties")).toBe( + "SELECT x AS ties FROM ties", + ) + expect(roundTrip("CREATE TABLE ties (ties INT, groups INT)")).toBe( + "CREATE TABLE ties (ties INT, groups INT)", + ) + expect( + roundTrip( + "SELECT sum(ties) OVER (ORDER BY groups ROWS 1 PRECEDING) FROM t", + ), + ).toBe("SELECT sum(ties) OVER (ORDER BY groups ROWS 1 PRECEDING) FROM t") + }) +}) + +describe("autocomplete", () => { + it("suggests GROUPS as a frame mode after ORDER BY in OVER and WINDOW", () => { + expect(getNextValidTokens("SELECT sum(x) OVER (ORDER BY ts ")).toContain( + "Groups", + ) + expect( + getNextValidTokens( + "SELECT sum(x) OVER w FROM t WINDOW w AS (ORDER BY ts ", + ), + ).toContain("Groups") + }) + + it("suggests GROUP and TIES after EXCLUDE", () => { + const tokens = getNextValidTokens( + "SELECT sum(x) OVER (ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW EXCLUDE ", + ) + expect(tokens).toEqual( + expect.arrayContaining(["Current", "Group", "Ties", "No"]), + ) + }) + + it("does not suggest TIES outside EXCLUDE", () => { + expect( + getNextValidTokens("SELECT sum(x) OVER (ORDER BY ts "), + ).not.toContain("Ties") + expect(getNextValidTokens("SELECT * FROM t ")).not.toContain("Ties") + }) +}) From 1fe1a17524cf7fb30d2639bf4064e60d587f82b8 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:07:14 +0300 Subject: [PATCH 16/36] feat: parse (expr).member, stacked unary / NOT operands and chained comparisons The server's ExpressionParser accepts these; sql-parser rejected them: - member access after `)`: `(x).a`, `(pg_catalog.pg_class()).relnamespace`, `pg_catalog.pg_class().y` (DOT dereference; DeclareTest:4542/4545, ExpressionParserTest:672). typeCastExpression: MANY(Dot identifier) gated on the previous token being `)`. New AST node memberAccess { expression, member }. - stacked prefix operators `- -a`, `~-a`, `-~-~1` (ExpressionParserTest:1863): unaryExpression OPTION -> MANY; one unary node per operator. toSql keeps a space in `- -a` so it does not become a `--` comment. - `NOT NOT x` (BooleanSubQueryPredicateTest:252) and NOT as a comparison operand `x = NOT y` (CompiledFilterIRSerializerTest:301): notExpression MANY(NOT); equality right operand may be `NOT `. - chained comparisons `a = 1 = true`, `a < b < c`, `a > 1 IS NULL`: equality/relational OPTION -> MANY, left-associative binary nodes (visitor orders operators/operands by offset). Tests: 3 server queries (fixture) + tests/expression-gaps.test.ts; red 13 failing -> all green, full suite 16784 passed. Corpus: fixed 13 (pg_class member access, NOT NOT) / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0. Suggestions: +IS after `price > 1 `, +NOT after `a = `, +LIKE/ILIKE after `a = 1 ` (all valid chain continuations). --- src/parser/ast.ts | 8 ++ src/parser/cst-types.d.ts | 4 + src/parser/parser.ts | 31 ++++- src/parser/toSql.ts | 6 + src/parser/visitor.ts | 178 ++++++++++++++++++----------- tests/expression-gaps.test.ts | 97 ++++++++++++++++ tests/fixtures/server-queries.json | 18 +++ 7 files changed, 269 insertions(+), 73 deletions(-) create mode 100644 tests/expression-gaps.test.ts diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 3ba161c..293b6d9 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -1260,6 +1260,7 @@ export type Expression = | CastExpression | TypeCastExpression | AtTimeZoneExpression + | MemberAccessExpression | InExpression | BetweenExpression | WithinExpression @@ -1375,6 +1376,13 @@ export interface TypeCastExpression extends AstNode { } /** ` AT TIME ZONE ''` (the server rewrites it to to_timezone(expr, tz)) */ +/** Member access on a parenthesised value: `(x).a` */ +export interface MemberAccessExpression extends AstNode { + type: "memberAccess" + expression: Expression + member: string +} + export interface AtTimeZoneExpression extends AstNode { type: "atTimeZone" expression: Expression diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index c65a1e7..06bac24 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -2510,6 +2510,8 @@ export type EqualityExpressionCstChildren = { RegexMatch?: IToken[]; RegexNotMatch?: IToken[]; RegexNotEquals?: IToken[]; + Not?: IToken[]; + notOperand?: NotExpressionCstNode[]; }; export interface RelationalExpressionCstNode extends CstNode { @@ -2648,6 +2650,8 @@ export interface TypeCastExpressionCstNode extends CstNode { export type TypeCastExpressionCstChildren = { primaryExpression: PrimaryExpressionCstNode[]; + Dot?: IToken[]; + member?: IdentifierCstNode[]; LBracket?: IToken[]; arraySubscript?: (ArraySubscriptCstNode)[]; Comma?: IToken[]; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index da635ac..1cacd07 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -4682,14 +4682,17 @@ class QuestDBParser extends CstParser { // Precedence 14: NOT (prefix) private notExpression = this.RULE("notExpression", () => { - this.OPTION(() => this.CONSUME(Not)) + // NOT may be stacked: `NOT NOT x` + this.MANY(() => this.CONSUME(Not)) this.SUBRULE(this.equalityExpression) }) // Precedence 13: =, !=, <>, ~, !~, LIKE, ILIKE + // Comparisons chain left-associatively (`a = 1 = true`), and the right + // operand may be a NOT expression (`x = NOT y`), as in ExpressionParser. private equalityExpression = this.RULE("equalityExpression", () => { this.SUBRULE(this.relationalExpression) - this.OPTION(() => { + this.MANY(() => { this.OR([ { ALT: () => this.CONSUME(Equals) }, { @@ -4706,14 +4709,22 @@ class QuestDBParser extends CstParser { { ALT: () => this.CONSUME(RegexNotMatch) }, { ALT: () => this.CONSUME(RegexNotEquals) }, ]) - this.SUBRULE1(this.relationalExpression) + this.OR1([ + { + ALT: () => { + this.CONSUME(Not) + this.SUBRULE(this.notExpression, { LABEL: "notOperand" }) + }, + }, + { ALT: () => this.SUBRULE1(this.relationalExpression) }, + ]) }) }) // Precedence 12: <, <=, >, >=, IS [NOT] NULL|TRUE|FALSE private relationalExpression = this.RULE("relationalExpression", () => { this.SUBRULE(this.setExpression) - this.OPTION(() => { + this.MANY(() => { this.OR([ { ALT: () => { @@ -4900,7 +4911,8 @@ class QuestDBParser extends CstParser { // Precedence 3: unary -, unary ~ (bitwise complement) private unaryExpression = this.RULE("unaryExpression", () => { - this.OPTION(() => { + // Prefix operators may be stacked: `- -a`, `~-a` + this.MANY(() => { this.OR([ { ALT: () => this.CONSUME(Minus) }, { ALT: () => this.CONSUME(RegexMatch) }, // ~ used as unary bitwise complement @@ -4912,6 +4924,15 @@ class QuestDBParser extends CstParser { // Precedence 2: :: (inline type cast), [] (array subscript) private typeCastExpression = this.RULE("typeCastExpression", () => { this.SUBRULE(this.primaryExpression) + // Member access on a parenthesised value: `(x).a`, `(f()).col` + // (ExpressionParser DOT dereference after `)`). + this.MANY3({ + GATE: () => this.LA(0).tokenType === RParen, + DEF: () => { + this.CONSUME(Dot) + this.SUBRULE(this.identifier, { LABEL: "member" }) + }, + }) // Postfix array subscript: expr[i], expr[i:j], expr[i, j] this.MANY(() => { this.CONSUME(LBracket) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 23dc396..2dd005a 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -1862,6 +1862,8 @@ function expressionToSql(expr: AST.Expression): string { return typeCastExprToSql(expr) case "atTimeZone": return `${expressionToSql(expr.expression)} AT TIME ZONE ${expressionToSql(expr.timeZone)}` + case "memberAccess": + return `${expressionToSql(expr.expression)}.${escapeIdentifier(expr.member)}` case "in": return inExprToSql(expr) case "between": @@ -1958,6 +1960,10 @@ function unaryExprToSql(expr: AST.UnaryExpression): string { if (expr.operator === "NOT") { return `NOT ${operand}` } + // `- -a`: keep a space so the two minuses do not lex as a `--` comment + if (expr.operator === "-" && operand.startsWith("-")) { + return `- ${operand}` + } return `${expr.operator}${operand}` } diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 89e2601..68ba515 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -3893,10 +3893,11 @@ class QuestDBVisitor extends BaseVisitor { } notExpression(ctx: NotExpressionCstChildren): AST.Expression { - const inner = this.visit(ctx.equalityExpression) as AST.Expression + let inner = this.visit(ctx.equalityExpression) as AST.Expression - if (ctx.Not) { - return { + // `NOT NOT x`: one unary node per NOT + for (let i = 0; i < (ctx.Not?.length ?? 0); i++) { + inner = { type: "unary", operator: "NOT", operand: inner, @@ -3907,70 +3908,105 @@ class QuestDBVisitor extends BaseVisitor { } equalityExpression(ctx: EqualityExpressionCstChildren): AST.Expression { - const left = this.visit(ctx.relationalExpression[0]) as AST.Expression + let result = this.visit(ctx.relationalExpression[0]) as AST.Expression + + // Operators and right operands, both in source order (left-associative + // chain: `a = 1 = true`). A right operand is either a relational + // expression or `NOT `. + const ops = [ + ...(ctx.Equals ?? []).map((t) => ({ t, op: "=" })), + ...(ctx.NotEquals ?? []).map((t) => ({ t, op: "!=" })), + ...(ctx.Like ?? []).map((t) => ({ t, op: "LIKE" })), + ...(ctx.Ilike ?? []).map((t) => ({ t, op: "ILIKE" })), + ...(ctx.RegexMatch ?? []).map((t) => ({ t, op: "~" })), + ...(ctx.RegexNotMatch ?? []).map((t) => ({ t, op: "!~" })), + ...(ctx.RegexNotEquals ?? []).map((t) => ({ t, op: "~=" })), + ].sort((a, b) => a.t.startOffset - b.t.startOffset) + const operands = [ + ...ctx.relationalExpression.slice(1).map((node) => ({ + offset: this.getFirstTokenOffset(node), + build: () => this.visit(node) as AST.Expression, + })), + ...(ctx.notOperand ?? []).map((node, i) => ({ + offset: ctx.Not![i].startOffset, + build: (): AST.Expression => ({ + type: "unary", + operator: "NOT", + operand: this.visit(node) as AST.Expression, + }), + })), + ].sort((a, b) => a.offset - b.offset) - if (ctx.relationalExpression && ctx.relationalExpression.length > 1) { - let operator = "" - if (ctx.Equals) operator = "=" - else if (ctx.NotEquals) { + ops.forEach(({ t, op }, i) => { + const operand = operands[i] + let operator = op + if (op === "!=") { // `<> ALL` / `!= ALL` keeps the operator as written - operator = ctx.All ? `${ctx.NotEquals[0].image} ALL` : "!=" - } else if (ctx.Like) operator = "LIKE" - else if (ctx.Ilike) operator = "ILIKE" - else if (ctx.RegexMatch) operator = "~" - else if (ctx.RegexNotMatch) operator = "!~" - else if (ctx.RegexNotEquals) operator = "~=" - - return { + const all = ctx.All?.some( + (a) => + a.startOffset > t.startOffset && a.startOffset < operand.offset, + ) + if (all) operator = `${t.image} ALL` + } + result = { type: "binary", operator, - left, - right: this.visit(ctx.relationalExpression[1]) as AST.Expression, + left: result, + right: operand.build(), } - } + }) - return left + return result } relationalExpression(ctx: RelationalExpressionCstChildren): AST.Expression { - const left = this.visit(ctx.setExpression[0]) as AST.Expression - - // Check for IS [NOT] TRUE / FALSE - if (ctx.Is && (ctx.True || ctx.False)) { - return { - type: "isBoolean", - expression: left, - value: !!ctx.True, - not: !!ctx.Not, - } - } - - // Check for IS [NOT] NULL - if (ctx.Is) { - return { - type: "isNull", - expression: left, - not: !!ctx.Not, - } - } - - // Check for relational operators (<, <=, >, >=) - if (ctx.setExpression && ctx.setExpression.length > 1) { - let operator = "" - if (ctx.LessThan) operator = "<" - else if (ctx.LessThanOrEqual) operator = "<=" - else if (ctx.GreaterThan) operator = ">" - else if (ctx.GreaterThanOrEqual) operator = ">=" - - return { - type: "binary", - operator, - left, - right: this.visit(ctx.setExpression[1]) as AST.Expression, + let result = this.visit(ctx.setExpression[0]) as AST.Expression + + // Left-associative chain of `< <= > >=` and IS [NOT] NULL|TRUE|FALSE, + // in source order. + const ops = [ + ...(ctx.LessThan ?? []).map((t) => ({ t, op: "<" })), + ...(ctx.LessThanOrEqual ?? []).map((t) => ({ t, op: "<=" })), + ...(ctx.GreaterThan ?? []).map((t) => ({ t, op: ">" })), + ...(ctx.GreaterThanOrEqual ?? []).map((t) => ({ t, op: ">=" })), + ...(ctx.Is ?? []).map((t) => ({ t, op: "IS" })), + ].sort((a, b) => a.t.startOffset - b.t.startOffset) + const isValues = [ + ...(ctx.Null ?? []).map((t) => ({ t, v: "NULL" })), + ...(ctx.True ?? []).map((t) => ({ t, v: "TRUE" })), + ...(ctx.False ?? []).map((t) => ({ t, v: "FALSE" })), + ].sort((a, b) => a.t.startOffset - b.t.startOffset) + let operandIdx = 1 + let isIdx = 0 + + for (const { t, op } of ops) { + if (op === "IS") { + const value = isValues[isIdx++] + const not = !!ctx.Not?.some( + (n) => + n.startOffset > t.startOffset && + n.startOffset < value.t.startOffset, + ) + result = + value.v === "NULL" + ? { type: "isNull", expression: result, not } + : { + type: "isBoolean", + expression: result, + value: value.v === "TRUE", + not, + } + } else { + result = { + type: "binary", + operator: op, + left: result, + right: this.visit(ctx.setExpression[operandIdx++]) as AST.Expression, + } } } - return left + return result } setExpression(ctx: SetExpressionCstChildren): AST.Expression { @@ -4193,20 +4229,17 @@ class QuestDBVisitor extends BaseVisitor { } unaryExpression(ctx: UnaryExpressionCstChildren): AST.Expression { - const inner = this.visit(ctx.typeCastExpression) as AST.Expression - - if (ctx.Minus) { - return { - type: "unary", - operator: "-", - operand: inner, - } - } - - if (ctx.RegexMatch) { - return { + let inner = this.visit(ctx.typeCastExpression) as AST.Expression + + // Stacked prefix operators (`- -a`, `~-a`): innermost is the last one + const ops = [ + ...(ctx.Minus ?? []).map((t) => ({ t, op: "-" })), + ...(ctx.RegexMatch ?? []).map((t) => ({ t, op: "~" })), // bitwise complement + ].sort((a, b) => b.t.startOffset - a.t.startOffset) + for (const { op } of ops) { + inner = { type: "unary", - operator: "~", + operator: op, operand: inner, } } @@ -4217,6 +4250,15 @@ class QuestDBVisitor extends BaseVisitor { typeCastExpression(ctx: TypeCastExpressionCstChildren): AST.Expression { let inner = this.visit(ctx.primaryExpression) as AST.Expression + // Member access on a parenthesised value: `(x).a` + for (const member of ctx.member ?? []) { + inner = { + type: "memberAccess", + expression: inner, + member: this.extractIdentifierName(member.children), + } + } + // Handle array subscripts: expr[i], expr[i:j], expr[i, j] // Group subscripts by their enclosing bracket pair so that // arr[1, 2] produces a single ArrayAccessExpression with 2 subscripts (nd-array access), diff --git a/tests/expression-gaps.test.ts b/tests/expression-gaps.test.ts new file mode 100644 index 0000000..12874fe --- /dev/null +++ b/tests/expression-gaps.test.ts @@ -0,0 +1,97 @@ +/** + * Expression forms the QuestDB server's ExpressionParser accepts: + * - `(expr).member` member access on a parenthesised value (DOT dereference + * after `)`; ExpressionParserTest:672, DeclareTest:4545) + * - `- -a`, `~-a`, `-~-~1` stacked prefix operators (ExpressionParserTest:1863) + * - `NOT NOT x`, `x = NOT y` NOT as a nested / right-hand operand + * (BooleanSubQueryPredicateTest:252, CompiledFilterIRSerializerTest:301) + * - `a = 1 = true` left-associative chained comparisons + * + * Statement-level server queries live in tests/fixtures/server-queries.json + * (feature "expression-gaps"). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst } from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const ok = (sql: string): AST.Statement[] => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast +} + +const roundTrip = (sql: string) => { + const out = toSql(ok(sql)) + // the output must re-parse to the same SQL + expect(toSql(ok(out))).toBe(out) + return out +} + +describe("expression gaps", () => { + it("member access on a parenthesised value", () => { + expect(roundTrip("SELECT (x).a FROM t")).toBe("SELECT (x).a FROM t") + expect( + roundTrip("SELECT case when (a).b then (b).c + 1 else -(b).c end FROM t"), + ).toBe("SELECT CASE WHEN (a).b THEN (b).c + 1 ELSE -(b).c END FROM t") + const sel = ok("SELECT (x).a FROM t")[0] as AST.SelectStatement + expect(sel.columns[0]).toMatchObject({ + expression: { + type: "memberAccess", + member: "a", + expression: { type: "paren" }, + }, + }) + }) + + it("stacked unary operators", () => { + expect(roundTrip("SELECT - -a FROM t")).toBe("SELECT - -a FROM t") + expect(roundTrip("SELECT ~-a FROM t")).toBe("SELECT ~-a FROM t") + expect(roundTrip("SELECT * FROM t WHERE -~-~1 > ~-1")).toBe( + "SELECT * FROM t WHERE -~-~1 > ~-1", + ) + }) + + it("NOT NOT and NOT as a comparison operand", () => { + expect(roundTrip("SELECT * FROM t WHERE NOT NOT x")).toBe( + "SELECT * FROM t WHERE NOT NOT x", + ) + expect(roundTrip("SELECT * FROM t WHERE x = NOT y")).toBe( + "SELECT * FROM t WHERE x = NOT y", + ) + expect( + roundTrip( + "SELECT * FROM t WHERE aboolean = true or not aboolean = not false", + ), + ).toBe("SELECT * FROM t WHERE aboolean = TRUE OR NOT aboolean = NOT FALSE") + }) + + it("chained comparisons are left-associative", () => { + const sel = ok( + "SELECT * FROM t WHERE a = 1 = true", + )[0] as AST.SelectStatement + expect(sel.where).toMatchObject({ + type: "binary", + operator: "=", + left: { type: "binary", operator: "=" }, + right: { type: "literal", value: true }, + }) + expect(roundTrip("SELECT * FROM t WHERE a = 1 = true")).toBe( + "SELECT * FROM t WHERE a = 1 = TRUE", + ) + expect(roundTrip("SELECT * FROM t WHERE a <> 1 != b")).toBe( + "SELECT * FROM t WHERE a != 1 != b", + ) + expect(roundTrip("SELECT * FROM t WHERE a < b < c")).toBe( + "SELECT * FROM t WHERE a < b < c", + ) + }) + + it("dotted names and plain expressions are unchanged", () => { + expect(roundTrip("SELECT t.a, -a, NOT b FROM t")).toBe( + "SELECT t.a, -a, NOT b FROM t", + ) + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index d1b427b..cddaab4 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1452,5 +1452,23 @@ "source": "questdb:core/src/test/java/io/questdb/test/cairo/lv/LiveViewCheckpointFunctionCompilerTest.java:712", "feature": "window-frames", "toSql": "SELECT ts, sym, sum(x) OVER (PARTITION BY sym ORDER BY ts ROWS BETWEEN 3 PRECEDING AND CURRENT ROW EXCLUDE TIES) AS s FROM base" + }, + { + "query": "DECLARE @x := CASE WHEN (v > 1) THEN (v).x ELSE 2 END SELECT @x FROM t", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:4542", + "feature": "expression-gaps", + "toSql": "DECLARE @x := CASE WHEN (v > 1) THEN (v).x ELSE 2 END SELECT @x FROM t" + }, + { + "query": "DECLARE @x := (pg_catalog.pg_class()).relnamespace SELECT @x x FROM long_sequence(1)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:4545", + "feature": "expression-gaps", + "toSql": "DECLARE @x := (pg_catalog.pg_class()).relnamespace SELECT @x AS x FROM long_sequence(1)" + }, + { + "query": "select * from t where not not (select b from x_true limit 1)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/BooleanSubQueryPredicateTest.java:252", + "feature": "expression-gaps", + "toSql": "SELECT * FROM t WHERE NOT NOT (SELECT b FROM x_true LIMIT 1)" } ] From 764aea09a838c2a5225992f10dcd3eb727f1a17f Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:11:28 +0300 Subject: [PATCH 17/36] feat: DECLARE AUDITED/OVERRIDABLE markers, trailing commas, implicit select after DECLARE/WITH, WITH/DECLARE set-op branches Server (questdb#7602; SqlParser.parseDeclare, isMarkedDeclarationAhead, parseQuery) accepts, sql-parser rejected: - `DECLARE [OVERRIDABLE] [AUDITED] @x := ...` markers in any order per variable (ViewAuditTest, DeclareTest). New non-reserved keyword `audited`; declareAssignment MANY(OVERRIDABLE|AUDITED). AST: DeclareAssignment.audited. toSql prints OVERRIDABLE before AUDITED. Duplicate markers are accepted (server rejects; looser is fine). - trailing comma in a value list `('a',)` (any parenthesised list) and after the last declaration (`DECLARE @lim := 1, audited WHERE ...`), gated on a `[markers] @var` lookahead so audited/overridable stay table names. - implicit select after a DECLARE / WITH prefix: `DECLARE @x := 1 (SELECT @x)`, `DECLARE @lim := 1 t WHERE x > @lim`, `(WITH c AS (...) c)`, top-level `WITH c AS (...) c`. selectStatement = DECLARE [WITH] body | WITH body | selectBody; new rule prefixedQueryBody (selectBody | implicitSelectStatement). - set-operation branch starting with WITH or DECLARE (WithClauseTest:649/654, DeclareTest:5616): setOperation alt -> selectStatement (later set ops nest into that branch; toSql prints them flat, which re-parses the same). Tests: 7 server queries (fixture) + tests/declare.test.ts; red 27 -> green, full suite 16818 passed. Corpus: fixed 121 / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0. Suggestions: +AUDITED after `DECLARE ` (OVERRIDABLE already there); table names after `DECLARE @x := 1 ` and `WITH .. AS (..) `; markers/WITH/SELECT/ tables after `DECLARE @x := 1, `; +DECLARE,WITH after `UNION ALL `. --- src/grammar/keywords.ts | 1 + src/parser/ast.ts | 1 + src/parser/cst-types.d.ts | 19 ++++++- src/parser/lexer.ts | 2 + src/parser/parser.ts | 81 +++++++++++++++++++++++++++--- src/parser/toSql.ts | 3 +- src/parser/tokens.ts | 2 + src/parser/visitor.ts | 22 +++++++- tests/declare.test.ts | 81 ++++++++++++++++++++++++++++++ tests/fixtures/server-queries.json | 42 ++++++++++++++++ 10 files changed, 242 insertions(+), 12 deletions(-) create mode 100644 tests/declare.test.ts diff --git a/src/grammar/keywords.ts b/src/grammar/keywords.ts index e8cdc7b..0654d1e 100644 --- a/src/grammar/keywords.ts +++ b/src/grammar/keywords.ts @@ -16,6 +16,7 @@ export const keywords: string[] = [ "at", "atomic", "attach", + "audited", "backup", "base", "batch", diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 293b6d9..b4a5c05 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -153,6 +153,7 @@ export interface DeclareAssignment extends AstNode { name: string value: Expression overridable?: boolean + audited?: boolean } export interface CreateTableStatement extends AstNode { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 06bac24..dc8181d 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -61,6 +61,7 @@ export type WithStatementCstChildren = { insertStatement?: InsertStatementCstNode[]; updateStatement?: UpdateStatementCstNode[]; selectBody?: SelectBodyCstNode[]; + implicitSelectStatement?: ImplicitSelectStatementCstNode[]; }; export interface SelectStatementCstNode extends CstNode { @@ -70,8 +71,19 @@ export interface SelectStatementCstNode extends CstNode { export type SelectStatementCstChildren = { declareClause?: DeclareClauseCstNode[]; - withClause?: WithClauseCstNode[]; - selectBody: SelectBodyCstNode[]; + withClause?: (WithClauseCstNode)[]; + prefixedQueryBody?: (PrefixedQueryBodyCstNode)[]; + selectBody?: SelectBodyCstNode[]; +}; + +export interface PrefixedQueryBodyCstNode extends CstNode { + name: "prefixedQueryBody"; + children: PrefixedQueryBodyCstChildren; +} + +export type PrefixedQueryBodyCstChildren = { + selectBody?: SelectBodyCstNode[]; + implicitSelectStatement?: ImplicitSelectStatementCstNode[]; }; export interface SelectBodyCstNode extends CstNode { @@ -145,6 +157,7 @@ export type SetOperationCstChildren = { Intersect?: IToken[]; All?: IToken[]; simpleSelect?: SimpleSelectCstNode[]; + selectStatement?: SelectStatementCstNode[]; implicitSelectBody?: ImplicitSelectBodyCstNode[]; }; @@ -776,6 +789,7 @@ export interface DeclareAssignmentCstNode extends CstNode { export type DeclareAssignmentCstChildren = { Overridable?: IToken[]; + Audited?: IToken[]; VariableReference: IToken[]; ColonEquals: IToken[]; expression: ExpressionCstNode[]; @@ -3133,6 +3147,7 @@ export interface ICstNodeVisitor extends ICstVisitor { statement(children: StatementCstChildren, param?: IN): OUT; withStatement(children: WithStatementCstChildren, param?: IN): OUT; selectStatement(children: SelectStatementCstChildren, param?: IN): OUT; + prefixedQueryBody(children: PrefixedQueryBodyCstChildren, param?: IN): OUT; selectBody(children: SelectBodyCstChildren, param?: IN): OUT; withClause(children: WithClauseCstChildren, param?: IN): OUT; cteDefinition(children: CteDefinitionCstChildren, param?: IN): OUT; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index bcaa9ff..dfeb09c 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -20,6 +20,7 @@ import { Asof, Assume, Attach, + Audited, Atomic, Backup, Base, @@ -362,6 +363,7 @@ export { Asof, Assume, Attach, + Audited, Atomic, Backup, Base, diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 1cacd07..83c6e69 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -334,6 +334,7 @@ import { BitOr, Within, Overridable, + Audited, LBracket, RBracket, Colon, @@ -658,6 +659,8 @@ class QuestDBParser extends CstParser { // SELECT after WITH: no DECLARE/WITH prefixes allowed here ALT: () => this.SUBRULE(this.selectBody), }, + // implicit select after WITH: `WITH c AS (...) c WHERE ...` + { ALT: () => this.SUBRULE(this.implicitSelectStatement) }, ]) }) @@ -665,10 +668,37 @@ class QuestDBParser extends CstParser { // SELECT Statement // ========================================================================== + // After a DECLARE or WITH prefix the query may also be an implicit select + // (`DECLARE @x := 1 (SELECT @x)`, `DECLARE @lim := 1 t WHERE x > @lim`, + // `(WITH c AS (...) c)`), as in the server's parseQuery. Without a prefix + // a plain SELECT is required (the implicit form is its own statement). private selectStatement = this.RULE("selectStatement", () => { - this.OPTION(() => this.SUBRULE(this.declareClause)) - this.OPTION2(() => this.SUBRULE(this.withClause)) - this.SUBRULE(this.selectBody) + this.OR([ + { + ALT: () => { + this.SUBRULE(this.declareClause) + this.OPTION(() => this.SUBRULE(this.withClause)) + this.SUBRULE(this.prefixedQueryBody) + }, + }, + { + ALT: () => { + this.SUBRULE1(this.withClause) + this.SUBRULE1(this.prefixedQueryBody) + }, + }, + { ALT: () => this.SUBRULE(this.selectBody) }, + ]) + }) + + private prefixedQueryBody = this.RULE("prefixedQueryBody", () => { + this.OR([ + { + GATE: () => this.LA(1).tokenType === Select, + ALT: () => this.SUBRULE(this.selectBody), + }, + { ALT: () => this.SUBRULE(this.implicitSelectStatement) }, + ]) }) private selectBody = this.RULE("selectBody", () => { @@ -746,9 +776,17 @@ class QuestDBParser extends CstParser { { ALT: () => this.CONSUME(Intersect) }, ]) this.OPTION(() => this.CONSUME(All)) - // Right operand: full SELECT or bare table/implicit form + // Right operand: full SELECT or bare table/implicit form; a branch may + // start with its own WITH or DECLARE (WithClauseTest, DeclareTest) this.OR1([ { ALT: () => this.SUBRULE(this.simpleSelect) }, + { + GATE: () => { + const t = this.LA(1).tokenType + return t === With || t === Declare + }, + ALT: () => this.SUBRULE(this.selectStatement), + }, { ALT: () => this.SUBRULE(this.implicitSelectBody) }, ]) }) @@ -1697,12 +1735,37 @@ class QuestDBParser extends CstParser { this.SUBRULE(this.declareAssignment) this.MANY(() => { this.CONSUME(Comma) - this.SUBRULE1(this.declareAssignment) + // A trailing comma may end the block: `DECLARE @lim := 1, t WHERE ...` + this.OPTION({ + GATE: () => this.isDeclarationAhead(), + DEF: () => this.SUBRULE1(this.declareAssignment), + }) }) }) + // `[AUDITED|OVERRIDABLE]* @var` ahead? The markers are also valid table + // names (`DECLARE @lim := 1, audited WHERE ...`); SqlParser + // isMarkedDeclarationAhead treats them as markers only before a variable + // (or SELECT / := for its error messages). + private isDeclarationAhead(): boolean { + let i = 1 + while ( + this.LA(i).tokenType === Audited || + this.LA(i).tokenType === Overridable + ) { + i++ + } + return this.LA(i).tokenType === VariableReference + } + private declareAssignment = this.RULE("declareAssignment", () => { - this.OPTION(() => this.CONSUME(Overridable)) + // DECLARE [OVERRIDABLE] [AUDITED] @x, markers in any order + this.MANY(() => { + this.OR([ + { ALT: () => this.CONSUME(Overridable) }, + { ALT: () => this.CONSUME(Audited) }, + ]) + }) this.CONSUME(VariableReference) this.CONSUME(ColonEquals) this.SUBRULE(this.expression) @@ -5059,11 +5122,15 @@ class QuestDBParser extends CstParser { }, { // Single expression or row constructor: (expr) or (expr, expr, ...) + // A trailing comma is allowed: `('a',)` (DECLARE value lists) ALT: () => { this.SUBRULE(this.expression) this.MANY2(() => { this.CONSUME1(Comma) - this.SUBRULE1(this.expression) + this.OPTION3({ + GATE: () => this.LA(1).tokenType !== RParen, + DEF: () => this.SUBRULE1(this.expression), + }) }) }, }, diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 2dd005a..ccc318b 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -1441,7 +1441,8 @@ function refreshMaterializedViewToSql( function declareClauseToSql(clause: AST.DeclareClause): string { const assignments = clause.assignments.map((a) => { - const prefix = a.overridable ? "OVERRIDABLE " : "" + const prefix = + (a.overridable ? "OVERRIDABLE " : "") + (a.audited ? "AUDITED " : "") return `${prefix}@${a.name} := ${expressionToSql(a.value)}` }) return `DECLARE ${assignments.join(", ")}` diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index db4b6e0..f2e6b33 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -307,6 +307,7 @@ export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ "Nocache", "Observation", "Overridable", + "Audited", "Owned", "Period", "Reindex", @@ -619,6 +620,7 @@ export const Others = getToken("Others") export const Outer = getToken("Outer") export const Over = getToken("Over") export const Overridable = getToken("Overridable") +export const Audited = getToken("Audited") export const Owned = getToken("Owned") export const Param = getToken("Param") export const Parameters = getToken("Parameters") diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 68ba515..85251d6 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -156,6 +156,7 @@ import type { SelectItemCstChildren, SelectListCstChildren, SelectBodyCstChildren, + PrefixedQueryBodyCstChildren, SelectStatementCstChildren, SetClauseCstChildren, SetExpressionCstChildren, @@ -431,6 +432,8 @@ class QuestDBVisitor extends BaseVisitor { inner = this.visit(ctx.updateStatement) as AST.UpdateStatement } else if (ctx.selectBody) { inner = this.visit(ctx.selectBody) as AST.SelectStatement + } else if (ctx.implicitSelectStatement) { + inner = this.visit(ctx.implicitSelectStatement) as AST.SelectStatement } else { throw new Error("withStatement: expected insert, update, or select") } @@ -444,7 +447,11 @@ class QuestDBVisitor extends BaseVisitor { // ========================================================================== selectStatement(ctx: SelectStatementCstChildren): AST.SelectStatement { - const result = this.visit(ctx.selectBody) as AST.SelectStatement + const result = ( + ctx.selectBody + ? this.visit(ctx.selectBody) + : this.visit(ctx.prefixedQueryBody!) + ) as AST.SelectStatement if (ctx.declareClause) { result.declare = this.visit(ctx.declareClause) as AST.DeclareClause @@ -457,6 +464,14 @@ class QuestDBVisitor extends BaseVisitor { return result } + prefixedQueryBody(ctx: PrefixedQueryBodyCstChildren): AST.SelectStatement { + return ( + ctx.selectBody + ? this.visit(ctx.selectBody) + : this.visit(ctx.implicitSelectStatement!) + ) as AST.SelectStatement + } + selectBody(ctx: SelectBodyCstChildren): AST.SelectStatement { const result = this.visit(ctx.simpleSelect) as AST.SelectStatement @@ -553,7 +568,9 @@ class QuestDBVisitor extends BaseVisitor { const select = ctx.simpleSelect ? (this.visit(ctx.simpleSelect) as AST.SelectStatement) - : (this.visit(ctx.implicitSelectBody!) as AST.SelectStatement) + : ctx.selectStatement + ? (this.visit(ctx.selectStatement) as AST.SelectStatement) + : (this.visit(ctx.implicitSelectBody!) as AST.SelectStatement) const result: AST.SetOperation = { type: "setOperation", @@ -1359,6 +1376,7 @@ class QuestDBVisitor extends BaseVisitor { (this.visit(ctx.expression) as AST.Expression), } if (ctx.Overridable) result.overridable = true + if (ctx.Audited) result.audited = true return result } diff --git a/tests/declare.test.ts b/tests/declare.test.ts new file mode 100644 index 0000000..c5ec55c --- /dev/null +++ b/tests/declare.test.ts @@ -0,0 +1,81 @@ +/** + * DECLARE forms the QuestDB server accepts (questdb#7602, SqlParser.parseDeclare, + * isMarkedDeclarationAhead): + * - `DECLARE [OVERRIDABLE] [AUDITED] @x := ...` markers in any order (per variable) + * - a trailing comma in a value list `('a',)` and after the last declaration + * - an implicit select after DECLARE / WITH: `DECLARE @x := 1 (SELECT @x)`, + * `DECLARE @lim := 1 t WHERE x > @lim`, `(WITH c AS (...) c)` + * - set-operation branches starting with WITH or DECLARE + * + * Statement-level server queries live in tests/fixtures/server-queries.json + * (feature "declare"). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const ok = (sql: string): AST.Statement[] => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast +} + +describe("DECLARE", () => { + it("AUDITED / OVERRIDABLE markers in any order", () => { + const sel = ok( + "DECLARE AUDITED OVERRIDABLE @x := 1, AUDITED @y := 2, @z := 3 SELECT @x", + )[0] as AST.SelectStatement + expect( + sel.declare!.assignments.map((a) => [a.audited, a.overridable]), + ).toEqual([ + [true, true], + [true, undefined], + [undefined, undefined], + ]) + expect(toSql(sel)).toBe( + "DECLARE OVERRIDABLE AUDITED @x := 1, AUDITED @y := 2, @z := 3 SELECT @x", + ) + }) + + it("audited / overridable stay usable as table names", () => { + for (const q of [ + "DECLARE @lim := 1 audited WHERE x > @lim", + "DECLARE @lim := 1 overridable WHERE x > @lim", + "DECLARE @lim := 1, overridable WHERE x > @lim", + "DECLARE @lim := 1 audited overridable WHERE overridable.x > @lim", + "SELECT audited FROM audited", + ]) { + const out = toSql(ok(q)) + expect(toSql(ok(out))).toBe(out) + } + }) + + it("trailing comma in a value list", () => { + ok("DECLARE @x := (2,) SELECT l FROM k WHERE l IN @x") + }) + + it("implicit select after DECLARE / WITH", () => { + expect(toSql(ok("DECLARE @x := abs(-1) (SELECT @x)"))).toBe( + "DECLARE @x := abs(-1) (SELECT @x)", + ) + expect(toSql(ok("SELECT * FROM (WITH c AS (SELECT 1 a) c)"))).toBe( + "SELECT * FROM (WITH c AS (SELECT 1 AS a) c)", + ) + }) + + it("set-operation branch starting with DECLARE", () => { + ok("SELECT 1 a UNION ALL DECLARE @x := 2 SELECT @x") + ok( + "CREATE TABLE c_union AS (SELECT 1 a UNION ALL DECLARE AUDITED @x := 2 SELECT @x)", + ) + }) + + it("suggests AUDITED and OVERRIDABLE after DECLARE", () => { + const names = getNextValidTokens("DECLARE ") + expect(names).toContain("Audited") + expect(names).toContain("Overridable") + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index cddaab4..9ec632e 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1470,5 +1470,47 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/BooleanSubQueryPredicateTest.java:252", "feature": "expression-gaps", "toSql": "SELECT * FROM t WHERE NOT NOT (SELECT b FROM x_true LIMIT 1)" + }, + { + "query": "CREATE VIEW v_cte AS (WITH c AS (DECLARE OVERRIDABLE AUDITED @x := 1 SELECT @x a) SELECT * FROM c)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/view/ViewAuditTest.java:193", + "feature": "declare", + "toSql": "CREATE VIEW v_cte AS (WITH c AS (DECLARE OVERRIDABLE AUDITED @x := 1 SELECT @x AS a) SELECT * FROM c)" + }, + { + "query": "CREATE VIEW v_bare AS SELECT * FROM (DECLARE AUDITED @x := 1 SELECT @x a)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/view/ViewAuditTest.java:192", + "feature": "declare", + "toSql": "CREATE VIEW v_bare AS SELECT * FROM (DECLARE AUDITED @x := 1 SELECT @x AS a)" + }, + { + "query": "SELECT 1 a UNION ALL DECLARE OVERRIDABLE AUDITED @x := 2 SELECT @x", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:5616", + "feature": "declare", + "toSql": "SELECT 1 AS a UNION ALL DECLARE OVERRIDABLE AUDITED @x := 2 SELECT @x" + }, + { + "query": "DECLARE @x := ('a',) SELECT s FROM k WHERE s IN @x", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:5409", + "feature": "declare", + "toSql": "DECLARE @x := ('a') SELECT s FROM k WHERE s IN @x" + }, + { + "query": "DECLARE @x := 1 (SELECT @x AS x)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:4332", + "feature": "declare", + "toSql": "DECLARE @x := 1 (SELECT @x AS x)" + }, + { + "query": "DECLARE @lim := 1, audited WHERE x > @lim", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:5867", + "feature": "declare", + "toSql": "DECLARE @lim := 1 audited WHERE x > @lim" + }, + { + "query": "WITH c AS (SELECT 'a'::SYMBOL x) SELECT * FROM c UNION ALL WITH c AS (SELECT 'b'::SYMBOL x) SELECT * FROM c UNION ALL SELECT * FROM c", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/WithClauseTest.java:654", + "feature": "declare", + "toSql": "WITH c AS (SELECT 'a'::SYMBOL AS x) SELECT * FROM c UNION ALL WITH c AS (SELECT 'b'::SYMBOL AS x) SELECT * FROM c UNION ALL SELECT * FROM c" } ] From c89a663c92d74295f6add75bd8493310c528f824 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:18:10 +0300 Subject: [PATCH 18/36] feat: SAMPLE BY [unit], FILL(, ...), ALIGN TO ... TIME ZONE / WITH OFFSET The server parses all three with expr() (SqlParser.expectSample, FILL and ALIGN TO handling in parseSelectClauses): - SAMPLE BY [one-letter unit]: `@q h` (was a silent 2-statement split), `2 h`, `(1m)`, `(10+20)m`, `(1+2)*10L m`, `$1 d`. sampleByClause = bare unit letter | expression + optional unit. The unit is optional here (server wants it unless the expression is a full period) - looser, so `SAMPLE BY $1` / `SAMPLE BY foo` now parse (tests updated). AST unchanged: duration = expression SQL [+ " " + unit]. - FILL values are expressions: -1, @f, true, PREV(abs(a)), 1+2, 100L, NaN, '5'. AST fill stays string[]: plain names upper-cased (PREV/LINEAR/NONE), PREV(...) upper-cased, NULL; others printed as SQL - string values now keep their quotes (FILL('5') printed FILL(5) before). - ALIGN TO CALENDAR TIME ZONE [WITH OFFSET ]: @tz, $1, rnd_str('foo','bar') (was a silent split). AST: timeZone/offset strings unchanged for string literals / names; new optional timeZoneExpression / offsetExpression for other values (toSql prints them). - Autocomplete: content assist filters the top level of those expressions (keeps literal / variable tokens) so SAMPLE BY / FILL( / TIME ZONE do not offer every column, function and operator; FILL( now offers PREV, LINEAR, NONE next to NULL. Tests: 8 server queries (fixture) + tests/sample-by-fill-align.test.ts; red 37 -> green; full suite 16872 passed. Corpus: fixed 61 / rtChanged 8 (2 former silent splits, 6 FILL('str') now keeping quotes) / regressed 0 / throws 0 / formatterChangedMeaning 0. Suggestions: +PREV,LINEAR,NONE after `FILL(` / `FILL(PREV, `; `(` fallback after `SAMPLE BY ` and `TIME ZONE `; clause keywords after `SAMPLE BY 2 ` (unit optional); `(` no longer offered after `FILL(PREV `. --- src/autocomplete/content-assist.ts | 71 ++++++++++++++++---- src/parser/ast.ts | 4 ++ src/parser/cst-types.d.ts | 16 ++--- src/parser/parser.ts | 60 ++++++++--------- src/parser/toSql.ts | 17 +++-- src/parser/visitor.ts | 72 ++++++++++++++------- tests/bind-variables.test.ts | 7 +- tests/fixtures/server-queries.json | 48 ++++++++++++++ tests/parser.test.ts | 9 +-- tests/sample-by-fill-align.test.ts | 100 +++++++++++++++++++++++++++++ 10 files changed, 309 insertions(+), 95 deletions(-) create mode 100644 tests/sample-by-fill-align.test.ts diff --git a/src/autocomplete/content-assist.ts b/src/autocomplete/content-assist.ts index 2ac214c..9cfd375 100644 --- a/src/autocomplete/content-assist.ts +++ b/src/autocomplete/content-assist.ts @@ -952,6 +952,44 @@ function contextKeywordSuggestions( return [...out] } +const QUIET_EXPRESSION_ANCHORS = [ + "sampleByClause", + "alignToClause", + "fillValue", +] +// Literal / variable / punctuation tokens stay valid there (no keyword noise) +const QUIET_EXPRESSION_KEPT_TOKENS = new Set([ + "DurationLiteral", + "NumberLiteral", + "LongLiteral", + "DecimalLiteral", + "FloatLiteral", + "StringLiteral", + "EscapeStringLiteral", + "VariableReference", + "BindVariable", + "Minus", + "LParen", +]) + +/** + * If the ruleStack is at the top level of the expression directly under + * SAMPLE BY, ALIGN TO (TIME ZONE / WITH OFFSET) or a FILL value, return that + * anchor rule. Nested expressions (function arguments, parentheses) are not + * top level. + */ +function quietExpressionAnchor(ruleStack: string[]): string | undefined { + for (let i = ruleStack.length - 2; i >= 0; i--) { + if (!QUIET_EXPRESSION_ANCHORS.includes(ruleStack[i])) continue + if (ruleStack[i + 1] !== "expression") return undefined + const nested = ruleStack + .slice(i + 2) + .some((r) => r === "expression" || r === "selectStatement") + return nested ? undefined : ruleStack[i] + } + return undefined +} + /** * Compute content assist suggestions, handling CTE context specially. * @@ -985,17 +1023,27 @@ function computeSuggestions(tokens: IToken[]): ComputeResult { // columnDefinition only takes DOUBLE PRECISION in CREATE TABLE (a rule // argument gates it); ALTER TABLE ADD COLUMN shares the rule, so drop // PRECISION there as well. - const suggestions = rawSuggestions.filter( - (s) => - !( - (s.ruleStack[s.ruleStack.length - 1] === - "timestampWithTimeZoneLiteral" && - (s.nextTokenType.name === "Timestamp" || - s.nextTokenType.name === "TimestampNs")) || - (s.nextTokenType.name === "Precision" && - s.ruleStack.includes("alterTableAction")) - ), - ) + // SAMPLE BY / FILL / ALIGN TO ... TIME ZONE / WITH OFFSET take any + // expression on the server, but at the top level of those expressions we + // keep the short lists offered before (nothing for SAMPLE BY and ALIGN TO, + // NULL / PREV / LINEAR / NONE for FILL) instead of every column, function + // and operator. + let fillOperandStart = false + const suggestions = rawSuggestions.filter((s) => { + const quiet = quietExpressionAnchor(s.ruleStack) + if (quiet === "fillValue" && s.nextTokenType.name === "Null") { + fillOperandStart = true + return true + } + return !( + (quiet && !QUIET_EXPRESSION_KEPT_TOKENS.has(s.nextTokenType.name)) || + (s.ruleStack[s.ruleStack.length - 1] === "timestampWithTimeZoneLiteral" && + (s.nextTokenType.name === "Timestamp" || + s.nextTokenType.name === "TimestampNs")) || + (s.nextTokenType.name === "Precision" && + s.ruleStack.includes("alterTableAction")) + ) + }) const result = suggestions.map((s) => s.nextTokenType) // Walk every IdentifierKeyword path and union the category flags valid at @@ -1050,6 +1098,7 @@ function computeSuggestions(tokens: IToken[]): ComputeResult { ) const contextKeywords = contextKeywordSuggestions(tokens, suggestions) + if (fillOperandStart) contextKeywords.push("PREV", "LINEAR", "NONE") const contextFunctions = contextFunctionSuggestions(tokens, suggestions) return { diff --git a/src/parser/ast.ts b/src/parser/ast.ts index b4a5c05..205acf4 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -1205,6 +1205,10 @@ export interface AlignToClause extends AstNode { mode: "firstObservation" | "calendar" timeZone?: string offset?: string + /** TIME ZONE value when it is not a string literal / plain name (@tz, $1, f(...)) */ + timeZoneExpression?: Expression + /** WITH OFFSET value when it is not a string literal / plain name */ + offsetExpression?: Expression } export interface LatestOnClause extends AstNode { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index dc8181d..07d45d7 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -507,11 +507,9 @@ export interface SampleByClauseCstNode extends CstNode { export type SampleByClauseCstChildren = { Sample: IToken[]; By: IToken[]; - DurationLiteral?: IToken[]; - VariableReference?: IToken[]; - BindVariable?: IToken[]; - sampleByUnit?: IToken[]; Identifier?: IToken[]; + sampleByExpression?: ExpressionCstNode[]; + sampleByUnit?: IToken[]; groupingSetsSpec?: GroupingSetsSpecCstNode[]; fromToClause?: FromToClauseCstNode[]; fillClause?: FillClauseCstNode[]; @@ -565,11 +563,7 @@ export interface FillValueCstNode extends CstNode { } export type FillValueCstChildren = { - Null?: IToken[]; - NumberLiteral?: IToken[]; - identifier?: (IdentifierCstNode)[]; - LParen?: IToken[]; - RParen?: IToken[]; + expression: ExpressionCstNode[]; }; export interface AlignToClauseCstNode extends CstNode { @@ -585,10 +579,10 @@ export type AlignToClauseCstChildren = { Calendar?: IToken[]; Time?: IToken[]; Zone?: IToken[]; - timeZoneValue?: TimeZoneValueCstNode[]; + timeZoneExpression?: ExpressionCstNode[]; With?: IToken[]; Offset?: IToken[]; - stringOrIdentifier?: StringOrIdentifierCstNode[]; + offsetExpression?: ExpressionCstNode[]; }; export interface FromToClauseCstNode extends CstNode { diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 83c6e69..e120bf0 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -1359,29 +1359,30 @@ class QuestDBParser extends CstParser { this.CONSUME(Sample) this.CONSUME(By) this.OR([ - { ALT: () => this.CONSUME(DurationLiteral) }, - { ALT: () => this.CONSUME(VariableReference) }, - { - // SAMPLE BY $1 d / :n h — a bind variable is not a full period, so - // the server requires a one-letter unit after it - // (SqlParser.expectSample: "one letter sample by period unit expected"). - ALT: () => { - this.CONSUME(BindVariable) - this.OR1([ - { - GATE: () => this.isSampleByPeriodLetter(this.LA(1).image), - ALT: () => this.CONSUME1(Identifier, { LABEL: "sampleByUnit" }), - }, - ]) - }, - }, // Bare single-letter period unit with no leading digit, e.g. `SAMPLE BY w` // (equivalent to `SAMPLE BY 1w`). Lexes as an Identifier; the GATE keeps // this alternative restricted to valid unit letters. { - GATE: () => this.isSampleByPeriodLetter(this.LA(1).image), + GATE: () => + this.LA(1).tokenType === Identifier && + this.isSampleByPeriodLetter(this.LA(1).image), ALT: () => this.CONSUME(Identifier), }, + { + // SAMPLE BY [unit letter] (SqlParser.expectSample): a full + // period (`1h`, `@unit`) or an expression followed by a one-letter + // unit: `@q h`, `2 h`, `(10+20)m`, `$1 d`. The unit is optional here + // (the server requires it unless the expression is a full period). + ALT: () => { + this.SUBRULE(this.expression, { LABEL: "sampleByExpression" }) + this.OPTION4({ + GATE: () => + this.LA(1).tokenType === Identifier && + this.isSampleByPeriodLetter(this.LA(1).image), + DEF: () => this.CONSUME1(Identifier, { LABEL: "sampleByUnit" }), + }) + }, + }, ]) // SAMPLE BY 1h ROLLUP(a) | CUBE(a) | GROUPING SETS ((a), ()): right after // the interval, without a plain-key prefix (questdb#6853). @@ -1456,21 +1457,10 @@ class QuestDBParser extends CstParser { this.CONSUME(RParen) }) + // FILL values are expressions on the server (NULL, PREV, LINEAR, NONE, + // PREV(col), -1, @var, 1+2, '5', ...). private fillValue = this.RULE("fillValue", () => { - this.OR([ - { ALT: () => this.CONSUME(Null) }, - { ALT: () => this.CONSUME(NumberLiteral) }, - { - ALT: () => { - this.SUBRULE(this.identifier) - this.OPTION(() => { - this.CONSUME(LParen) - this.SUBRULE1(this.identifier) - this.CONSUME(RParen) - }) - }, - }, - ]) + this.SUBRULE(this.expression) }) private alignToClause = this.RULE("alignToClause", () => { @@ -1486,17 +1476,17 @@ class QuestDBParser extends CstParser { { ALT: () => { this.CONSUME(Calendar) - // Optional TIME ZONE clause + // Optional TIME ZONE (string, @var, $1, function call, ...) this.OPTION(() => { this.CONSUME(Time) this.CONSUME(Zone) - this.SUBRULE(this.timeZoneValue) + this.SUBRULE(this.expression, { LABEL: "timeZoneExpression" }) }) - // Optional WITH OFFSET clause (can appear with or without TIME ZONE) + // Optional WITH OFFSET (with or without TIME ZONE) this.OPTION1(() => { this.CONSUME(With) this.CONSUME(Offset) - this.SUBRULE(this.stringOrIdentifier) + this.SUBRULE1(this.expression, { LABEL: "offsetExpression" }) }) }, }, diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index ccc318b..2c08ef4 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -224,13 +224,20 @@ function selectToSql(stmt: AST.SelectStatement): string { parts.push("FIRST OBSERVATION") } else { parts.push("CALENDAR") - if (stmt.sampleBy.alignTo.timeZone) { + const align = stmt.sampleBy.alignTo + if (align.timeZoneExpression) { parts.push("TIME ZONE") - parts.push(escapeString(stmt.sampleBy.alignTo.timeZone)) + parts.push(expressionToSql(align.timeZoneExpression)) + } else if (align.timeZone) { + parts.push("TIME ZONE") + parts.push(escapeString(align.timeZone)) } - if (stmt.sampleBy.alignTo.offset) { + if (align.offsetExpression) { + parts.push("WITH OFFSET") + parts.push(expressionToSql(align.offsetExpression)) + } else if (align.offset) { parts.push("WITH OFFSET") - parts.push(escapeString(stmt.sampleBy.alignTo.offset)) + parts.push(escapeString(align.offset)) } } } @@ -1838,7 +1845,7 @@ function pivotToSql(stmt: AST.PivotStatement): string { // Expressions // ============================================================================= -function expressionToSql(expr: AST.Expression): string { +export function expressionToSql(expr: AST.Expression): string { if (!expr) return "" switch (expr.type) { case "binary": diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 85251d6..c8063b4 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -9,6 +9,7 @@ type CstChildrenRecord = Record import { parser } from "./parser" import * as AST from "./ast" +import { expressionToSql } from "./toSql" import type { AddUserStatementCstChildren, AdditiveExpressionCstChildren, @@ -1063,13 +1064,9 @@ class QuestDBVisitor extends BaseVisitor { sampleByClause(ctx: SampleByClauseCstChildren): AST.SampleByClause { const result: AST.SampleByClause = { type: "sampleBy", - duration: ctx.DurationLiteral - ? this.tokenImage(ctx.DurationLiteral[0]) - : ctx.VariableReference - ? this.tokenImage(ctx.VariableReference[0]) - : ctx.BindVariable - ? `${this.tokenImage(ctx.BindVariable[0])} ${this.tokenImage(ctx.sampleByUnit![0])}` - : this.tokenImage(ctx.Identifier![0]), + duration: ctx.sampleByExpression + ? this.sampleByDuration(ctx.sampleByExpression[0], ctx.sampleByUnit) + : this.tokenImage(ctx.Identifier![0]), } if (ctx.groupingSetsSpec) { result.groupingSets = this.visit(ctx.groupingSetsSpec) as AST.GroupingSets @@ -1092,20 +1089,29 @@ class QuestDBVisitor extends BaseVisitor { return ctx.fillValue.map((v: CstNode) => this.visit(v) as string) } + /** SAMPLE BY text: the expression as SQL, plus the unit letter if any */ + private sampleByDuration(node: CstNode, unit: IToken[] | undefined): string { + // A lone duration literal keeps its image (`1h`, `30m`, `1d`) + const bare = this.bareDurationLiteral(node) + const text = bare + ? String(bare.value) + : expressionToSql(this.visit(node) as AST.Expression) + return unit ? `${text} ${this.tokenImage(unit[0])}` : text + } + + // FILL values: plain names (PREV, LINEAR, NONE, NULL) are upper-cased and + // PREV(...) keeps an upper-case name, as before; other values are printed + // as expressions (-1, @f, 1 + 2, '5'). fillValue(ctx: FillValueCstChildren): string { - if (ctx.Null) return "NULL" - if (ctx.NumberLiteral) return this.tokenImage(ctx.NumberLiteral[0]) - if (ctx.identifier) { - const name = this.extractIdentifierName( - ctx.identifier[0].children, - ).toUpperCase() - if (ctx.LParen && ctx.identifier[1]) { - const column = this.extractIdentifierName(ctx.identifier[1].children) - return `${name}(${column})` - } - return name + const expr = this.visit(ctx.expression) as AST.Expression + if (expr.type === "column" && expr.name.parts.length === 1) { + return expr.name.parts[0].toUpperCase() } - return "" + if (expr.type === "literal" && expr.literalType === "null") return "NULL" + if (expr.type === "function" && expr.name.toUpperCase() === "PREV") { + return expressionToSql({ ...expr, name: "PREV" }) + } + return expressionToSql(expr) } alignToClause(ctx: AlignToClauseCstChildren): AST.AlignToClause { @@ -1113,11 +1119,29 @@ class QuestDBVisitor extends BaseVisitor { type: "alignTo", mode: ctx.Calendar ? "calendar" : "firstObservation", } - if (ctx.Zone && ctx.timeZoneValue) { - result.timeZone = this.extractMaybeString(ctx.timeZoneValue[0]) - } - if (ctx.Offset && ctx.stringOrIdentifier) { - result.offset = this.extractMaybeString(ctx.stringOrIdentifier[0]) + // A string literal or plain name stays a string; anything else (@tz, $1, + // f(...)) is kept as an expression too. + const value = ( + node: CstNode, + ): { text: string; expression?: AST.Expression } => { + const expr = this.visit(node) as AST.Expression + if (expr.type === "literal" && expr.literalType === "string") { + return { text: String(expr.value) } + } + if (expr.type === "column" && expr.name.parts.length === 1) { + return { text: expr.name.parts[0] } + } + return { text: expressionToSql(expr), expression: expr } + } + if (ctx.timeZoneExpression) { + const v = value(ctx.timeZoneExpression[0]) + result.timeZone = v.text + if (v.expression) result.timeZoneExpression = v.expression + } + if (ctx.offsetExpression) { + const v = value(ctx.offsetExpression[0]) + result.offset = v.text + if (v.expression) result.offsetExpression = v.expression } return result } diff --git a/tests/bind-variables.test.ts b/tests/bind-variables.test.ts index defa197..f16fcac 100644 --- a/tests/bind-variables.test.ts +++ b/tests/bind-variables.test.ts @@ -249,11 +249,8 @@ describe("round trip", () => { describe("errors", () => { it.each([ - // SqlParser.expectSample: a bind variable interval needs a unit letter - "SELECT * FROM t SAMPLE BY $1", - "SELECT * FROM t SAMPLE BY $1 FILL(NULL)", - // not a sample by period letter - "SELECT * FROM t SAMPLE BY $1 x", + // (SAMPLE BY $1 without a unit letter is accepted: SAMPLE BY takes any + // expression and the unit stays optional, looser than SqlParser.expectSample) // a bare colon is not a value "SELECT * FROM t WHERE a = : sym", // a colon after a completed operand is not a bind variable diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 9ec632e..8976d74 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1512,5 +1512,53 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/WithClauseTest.java:654", "feature": "declare", "toSql": "WITH c AS (SELECT 'a'::SYMBOL AS x) SELECT * FROM c UNION ALL WITH c AS (SELECT 'b'::SYMBOL AS x) SELECT * FROM c UNION ALL SELECT * FROM c" + }, + { + "query": "SELECT ts, percentile_cont(v, 0.25) p FROM t SAMPLE BY 1h FILL(-1)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/groupby/PercentileContDoubleGroupByFunctionFactoryTest.java:530", + "feature": "sample-by-fill-align", + "toSql": "SELECT ts, percentile_cont(v, 0.25) AS p FROM t SAMPLE BY 1h FILL(-1)" + }, + { + "query": "SELECT ts, sum(a) FROM t SAMPLE BY 1h FILL(PREV(abs(a))) ALIGN TO CALENDAR", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/groupby/SampleByFillPrevTest.java:1988", + "feature": "sample-by-fill-align", + "toSql": "SELECT ts, sum(a) FROM t SAMPLE BY 1h FILL(PREV(abs(a))) ALIGN TO CALENDAR" + }, + { + "query": "DECLARE @f := (SELECT 1) SELECT ts, count() FROM k SAMPLE BY 12h FILL(@f)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:2028", + "feature": "sample-by-fill-align", + "toSql": "DECLARE @f := (SELECT 1) SELECT ts, count() FROM k SAMPLE BY 12h FILL(@f)" + }, + { + "query": "select b, sum(a), k k1, k from x y sample by 3h align to calendar time zone rnd_str('foo','bar') with offset '00:15'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SampleBySqlParserTest.java:57", + "feature": "sample-by-fill-align", + "toSql": "SELECT b, sum(a), k AS k1, k FROM x AS y SAMPLE BY 3h ALIGN TO CALENDAR TIME ZONE rnd_str('foo', 'bar') WITH OFFSET '00:15'" + }, + { + "query": "DECLARE @tz := 'Antarctica/McMurdo' SELECT timestamp, symbol, avg(price) FROM trades SAMPLE BY 1h ALIGN TO CALENDAR TIME ZONE @tz", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:1036", + "feature": "sample-by-fill-align", + "toSql": "DECLARE @tz := 'Antarctica/McMurdo' SELECT timestamp, symbol, avg(price) FROM trades SAMPLE BY 1h ALIGN TO CALENDAR TIME ZONE @tz" + }, + { + "query": "DECLARE @q := (SELECT 1) SELECT ts, count() FROM k SAMPLE BY @q h", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:3750", + "feature": "sample-by-fill-align", + "toSql": "DECLARE @q := (SELECT 1) SELECT ts, count() FROM k SAMPLE BY @q h" + }, + { + "query": "select sum(a), k from x sample by (1+2)*10L m align to calendar", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/groupby/SampleByNanoTimestampTest.java:16610", + "feature": "sample-by-fill-align", + "toSql": "SELECT sum(a), k FROM x SAMPLE BY (1 + 2) * 10L m ALIGN TO CALENDAR" + }, + { + "query": "select timestamp + 60000000 as 'timestamp', timestamp, count() from t where name = 'a' sample by (1m)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlOptimiserTest.java:4550", + "feature": "sample-by-fill-align", + "toSql": "SELECT timestamp + 60000000 AS timestamp, timestamp, count() FROM t WHERE name = 'a' SAMPLE BY (1m)" } ] diff --git a/tests/parser.test.ts b/tests/parser.test.ts index 53d9159..f89f0bf 100644 --- a/tests/parser.test.ts +++ b/tests/parser.test.ts @@ -7944,10 +7944,11 @@ orders PIVOT (sum(amount) FOR status IN ('open'))` if (s.type === "select") expect(s.sampleBy?.duration).toBe(u) }) } - it("rejects an arbitrary identifier unit", () => { - expect( - parseToAst("SELECT ts, avg(x) FROM t SAMPLE BY foo").errors.length, - ).toBeGreaterThan(0) + it("parses any other name as a SAMPLE BY expression", () => { + // SAMPLE BY takes an expression (SqlParser.expectSample); the server + // then wants a full period or a unit letter, sql-parser stays looser. + const r = parseToAst("SELECT ts, avg(x) FROM t SAMPLE BY foo") + expect(r.errors).toHaveLength(0) }) }) diff --git a/tests/sample-by-fill-align.test.ts b/tests/sample-by-fill-align.test.ts new file mode 100644 index 0000000..a32e22c --- /dev/null +++ b/tests/sample-by-fill-align.test.ts @@ -0,0 +1,100 @@ +/** + * SAMPLE BY forms the QuestDB server accepts (SqlParser.expectSample and the + * FILL / ALIGN TO parsing after it): + * - SAMPLE BY [one-letter unit]: `@q h`, `2 h`, `(1m)`, `(10+20)m`, `$1 d` + * - FILL(, ...): `-1`, `@f`, `true`, `PREV(abs(a))`, `1+2`, `100L`, `NaN`, `'5'` + * - ALIGN TO CALENDAR TIME ZONE [WITH OFFSET ] + * + * Statement-level server queries live in tests/fixtures/server-queries.json + * (feature "sample-by-fill-align"). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, createAutocompleteProvider } from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const ok = (sql: string): AST.SelectStatement => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + const out = toSql(result.ast) + // canonical output re-parses to itself + const again = parseToAst(out) + expect(again.errors).toEqual([]) + expect(toSql(again.ast)).toBe(out) + return result.ast[0] as AST.SelectStatement +} + +describe("SAMPLE BY expression + unit", () => { + it.each([ + ["SELECT count() FROM t SAMPLE BY @q h", "@q h"], + ["SELECT count() FROM t SAMPLE BY 2 h", "2 h"], + ["SELECT count() FROM t SAMPLE BY (1m)", "(1m)"], + ["SELECT count() FROM t SAMPLE BY (10+20)m", "(10 + 20) m"], + ["SELECT count() FROM t SAMPLE BY $1 d", "$1 d"], + ["SELECT count() FROM t SAMPLE BY 1h", "1h"], + ["SELECT count() FROM t SAMPLE BY w", "w"], + ["SELECT count() FROM t SAMPLE BY @unit", "@unit"], + ])("%s", (sql, duration) => { + expect(ok(sql).sampleBy!.duration).toBe(duration) + }) +}) + +describe("FILL with any expression", () => { + it.each([ + ["FILL(-1)", ["-1"]], + ["FILL(prev, -1, -1.5, prev)", ["PREV", "-1", "-1.5", "PREV"]], + ["FILL(@f)", ["@f"]], + ["FILL(true)", ["TRUE"]], + ["FILL(prev(abs(a)))", ["PREV(abs(a))"]], + ["FILL(PREV(a))", ["PREV(a)"]], + ["FILL(1+2)", ["1 + 2"]], + ["FILL(100L)", ["100L"]], + ["FILL(NaN)", ["NaN"]], + ["FILL('5')", ["'5'"]], + ["FILL(NULL, LINEAR, NONE)", ["NULL", "LINEAR", "NONE"]], + ])("%s", (fill, values) => { + expect( + ok(`SELECT count() FROM t SAMPLE BY 1h ${fill}`).sampleBy!.fill, + ).toEqual(values) + }) + + it("keeps FILL keyword suggestions", () => { + const labels = (sql: string) => + createAutocompleteProvider({ tables: [], columns: {} }) + .getSuggestions(sql, sql.length) + .map((s) => s.label) + const fill = labels("SELECT count() FROM t SAMPLE BY 1h FILL(") + for (const k of ["NULL", "PREV", "LINEAR", "NONE"]) + expect(fill).toContain(k) + // no function / operator noise at the top level of FILL / SAMPLE BY + expect(fill).not.toContain("CASE") + expect(labels("SELECT count() FROM t SAMPLE BY ")).not.toContain("CASE") + }) +}) + +describe("ALIGN TO CALENDAR TIME ZONE / WITH OFFSET expressions", () => { + it("keeps string values", () => { + const s = ok( + "SELECT count() FROM t SAMPLE BY 1h ALIGN TO CALENDAR TIME ZONE 'Europe/London' WITH OFFSET '00:15'", + ) + expect(s.sampleBy!.alignTo).toEqual({ + type: "alignTo", + mode: "calendar", + timeZone: "Europe/London", + offset: "00:15", + }) + }) + + it.each([ + "TIME ZONE @tz", + "WITH OFFSET @o", + "TIME ZONE concat('Europe', '/London')", + "TIME ZONE $1 WITH OFFSET $2", + ])("%s", (align) => { + const sql = `SELECT count() FROM t SAMPLE BY 1h ALIGN TO CALENDAR ${align}` + ok(sql) + expect(toSql(parseToAst(sql).ast)).toBe(sql) + }) +}) From e1ef5c03744d91302e604ee37d55dbf99f25f010 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:27:15 +0300 Subject: [PATCH 19/36] feat: accept server clause orders (LATEST before WHERE/SAMPLE BY, PIVOT before SAMPLE/GROUP BY), LATEST ON @var, open-ended LIMIT Server (SqlParser.parseFromClause): deprecated LATEST BY right after the table, then joins -> WHERE -> LATEST ON -> PIVOT -> SAMPLE BY -> GROUP BY -> WINDOW -> SUBSAMPLE -> ORDER BY -> LIMIT. LATEST ON/BY operands go through expectLiteral with decls, so @variables are allowed. LIMIT accepts `5,` and `,5` (missing lower bound defaults to 0, SqlCodeGenerator.getLoFunction). Before: `... LATEST ON ts PARTITION BY s SAMPLE BY 1d`, `... PIVOT (...) GROUP BY|SAMPLE BY ...`, `FROM t LATEST BY s WHERE ...`, `PARTITION BY @c`, `LIMIT 5,` and `LIMIT ,5` were errors. Now: simpleSelect and implicitSelectBody share selectClauseChain(), a set of alternatives covering the server order AND every order accepted before (SAMPLE BY -> LATEST, GROUP BY -> PIVOT, WHERE -> LATEST BY); written as alternatives so autocomplete does not offer a clause twice. New rule latestOnColumn (columnRef | @var; AST keeps a one-part name "@var", printed raw). limitClause: `LIMIT ,5` -> AST lowerBound literal 0 (toSql `LIMIT 0, 5`), `LIMIT 5,` -> `LIMIT 5`. AST: optional SelectStatement.pivotBeforeGroupBy so toSql keeps PIVOT before SAMPLE/GROUP BY. toSql now prints LATEST BY before WHERE and LATEST ON before SAMPLE BY (server order; same AST meaning). Tests: 8 server queries (LatestByHoistEquivalenceTest:139, LatestByTest:1472, QueryExecutionTimeoutTest:331, SqlParserTest:8285, DeclareTest:2005/875, LimitTest:176/192) + tests/clause-order.test.ts. Red 26 -> green; full suite 16910 passed. Corpus: fixed 18 / rtChanged 1 (`(tab where x = 100 latest by x)` now prints LATEST BY before WHERE) / regressed 0 / throws 0 / fmtChangedMeaning 0. Suggestions: +WHERE,SAMPLE after `LATEST BY|ON ... ` (WHERE is only valid after LATEST BY; ruleStack can't tell them apart), +SAMPLE,GROUP after `PIVOT (...) `, +UNION/EXCEPT/INTERSECT after `LIMIT 5, ` (open upper bound). --- src/parser/ast.ts | 5 + src/parser/cst-types.d.ts | 48 +++++---- src/parser/parser.ts | 162 +++++++++++++++++++++++------ src/parser/toSql.ts | 42 +++++--- src/parser/visitor.ts | 50 ++++++++- tests/clause-order.test.ts | 87 ++++++++++++++++ tests/fixtures/server-queries.json | 48 +++++++++ 7 files changed, 373 insertions(+), 69 deletions(-) create mode 100644 tests/clause-order.test.ts diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 205acf4..4e0726f 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -79,6 +79,11 @@ export interface SelectStatement extends AstNode { /** GROUP BY ROLLUP(...) / CUBE(...) / GROUPING SETS (...), after any plain keys */ groupingSets?: GroupingSets pivot?: PivotClause + /** + * PIVOT written before SAMPLE BY / GROUP BY (server order): those clauses + * then apply to the pivoted result. + */ + pivotBeforeGroupBy?: boolean /** Named window definitions: SELECT ... WINDOW w AS (...) [, w2 AS (...)] */ namedWindows?: NamedWindow[] /** SUBSAMPLE method(args): server-side downsampling, before ORDER BY */ diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 07d45d7..2dc12dd 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -132,14 +132,14 @@ export type SimpleSelectCstChildren = { selectList: SelectListCstNode[]; From?: IToken[]; fromClause?: FromClauseCstNode[]; - whereClause?: WhereClauseCstNode[]; - sampleByClause?: SampleByClauseCstNode[]; - latestOnClause?: LatestOnClauseCstNode[]; - groupByClause?: GroupByClauseCstNode[]; - Pivot?: IToken[]; - LParen?: IToken[]; - pivotBody?: PivotBodyCstNode[]; - RParen?: IToken[]; + latestOnClause?: (LatestOnClauseCstNode)[]; + whereClause?: (WhereClauseCstNode)[]; + Pivot?: (IToken)[]; + LParen?: (IToken)[]; + pivotBody?: (PivotBodyCstNode)[]; + RParen?: (IToken)[]; + sampleByClause?: (SampleByClauseCstNode)[]; + groupByClause?: (GroupByClauseCstNode)[]; windowClause?: WindowClauseCstNode[]; subsampleClause?: SubsampleClauseCstNode[]; orderByClause?: OrderByClauseCstNode[]; @@ -266,14 +266,14 @@ export interface ImplicitSelectBodyCstNode extends CstNode { export type ImplicitSelectBodyCstChildren = { fromClause: FromClauseCstNode[]; - whereClause?: WhereClauseCstNode[]; - sampleByClause?: SampleByClauseCstNode[]; - latestOnClause?: LatestOnClauseCstNode[]; - groupByClause?: GroupByClauseCstNode[]; - Pivot?: IToken[]; - LParen?: IToken[]; - pivotBody?: PivotBodyCstNode[]; - RParen?: IToken[]; + latestOnClause?: (LatestOnClauseCstNode)[]; + whereClause?: (WhereClauseCstNode)[]; + Pivot?: (IToken)[]; + LParen?: (IToken)[]; + pivotBody?: (PivotBodyCstNode)[]; + RParen?: (IToken)[]; + sampleByClause?: (SampleByClauseCstNode)[]; + groupByClause?: (GroupByClauseCstNode)[]; subsampleClause?: SubsampleClauseCstNode[]; orderByClause?: OrderByClauseCstNode[]; limitClause?: LimitClauseCstNode[]; @@ -524,12 +524,22 @@ export interface LatestOnClauseCstNode extends CstNode { export type LatestOnClauseCstChildren = { Latest: IToken[]; On?: IToken[]; - columnRef?: (ColumnRefCstNode)[]; + latestOnColumn?: (LatestOnColumnCstNode)[]; Partition?: IToken[]; By?: (IToken)[]; Comma?: (IToken)[]; }; +export interface LatestOnColumnCstNode extends CstNode { + name: "latestOnColumn"; + children: LatestOnColumnCstChildren; +} + +export type LatestOnColumnCstChildren = { + VariableReference?: IToken[]; + columnRef?: ColumnRefCstNode[]; +}; + export interface SubsampleClauseCstNode extends CstNode { name: "subsampleClause"; children: SubsampleClauseCstChildren; @@ -690,7 +700,8 @@ export interface LimitClauseCstNode extends CstNode { export type LimitClauseCstChildren = { Limit: IToken[]; - expression: (ExpressionCstNode)[]; + openLowerBound?: IToken[]; + expression?: (ExpressionCstNode)[]; Comma?: IToken[]; }; @@ -3173,6 +3184,7 @@ export interface ICstNodeVisitor extends ICstVisitor { whereClause(children: WhereClauseCstChildren, param?: IN): OUT; sampleByClause(children: SampleByClauseCstChildren, param?: IN): OUT; latestOnClause(children: LatestOnClauseCstChildren, param?: IN): OUT; + latestOnColumn(children: LatestOnColumnCstChildren, param?: IN): OUT; subsampleClause(children: SubsampleClauseCstChildren, param?: IN): OUT; fillClause(children: FillClauseCstChildren, param?: IN): OUT; fillValue(children: FillValueCstChildren, param?: IN): OUT; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index e120bf0..35f39a2 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -741,17 +741,8 @@ class QuestDBParser extends CstParser { this.OPTION1(() => { this.CONSUME(From) this.SUBRULE(this.fromClause) - this.OPTION2(() => this.SUBRULE(this.whereClause)) - this.OPTION3(() => this.SUBRULE(this.sampleByClause)) - this.OPTION4(() => this.SUBRULE(this.latestOnClause)) - this.OPTION5(() => this.SUBRULE(this.groupByClause)) - // PIVOT clause: SELECT * FROM t PIVOT (agg FOR col IN (...)) - this.OPTION9(() => { - this.CONSUME(Pivot) - this.CONSUME(LParen) - this.SUBRULE(this.pivotBody) - this.CONSUME(RParen) - }) + // WHERE / LATEST / PIVOT / SAMPLE BY / GROUP BY + this.selectClauseChain() // Named window clause: WINDOW w AS (...) [, w2 AS (...)] // Standard SQL places WINDOW between HAVING/GROUP BY and ORDER BY. // GATE: disambiguate from WINDOW JOIN (which lives inside fromClause, @@ -769,6 +760,101 @@ class QuestDBParser extends CstParser { }) }) + // WHERE / LATEST ON|BY / PIVOT / SAMPLE BY / GROUP BY of simpleSelect and + // implicitSelectBody (inlined into the calling rule, so the CST children + // stay on that rule). Server order (SqlParser.parseFromClause): + // [LATEST BY (deprecated)] WHERE, LATEST ON, PIVOT, SAMPLE BY, GROUP BY + // The orders accepted before are kept: SAMPLE BY -> LATEST and + // GROUP BY -> PIVOT. Written as alternatives (not a run of OPTIONs) so that + // autocomplete never offers a clause that was already used. + // Indexes start at 20 to stay clear of the calling rules' own indexes. + private selectClauseChain(): void { + // Every call site needs its own fixed occurrence index (the same at + // recording time and at parse time), hence the explicit numbers. + const latest = (i: number) => this.subrule(i, this.latestOnClause) + const where = (i: number) => this.subrule(i, this.whereClause) + const sample = (i: number) => this.subrule(i, this.sampleByClause) + const group = (i: number) => this.subrule(i, this.groupByClause) + const pivot = (i: number) => { + this.consume(i, Pivot) + this.consume(i, LParen) + this.subrule(i, this.pivotBody) + this.consume(i, RParen) + } + // Each helper call gets its own base `b` (spaced by 10) and uses each + // subrule at most once with index b, and option indexes b..b+2. + // PIVOT [SAMPLE BY] [GROUP BY] + const fromPivot = (b: number) => { + pivot(b) + this.option(b, () => sample(b)) + this.option(b + 1, () => group(b)) + } + // SAMPLE BY [LATEST] [GROUP BY] [PIVOT] + const fromSample = (b: number) => { + sample(b) + this.option(b, () => latest(b)) + this.option(b + 1, () => group(b)) + this.option(b + 2, () => pivot(b)) + } + // GROUP BY [PIVOT] + const fromGroup = (b: number) => { + group(b) + this.option(b, () => pivot(b)) + } + // after LATEST: PIVOT [SAMPLE BY] [GROUP BY] | SAMPLE BY [GROUP BY] + // | GROUP BY [PIVOT] + const afterLatest = (b: number): void => { + this.option(b, () => { + this.or(b, [ + { ALT: () => fromPivot(b + 10) }, + { + ALT: () => { + sample(b + 20) + this.option(b + 20, () => group(b + 20)) + }, + }, + { ALT: () => fromGroup(b + 30) }, + ]) + }) + } + const withoutLatest = (b: number): void => { + this.or(b, [ + { ALT: () => fromPivot(b + 10) }, + { ALT: () => fromSample(b + 20) }, + { ALT: () => fromGroup(b + 30) }, + ]) + } + this.option(20, () => { + this.or(20, [ + { + // LATEST BY|ON [WHERE] ... (LATEST BY right after the table) + ALT: () => { + latest(20) + this.option(21, () => where(20)) + afterLatest(30) + }, + }, + { + ALT: () => { + where(21) + this.option(22, () => { + this.or(21, [ + { + ALT: () => { + latest(21) + afterLatest(70) + }, + }, + { ALT: () => withoutLatest(110) }, + ]) + }) + }, + }, + { ALT: () => withoutLatest(150) }, + ]) + }) + } + private setOperation = this.RULE("setOperation", () => { this.OR([ { ALT: () => this.CONSUME(Union) }, @@ -966,17 +1052,8 @@ class QuestDBParser extends CstParser { // QuestDB treats these as implicit SELECT * FROM ... private implicitSelectBody = this.RULE("implicitSelectBody", () => { this.SUBRULE(this.fromClause) - this.OPTION(() => this.SUBRULE(this.whereClause)) - this.OPTION1(() => this.SUBRULE(this.sampleByClause)) - this.OPTION2(() => this.SUBRULE(this.latestOnClause)) - this.OPTION3(() => this.SUBRULE(this.groupByClause)) - // PIVOT clause, as in simpleSelect: (t PIVOT (...) ORDER BY ...) - this.OPTION7(() => { - this.CONSUME(Pivot) - this.CONSUME(LParen) - this.SUBRULE(this.pivotBody) - this.CONSUME(RParen) - }) + // WHERE / LATEST / PIVOT / SAMPLE BY / GROUP BY, as in simpleSelect + this.selectClauseChain() this.OPTION6(() => this.SUBRULE(this.subsampleClause)) this.OPTION4(() => this.SUBRULE(this.orderByClause)) this.OPTION5(() => this.SUBRULE(this.limitClause)) @@ -1405,13 +1482,13 @@ class QuestDBParser extends CstParser { GATE: () => this.LA(1).tokenType === On, ALT: () => { this.CONSUME(On) - this.SUBRULE(this.columnRef) + this.SUBRULE(this.latestOnColumn) this.CONSUME(Partition) this.CONSUME(By) - this.SUBRULE1(this.columnRef) + this.SUBRULE1(this.latestOnColumn) this.MANY(() => { this.CONSUME(Comma) - this.SUBRULE2(this.columnRef) + this.SUBRULE2(this.latestOnColumn) }) }, }, @@ -1419,16 +1496,24 @@ class QuestDBParser extends CstParser { // LATEST BY col [, col...] ALT: () => { this.CONSUME1(By) - this.SUBRULE3(this.columnRef) + this.SUBRULE3(this.latestOnColumn) this.MANY1(() => { this.CONSUME1(Comma) - this.SUBRULE4(this.columnRef) + this.SUBRULE4(this.latestOnColumn) }) }, }, ]) }) + // LATEST ON/BY operand: column or @variable (expectLiteral with decls) + private latestOnColumn = this.RULE("latestOnColumn", () => { + this.OR([ + { ALT: () => this.CONSUME(VariableReference) }, + { ALT: () => this.SUBRULE(this.columnRef) }, + ]) + }) + // SUBSAMPLE method(arg, ...): server-side downsampling that returns original // rows (uniform, cadence, m4, minmax, lttb, sdt). The method name goes // through `identifier`, as QuestDB's parser reads any word here and the @@ -1626,11 +1711,24 @@ class QuestDBParser extends CstParser { private limitClause = this.RULE("limitClause", () => { this.CONSUME(Limit) - this.SUBRULE(this.expression) - this.OPTION(() => { - this.CONSUME(Comma) - this.SUBRULE1(this.expression) - }) + // Open-ended limits `LIMIT 5,` and `LIMIT ,5` are accepted by the server + this.OR([ + { + ALT: () => { + this.CONSUME1(Comma, { LABEL: "openLowerBound" }) + this.SUBRULE2(this.expression) + }, + }, + { + ALT: () => { + this.SUBRULE(this.expression) + this.OPTION(() => { + this.CONSUME(Comma) + this.OPTION1(() => this.SUBRULE1(this.expression)) + }) + }, + }, + ]) }) // ========================================================================== diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 2c08ef4..3a1f9e4 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -191,12 +191,27 @@ function selectToSql(stmt: AST.SelectStatement): string { } } + // deprecated LATEST BY: right after the table (server order) + if (stmt.latestOn && !stmt.latestOn.timestamp) { + parts.push(latestOnToSql(stmt.latestOn)) + } + // WHERE if (stmt.where) { parts.push("WHERE") parts.push(expressionToSql(stmt.where)) } + // LATEST ON (server order: after WHERE, before PIVOT / SAMPLE BY) + if (stmt.latestOn?.timestamp) { + parts.push(latestOnToSql(stmt.latestOn)) + } + + // PIVOT written before SAMPLE BY / GROUP BY + if (stmt.pivot && stmt.pivotBeforeGroupBy) { + parts.push(pivotClauseToSql(stmt.pivot)) + } + // SAMPLE BY (QuestDB) if (stmt.sampleBy) { parts.push("SAMPLE BY") @@ -243,19 +258,6 @@ function selectToSql(stmt: AST.SelectStatement): string { } } - // LATEST ON / LATEST BY (QuestDB) - if (stmt.latestOn) { - if (stmt.latestOn.timestamp) { - parts.push("LATEST ON") - parts.push(qualifiedNameToSql(stmt.latestOn.timestamp)) - parts.push("PARTITION BY") - parts.push(stmt.latestOn.partitionBy.map(qualifiedNameToSql).join(", ")) - } else { - parts.push("LATEST BY") - parts.push(stmt.latestOn.partitionBy.map(qualifiedNameToSql).join(", ")) - } - } - // GROUP BY [plain keys][, ROLLUP(...) | CUBE(...) | GROUPING SETS (...)] const groupByItems = (stmt.groupBy ?? []).map(expressionToSql) if (stmt.groupingSets) { @@ -267,7 +269,7 @@ function selectToSql(stmt: AST.SelectStatement): string { } // PIVOT (inline pivot clause in SELECT) - if (stmt.pivot) { + if (stmt.pivot && !stmt.pivotBeforeGroupBy) { parts.push(pivotClauseToSql(stmt.pivot)) } @@ -2225,6 +2227,18 @@ function arrayAccessToSql(expr: AST.ArrayAccessExpression): string { // Helpers // ============================================================================= +function latestOnToSql(latest: AST.LatestOnClause): string { + // @variables are kept as one-part names and printed as written + const name = (n: AST.QualifiedName) => + n.parts.length === 1 && n.parts[0].startsWith("@") + ? n.parts[0] + : qualifiedNameToSql(n) + const keys = latest.partitionBy.map(name).join(", ") + return latest.timestamp + ? `LATEST ON ${name(latest.timestamp)} PARTITION BY ${keys}` + : `LATEST BY ${keys}` +} + function qualifiedNameToSql(name: AST.QualifiedName): string { if (!name || !name.parts) return "" return name.parts.map(escapeIdentifier).join(".") diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index c8063b4..199901f 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -115,6 +115,7 @@ import type { IntervalValueCstChildren, JoinClauseCstChildren, LatestOnClauseCstChildren, + LatestOnColumnCstChildren, LimitClauseCstChildren, LiteralCstChildren, TimestampWithTimeZoneLiteralCstChildren, @@ -540,6 +541,7 @@ class QuestDBVisitor extends BaseVisitor { pivots: body.pivots, groupBy: body.groupBy, } + this.markPivotPosition(result, ctx) } if (ctx.windowClause) { @@ -771,6 +773,7 @@ class QuestDBVisitor extends BaseVisitor { pivots: body.pivots, groupBy: body.groupBy, } + this.markPivotPosition(result, ctx) } } if (ctx.subsampleClause) { @@ -1159,23 +1162,47 @@ class QuestDBVisitor extends BaseVisitor { } latestOnClause(ctx: LatestOnClauseCstChildren): AST.LatestOnClause { - const columnRefs = ctx.columnRef!.map( - (c: CstNode) => this.visit(c) as AST.ColumnRef, + const names = (ctx.latestOnColumn ?? []).map( + (c: CstNode) => this.visit(c) as AST.QualifiedName, ) if (ctx.On) { // LATEST ON timestamp PARTITION BY col1, col2, ... return { type: "latestOn", - timestamp: columnRefs[0].name, - partitionBy: columnRefs.slice(1).map((c: AST.ColumnRef) => c.name), + timestamp: names[0], + partitionBy: names.slice(1), } } // LATEST BY col1, col2, ... return { type: "latestOn", - partitionBy: columnRefs.map((c: AST.ColumnRef) => c.name), + partitionBy: names, + } + } + + // Column or @variable; a variable is kept as a one-part name "@var" + latestOnColumn(ctx: LatestOnColumnCstChildren): AST.QualifiedName { + if (ctx.VariableReference) { + return { + type: "qualifiedName", + parts: [ctx.VariableReference[0].image], + } } + return (this.visit(ctx.columnRef!) as AST.ColumnRef).name + } + + /** Records a PIVOT written before SAMPLE BY / GROUP BY (server order). */ + private markPivotPosition( + result: AST.SelectStatement, + ctx: SimpleSelectCstChildren | ImplicitSelectBodyCstChildren, + ): void { + const pivotOffset = ctx.Pivot?.[0]?.startOffset + if (pivotOffset === undefined) return + const later = [ctx.sampleByClause?.[0], ctx.groupByClause?.[0]].some( + (c) => c !== undefined && this.getFirstTokenOffset(c) > pivotOffset, + ) + if (later) result.pivotBeforeGroupBy = true } subsampleClause(ctx: SubsampleClauseCstChildren): AST.SubsampleClause { @@ -1260,9 +1287,22 @@ class QuestDBVisitor extends BaseVisitor { } limitClause(ctx: LimitClauseCstChildren): AST.LimitClause { + if (!ctx.expression) { + // Only reachable from error recovery (`LIMIT;`) + throw new Error("LIMIT expression expected") + } const expressions = ctx.expression.map( (e: CstNode) => this.visit(e) as AST.Expression, ) + // `LIMIT ,5`: the server defaults the missing lower bound to 0 + if (ctx.openLowerBound) { + expressions.unshift({ + type: "literal", + value: 0, + literalType: "number", + raw: "0", + }) + } const result: AST.LimitClause = { type: "limit", diff --git a/tests/clause-order.test.ts b/tests/clause-order.test.ts new file mode 100644 index 0000000..28de9e7 --- /dev/null +++ b/tests/clause-order.test.ts @@ -0,0 +1,87 @@ +/** + * Server clause order (SqlParser.parseFromClause): + * joins -> WHERE -> LATEST ON -> PIVOT -> SAMPLE BY -> GROUP BY -> WINDOW + * -> SUBSAMPLE -> ORDER BY -> LIMIT + * plus the deprecated `LATEST BY` right after the table (before WHERE), + * `@var` operands in LATEST ON, and open-ended `LIMIT 5,` / `LIMIT ,5`. + * The orders sql-parser accepted before (SAMPLE BY -> LATEST ON, + * GROUP BY -> PIVOT, WHERE -> LATEST BY) must keep parsing. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst } from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const ok = (sql: string): AST.SelectStatement => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] as AST.SelectStatement +} + +const roundTrip = (sql: string, expected: string) => { + const out = toSql(ok(sql)) + expect(out).toBe(expected) + expect(toSql(ok(out))).toBe(expected) +} + +describe("clause order", () => { + it("LATEST ON before SAMPLE BY", () => { + roundTrip( + "SELECT ts, count() FROM t WHERE x > 1 LATEST ON ts PARTITION BY sym SAMPLE BY 1d", + "SELECT ts, count() FROM t WHERE x > 1 LATEST ON ts PARTITION BY sym SAMPLE BY 1d", + ) + }) + + it("PIVOT before GROUP BY / SAMPLE BY keeps its position", () => { + const sel = ok( + "SELECT * FROM t PIVOT (sum(p) FOR y IN (2000, 2010)) GROUP BY x", + ) + expect(sel.pivot).toBeDefined() + expect(sel.groupBy).toHaveLength(1) + roundTrip( + "SELECT * FROM t PIVOT (sum(p) FOR y IN (2000, 2010)) GROUP BY x", + "SELECT * FROM t PIVOT (sum(p) FOR y IN (2000, 2010)) GROUP BY x", + ) + roundTrip( + "SELECT * FROM t WHERE a > 1 LATEST ON ts PARTITION BY s PIVOT (sum(p) FOR y IN (1)) SAMPLE BY 1h", + "SELECT * FROM t WHERE a > 1 LATEST ON ts PARTITION BY s PIVOT (sum(p) FOR y IN (1)) SAMPLE BY 1h", + ) + // implicit select body + roundTrip( + "SELECT * FROM (t PIVOT (sum(p) FOR y IN (1)) GROUP BY x)", + "SELECT * FROM (t PIVOT (sum(p) FOR y IN (1)) GROUP BY x)", + ) + }) + + it("deprecated LATEST BY before WHERE", () => { + const sel = ok("SELECT * FROM t LATEST BY s WHERE x > 1") + expect(sel.latestOn?.partitionBy[0].parts).toEqual(["s"]) + expect(sel.where).toBeDefined() + roundTrip( + "SELECT * FROM (t LATEST BY s WHERE x > 1)", + "SELECT * FROM (t LATEST BY s WHERE x > 1)", + ) + }) + + it("LATEST ON with @variables", () => { + roundTrip( + "SELECT * FROM t LATEST ON @ts PARTITION BY @a, b", + "SELECT * FROM t LATEST ON @ts PARTITION BY @a, b", + ) + }) + + it("open-ended LIMIT", () => { + expect(ok("SELECT * FROM t LIMIT 5,").limit?.upperBound).toBeUndefined() + roundTrip("SELECT * FROM t LIMIT 5,", "SELECT * FROM t LIMIT 5") + roundTrip("SELECT * FROM t LIMIT ,5", "SELECT * FROM t LIMIT 0, 5") + roundTrip("SELECT * FROM t LIMIT 1, 5", "SELECT * FROM t LIMIT 1, 5") + }) + + it("previously accepted orders still parse", () => { + ok("SELECT ts, count() FROM t SAMPLE BY 1d LATEST ON ts PARTITION BY s") + ok("SELECT * FROM t WHERE x > 1 LATEST BY s") + ok("SELECT * FROM t GROUP BY x PIVOT (sum(p) FOR y IN (1))") + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 8976d74..6641afb 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1560,5 +1560,53 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlOptimiserTest.java:4550", "feature": "sample-by-fill-align", "toSql": "SELECT timestamp + 60000000 AS timestamp, timestamp, count() FROM t WHERE name = 'a' SAMPLE BY (1m)" + }, + { + "query": "SELECT ts, count() FROM (SELECT * FROM t) LATEST ON ts PARTITION BY sym SAMPLE BY 1d", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/LatestByHoistEquivalenceTest.java:139", + "feature": "clause-order", + "toSql": "SELECT ts, count() FROM (SELECT * FROM t) LATEST ON ts PARTITION BY sym SAMPLE BY 1d" + }, + { + "query": "select ts, count() c from (select * from nb) latest on ts partition by s sample by 1d", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/LatestByTest.java:1472", + "feature": "clause-order", + "toSql": "SELECT ts, count() AS c FROM (SELECT * FROM nb) LATEST ON ts PARTITION BY s SAMPLE BY 1d" + }, + { + "query": "select * from x latest by b where b = 'HNR'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/QueryExecutionTimeoutTest.java:331", + "feature": "clause-order", + "toSql": "SELECT * FROM x LATEST BY b WHERE b = 'HNR'" + }, + { + "query": "select * from x latest by b where b = 'PEHN' and a < 22 and test_match()", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:8285", + "feature": "clause-order", + "toSql": "SELECT * FROM x LATEST BY b WHERE b = 'PEHN' AND a < 22 AND test_match()" + }, + { + "query": "DECLARE @c := (SELECT 1) SELECT * FROM k LATEST ON ts PARTITION BY @c", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:2005", + "feature": "clause-order", + "toSql": "DECLARE @c := (SELECT 1) SELECT * FROM k LATEST ON ts PARTITION BY @c" + }, + { + "query": "DECLARE @ts := timestamp, @sym := symbol SELECT * FROM trades LATEST ON @ts PARTITION BY @sym;", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:875", + "feature": "clause-order", + "toSql": "DECLARE @ts := timestamp, @sym := symbol SELECT * FROM trades LATEST ON @ts PARTITION BY @sym" + }, + { + "query": "select * from y limit 5,", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/LimitTest.java:176", + "feature": "clause-order", + "toSql": "SELECT * FROM y LIMIT 5" + }, + { + "query": "select * from y limit ,5", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/LimitTest.java:192", + "feature": "clause-order", + "toSql": "SELECT * FROM y LIMIT 0, 5" } ] From 731cf418fc25db3ae52a78023cbeee2457ababf8 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:32:34 +0300 Subject: [PATCH 20/36] feat: PIVOT alias after `)`, FOR IN, IN values aliased without AS Server (SqlParser.parsePivot): after the closing `)` an optional table alias (setModelAliasAndGetOptTok, `[AS] alias` unless a tableAliasStop word); the FOR expression stops at its top-level IN (setStopOnTopINOperator), so `FOR year(ts) IN`, `FOR year - 2000 IN`, `FOR (SELECT 'NL')::VARCHAR IN`; IN-list values take an alias with or without AS (`2010 D2`). Before: `PIVOT (...) p` silently split into a second statement `p`, `PIVOT (...) AS p` errored in SELECT form; FOR accepted only a column reference; `IN (2010 D2)` errored. Now: PIVOT clause (simpleSelect / implicitSelectBody) and pivotStatement take `[AS] alias`; implicit alias gated like table aliases (new implicitTableAliasAllowed(), the former tableRef gate) and additionally not when the next token is PIVOT/WHERE, so consecutive statements without `;` (`t PIVOT (...)\nu PIVOT (...)`) still split. AST: optional PivotClause.alias (toSql ` AS p`). FOR operand = bitOrExpression (the level right below IN/BETWEEN/comparisons; no GATE so autocomplete is not misled). pivotInValue: optional AS. PIVOT inside sub-query/CTE bodies already worked (implicit-select-bodies). Tests: 3 server queries (PivotTest:2914, 2934, 415) + tests/pivot.test.ts (alias forms, clause words after `)`, FOR expressions, IN aliases; no one-line server test uses a PIVOT alias). Red 13 -> green; full suite 16927 passed. Corpus: fixed 1 / rtChanged 0 / regressed 0 / throws 0 / fmtChangedMeaning 0 (most PivotTest queries are text blocks, not in the corpus). Suggestions: +AS after `PIVOT (...) `; clause keywords after `PIVOT (...) p `; expression starts after `FOR `. --- src/parser/ast.ts | 2 + src/parser/cst-types.d.ts | 6 ++- src/parser/parser.ts | 69 +++++++++++++++++++-------- src/parser/toSql.ts | 3 +- src/parser/visitor.ts | 14 ++++-- tests/fixtures/server-queries.json | 18 +++++++ tests/pivot.test.ts | 76 ++++++++++++++++++++++++++++++ 7 files changed, 163 insertions(+), 25 deletions(-) create mode 100644 tests/pivot.test.ts diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 4e0726f..244536b 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -1043,6 +1043,8 @@ export interface PivotClause extends AstNode { aggregations: PivotAggregation[] pivots: PivotForClause[] groupBy?: Expression[] + /** Table alias after the closing parenthesis: `PIVOT (...) [AS] p` */ + alias?: string } export interface PivotAggregation extends AstNode { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 2dc12dd..b07066c 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -138,6 +138,8 @@ export type SimpleSelectCstChildren = { LParen?: (IToken)[]; pivotBody?: (PivotBodyCstNode)[]; RParen?: (IToken)[]; + As?: (IToken)[]; + pivotAlias?: (IdentifierCstNode)[]; sampleByClause?: (SampleByClauseCstNode)[]; groupByClause?: (GroupByClauseCstNode)[]; windowClause?: WindowClauseCstNode[]; @@ -272,6 +274,8 @@ export type ImplicitSelectBodyCstChildren = { LParen?: (IToken)[]; pivotBody?: (PivotBodyCstNode)[]; RParen?: (IToken)[]; + As?: (IToken)[]; + pivotAlias?: (IdentifierCstNode)[]; sampleByClause?: (SampleByClauseCstNode)[]; groupByClause?: (GroupByClauseCstNode)[]; subsampleClause?: SubsampleClauseCstNode[]; @@ -2455,7 +2459,7 @@ export interface PivotForClauseCstNode extends CstNode { } export type PivotForClauseCstChildren = { - columnRef: ColumnRefCstNode[]; + bitOrExpression: BitOrExpressionCstNode[]; In: IToken[]; LParen: IToken[]; pivotInValue?: (PivotInValueCstNode)[]; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 35f39a2..f1ba148 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -446,6 +446,32 @@ class QuestDBParser extends CstParser { // True while parsing an EXPIRE ROWS WHEN predicate (see expireRowsClause). private inExpireRowsPredicate = false + // Implicit PIVOT alias: as for tables, but a word followed by PIVOT / + // WHERE is the next (implicit) statement, `t PIVOT (...)\nu PIVOT (...)`. + private pivotAliasAllowed(): boolean { + if (this.LA(1).tokenType === As) return true + const la2 = this.LA(2).tokenType + return la2 !== Pivot && la2 !== Where && this.implicitTableAliasAllowed() + } + + // Implicit (no AS) table alias allowed at LA(1)? Mirrors the server's + // tableAliasStop for the words that are identifiers here. + private implicitTableAliasAllowed(): boolean { + const la1 = this.LA(1).tokenType + // Don't consume LATEST as alias when followed by ON or BY (latestOnClause) + if (la1 === Latest) { + const la2 = this.LA(2).tokenType + return la2 !== On && la2 !== By + } + // Don't consume OWNED as alias when followed by BY (OWNED BY clause) + if (la1 === Owned) return this.LA(2).tokenType !== By + // If next token is AS, always allow (explicit alias) + if (la1 === As) return true + // Don't consume identifier as alias if followed by LParen — + // it's a function call starting a new statement, not an alias + return this.LA(2).tokenType !== LParen + } + findReSyncTokenType(): TokenType { if (this._semicolonBoundedRecovery) { const currIdx = (this as unknown as CstParserInternals).currIdx @@ -775,11 +801,19 @@ class QuestDBParser extends CstParser { const where = (i: number) => this.subrule(i, this.whereClause) const sample = (i: number) => this.subrule(i, this.sampleByClause) const group = (i: number) => this.subrule(i, this.groupByClause) + // PIVOT (...) [[AS] alias] (parsePivot -> setModelAliasAndGetOptTok) const pivot = (i: number) => { this.consume(i, Pivot) this.consume(i, LParen) this.subrule(i, this.pivotBody) this.consume(i, RParen) + this.option(i + 5, { + GATE: () => this.pivotAliasAllowed(), + DEF: () => { + this.option(i + 6, () => this.consume(i, As)) + this.subrule(i, this.identifier, { LABEL: "pivotAlias" }) + }, + }) } // Each helper call gets its own base `b` (spaced by 10) and uses each // subrule at most once with index b, and option indexes b..b+2. @@ -1120,21 +1154,7 @@ class QuestDBParser extends CstParser { // Optional alias — but NOT if the next token starts a clause // (e.g., LATEST ON/BY should not be consumed as alias) this.OPTION({ - GATE: () => { - const la1 = this.LA(1).tokenType - // Don't consume LATEST as alias when followed by ON or BY (latestOnClause) - if (la1 === Latest) { - const la2 = this.LA(2).tokenType - return la2 !== On && la2 !== By - } - // Don't consume OWNED as alias when followed by BY (OWNED BY clause) - if (la1 === Owned) return this.LA(2).tokenType !== By - // If next token is AS, always allow (explicit alias) - if (la1 === As) return true - // Don't consume identifier as alias if followed by LParen — - // it's a function call starting a new statement, not an alias - return this.LA(2).tokenType !== LParen - }, + GATE: () => this.implicitTableAliasAllowed(), DEF: () => { this.OR1([ { @@ -4732,9 +4752,12 @@ class QuestDBParser extends CstParser { this.CONSUME(LParen) this.SUBRULE(this.pivotBody) this.CONSUME2(RParen) - this.OPTION2(() => { - this.CONSUME(As) - this.SUBRULE6(this.identifier) + this.OPTION2({ + GATE: () => this.pivotAliasAllowed(), + DEF: () => { + this.OPTION3(() => this.CONSUME(As)) + this.SUBRULE6(this.identifier) + }, }) this.OPTION4(() => this.SUBRULE(this.orderByClause)) this.OPTION5(() => this.SUBRULE(this.limitClause)) @@ -4777,7 +4800,12 @@ class QuestDBParser extends CstParser { private pivotForClause = this.RULE("pivotForClause", () => { // FOR keyword is consumed by pivotBody (required on first, optional on subsequent) - this.SUBRULE(this.columnRef) + // FOR : the server stops the expression at its top-level IN + // (setStopOnTopINOperator). bitOrExpression is the expression level just + // below IN/BETWEEN/comparisons, so `FOR year(ts) IN`, `FOR year - 2000 IN` + // and `FOR (SELECT 'NL')::VARCHAR IN` parse without a GATE (content + // assist ignores GATEs and would explore `year IN (...)` as an operator). + this.SUBRULE(this.bitOrExpression) this.CONSUME(In) this.CONSUME(LParen) this.OR([ @@ -4799,8 +4827,9 @@ class QuestDBParser extends CstParser { // 'BTC-USD' AS bitcoin private pivotInValue = this.RULE("pivotInValue", () => { this.SUBRULE(this.expression) + // alias with or without AS: `2000 AS y2000`, `2000 y2000` this.OPTION(() => { - this.CONSUME(As) + this.OPTION1(() => this.CONSUME(As)) this.SUBRULE(this.identifier) }) }) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 3a1f9e4..59cb36b 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -1753,7 +1753,8 @@ function pivotClauseToSql(pivot: AST.PivotClause): string { if (pivot.groupBy && pivot.groupBy.length > 0) { pivotParts.push(`GROUP BY ${pivot.groupBy.map(expressionToSql).join(", ")}`) } - return `PIVOT (${pivotParts.join(" ")})` + const alias = pivot.alias ? ` AS ${escapeIdentifier(pivot.alias)}` : "" + return `PIVOT (${pivotParts.join(" ")})${alias}` } // ============================================================================= diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 199901f..07bc632 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -1192,11 +1192,19 @@ class QuestDBVisitor extends BaseVisitor { return (this.visit(ctx.columnRef!) as AST.ColumnRef).name } - /** Records a PIVOT written before SAMPLE BY / GROUP BY (server order). */ + /** + * Records the PIVOT alias and a PIVOT written before SAMPLE BY / GROUP BY + * (server order). + */ private markPivotPosition( result: AST.SelectStatement, ctx: SimpleSelectCstChildren | ImplicitSelectBodyCstChildren, ): void { + if (ctx.pivotAlias && result.pivot) { + result.pivot.alias = this.extractIdentifierName( + ctx.pivotAlias[0].children, + ) + } const pivotOffset = ctx.Pivot?.[0]?.startOffset if (pivotOffset === undefined) return const later = [ctx.sampleByClause?.[0], ctx.groupByClause?.[0]].some( @@ -3906,7 +3914,7 @@ class QuestDBVisitor extends BaseVisitor { pivotForClause(ctx: PivotForClauseCstChildren): AST.PivotForClause { const result: AST.PivotForClause = { type: "pivotFor", - expression: this.visit(ctx.columnRef) as AST.ColumnRef, + expression: this.visit(ctx.bitOrExpression) as AST.Expression, in: { type: "pivotIn", }, @@ -3926,7 +3934,7 @@ class QuestDBVisitor extends BaseVisitor { type: "pivotInValue", expression: this.visit(ctx.expression) as AST.Expression, } - if (ctx.As && ctx.identifier) { + if (ctx.identifier) { result.alias = this.extractIdentifierName(ctx.identifier[0].children) } return result diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 6641afb..a9e73c8 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1608,5 +1608,23 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/LimitTest.java:192", "feature": "clause-order", "toSql": "SELECT * FROM y LIMIT 0, 5" + }, + { + "query": "SELECT * FROM cities\nPIVOT (\n SUM(population)\n FOR year - 2000 IN (0, 10)\n GROUP BY country\n) ORDER BY country\n", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/PivotTest.java:2914", + "feature": "pivot", + "toSql": "SELECT * FROM cities PIVOT (SUM(population) FOR year - 2000 IN (0, 10) GROUP BY country) ORDER BY country" + }, + { + "query": "cities\nPIVOT (\n SUM(population)\n FOR\n year IN (2000 as D1, 2010 D2, 2020 as D3)\n GROUP BY country\n);\n", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/PivotTest.java:2934", + "feature": "pivot", + "toSql": "cities PIVOT (SUM(population) FOR year IN (2000 AS D1, 2010 AS D2, 2020 AS D3) GROUP BY country)" + }, + { + "query": "SELECT * FROM cities PIVOT (SUM(population) FOR (SELECT 'NL')::VARCHAR IN ('NL') GROUP BY year)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/PivotTest.java:415", + "feature": "pivot", + "toSql": "SELECT * FROM cities PIVOT (SUM(population) FOR (SELECT 'NL')::VARCHAR IN ('NL') GROUP BY year)" } ] diff --git a/tests/pivot.test.ts b/tests/pivot.test.ts new file mode 100644 index 0000000..7d461a0 --- /dev/null +++ b/tests/pivot.test.ts @@ -0,0 +1,76 @@ +/** + * PIVOT forms from the server's parsePivot: + * - table alias after `)`: `PIVOT (...) p` / `PIVOT (...) AS p` + * - `FOR IN`: the FOR expression stops at the top-level IN + * - IN values aliased with or without AS: `IN (2000 y2000)` + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const ok = (sql: string): AST.Statement => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +const roundTrip = (sql: string, expected: string) => { + const out = toSql(ok(sql)) + expect(out).toBe(expected) + expect(toSql(ok(out))).toBe(expected) +} + +describe("PIVOT", () => { + it("alias after the closing parenthesis", () => { + const sel = ok( + "SELECT * FROM cities PIVOT (sum(p) FOR year IN (2000, 2010)) p", + ) as AST.SelectStatement + expect(sel.pivot?.alias).toBe("p") + roundTrip( + "SELECT * FROM cities PIVOT (sum(p) FOR year IN (2000, 2010)) AS p ORDER BY country", + "SELECT * FROM cities PIVOT (sum(p) FOR year IN (2000, 2010)) AS p ORDER BY country", + ) + roundTrip( + "cities PIVOT (sum(p) FOR year IN (2000, 2010)) p", + "cities PIVOT (sum(p) FOR year IN (2000, 2010)) AS p", + ) + }) + + it("clause words after `)` are not taken as an alias", () => { + ok("SELECT * FROM t PIVOT (sum(v) FOR c IN ('a','b')) SUBSAMPLE lttb(a, 5)") + ok("SELECT * FROM t PIVOT (sum(v) FOR c IN ('a')) LIMIT 5") + ok("SELECT * FROM t PIVOT (sum(v) FOR c IN ('a')) GROUP BY x") + ok("SELECT * FROM t PIVOT (sum(v) FOR c IN ('a')) SAMPLE BY 1h") + }) + + it("FOR IN", () => { + roundTrip( + "SELECT * FROM t PIVOT (sum(p) FOR year(ts) IN (2000))", + "SELECT * FROM t PIVOT (sum(p) FOR year(ts) IN (2000))", + ) + roundTrip( + "SELECT * FROM t PIVOT (sum(p) FOR year - 2000 IN (0, 10) a IN ('x'))", + "SELECT * FROM t PIVOT (sum(p) FOR year - 2000 IN (0, 10) a IN ('x'))", + ) + // IN still works as an operator inside parentheses / WHERE + ok("SELECT * FROM t WHERE a IN (1, 2) PIVOT (sum(p) FOR y IN (1))") + }) + + it("IN values aliased without AS", () => { + roundTrip( + "SELECT * FROM t PIVOT (sum(p) FOR year IN (2000 y2000, 2010 AS y2010))", + "SELECT * FROM t PIVOT (sum(p) FOR year IN (2000 AS y2000, 2010 AS y2010))", + ) + }) + + it("autocomplete after PIVOT (...) still offers clause keywords", () => { + const names = getNextValidTokens( + "SELECT * FROM t PIVOT (sum(v) FOR c IN ('a')) ", + ) + expect(names).toContain("Order") + expect(names).toContain("As") + }) +}) From 217346a53de46cc75167fcc7727d2aded2c4e4ed Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:35:34 +0300 Subject: [PATCH 21/36] feat: join clause words after the joined table, HORIZON JOIN optional alias / sub-query / + offsets, UNNEST (cols) without alias Server (SqlParser.parseJoin, setModelAliasAndGetOptTok, tableAliasStop, expectIntervalLiteral, parseUnnest0): - `tolerance` and `range` stop an implicit table alias, so `ASOF|LT JOIN t2 TOLERANCE 2s` and `WINDOW JOIN p RANGE BETWEEN ...` need no alias/ON; EXCLUDE/INCLUDE PREVAILING likewise. - HORIZON JOIN takes a table or parenthesised sub-query (expectTableNameOrSubQuery) with an optional alias; offsets may be signed with `-` or `+`. - UNNEST(...) (v): column aliases without a table alias. Before: TOLERANCE/RANGE were taken as the table alias (error at `2s` / BETWEEN); HORIZON JOIN required `tableName alias`; `+1m` errored; `UNNEST(x)(v)` silently split into a 2nd statement `(v)`. Now: the tableRef alias gate is a method (implicitTableAliasAllowed, shared with the PIVOT alias) that also refuses TOLERANCE, RANGE and EXCLUDE/INCLUDE before PREVAILING (still fine as explicit `AS` aliases and identifiers). horizonJoin uses tableRef (RANGE refused by the gate, `LIST (` by the LParen check); AST unchanged (JoinClause.table is the tableRef). horizonOffset accepts `+` (AST string "+1m"). unnestClause: column list independent of the alias (alias labelled unnestAlias in the CST). Tests: 7 server queries (AsOfJoinTest:1987/5251, SqlParserTest:14994/14224, DeclareTest:3753, BooleanSubQueryRuntimeGateTest:321, HorizonJoinProjectionTest:1615) + tests/join-aliases.test.ts (no server test uses `+` offsets). Red 20 -> green; full suite 16958 passed. Corpus: fixed 10 / rtChanged 2 (former silent UNNEST(..)(val) splits) / regressed 0 / throws 0 / fmtChangedMeaning 0. Suggestions: after `HORIZON JOIN q ` ON/RANGE/LIST/joins/TIMESTAMP (was an error state: mandatory alias); after `HORIZON JOIN q ON (sym) ` operators + RANGE/LIST. --- src/parser/cst-types.d.ts | 9 ++-- src/parser/parser.ts | 66 +++++++++++++++------------- src/parser/visitor.ts | 40 ++++++----------- tests/fixtures/server-queries.json | 42 ++++++++++++++++++ tests/join-aliases.test.ts | 70 ++++++++++++++++++++++++++++++ 5 files changed, 165 insertions(+), 62 deletions(-) create mode 100644 tests/join-aliases.test.ts diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index b07066c..591c6ee 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -258,6 +258,7 @@ export type UnnestClauseCstChildren = { With?: IToken[]; Ordinality?: IToken[]; As?: IToken[]; + unnestAlias?: (IdentifierCstNode)[]; identifier?: (IdentifierCstNode)[]; }; @@ -419,10 +420,7 @@ export interface HorizonJoinCstNode extends CstNode { export type HorizonJoinCstChildren = { Horizon: IToken[]; Join: IToken[]; - tableName: TableNameCstNode[]; - As?: (IToken)[]; - identifier?: (IdentifierCstNode)[]; - Identifier?: IToken[]; + tableRef: TableRefCstNode[]; On?: IToken[]; expression?: ExpressionCstNode[]; Range?: IToken[]; @@ -430,6 +428,8 @@ export type HorizonJoinCstChildren = { horizonOffset?: (HorizonOffsetCstNode)[]; To?: IToken[]; Step?: IToken[]; + As?: (IToken)[]; + identifier?: (IdentifierCstNode)[]; List?: IToken[]; LParen?: IToken[]; Comma?: IToken[]; @@ -443,6 +443,7 @@ export interface HorizonOffsetCstNode extends CstNode { export type HorizonOffsetCstChildren = { Minus?: IToken[]; + Plus?: IToken[]; DurationLiteral?: IToken[]; NumberLiteral?: IToken[]; }; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index f1ba148..3155508 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -467,6 +467,14 @@ class QuestDBParser extends CstParser { if (la1 === Owned) return this.LA(2).tokenType !== By // If next token is AS, always allow (explicit alias) if (la1 === As) return true + // Join clause words (server tableAliasStop): `ASOF JOIN t TOLERANCE 2s`, + // `WINDOW JOIN t RANGE BETWEEN ...`, `... EXCLUDE PREVAILING` + if (la1 === Tolerance || la1 === Range) return false + if ( + (la1 === Exclude || la1 === Include) && + this.LA(2).tokenType === Prevailing + ) + return false // Don't consume identifier as alias if followed by LParen — // it's a function call starting a new statement, not an alias return this.LA(2).tokenType !== LParen @@ -1057,27 +1065,28 @@ class QuestDBParser extends CstParser { this.CONSUME(With) this.CONSUME(Ordinality) }) - // Optional alias + column aliases + // Optional alias + column aliases; the column alias list may also come + // without a table alias: UNNEST(...) (v) (server parseUnnest0) this.OPTION1(() => { this.OR([ { ALT: () => { this.CONSUME(As) - this.SUBRULE(this.identifier) + this.SUBRULE(this.identifier, { LABEL: "unnestAlias" }) }, }, - { ALT: () => this.SUBRULE1(this.identifier) }, + { ALT: () => this.SUBRULE1(this.identifier, { LABEL: "unnestAlias" }) }, ]) - // Optional column alias list: (col1, col2, ...) - this.OPTION2(() => { - this.CONSUME1(LParen) - this.SUBRULE2(this.identifier) - this.MANY1(() => { - this.CONSUME1(Comma) - this.SUBRULE3(this.identifier) - }) - this.CONSUME1(RParen) + }) + // Optional column alias list: (col1, col2, ...) + this.OPTION2(() => { + this.CONSUME1(LParen) + this.SUBRULE2(this.identifier) + this.MANY1(() => { + this.CONSUME1(Comma) + this.SUBRULE3(this.identifier) }) + this.CONSUME1(RParen) }) }) @@ -1269,26 +1278,15 @@ class QuestDBParser extends CstParser { }) }) - // HORIZON JOIN: tableName alias [ON expr] (RANGE FROM/TO/STEP | LIST (...)) AS alias - // Uses tableName + custom alias instead of tableRef to avoid ambiguity - // between implicit keyword aliases and RANGE/LIST/ON keywords. + // HORIZON JOIN: tableRef [ON expr] (RANGE FROM/TO/STEP | LIST (...)) AS alias + // The target is a table or a parenthesised sub-query with an optional alias + // (server parseJoin: expectTableNameOrSubQuery + setModelAliasAndGetOptTok). + // tableRef's alias gate refuses RANGE; `LIST (` is refused by its LParen + // check. private horizonJoin = this.RULE("horizonJoin", () => { this.CONSUME(Horizon) this.CONSUME(Join) - this.SUBRULE(this.tableName) - // Table alias (mandatory): explicit (AS ) or implicit (bare Identifier only, not keywords) - this.OR1([ - { - ALT: () => { - this.CONSUME(As) - this.SUBRULE(this.identifier) - }, - }, - { - // Implicit alias: only base Identifier, never keywords like RANGE/LIST - ALT: () => this.CONSUME(Identifier), - }, - ]) + this.SUBRULE(this.tableRef) this.OPTION(() => { this.CONSUME(On) this.SUBRULE(this.expression) @@ -1330,9 +1328,15 @@ class QuestDBParser extends CstParser { }) }) - // Horizon offset value: optional minus sign + DurationLiteral | NumberLiteral + // Horizon offset value: optional sign + DurationLiteral | NumberLiteral + // (server expectIntervalLiteral accepts `-` and `+`) private horizonOffset = this.RULE("horizonOffset", () => { - this.OPTION(() => this.CONSUME(Minus)) + this.OPTION(() => { + this.OR1([ + { ALT: () => this.CONSUME(Minus) }, + { ALT: () => this.CONSUME(Plus) }, + ]) + }) this.OR([ { ALT: () => this.CONSUME(DurationLiteral) }, { ALT: () => this.CONSUME(NumberLiteral) }, diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 07bc632..0df77c3 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -727,14 +727,13 @@ class QuestDBVisitor extends BaseVisitor { type: "tableRef", table: unnest, } - const identifiers = ctx.identifier || [] - if (identifiers.length > 0) { - result.alias = this.extractIdentifierName(identifiers[0].children) + if (ctx.unnestAlias) { + result.alias = this.extractIdentifierName(ctx.unnestAlias[0].children) } - if (identifiers.length > 1) { - result.columnAliases = identifiers - .slice(1) - .map((id: CstNode) => this.extractIdentifierName(id.children)) + if (ctx.identifier && ctx.identifier.length > 0) { + result.columnAliases = ctx.identifier.map((id: CstNode) => + this.extractIdentifierName(id.children), + ) } return result } @@ -947,21 +946,7 @@ class QuestDBVisitor extends BaseVisitor { } horizonJoin(ctx: HorizonJoinCstChildren): AST.JoinClause { - // Build TableRef from tableName + alias - const tableRef: AST.TableRef = { - type: "tableRef", - table: this.visit(ctx.tableName) as AST.QualifiedName, - } - if (ctx.Identifier) { - // Implicit alias (bare identifier, not a keyword) - tableRef.alias = ctx.Identifier[0].image - } else if (ctx.identifier && ctx.identifier.length > 0) { - // Explicit alias (AS ): the FIRST identifier is the table alias. - // (The horizon alias, when present, is the last identifier.) - tableRef.alias = ( - this.visit(ctx.identifier[0]) as AST.QualifiedName - ).parts[0] - } + const tableRef = this.visit(ctx.tableRef) as AST.TableRef const result: AST.JoinClause = { type: "join", @@ -985,17 +970,18 @@ class QuestDBVisitor extends BaseVisitor { (o) => this.visit(o) as string, ) } - // Horizon alias (the trailing `AS `) exists only with a RANGE/LIST clause, - // and is the LAST identifier. Non-last joins in a chain have none (#6881). + // Horizon alias (the trailing `AS `) exists only with a RANGE/LIST + // clause. Non-last joins in a chain have none (#6881). if ((ctx.Range || ctx.List) && ctx.identifier) { - const lastId = ctx.identifier[ctx.identifier.length - 1] - result.horizonAlias = (this.visit(lastId) as AST.QualifiedName).parts[0] + result.horizonAlias = ( + this.visit(ctx.identifier[0]) as AST.QualifiedName + ).parts[0] } return result } horizonOffset(ctx: HorizonOffsetCstChildren): string { - const sign = ctx.Minus ? "-" : "" + const sign = ctx.Minus ? "-" : ctx.Plus ? "+" : "" if (ctx.DurationLiteral) { return sign + ctx.DurationLiteral[0].image } diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index a9e73c8..ee5fcdd 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1626,5 +1626,47 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/PivotTest.java:415", "feature": "pivot", "toSql": "SELECT * FROM cities PIVOT (SUM(population) FOR (SELECT 'NL')::VARCHAR IN ('NL') GROUP BY year)" + }, + { + "query": "SELECT * FROM t1 ASOF JOIN t2 TOLERANCE 2s;", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/join/AsOfJoinTest.java:1987", + "feature": "join-aliases", + "toSql": "SELECT * FROM t1 ASOF JOIN t2 TOLERANCE 2s" + }, + { + "query": "SELECT * FROM t1 LT JOIN t2 TOLERANCE 2s;", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/join/AsOfJoinTest.java:5251", + "feature": "join-aliases", + "toSql": "SELECT * FROM t1 LT JOIN t2 TOLERANCE 2s" + }, + { + "query": "select t.* from trades t WINDOW JOIN quotes range between 2 second preceding and 1 second preceding order by t.timestamp", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:14994", + "feature": "join-aliases", + "toSql": "SELECT t.* FROM trades AS t WINDOW JOIN quotes RANGE BETWEEN 2 SECOND PRECEDING AND 1 SECOND PRECEDING ORDER BY t.timestamp" + }, + { + "query": "DECLARE @q := (SELECT 3_600_000_000L) SELECT k.ts, sum(p.v) FROM k WINDOW JOIN p RANGE BETWEEN @q PRECEDING AND CURRENT ROW", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/DeclareTest.java:3753", + "feature": "join-aliases", + "toSql": "DECLARE @q := (SELECT 3_600_000_000L) SELECT k.ts, sum(p.v) FROM k WINDOW JOIN p RANGE BETWEEN @q PRECEDING AND CURRENT ROW" + }, + { + "query": "explain select h.offset, avg(ws.v) from (select * from wm where now() < '2100-01-01') wm horizon join ws on (wm.k = ws.k) range from 0s to 2s step 1s as h", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/BooleanSubQueryRuntimeGateTest.java:321", + "feature": "join-aliases", + "toSql": "EXPLAIN SELECT h.offset, avg(ws.v) FROM (SELECT * FROM wm WHERE now() < '2100-01-01') AS wm HORIZON JOIN ws ON (wm.k = ws.k) RANGE FROM 0s TO 2s STEP 1s AS h" + }, + { + "query": "SELECT t.sym, q.id FROM trades t HORIZON JOIN (quotes WHERE bid > 50) q ON (sym) LIST (0s) AS h", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/HorizonJoinProjectionTest.java:1615", + "feature": "join-aliases", + "toSql": "SELECT t.sym, q.id FROM trades AS t HORIZON JOIN (quotes WHERE bid > 50) AS q ON (sym) LIST (0s) AS h" + }, + { + "query": "SELECT val FROM t, UNNEST(t.arr)(val) WHERE val > 1.0", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:14224", + "feature": "join-aliases", + "toSql": "SELECT val FROM t, UNNEST(t.arr)(val) WHERE val > 1.0" } ] diff --git a/tests/join-aliases.test.ts b/tests/join-aliases.test.ts new file mode 100644 index 0000000..4cdd40c --- /dev/null +++ b/tests/join-aliases.test.ts @@ -0,0 +1,70 @@ +/** + * Join forms where a clause word follows the joined table directly + * (server tableAliasStop: TOLERANCE, RANGE; EXCLUDE/INCLUDE PREVAILING), + * HORIZON JOIN with an optional alias / sub-query target / `+` offsets, + * and UNNEST column aliases without a table alias. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst } from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const ok = (sql: string): AST.SelectStatement => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] as AST.SelectStatement +} + +const roundTrip = (sql: string, expected: string) => { + const out = toSql(ok(sql)) + expect(out).toBe(expected) + expect(toSql(ok(out))).toBe(expected) +} + +describe("join aliases", () => { + it("TOLERANCE / RANGE / EXCLUDE right after the joined table", () => { + const sel = ok("SELECT * FROM a ASOF JOIN b TOLERANCE 10s") + expect(sel.from![0].joins![0].table.alias).toBeUndefined() + roundTrip( + "SELECT * FROM a WINDOW JOIN p RANGE BETWEEN 1 MINUTE PRECEDING AND CURRENT ROW EXCLUDE PREVAILING", + "SELECT * FROM a WINDOW JOIN p RANGE BETWEEN 1 MINUTE PRECEDING AND CURRENT ROW EXCLUDE PREVAILING", + ) + // the words still work as explicit aliases and identifiers + ok("SELECT * FROM a ASOF JOIN b AS tolerance TOLERANCE 10s") + ok("SELECT tolerance, range FROM t") + expect(ok("SELECT * FROM t exclude").from![0].alias).toBe("exclude") + }) + + it("HORIZON JOIN alias optional, sub-query target, + offsets", () => { + const sel = ok( + "SELECT * FROM t HORIZON JOIN quotes ON (sym) LIST (1s, 5s) AS h", + ) + const join = sel.from![0].joins![0] + expect(join.table.alias).toBeUndefined() + expect(join.horizonAlias).toBe("h") + roundTrip( + "SELECT * FROM t HORIZON JOIN (quotes WHERE x > 0) q ON (sym) LIST (0s, 1s) AS h", + "SELECT * FROM t HORIZON JOIN (quotes WHERE x > 0) AS q ON (sym) LIST (0s, 1s) AS h", + ) + roundTrip( + "SELECT * FROM t HORIZON JOIN q RANGE FROM -1m TO +1m STEP 1m AS h", + "SELECT * FROM t HORIZON JOIN q RANGE FROM -1m TO +1m STEP 1m AS h", + ) + roundTrip( + "SELECT * FROM t HORIZON JOIN q AS x LIST (+1s) AS h", + "SELECT * FROM t HORIZON JOIN q AS x LIST (+1s) AS h", + ) + }) + + it("UNNEST column aliases without a table alias", () => { + const sel = ok("SELECT * FROM UNNEST(ARRAY[1.0,2.0]) (v)") + expect(sel.from![0].alias).toBeUndefined() + expect(sel.from![0].columnAliases).toEqual(["v"]) + roundTrip( + "SELECT * FROM UNNEST(ARRAY[1.0, 2.0]) u(v)", + "SELECT * FROM UNNEST(ARRAY[1.0, 2.0]) AS u(v)", + ) + }) +}) From 833610344e5554ab862862d57ec21398245d4cd8 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:40:23 +0300 Subject: [PATCH 22/36] feat: schema-qualified table functions in FROM/JOIN, UNION DISTINCT Server: FROM/JOIN sources may be schema-qualified function calls (`pg_catalog.pg_class()`, `information_schema.tables()`, used throughout the catalogue tests); parseDml accepts `UNION DISTINCT` as plain UNION. Before: `FROM information_schema.tables()` errored at `(`; `UNION DISTINCT` errored. Now: tableFunctionName = identifier (Dot identifier)*; the AST name is the dotted string ("pg_catalog.pg_class"). In tableRef the tableName alternative now comes before tableFunctionCall, gated off only when `name[.name]* (` follows AND the function call backtrack succeeds (so inputs that parsed before as a table keep doing so). Order matters for content assist: it ignores GATEs and prunes later alternatives once a path consumes the input, so with the function alternative first `FROM sys.t ` only offered `(` (docs autocomplete walkthrough). setOperation: UNION [ALL | DISTINCT], EXCEPT/INTERSECT [ALL] (no DISTINCT suggestion after EXCEPT/INTERSECT); AST unchanged (DISTINCT == UNION; toSql prints UNION). Tests: 4 server queries (InformationSchemaTablesFunctionFactoryTest:42, LiveViewTest:1193, InformationSchemaColumnsFunctionFactoryTest:78, SqlParserTest:13928) + tests/from-sources.test.ts (pg_class/pg_namespace JOIN, _pg_expandarray). Red 13 -> green; full suite 16976 passed. Corpus: fixed 13 / rtChanged 0 / regressed 0 / throws 0 / fmtChangedMeaning 0. Suggestions: +DISTINCT after `UNION `; alias/join keywords after `FROM pg_catalog.pg_class() ` (was an error state). --- src/parser/cst-types.d.ts | 8 ++-- src/parser/parser.ts | 67 ++++++++++++++++++++++++++---- src/parser/visitor.ts | 5 ++- tests/fixtures/server-queries.json | 24 +++++++++++ tests/from-sources.test.ts | 56 +++++++++++++++++++++++++ 5 files changed, 146 insertions(+), 14 deletions(-) create mode 100644 tests/from-sources.test.ts diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 591c6ee..391ca70 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -155,9 +155,10 @@ export interface SetOperationCstNode extends CstNode { export type SetOperationCstChildren = { Union?: IToken[]; + All?: (IToken)[]; + Distinct?: IToken[]; Except?: IToken[]; Intersect?: IToken[]; - All?: IToken[]; simpleSelect?: SimpleSelectCstNode[]; selectStatement?: SelectStatementCstNode[]; implicitSelectBody?: ImplicitSelectBodyCstNode[]; @@ -305,10 +306,10 @@ export type TableRefCstChildren = { RParen?: (IToken)[]; showStatement?: ShowStatementCstNode[]; implicitSelectStatement?: ImplicitSelectStatementCstNode[]; - tableFunctionCall?: TableFunctionCallCstNode[]; VariableReference?: IToken[]; StringLiteral?: IToken[]; tableName?: TableNameCstNode[]; + tableFunctionCall?: TableFunctionCallCstNode[]; Timestamp?: IToken[]; columnRef?: ColumnRefCstNode[]; As?: IToken[]; @@ -347,7 +348,8 @@ export interface TableFunctionNameCstNode extends CstNode { } export type TableFunctionNameCstChildren = { - identifier: IdentifierCstNode[]; + identifier: (IdentifierCstNode)[]; + Dot?: IToken[]; }; export interface JoinClauseCstNode extends CstNode { diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 3155508..a31cd45 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -446,6 +446,25 @@ class QuestDBParser extends CstParser { // True while parsing an EXPIRE ROWS WHEN predicate (see expireRowsClause). private inExpireRowsPredicate = false + // `name [. name]* (` ahead: a (schema-qualified) table function call + private isTableFunctionCallAhead(): boolean { + let i = 1 + for (;;) { + const tok = this.LA(i) + const t = tok.tokenType + const isName = + t === Identifier || + t === QuotedIdentifier || + t === StringLiteral || + tokenMatcher(tok, IdentifierKeyword) + if (!isName) return false + const next = this.LA(i + 1).tokenType + if (next === LParen) return true + if (next !== Dot) return false + i += 2 + } + } + // Implicit PIVOT alias: as for tables, but a word followed by PIVOT / // WHERE is the next (implicit) statement, `t PIVOT (...)\nu PIVOT (...)`. private pivotAliasAllowed(): boolean { @@ -899,11 +918,28 @@ class QuestDBParser extends CstParser { private setOperation = this.RULE("setOperation", () => { this.OR([ - { ALT: () => this.CONSUME(Union) }, - { ALT: () => this.CONSUME(Except) }, - { ALT: () => this.CONSUME(Intersect) }, + { + ALT: () => { + this.CONSUME(Union) + // UNION DISTINCT is plain UNION on the server (parseDml) + this.OPTION1(() => { + this.OR2([ + { ALT: () => this.CONSUME(All) }, + { ALT: () => this.CONSUME(Distinct) }, + ]) + }) + }, + }, + { + ALT: () => { + this.OR3([ + { ALT: () => this.CONSUME(Except) }, + { ALT: () => this.CONSUME(Intersect) }, + ]) + this.OPTION(() => this.CONSUME1(All)) + }, + }, ]) - this.OPTION(() => this.CONSUME(All)) // Right operand: full SELECT or bare table/implicit form; a branch may // start with its own WITH or DECLARE (WithClauseTest, DeclareTest) this.OR1([ @@ -1112,6 +1148,7 @@ class QuestDBParser extends CstParser { }) private tableRef = this.RULE("tableRef", () => { + const isTableFunctionCall = this.BACKTRACK(this.tableFunctionCall) this.OR([ { // Standard subquery: (SELECT ...), (WITH ...), (DECLARE ...) @@ -1142,16 +1179,23 @@ class QuestDBParser extends CstParser { this.CONSUME2(RParen) }, }, - { - GATE: this.BACKTRACK(this.tableFunctionCall), - ALT: () => this.SUBRULE(this.tableFunctionCall), - }, { // @variable reference as table source (DECLARE @subquery := (...); SELECT * FROM @subquery) ALT: () => this.CONSUME(VariableReference), }, { ALT: () => this.CONSUME(StringLiteral) }, - { ALT: () => this.SUBRULE(this.tableName) }, + // tableName before tableFunctionCall: content assist ignores GATEs and + // prunes later alternatives once one path consumes the input, so the + // table path must come first or `FROM s.t ` would only offer `(`. + { + GATE: () => + !this.isTableFunctionCallAhead() || !isTableFunctionCall.call(this), + ALT: () => this.SUBRULE(this.tableName), + }, + { + GATE: isTableFunctionCall, + ALT: () => this.SUBRULE(this.tableFunctionCall), + }, ]) // Optional TIMESTAMP designation on subquery/table results this.OPTION2(() => { @@ -1205,8 +1249,13 @@ class QuestDBParser extends CstParser { this.CONSUME(RParen) }) + // Optionally schema-qualified: pg_catalog.pg_class(), information_schema.tables() private tableFunctionName = this.RULE("tableFunctionName", () => { this.SUBRULE(this.identifier) + this.MANY(() => { + this.CONSUME(Dot) + this.SUBRULE1(this.identifier) + }) }) // ---- Join clause: dispatches to type-specific sub-rules so that each diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 0df77c3..222abca 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -860,8 +860,9 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.tableFunctionName) { const fnName = ctx.tableFunctionName[0] if (fnName.children?.identifier) { - const id = this.visit(fnName.children.identifier) as AST.QualifiedName - name = id.parts.join(".") + name = (fnName.children.identifier as CstNode[]) + .map((i) => (this.visit(i) as AST.QualifiedName).parts.join(".")) + .join(".") } else { name = "unknown" } diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index ee5fcdd..79df371 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1668,5 +1668,29 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:14224", "feature": "join-aliases", "toSql": "SELECT val FROM t, UNNEST(t.arr)(val) WHERE val > 1.0" + }, + { + "query": "select * from information_schema.tables() order by table_name", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/catalogue/InformationSchemaTablesFunctionFactoryTest.java:42", + "feature": "from-sources", + "toSql": "SELECT * FROM information_schema.tables() ORDER BY table_name" + }, + { + "query": "SELECT table_type, is_insertable_into FROM information_schema.tables()", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/lv/LiveViewTest.java:1193", + "feature": "from-sources", + "toSql": "SELECT table_type, is_insertable_into FROM information_schema.tables()" + }, + { + "query": "SELECT table_name, column_name FROM information_schema.columns() ORDER BY table_name, column_name", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/catalogue/InformationSchemaColumnsFunctionFactoryTest.java:78", + "feature": "from-sources", + "toSql": "SELECT table_name, column_name FROM information_schema.columns() ORDER BY table_name, column_name" + }, + { + "query": "select * from a union distinct select * from b union select * from c", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:13928", + "feature": "from-sources", + "toSql": "SELECT * FROM a UNION SELECT * FROM b UNION SELECT * FROM c" } ] diff --git a/tests/from-sources.test.ts b/tests/from-sources.test.ts new file mode 100644 index 0000000..f026c98 --- /dev/null +++ b/tests/from-sources.test.ts @@ -0,0 +1,56 @@ +/** + * Schema-qualified table functions in FROM / JOIN + * (`pg_catalog.pg_class()`, `information_schema.tables()`) and + * `UNION DISTINCT` (server parseDml: same as UNION). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst } from "../src/index" +import { toSql } from "../src/parser/toSql" +import type * as AST from "../src/parser/ast" + +const ok = (sql: string): AST.SelectStatement => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] as AST.SelectStatement +} + +const roundTrip = (sql: string, expected: string) => { + const out = toSql(ok(sql)) + expect(out).toBe(expected) + expect(toSql(ok(out))).toBe(expected) +} + +describe("FROM sources", () => { + it("schema-qualified table functions", () => { + const sel = ok( + "SELECT * FROM pg_catalog.pg_class() c JOIN pg_catalog.pg_namespace() n ON n.oid = c.relnamespace", + ) + const from = sel.from![0] + expect(from.table).toMatchObject({ + type: "tableFunctionCall", + name: "pg_catalog.pg_class", + }) + expect(from.alias).toBe("c") + roundTrip( + "SELECT * FROM pg_catalog.pg_class() c JOIN pg_catalog.pg_namespace() n ON n.oid = c.relnamespace", + "SELECT * FROM pg_catalog.pg_class() AS c JOIN pg_catalog.pg_namespace() AS n ON n.oid = c.relnamespace", + ) + roundTrip( + "SELECT * FROM information_schema._pg_expandarray(x)", + "SELECT * FROM information_schema._pg_expandarray(x)", + ) + // qualified table names still parse as tables + expect(ok("SELECT * FROM s.t x").from![0].table).toMatchObject({ + type: "qualifiedName", + }) + }) + + it("UNION DISTINCT", () => { + roundTrip( + "SELECT a FROM t UNION DISTINCT SELECT a FROM u", + "SELECT a FROM t UNION SELECT a FROM u", + ) + }) +}) From 84c258f2e6b84d884e35f0e7dc2d65a7d705171a Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:45:28 +0300 Subject: [PATCH 23/36] feat: CREATE/ALTER TABLE [NOT] NULL column modifiers, CTAS CAST symbol options, CAST/INDEX in any order Server: parseCreateTableColumns skips `[PRECISION] [NOT] NULL` after a column type / PARQUET(...); SqlCompilerImpl.addColumnWithType skips `[NOT] NULL` (ALTER TABLE ADD COLUMN, ALTER COLUMN TYPE); parseCreateTableCastDef reads `SYMBOL [CAPACITY n] [CACHE|NOCACHE]`; parseCreateTable loops over `, CAST(..)` and `, INDEX(..)` in any order. Before: all of these errored. Now: new rule columnNullModifier (`[PRECISION] [NOT] NULL`, may match nothing; PRECISION gated to CREATE TABLE and filtered from autocomplete) used by columnDefinition and ALTER COLUMN TYPE; visitor ignores it (toSql drops the modifiers, like the server). castDefinition takes CAPACITY/CACHE/NOCACHE (AST: optional CastDefinition.symbolCapacity / cache, printed by toSql). CTAS `, CAST | , INDEX` in one loop (toSql still prints casts before indexes). Tests: 6 server queries (HashJoinTest:136, SqlParserTest:3045/2889/2839, AlterTableAddColumnTest:295, CreateTableTest:468) + tests/create-table-options.test.ts. Red 21 -> green; full suite 17003 passed. Corpus: fixed 18 / rtChanged 0 / regressed 0 / throws 0 / fmtChangedMeaning 0. Suggestions: +NOT, NULL after a column type (CREATE TABLE, ADD COLUMN, ALTER COLUMN TYPE); +CAPACITY/CACHE/NOCACHE inside CTAS CAST(.. AS type. --- src/autocomplete/content-assist.ts | 3 +- src/parser/ast.ts | 4 ++ src/parser/cst-types.d.ts | 18 +++++++++ src/parser/parser.ts | 42 +++++++++++++++++--- src/parser/toSql.ts | 9 ++++- src/parser/visitor.ts | 13 ++++++- tests/create-table-options.test.ts | 61 ++++++++++++++++++++++++++++++ tests/fixtures/server-queries.json | 36 ++++++++++++++++++ 8 files changed, 178 insertions(+), 8 deletions(-) create mode 100644 tests/create-table-options.test.ts diff --git a/src/autocomplete/content-assist.ts b/src/autocomplete/content-assist.ts index 9cfd375..8991590 100644 --- a/src/autocomplete/content-assist.ts +++ b/src/autocomplete/content-assist.ts @@ -1041,7 +1041,8 @@ function computeSuggestions(tokens: IToken[]): ComputeResult { (s.nextTokenType.name === "Timestamp" || s.nextTokenType.name === "TimestampNs")) || (s.nextTokenType.name === "Precision" && - s.ruleStack.includes("alterTableAction")) + (s.ruleStack.includes("alterTableAction") || + s.ruleStack[s.ruleStack.length - 1] === "columnNullModifier")) ) }) const result = suggestions.map((s) => s.nextTokenType) diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 244536b..ac11c93 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -206,6 +206,10 @@ export interface CastDefinition extends AstNode { type: "castDefinition" column: QualifiedName dataType: string + /** SYMBOL CAPACITY n */ + symbolCapacity?: number + /** SYMBOL CACHE (true) / NOCACHE (false) */ + cache?: boolean } export interface IndexDefinition extends AstNode { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 391ca70..516fb03 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -1178,6 +1178,18 @@ export type ColumnDefinitionCstChildren = { Index?: IToken[]; indexTypeOptions?: IndexTypeOptionsCstNode[]; parquetConfig?: ParquetConfigCstNode[]; + columnNullModifier: ColumnNullModifierCstNode[]; +}; + +export interface ColumnNullModifierCstNode extends CstNode { + name: "columnNullModifier"; + children: ColumnNullModifierCstChildren; +} + +export type ColumnNullModifierCstChildren = { + Precision?: IToken[]; + Not?: IToken[]; + Null?: IToken[]; }; export interface ParquetConfigCstNode extends CstNode { @@ -1301,6 +1313,10 @@ export type CastDefinitionCstChildren = { As: IToken[]; doublePrecisionType?: DoublePrecisionTypeCstNode[]; dataType?: DataTypeCstNode[]; + Capacity?: IToken[]; + NumberLiteral?: IToken[]; + Cache?: IToken[]; + Nocache?: IToken[]; RParen: IToken[]; }; @@ -1639,6 +1655,7 @@ export type AlterTableActionCstChildren = { NumberLiteral?: (IToken)[]; Cache?: (IToken)[]; Nocache?: (IToken)[]; + columnNullModifier?: ColumnNullModifierCstNode[]; Index?: (IToken)[]; indexTypeOptions?: IndexTypeOptionsCstNode[]; Symbol?: IToken[]; @@ -3231,6 +3248,7 @@ export interface ICstNodeVisitor extends ICstVisitor { materializedViewPeriod(children: MaterializedViewPeriodCstChildren, param?: IN): OUT; materializedViewPartition(children: MaterializedViewPartitionCstChildren, param?: IN): OUT; columnDefinition(children: ColumnDefinitionCstChildren, param?: IN): OUT; + columnNullModifier(children: ColumnNullModifierCstChildren, param?: IN): OUT; parquetConfig(children: ParquetConfigCstChildren, param?: IN): OUT; parquetEncoding(children: ParquetEncodingCstChildren, param?: IN): OUT; parquetCompression(children: ParquetCompressionCstChildren, param?: IN): OUT; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index a31cd45..26a51c4 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -2031,13 +2031,14 @@ class QuestDBParser extends CstParser { { ALT: () => this.SUBRULE(this.implicitSelectStatement) }, ]) this.CONSUME(RParen) + // `, CAST(...)` and `, INDEX(...)` in any order (server + // SqlParser.parseCreateTable loops over both) this.MANY2(() => { this.CONSUME(Comma) - this.SUBRULE(this.castDefinition) - }) - this.MANY3(() => { - this.CONSUME1(Comma) - this.SUBRULE(this.indexDefinition) + this.OR8([ + { ALT: () => this.SUBRULE(this.castDefinition) }, + { ALT: () => this.SUBRULE(this.indexDefinition) }, + ]) }) }, }, @@ -2637,6 +2638,25 @@ class QuestDBParser extends CstParser { }) // Optional PARQUET config this.OPTION4(() => this.SUBRULE(this.parquetConfig)) + this.SUBRULE(this.columnNullModifier, { ARGS: [allowDoublePrecision] }) + }, + ) + + // `[PRECISION] [NOT] NULL` after a column type: parsed and ignored by the + // server (SqlParser.parseCreateTableColumns, SqlCompilerImpl.addColumnWithType). + // May match nothing. PRECISION only in CREATE TABLE (allowPrecision); + // autocomplete drops PRECISION here (only offered after DOUBLE). + private columnNullModifier = this.RULE( + "columnNullModifier", + (allowPrecision?: boolean) => { + this.OPTION({ + GATE: () => !!allowPrecision, + DEF: () => this.CONSUME(Precision), + }) + this.OPTION1(() => { + this.OPTION2(() => this.CONSUME(Not)) + this.CONSUME(Null) + }) }, ) @@ -2801,6 +2821,17 @@ class QuestDBParser extends CstParser { { ALT: () => this.SUBRULE(this.dataType) }, ], }) + // SYMBOL options (SqlParser.parseCreateTableCastDef) + this.OPTION(() => { + this.CONSUME(Capacity) + this.CONSUME(NumberLiteral) + }) + this.OPTION1(() => { + this.OR1([ + { ALT: () => this.CONSUME(Cache) }, + { ALT: () => this.CONSUME(Nocache) }, + ]) + }) this.CONSUME(RParen) }) @@ -3346,6 +3377,7 @@ class QuestDBParser extends CstParser { // Optional CACHE/NOCACHE this.OPTION5(() => this.CONSUME1(Cache)) this.OPTION6(() => this.CONSUME1(Nocache)) + this.SUBRULE(this.columnNullModifier) }, }, { diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 59cb36b..0a6bd84 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -715,7 +715,14 @@ function createTableToSql(stmt: AST.CreateTableStatement): string { let asSql = `AS (${selectToSql(stmt.asSelect)})` if (stmt.casts && stmt.casts.length > 0) { for (const cast of stmt.casts) { - asSql += `, CAST(${qualifiedNameToSql(cast.column)} AS ${cast.dataType})` + let castSql = `${qualifiedNameToSql(cast.column)} AS ${cast.dataType}` + if (cast.symbolCapacity !== undefined) { + castSql += ` CAPACITY ${cast.symbolCapacity}` + } + if (cast.cache !== undefined) { + castSql += cast.cache ? " CACHE" : " NOCACHE" + } + asSql += `, CAST(${castSql})` } } if (stmt.indexes && stmt.indexes.length > 0) { diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 222abca..45aeb5e 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -1780,11 +1780,22 @@ class QuestDBVisitor extends BaseVisitor { castDefinition(ctx: CastDefinitionCstChildren): AST.CastDefinition { const colRef = this.visit(ctx.columnRef) as AST.ColumnRef - return { + const result: AST.CastDefinition = { type: "castDefinition", column: colRef.name ?? colRef, dataType: this.visit(ctx.doublePrecisionType ?? ctx.dataType!) as string, } + if (ctx.NumberLiteral) { + result.symbolCapacity = tokenInt(ctx.NumberLiteral[0].image) + } + if (ctx.Cache) result.cache = true + else if (ctx.Nocache) result.cache = false + return result + } + + // `[PRECISION] [NOT] NULL` is ignored (as on the server) + columnNullModifier(): undefined { + return undefined } indexDefinition(ctx: IndexDefinitionCstChildren): AST.IndexDefinition { diff --git a/tests/create-table-options.test.ts b/tests/create-table-options.test.ts new file mode 100644 index 0000000..e8e7377 --- /dev/null +++ b/tests/create-table-options.test.ts @@ -0,0 +1,61 @@ +/** + * CREATE TABLE / ALTER TABLE column modifiers ignored by the server + * (`[PRECISION] [NOT] NULL`), CTAS `CAST(col AS SYMBOL [CAPACITY n] + * [CACHE|NOCACHE])` and CAST/INDEX clauses in any order. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" + +const ok = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +describe("create table options", () => { + it("[PRECISION] [NOT] NULL after column types", () => { + ok( + "CREATE TABLE t (a INT NOT NULL, b STRING NULL, ts TIMESTAMP) TIMESTAMP(ts)", + ) + ok( + "CREATE TABLE t (a DOUBLE PRECISION NOT NULL, b VARCHAR PARQUET(PLAIN) NULL)", + ) + ok("CREATE TABLE t (a INT PRECISION NULL, b LONG)") + ok("ALTER TABLE t ADD COLUMN a INT NOT NULL, b STRING NULL") + ok("ALTER TABLE t ALTER COLUMN x TYPE VARCHAR NULL") + ok("ALTER TABLE t ALTER COLUMN x TYPE VARCHAR NOT NULL") + }) + + it("CTAS cast symbol options and CAST/INDEX in any order", () => { + const stmt = ok( + "CREATE TABLE t AS (SELECT * FROM x), INDEX(s), CAST(a AS LONG), CAST(s AS SYMBOL CAPACITY 64 CACHE)", + ) as { casts?: unknown[]; indexes?: unknown[] } + expect(stmt.casts).toHaveLength(2) + expect(stmt.casts![1]).toMatchObject({ + dataType: "SYMBOL", + symbolCapacity: 64, + cache: true, + }) + expect(stmt.indexes).toHaveLength(1) + }) + + it("autocomplete", () => { + const afterType = getNextValidTokens("CREATE TABLE t (a INT ") + expect(afterType).toContain("Not") + expect(afterType).toContain("Null") + expect(afterType).not.toContain("Precision") + expect(getNextValidTokens("CREATE TABLE t (a DOUBLE ")).toContain( + "Precision", + ) + const afterSym = getNextValidTokens( + "CREATE TABLE t AS (SELECT * FROM x), CAST(s AS SYMBOL ", + ) + expect(afterSym).toContain("Capacity") + expect(afterSym).toContain("Nocache") + expect( + getNextValidTokens("CREATE TABLE t AS (SELECT * FROM x), INDEX(s), "), + ).toContain("Cast") + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 79df371..9f9683e 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1692,5 +1692,41 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:13928", "feature": "from-sources", "toSql": "SELECT * FROM a UNION SELECT * FROM b UNION SELECT * FROM c" + }, + { + "query": "create table weather_data_historical ( sensor_time timestamp not null, sensor_day symbol, min_temperature_out float, max_temperature_out float, avg_temperature_out float, snow_height float, rain_acc_24h float, max_wind_gust_speed float ) TIMESTAMP(sensor_time)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/join/HashJoinTest.java:136", + "feature": "create-table-options", + "toSql": "CREATE TABLE weather_data_historical (sensor_time TIMESTAMP, sensor_day SYMBOL, min_temperature_out FLOAT, max_temperature_out FLOAT, avg_temperature_out FLOAT, snow_height FLOAT, rain_acc_24h FLOAT, max_wind_gust_speed FLOAT) TIMESTAMP(sensor_time)" + }, + { + "query": "CREATE TABLE \"quickstart-events4\" ( \"flag\" BOOLEAN NOT NULL, \"id8\" SMALLINT NOT NULL, \"id16\" SMALLINT NOT NULL, \"id32\" INT NOT NULL, \"id64\" BIGINT NOT NULL, \"idFloat\" REAL NOT NULL, \"idDouble\" DOUBLE PRECISION NOT NULL, \"idBytes\" BYTEA NOT NULL, \"msg\" TEXT NULL)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:3045", + "feature": "create-table-options", + "toSql": "CREATE TABLE \"quickstart-events4\" (flag BOOLEAN, id8 SMALLINT, id16 SMALLINT, id32 INT, id64 BIGINT, idFloat REAL, idDouble DOUBLE PRECISION, idBytes BYTEA, msg TEXT)" + }, + { + "query": "alter table x add column \"mycol\" int not null", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/AlterTableAddColumnTest.java:295", + "feature": "create-table-options", + "toSql": "ALTER TABLE x ADD COLUMN mycol INT" + }, + { + "query": "create table x as (tab), cast(a as double), cast(c as symbol capacity 20 nocache), index(c capacity 100000000)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:2889", + "feature": "create-table-options", + "toSql": "CREATE TABLE x AS (tab), CAST(a AS DOUBLE), CAST(c AS SYMBOL CAPACITY 20 NOCACHE), INDEX(c CAPACITY 100000000)" + }, + { + "query": "create table x as (tab), cast(a as double), cast(c as symbol capacity 16)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserTest.java:2839", + "feature": "create-table-options", + "toSql": "CREATE TABLE x AS (tab), CAST(a AS DOUBLE), CAST(c AS SYMBOL CAPACITY 16)" + }, + { + "query": "create table new as (select * from old), index(s), cast(s as symbol), cast(ts as date)", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/CreateTableTest.java:468", + "feature": "create-table-options", + "toSql": "CREATE TABLE new AS (SELECT * FROM old), CAST(s AS SYMBOL), CAST(ts AS DATE), INDEX(s)" } ] From a9922f70adb8a93432dc0dd964704608785b3e3f Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:49:03 +0300 Subject: [PATCH 24/36] feat: CREATE MATERIALIZED VIEW TTL without PARTITION BY, bare/DEFERRED/INCREMENTAL REFRESH, one-letter TTL units MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server (SqlParser.parseCreateMatView): REFRESH [IMMEDIATE | MANUAL | EVERY .. | INCREMENTAL] [DEFERRED] [PERIOD(..)] AS — the mode is optional and INCREMENTAL is the obsolete name of IMMEDIATE; TTL is parsed by SqlParserCallback.parseTtlSettings independently of PARTITION BY; parseTtlHoursOrMonths accepts ` ` where unit goes through PartitionBy.ttlUnitFromString (HOUR(S)..YEAR(S) and h/d/w/m/y). Before: `.. SAMPLE BY 30s) TTL 12 HOURS`, `REFRESH AS`, `REFRESH DEFERRED`, `REFRESH INCREMENTAL`, `TTL 2 h` all errored. Now: new rule ttlClause (DurationLiteral | number + unit word | number + one-letter identifier, gated) shared by CREATE TABLE and CREATE MATERIALIZED VIEW; the mat-view TTL moved out of materializedViewPartition into its own optional clause after PARTITION BY (AST unchanged: stmt.ttl). REFRESH takes an optional materializedViewRefresh (PERIOD handled by the existing following option); materializedViewRefresh accepts INCREMENTAL (AST mode "immediate", not suggested) and bare DEFERRED (also reachable from ALTER MATERIALIZED VIEW SET REFRESH — looser than the server there). Tests: 3 server queries (StoragePolicyWriterTest:2510/2637, MatViewReloadOnRestartTest:970) + tests/mat-view-create-options.test.ts. Red 12 -> green; full suite 17018 passed. Corpus: fixed 4 / rtChanged 0 / regressed 0 / throws 0 / fmtChangedMeaning 0. Suggestions: +TTL after the mat-view body; +DEFERRED, AS after `REFRESH `; +DEFERRED after ALTER .. SET REFRESH; unit words after `TTL 2 ` in mat views. --- src/autocomplete/content-assist.ts | 5 +- src/parser/cst-types.d.ts | 41 +++++-------- src/parser/parser.ts | 85 ++++++++++----------------- src/parser/visitor.ts | 38 +++++++++--- tests/fixtures/server-queries.json | 19 ++++++ tests/mat-view-create-options.test.ts | 79 +++++++++++++++++++++++++ 6 files changed, 181 insertions(+), 86 deletions(-) create mode 100644 tests/mat-view-create-options.test.ts diff --git a/src/autocomplete/content-assist.ts b/src/autocomplete/content-assist.ts index 8991590..38d073e 100644 --- a/src/autocomplete/content-assist.ts +++ b/src/autocomplete/content-assist.ts @@ -1042,7 +1042,10 @@ function computeSuggestions(tokens: IToken[]): ComputeResult { s.nextTokenType.name === "TimestampNs")) || (s.nextTokenType.name === "Precision" && (s.ruleStack.includes("alterTableAction") || - s.ruleStack[s.ruleStack.length - 1] === "columnNullModifier")) + s.ruleStack[s.ruleStack.length - 1] === "columnNullModifier")) || + // INCREMENTAL is an obsolete synonym of IMMEDIATE in REFRESH modes + (s.nextTokenType.name === "Incremental" && + s.ruleStack[s.ruleStack.length - 1] === "materializedViewRefresh") ) }) const result = suggestions.map((s) => s.nextTokenType) diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 516fb03..80912e6 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -875,19 +875,7 @@ export type CreateTableBodyCstChildren = { Partition?: IToken[]; By?: (IToken)[]; partitionPeriod?: PartitionPeriodCstNode[]; - Ttl?: IToken[]; - DurationLiteral?: IToken[]; - NumberLiteral?: IToken[]; - Hours?: IToken[]; - Days?: IToken[]; - Weeks?: IToken[]; - Months?: IToken[]; - Years?: IToken[]; - Hour?: IToken[]; - Day?: IToken[]; - Week?: IToken[]; - Month?: IToken[]; - Year?: IToken[]; + ttlClause?: TtlClauseCstNode[]; optionalStoragePolicy: OptionalStoragePolicyCstNode[]; optionalTableFormat: (OptionalTableFormatCstNode)[]; Bypass?: IToken[]; @@ -1043,8 +1031,8 @@ export type CreateMaterializedViewBodyCstChildren = { With?: IToken[]; Base?: IToken[]; Refresh?: IToken[]; - materializedViewPeriod?: (MaterializedViewPeriodCstNode)[]; materializedViewRefresh?: MaterializedViewRefreshCstNode[]; + materializedViewPeriod?: MaterializedViewPeriodCstNode[]; As: IToken[]; LParen?: (IToken)[]; selectStatement?: (SelectStatementCstNode)[]; @@ -1055,6 +1043,7 @@ export type CreateMaterializedViewBodyCstChildren = { Timestamp?: IToken[]; columnRef?: ColumnRefCstNode[]; materializedViewPartition?: MaterializedViewPartitionCstNode[]; + ttlClause?: TtlClauseCstNode[]; optionalStoragePolicy: OptionalStoragePolicyCstNode[]; optionalExpireRows: OptionalExpireRowsCstNode[]; In?: IToken[]; @@ -1108,6 +1097,7 @@ export interface MaterializedViewRefreshCstNode extends CstNode { export type MaterializedViewRefreshCstChildren = { Immediate?: IToken[]; Manual?: IToken[]; + Incremental?: IToken[]; Deferred?: (IToken)[]; Every?: IToken[]; intervalValue?: IntervalValueCstNode[]; @@ -1147,19 +1137,19 @@ export type MaterializedViewPartitionCstChildren = { Partition: IToken[]; By: IToken[]; partitionPeriod: PartitionPeriodCstNode[]; - Ttl?: IToken[]; +}; + +export interface TtlClauseCstNode extends CstNode { + name: "ttlClause"; + children: TtlClauseCstChildren; +} + +export type TtlClauseCstChildren = { + Ttl: IToken[]; DurationLiteral?: IToken[]; NumberLiteral?: IToken[]; - Hours?: IToken[]; - Days?: IToken[]; - Weeks?: IToken[]; - Months?: IToken[]; - Years?: IToken[]; - Hour?: IToken[]; - Day?: IToken[]; - Week?: IToken[]; - Month?: IToken[]; - Year?: IToken[]; + storagePolicyTimeUnit?: StoragePolicyTimeUnitCstNode[]; + Identifier?: IToken[]; }; export interface ColumnDefinitionCstNode extends CstNode { @@ -3247,6 +3237,7 @@ export interface ICstNodeVisitor extends ICstVisitor { materializedViewRefresh(children: MaterializedViewRefreshCstChildren, param?: IN): OUT; materializedViewPeriod(children: MaterializedViewPeriodCstChildren, param?: IN): OUT; materializedViewPartition(children: MaterializedViewPartitionCstChildren, param?: IN): OUT; + ttlClause(children: TtlClauseCstChildren, param?: IN): OUT; columnDefinition(children: ColumnDefinitionCstChildren, param?: IN): OUT; columnNullModifier(children: ColumnNullModifierCstChildren, param?: IN): OUT; parquetConfig(children: ParquetConfigCstChildren, param?: IN): OUT; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 26a51c4..7393c18 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -2086,29 +2086,7 @@ class QuestDBParser extends CstParser { }) // Optional TTL (must come before WAL per Java order) - this.OPTION4(() => { - this.CONSUME(Ttl) - this.OR2([ - { ALT: () => this.CONSUME1(DurationLiteral) }, - { - ALT: () => { - this.CONSUME1(NumberLiteral) - this.OR5([ - { ALT: () => this.CONSUME(Hours) }, - { ALT: () => this.CONSUME(Days) }, - { ALT: () => this.CONSUME(Weeks) }, - { ALT: () => this.CONSUME(Months) }, - { ALT: () => this.CONSUME(Years) }, - { ALT: () => this.CONSUME1(Hour) }, - { ALT: () => this.CONSUME1(Day) }, - { ALT: () => this.CONSUME1(Week) }, - { ALT: () => this.CONSUME1(Month) }, - { ALT: () => this.CONSUME1(Year) }, - ]) - }, - }, - ]) - }) + this.OPTION4(() => this.SUBRULE(this.ttlClause)) this.SUBRULE(this.optionalStoragePolicy) @@ -2353,15 +2331,11 @@ class QuestDBParser extends CstParser { this.CONSUME(Base) this.SUBRULE1(this.stringOrQualifiedName) }) + // REFRESH [mode] [DEFERRED] [PERIOD(...)] — mode is optional + // (SqlParser.parseCreateMatView); PERIOD is matched by OPTION3. this.OPTION2(() => { this.CONSUME(Refresh) - this.OR1([ - { - GATE: () => this.LA(1).tokenType === Period, - ALT: () => this.SUBRULE(this.materializedViewPeriod), - }, - { ALT: () => this.SUBRULE(this.materializedViewRefresh) }, - ]) + this.OPTION5(() => this.SUBRULE(this.materializedViewRefresh)) }) this.OPTION3(() => { this.SUBRULE1(this.materializedViewPeriod) @@ -2408,6 +2382,8 @@ class QuestDBParser extends CstParser { this.OPTION7(() => { this.SUBRULE(this.materializedViewPartition) }) + // TTL does not require PARTITION BY (SqlParserCallback.parseTtlSettings) + this.OPTION4(() => this.SUBRULE(this.ttlClause)) this.SUBRULE(this.optionalStoragePolicy) this.SUBRULE(this.optionalExpireRows) this.OPTION8(() => { @@ -2519,10 +2495,14 @@ class QuestDBParser extends CstParser { this.OR1([ { ALT: () => this.CONSUME(Immediate) }, { ALT: () => this.CONSUME(Manual) }, + // obsolete synonym of IMMEDIATE (not suggested) + { ALT: () => this.CONSUME(Incremental) }, ]) this.OPTION(() => this.CONSUME(Deferred)) }, }, + // REFRESH DEFERRED (mode defaults to IMMEDIATE) + { ALT: () => this.CONSUME2(Deferred) }, { ALT: () => { this.CONSUME(Every) @@ -2578,32 +2558,31 @@ class QuestDBParser extends CstParser { this.CONSUME(Partition) this.CONSUME(By) this.SUBRULE(this.partitionPeriod) - this.OPTION(() => { - this.CONSUME(Ttl) - this.OR1([ - { ALT: () => this.CONSUME(DurationLiteral) }, - { - ALT: () => { - this.CONSUME(NumberLiteral) - this.OR2([ - { ALT: () => this.CONSUME(Hours) }, - { ALT: () => this.CONSUME(Days) }, - { ALT: () => this.CONSUME(Weeks) }, - { ALT: () => this.CONSUME(Months) }, - { ALT: () => this.CONSUME(Years) }, - { ALT: () => this.CONSUME(Hour) }, - { ALT: () => this.CONSUME(Day) }, - { ALT: () => this.CONSUME(Week) }, - { ALT: () => this.CONSUME(Month) }, - { ALT: () => this.CONSUME(Year) }, - ]) - }, - }, - ]) - }) }, ) + // TTL | TTL where unit is HOUR(S)/DAY(S)/WEEK(S)/ + // MONTH(S)/YEAR(S) or a single letter h/d/w/m/y + // (SqlParser.parseTtlHoursOrMonths, PartitionBy.ttlUnitFromString). + private ttlClause = this.RULE("ttlClause", () => { + this.CONSUME(Ttl) + this.OR([ + { ALT: () => this.CONSUME(DurationLiteral) }, + { + ALT: () => { + this.CONSUME(NumberLiteral) + this.OR1([ + { ALT: () => this.SUBRULE(this.storagePolicyTimeUnit) }, + { + GATE: () => /^[hdwmy]$/i.test(this.LA(1).image), + ALT: () => this.CONSUME(Identifier), + }, + ]) + }, + }, + ]) + }) + // allowDoublePrecision: CREATE TABLE columns (SqlParser.toColumnType skips // PRECISION after DOUBLE). ALTER TABLE ADD COLUMN does not // (SqlCompilerImpl.addColumnWithType); autocomplete drops PRECISION there. diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 45aeb5e..7bd92f1 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -173,6 +173,7 @@ import type { StoragePolicyCstChildren, StoragePolicyTimeUnitCstChildren, StoragePolicyTtlCstChildren, + TtlClauseCstChildren, HorizonJoinCstChildren, HorizonOffsetCstChildren, StandardJoinCstChildren, @@ -1548,8 +1549,8 @@ class QuestDBVisitor extends BaseVisitor { result.wal = true } - if (ctx.Ttl && (ctx.NumberLiteral || ctx.DurationLiteral)) { - result.ttl = this.extractTtl(ctx) + if (ctx.ttlClause) { + result.ttl = this.visit(ctx.ttlClause) as AST.CreateTableStatement["ttl"] } if (ctx.optionalStoragePolicy) { @@ -2038,7 +2039,11 @@ class QuestDBVisitor extends BaseVisitor { ttl?: AST.CreateMaterializedViewStatement["ttl"] } if (partition.partitionBy) result.partitionBy = partition.partitionBy - if (partition.ttl) result.ttl = partition.ttl + } + if (ctx.ttlClause) { + result.ttl = this.visit( + ctx.ttlClause, + ) as AST.CreateMaterializedViewStatement["ttl"] } if (ctx.optionalStoragePolicy) { const sp = this.visit(ctx.optionalStoragePolicy) as @@ -2121,7 +2126,8 @@ class QuestDBVisitor extends BaseVisitor { const result: AST.MaterializedViewRefresh = { type: "materializedViewRefresh", } - if (ctx.Immediate) result.mode = "immediate" + // INCREMENTAL is the obsolete name of IMMEDIATE + if (ctx.Immediate || ctx.Incremental) result.mode = "immediate" if (ctx.Manual) result.mode = "manual" if (ctx.intervalValue) { result.every = this.extractMaybeString(ctx.intervalValue[0]) @@ -2173,12 +2179,16 @@ class QuestDBVisitor extends BaseVisitor { | "DAY" | "HOUR" } - if (ctx.Ttl && (ctx.NumberLiteral || ctx.DurationLiteral)) { - result.ttl = this.extractTtl(ctx) - } return result } + ttlClause(ctx: TtlClauseCstChildren): { + value: number + unit: "HOURS" | "DAYS" | "WEEKS" | "MONTHS" | "YEARS" + } { + return this.extractTtl(ctx) + } + createLiveViewBody( ctx: CreateLiveViewBodyCstChildren, ): AST.CreateLiveViewStatement { @@ -5236,6 +5246,20 @@ class QuestDBVisitor extends BaseVisitor { unit = this.visit(ctx.timeUnit as CstNode[]) as typeof unit } else if (ctx.storagePolicyTimeUnit) { unit = this.visit(ctx.storagePolicyTimeUnit as CstNode[]) as typeof unit + } else if (ctx.Identifier) { + // single-letter unit: TTL 2 h + const LETTER_UNITS: Record< + string, + "HOURS" | "DAYS" | "WEEKS" | "MONTHS" | "YEARS" + > = { + h: "HOURS", + d: "DAYS", + w: "WEEKS", + m: "MONTHS", + y: "YEARS", + } + unit = + LETTER_UNITS[(ctx.Identifier[0] as IToken).image.toLowerCase()] ?? unit // Check plural forms first (TTL units) before singular (which may be PARTITION BY units) } else if (ctx.Hours) unit = "HOURS" else if (ctx.Days) unit = "DAYS" diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 9f9683e..cd7bfc6 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1728,5 +1728,24 @@ "source": "questdb:core/src/test/java/io/questdb/test/cairo/CreateTableTest.java:468", "feature": "create-table-options", "toSql": "CREATE TABLE new AS (SELECT * FROM old), CAST(s AS SYMBOL), CAST(ts AS DATE), INDEX(s)" + }, + { + "query": "CREATE MATERIALIZED VIEW mv15 AS (SELECT ts, avg(x) FROM t15 SAMPLE BY 1m) TTL 5H", + "source": "questdb-enterprise:src/test/java/com/questdb/cairo/cold/storage/StoragePolicyWriterTest.java:2510", + "feature": "mat-view-create-options", + "toSql": "CREATE MATERIALIZED VIEW mv15 AS (SELECT ts, avg(x) FROM t15 SAMPLE BY 1m) TTL 5 HOURS" + }, + { + "query": "CREATE MATERIALIZED VIEW mv15 AS (SELECT ts, avg(x) FROM t15 SAMPLE BY 2h) TTL 24 MONTHS", + "source": "questdb-enterprise:src/test/java/com/questdb/cairo/cold/storage/StoragePolicyWriterTest.java:2637", + "feature": "mat-view-create-options", + "toSql": "CREATE MATERIALIZED VIEW mv15 AS (SELECT ts, avg(x) FROM t15 SAMPLE BY 2h) TTL 24 MONTHS" + }, + { + "query": "create materialized view price_1h refresh incremental period(length 1d) as select sym, last(price) as price, ts from base_price sample by 1h;", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewReloadOnRestartTest.java:970", + "feature": "mat-view-create-options", + "toSql": "CREATE MATERIALIZED VIEW price_1h REFRESH IMMEDIATE PERIOD (LENGTH 1d) AS SELECT sym, last(price) AS price, ts FROM base_price SAMPLE BY 1h", + "skipAutocomplete": "INCREMENTAL is an obsolete synonym of IMMEDIATE and intentionally not suggested" } ] diff --git a/tests/mat-view-create-options.test.ts b/tests/mat-view-create-options.test.ts new file mode 100644 index 0000000..6a852d6 --- /dev/null +++ b/tests/mat-view-create-options.test.ts @@ -0,0 +1,79 @@ +/** + * CREATE MATERIALIZED VIEW: TTL without PARTITION BY, REFRESH without a mode / + * DEFERRED / INCREMENTAL (obsolete IMMEDIATE), TTL with a one-letter unit. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" + +const ok = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +const q = "AS (SELECT ts, avg(v) FROM t SAMPLE BY 30s)" + +describe("mat view create options", () => { + it("TTL without PARTITION BY", () => { + const mv = ok(`CREATE MATERIALIZED VIEW test ${q} TTL 12 HOURS`) + expect(mv).toMatchObject({ ttl: { value: 12, unit: "HOURS" } }) + ok(`CREATE MATERIALIZED VIEW test ${q} TIMESTAMP(ts) TTL 1 MONTH`) + expect( + toSql( + ok(`CREATE MATERIALIZED VIEW test ${q} PARTITION BY DAY TTL 3 DAYS`), + ), + ).toBe(`CREATE MATERIALIZED VIEW test ${q} PARTITION BY DAY TTL 3 DAYS`) + }) + + it("TTL with a one-letter unit", () => { + expect( + ok(`CREATE MATERIALIZED VIEW test ${q} PARTITION BY DAY TTL 2 h`), + ).toMatchObject({ ttl: { value: 2, unit: "HOURS" } }) + expect( + ok( + "CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY TTL 1 d", + ), + ).toMatchObject({ ttl: { value: 1, unit: "DAYS" } }) + ok( + "CREATE TABLE t (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY TTL 1 M WAL", + ) + }) + + it("REFRESH without mode, DEFERRED, INCREMENTAL", () => { + ok(`CREATE MATERIALIZED VIEW test REFRESH ${q}`) + expect( + ok(`CREATE MATERIALIZED VIEW test REFRESH DEFERRED ${q}`), + ).toMatchObject({ refresh: { deferred: true } }) + ok(`CREATE MATERIALIZED VIEW test REFRESH DEFERRED PERIOD(LENGTH 1d) ${q}`) + expect( + ok(`CREATE MATERIALIZED VIEW test REFRESH INCREMENTAL ${q}`), + ).toMatchObject({ refresh: { mode: "immediate" } }) + ok( + `CREATE MATERIALIZED VIEW test REFRESH INCREMENTAL PERIOD(LENGTH 1d) ${q}`, + ) + ok(`CREATE MATERIALIZED VIEW test REFRESH PERIOD(LENGTH 1d) ${q}`) + }) + + it("autocomplete", () => { + const afterRefresh = getNextValidTokens( + "CREATE MATERIALIZED VIEW v REFRESH ", + ) + expect(afterRefresh).toEqual( + expect.arrayContaining([ + "Immediate", + "Manual", + "Every", + "Deferred", + "Period", + "As", + ]), + ) + expect(afterRefresh).not.toContain("Incremental") + expect(getNextValidTokens(`CREATE MATERIALIZED VIEW v ${q} `)).toContain( + "Ttl", + ) + }) +}) From cd66977ef07a4a3b2bc25d646012c1e69c0c6c26 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:52:00 +0300 Subject: [PATCH 25/36] feat: ALTER TABLE ADD without COLUMN, per-column IF NOT EXISTS, RENAME COLUMN lists, TYPE SYMBOL .. INDEX Server (SqlCompilerImpl): alterTableAddColumn ignores an optional COLUMN and reads `[IF NOT EXISTS] name type` per comma-separated column; alterTableRenameColumn loops over `old TO new` pairs; addColumnWithType (also used by ALTER COLUMN .. TYPE) reads SYMBOL [CAPACITY n] [CACHE|NOCACHE] [INDEX [TYPE ..] [CAPACITY n]]. Before: `ADD x INT` (~40 server tests) and rename lists errored; `TYPE SYMBOL INDEX` silently split into a second statement `INDEX`. Now: ADD -> new rule alterTableAddColumns ([COLUMN] addColumnItem {, addColumnItem}); addColumnItem = [IF NOT EXISTS] columnDefinition. AST: first-column IF NOT EXISTS stays AddColumnAction.ifNotExists; later columns get ColumnDefinition.ifNotExists (toSql prints it before the column). RENAME COLUMN takes `, old TO new` pairs (AST RenameColumnAction.additionalRenames). ALTER COLUMN TYPE gets alterColumnTypeIndex (optional INDEX + indexTypeOptions; AST AlterColumnAction.indexed/indexType/indexInclude/ indexCapacity) before the [NOT] NULL modifier. Tests: 7 server queries (CompositeColumnMappingTest:87, AlterTableChangeColumnTypeTest:933, AggregateTest:2198, AlterTableAddColumnTest:82/324, TableWriterAsyncCmdTest:406, ent HttpCallbackTest:33) + tests/alter-table-columns.test.ts. Red 22 -> green; full suite 17050 passed. Corpus: fixed 34 / rtChanged 3 (TYPE SYMBOL INDEX splits fixed) / regressed 0 / throws 0 / fmtChangedMeaning 0. Suggestions: +IF after `ADD ` and `ADD x INT, `; column names after `RENAME COLUMN a TO b, `; +INDEX after `TYPE SYMBOL `. --- src/parser/ast.ts | 8 +++ src/parser/cst-types.d.ts | 50 +++++++++++++++--- src/parser/parser.ts | 51 +++++++++++++----- src/parser/toSql.ts | 28 +++++++++- src/parser/visitor.ts | 85 +++++++++++++++++++++++++----- tests/alter-table-columns.test.ts | 67 +++++++++++++++++++++++ tests/fixtures/server-queries.json | 42 +++++++++++++++ 7 files changed, 297 insertions(+), 34 deletions(-) create mode 100644 tests/alter-table-columns.test.ts diff --git a/src/parser/ast.ts b/src/parser/ast.ts index ac11c93..fe9e673 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -381,6 +381,8 @@ export interface ColumnDefinition extends AstNode { type: "columnDefinition" name: string dataType: string + /** ALTER TABLE ADD COLUMN a INT, IF NOT EXISTS b INT (columns after the first) */ + ifNotExists?: boolean /** SYMBOL CAPACITY value */ symbolCapacity?: number /** SYMBOL CACHE/NOCACHE */ @@ -604,6 +606,8 @@ export interface RenameColumnAction { actionType: "renameColumn" oldName: string newName: string + /** further `, old TO new` pairs */ + additionalRenames?: { oldName: string; newName: string }[] } export interface AlterColumnAction { @@ -623,6 +627,10 @@ export interface AlterColumnAction { indexType?: "posting" | "posting_delta" | "posting_ef" | "bitmap" | "none" indexInclude?: string[] cache?: boolean + /** TYPE SYMBOL ... INDEX */ + indexed?: boolean + /** TYPE SYMBOL ... INDEX CAPACITY n */ + indexCapacity?: number /** PARQUET config for SET PARQUET(...) */ parquetConfig?: ParquetConfig } diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 80912e6..736b558 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -1620,14 +1620,11 @@ export interface AlterTableActionCstNode extends CstNode { export type AlterTableActionCstChildren = { Add?: (IToken)[]; - Column?: (IToken)[]; - If?: IToken[]; - Not?: IToken[]; - Exists?: IToken[]; - columnDefinition?: (ColumnDefinitionCstNode)[]; - Comma?: (IToken)[]; + alterTableAddColumns?: AlterTableAddColumnsCstNode[]; Drop?: (IToken)[]; + Column?: (IToken)[]; columnRef?: (ColumnRefCstNode)[]; + Comma?: (IToken)[]; Partition?: (IToken)[]; List?: (IToken)[]; StringLiteral?: (IToken)[]; @@ -1636,8 +1633,8 @@ export type AlterTableActionCstChildren = { Storage?: (IToken)[]; Policy?: (IToken)[]; Rename?: IToken[]; - To?: IToken[]; - identifier?: IdentifierCstNode[]; + To?: (IToken)[]; + identifier?: (IdentifierCstNode)[]; Alter?: IToken[]; Type?: (IToken)[]; dataType?: DataTypeCstNode[]; @@ -1645,6 +1642,7 @@ export type AlterTableActionCstChildren = { NumberLiteral?: (IToken)[]; Cache?: (IToken)[]; Nocache?: (IToken)[]; + alterColumnTypeIndex?: AlterColumnTypeIndexCstNode[]; columnNullModifier?: ColumnNullModifierCstNode[]; Index?: (IToken)[]; indexTypeOptions?: IndexTypeOptionsCstNode[]; @@ -1681,6 +1679,39 @@ export type AlterTableActionCstChildren = { convertPartitionTarget?: ConvertPartitionTargetCstNode[]; }; +export interface AlterTableAddColumnsCstNode extends CstNode { + name: "alterTableAddColumns"; + children: AlterTableAddColumnsCstChildren; +} + +export type AlterTableAddColumnsCstChildren = { + Column?: IToken[]; + addColumnItem: (AddColumnItemCstNode)[]; + Comma?: IToken[]; +}; + +export interface AddColumnItemCstNode extends CstNode { + name: "addColumnItem"; + children: AddColumnItemCstChildren; +} + +export type AddColumnItemCstChildren = { + If?: IToken[]; + Not?: IToken[]; + Exists?: IToken[]; + columnDefinition: ColumnDefinitionCstNode[]; +}; + +export interface AlterColumnTypeIndexCstNode extends CstNode { + name: "alterColumnTypeIndex"; + children: AlterColumnTypeIndexCstChildren; +} + +export type AlterColumnTypeIndexCstChildren = { + Index?: IToken[]; + indexTypeOptions?: IndexTypeOptionsCstNode[]; +}; + export interface AlterTableForceActionCstNode extends CstNode { name: "alterTableForceAction"; children: AlterTableForceActionCstChildren; @@ -3268,6 +3299,9 @@ export interface ICstNodeVisitor extends ICstVisitor { memoryLimit(children: MemoryLimitCstChildren, param?: IN): OUT; alterTableStatement(children: AlterTableStatementCstChildren, param?: IN): OUT; alterTableAction(children: AlterTableActionCstChildren, param?: IN): OUT; + alterTableAddColumns(children: AlterTableAddColumnsCstChildren, param?: IN): OUT; + addColumnItem(children: AddColumnItemCstChildren, param?: IN): OUT; + alterColumnTypeIndex(children: AlterColumnTypeIndexCstChildren, param?: IN): OUT; alterTableForceAction(children: AlterTableForceActionCstChildren, param?: IN): OUT; convertPartitionTarget(children: ConvertPartitionTargetCstChildren, param?: IN): OUT; alterMaterializedViewStatement(children: AlterMaterializedViewStatementCstChildren, param?: IN): OUT; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 7393c18..69bd941 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -3260,21 +3260,11 @@ class QuestDBParser extends CstParser { private alterTableAction = this.RULE("alterTableAction", () => { this.OR([ - // ADD COLUMN + // ADD [COLUMN] [IF NOT EXISTS] col type [, [IF NOT EXISTS] col type ...] { ALT: () => { this.CONSUME(Add) - this.CONSUME(Column) - this.OPTION(() => { - this.CONSUME(If) - this.CONSUME(Not) - this.CONSUME(Exists) - }) - this.SUBRULE(this.columnDefinition) - this.MANY(() => { - this.CONSUME(Comma) - this.SUBRULE1(this.columnDefinition) - }) + this.SUBRULE(this.alterTableAddColumns) }, }, // DROP (COLUMN ... | PARTITION ... | STORAGE POLICY) @@ -3335,6 +3325,13 @@ class QuestDBParser extends CstParser { this.SUBRULE2(this.columnRef) this.CONSUME(To) this.SUBRULE3(this.identifier) + // `, old TO new` pairs (SqlCompilerImpl.alterTableRenameColumn) + this.MANY6(() => { + this.CONSUME6(Comma) + this.SUBRULE4(this.columnRef) + this.CONSUME1(To) + this.SUBRULE4(this.identifier) + }) }, }, // ALTER COLUMN (for TYPE, ADD INDEX, CACHE/NOCACHE) @@ -3356,6 +3353,7 @@ class QuestDBParser extends CstParser { // Optional CACHE/NOCACHE this.OPTION5(() => this.CONSUME1(Cache)) this.OPTION6(() => this.CONSUME1(Nocache)) + this.SUBRULE(this.alterColumnTypeIndex) this.SUBRULE(this.columnNullModifier) }, }, @@ -3573,6 +3571,35 @@ class QuestDBParser extends CstParser { ]) }) + // ALTER TABLE ADD [COLUMN] [IF NOT EXISTS] col type [, [IF NOT EXISTS] col type] + // (SqlCompilerImpl.alterTableAddColumn: COLUMN optional, IF NOT EXISTS per column) + private alterTableAddColumns = this.RULE("alterTableAddColumns", () => { + this.OPTION(() => this.CONSUME(Column)) + this.SUBRULE(this.addColumnItem) + this.MANY(() => { + this.CONSUME(Comma) + this.SUBRULE1(this.addColumnItem) + }) + }) + + private addColumnItem = this.RULE("addColumnItem", () => { + this.OPTION(() => { + this.CONSUME(If) + this.CONSUME(Not) + this.CONSUME(Exists) + }) + this.SUBRULE(this.columnDefinition) + }) + + // ALTER COLUMN .. TYPE SYMBOL .. [INDEX [TYPE ..] [CAPACITY n]] + // (SqlCompilerImpl.addColumnWithType). May match nothing. + private alterColumnTypeIndex = this.RULE("alterColumnTypeIndex", () => { + this.OPTION(() => { + this.CONSUME(Index) + this.SUBRULE(this.indexTypeOptions) + }) + }) + // FORCE DROP PARTITION LIST '

' [, ...] // FORCE REINDEX PARQUET [COLUMN ] [PARTITION LIST '

' [, ...]] private alterTableForceAction = this.RULE("alterTableForceAction", () => { diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 0a6bd84..5fe41ec 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -899,7 +899,15 @@ function alterTableToSql(stmt: AST.AlterTableStatement): string { if (action.ifNotExists) { parts.push("IF NOT EXISTS") } - parts.push(action.columns.map(columnDefToSql).join(", ")) + parts.push( + action.columns + .map( + (c, i) => + (i > 0 && c.ifNotExists ? "IF NOT EXISTS " : "") + + columnDefToSql(c), + ) + .join(", "), + ) break } case "dropColumn": @@ -911,6 +919,14 @@ function alterTableToSql(stmt: AST.AlterTableStatement): string { parts.push(escapeIdentifier(action.oldName)) parts.push("TO") parts.push(escapeIdentifier(action.newName)) + for (const r of action.additionalRenames ?? []) { + parts[parts.length - 1] += "," + parts.push( + escapeIdentifier(r.oldName), + "TO", + escapeIdentifier(r.newName), + ) + } break case "alterColumn": parts.push("ALTER COLUMN") @@ -927,6 +943,16 @@ function alterTableToSql(stmt: AST.AlterTableStatement): string { } else if (action.cache === false) { parts.push("NOCACHE") } + if (action.indexed) { + parts.push( + "INDEX" + + indexOptionsToSql( + action.indexType, + action.indexInclude, + action.indexCapacity, + ), + ) + } } else if (action.alterType === "addIndex") { parts.push( "ADD INDEX" + diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 7bd92f1..d7fc2b7 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -174,6 +174,9 @@ import type { StoragePolicyTimeUnitCstChildren, StoragePolicyTtlCstChildren, TtlClauseCstChildren, + AlterTableAddColumnsCstChildren, + AddColumnItemCstChildren, + AlterColumnTypeIndexCstChildren, HorizonJoinCstChildren, HorizonOffsetCstChildren, StandardJoinCstChildren, @@ -1794,6 +1797,44 @@ class QuestDBVisitor extends BaseVisitor { return result } + alterTableAddColumns( + ctx: AlterTableAddColumnsCstChildren, + ): AST.AddColumnAction { + const columns = ctx.addColumnItem.map( + (c) => this.visit(c) as AST.ColumnDefinition, + ) + const result: AST.AddColumnAction = { actionType: "addColumn", columns } + // IF NOT EXISTS on the first column is the action-level flag + if (columns[0].ifNotExists) { + delete columns[0].ifNotExists + result.ifNotExists = true + } + return result + } + + addColumnItem(ctx: AddColumnItemCstChildren): AST.ColumnDefinition { + const col = this.visit(ctx.columnDefinition) as AST.ColumnDefinition + if (ctx.If) col.ifNotExists = true + return col + } + + alterColumnTypeIndex(ctx: AlterColumnTypeIndexCstChildren): + | { + capacity?: number + indexType?: AST.AlterColumnAction["indexType"] + include?: string[] + } + | undefined { + if (!ctx.Index) return undefined + return ctx.indexTypeOptions + ? (this.visit(ctx.indexTypeOptions) as { + capacity?: number + indexType?: AST.AlterColumnAction["indexType"] + include?: string[] + }) + : {} + } + // `[PRECISION] [NOT] NULL` is ignored (as on the server) columnNullModifier(): undefined { return undefined @@ -2686,18 +2727,9 @@ class QuestDBVisitor extends BaseVisitor { return { actionType: "disableStoragePolicy" } } - // ADD COLUMN - if (ctx.Add && ctx.columnDefinition) { - const result: AST.AddColumnAction = { - actionType: "addColumn", - columns: ctx.columnDefinition.map( - (c: CstNode) => this.visit(c) as AST.ColumnDefinition, - ), - } - if (ctx.If) { - result.ifNotExists = true - } - return result + // ADD [COLUMN] + if (ctx.alterTableAddColumns) { + return this.visit(ctx.alterTableAddColumns) as AST.AddColumnAction } // DROP COLUMN (when Drop token exists but no Partition and no Alter — Alter + Drop = ALTER COLUMN DROP INDEX) @@ -2716,11 +2748,23 @@ class QuestDBVisitor extends BaseVisitor { const oldRef = this.visit(ctx.columnRef![0]) as AST.ColumnRef const oldName = oldRef.name.parts[oldRef.name.parts.length - 1] const newName = this.extractIdentifierName(ctx.identifier![0].children) - return { + const result: AST.RenameColumnAction = { actionType: "renameColumn", oldName, newName, } + if (ctx.columnRef!.length > 1) { + result.additionalRenames = ctx.columnRef!.slice(1).map((c, i) => { + const ref = this.visit(c) as AST.ColumnRef + return { + oldName: ref.name.parts[ref.name.parts.length - 1], + newName: this.extractIdentifierName( + ctx.identifier![i + 1].children, + ), + } + }) + } + return result } // ALTER COLUMN @@ -2794,6 +2838,21 @@ class QuestDBVisitor extends BaseVisitor { if (alterType === "type") { if (ctx.Cache) result.cache = true else if (ctx.Nocache) result.cache = false + const idx = ctx.alterColumnTypeIndex + ? (this.visit(ctx.alterColumnTypeIndex) as + | { + capacity?: number + indexType?: AST.AlterColumnAction["indexType"] + include?: string[] + } + | undefined) + : undefined + if (idx) { + result.indexed = true + if (idx.indexType) result.indexType = idx.indexType + if (idx.include) result.indexInclude = idx.include + if (idx.capacity !== undefined) result.indexCapacity = idx.capacity + } } return result } diff --git a/tests/alter-table-columns.test.ts b/tests/alter-table-columns.test.ts new file mode 100644 index 0000000..5e6d5af --- /dev/null +++ b/tests/alter-table-columns.test.ts @@ -0,0 +1,67 @@ +/** + * ALTER TABLE: ALTER COLUMN .. TYPE SYMBOL .. INDEX, ADD without COLUMN, + * RENAME COLUMN lists, per-column IF NOT EXISTS + * (SqlCompilerImpl.alterTableAddColumn / addColumnWithType / alterTableRenameColumn). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" + +const ok = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} +const rt = (sql: string, expected: string) => { + expect(toSql(ok(sql))).toBe(expected) + expect(toSql(ok(expected))).toBe(expected) +} + +describe("alter table columns", () => { + it("TYPE SYMBOL .. INDEX", () => { + rt( + "ALTER TABLE t ALTER COLUMN s TYPE SYMBOL CAPACITY 256 NOCACHE INDEX TYPE BITMAP CAPACITY 512", + "ALTER TABLE t ALTER COLUMN s TYPE SYMBOL CAPACITY 256 NOCACHE INDEX TYPE BITMAP CAPACITY 512", + ) + rt( + "ALTER TABLE t ALTER COLUMN s TYPE SYMBOL CACHE INDEX", + "ALTER TABLE t ALTER COLUMN s TYPE SYMBOL CACHE INDEX", + ) + }) + + it("ADD without COLUMN, IF NOT EXISTS per column", () => { + rt( + "ALTER TABLE t ADD x INT, y DOUBLE", + "ALTER TABLE t ADD COLUMN x INT, y DOUBLE", + ) + rt( + "ALTER TABLE t ADD IF NOT EXISTS x INT", + "ALTER TABLE t ADD COLUMN IF NOT EXISTS x INT", + ) + rt( + "ALTER TABLE t ADD COLUMN x INT, IF NOT EXISTS y INT", + "ALTER TABLE t ADD COLUMN x INT, IF NOT EXISTS y INT", + ) + }) + + it("RENAME COLUMN list", () => { + expect(ok("ALTER TABLE t RENAME COLUMN a TO b, c TO d")).toMatchObject({ + action: { + oldName: "a", + newName: "b", + additionalRenames: [{ oldName: "c", newName: "d" }], + }, + }) + }) + + it("autocomplete", () => { + expect(getNextValidTokens("ALTER TABLE t ADD ")).toEqual( + expect.arrayContaining(["Column", "If"]), + ) + expect( + getNextValidTokens("ALTER TABLE t ALTER COLUMN s TYPE SYMBOL "), + ).toContain("Index") + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index cd7bfc6..53ebf89 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1747,5 +1747,47 @@ "feature": "mat-view-create-options", "toSql": "CREATE MATERIALIZED VIEW price_1h REFRESH IMMEDIATE PERIOD (LENGTH 1d) AS SELECT sym, last(price) AS price, ts FROM base_price SAMPLE BY 1h", "skipAutocomplete": "INCREMENTAL is an obsolete synonym of IMMEDIATE and intentionally not suggested" + }, + { + "query": "ALTER TABLE t ALTER COLUMN s TYPE SYMBOL INDEX", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/composite/CompositeColumnMappingTest.java:87", + "feature": "alter-table-columns", + "toSql": "ALTER TABLE t ALTER COLUMN s TYPE SYMBOL INDEX" + }, + { + "query": "alter table x alter column ik type symbol index", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/AlterTableChangeColumnTypeTest.java:933", + "feature": "alter-table-columns", + "toSql": "ALTER TABLE x ALTER COLUMN ik TYPE SYMBOL INDEX" + }, + { + "query": "alter table tab add val2 DOUBLE", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/AggregateTest.java:2198", + "feature": "alter-table-columns", + "toSql": "ALTER TABLE tab ADD COLUMN val2 DOUBLE" + }, + { + "query": "alter table x add mycol int not null, mycol2 int", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/AlterTableAddColumnTest.java:82", + "feature": "alter-table-columns", + "toSql": "ALTER TABLE x ADD COLUMN mycol INT, mycol2 INT" + }, + { + "query": "alter table x add \"spa ce\" string", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/AlterTableAddColumnTest.java:324", + "feature": "alter-table-columns", + "toSql": "ALTER TABLE x ADD COLUMN \"spa ce\" STRING" + }, + { + "query": "alter table product rename column name to name1, timestamp to timestamp1", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/TableWriterAsyncCmdTest.java:406", + "feature": "alter-table-columns", + "toSql": "ALTER TABLE product RENAME COLUMN name TO name1, timestamp TO timestamp1" + }, + { + "query": "alter table test rename column col1 to colA, col2 to colB", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/integration/acl/HttpCallbackTest.java:33", + "feature": "alter-table-columns", + "toSql": "ALTER TABLE test RENAME COLUMN col1 TO colA, col2 TO colB" } ] From 21c786ec01819a99eec0d6c4e3d008f122e6ce1e Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:54:13 +0300 Subject: [PATCH 26/36] feat: CONVERT PARTITION TO PARQUET|NATIVE LIST .. [WITH (..)] Server (SqlCompilerImpl CONVERT branch + alterTableDropConvertDetachOrAttachPartition[ByList]): CONVERT PARTITION TO PARQUET|NATIVE then LIST '

' [, ...] | WHERE , then for PARQUET an optional WITH (bloom_filter_columns = .., fpp = ..). Before: the server form (~500 server tests) silently split into `CONVERT PARTITION TO PARQUET; LIST AS "2024-01-01"` (LIST was only accepted before TO); with WITH it errored. Now: convertPartitionTarget accepts LIST after the target (the old LIST-before-TO form still parses). AST: ConvertPartitionAction.listAfterTarget so toSql keeps the order. Unquoted partition names are not added (they lex as arithmetic; not trivial). Tests: 5 server queries (AdaptiveParquetColumnConversionRestartTest:75, ArrayTest:2399, ParquetWriteTest:2774, DataSyncFailureCleanupTest:62, CoveringIndexParquetNativeRoundTripTest:273) + tests/convert-partition.test.ts. Red 17 -> green; full suite 17073 passed. Corpus: fixed 11 / rtChanged 195 (all `TO .. LIST '..'` splits fixed) / regressed 2 (`.. TO PARQUET LIST` / `.. TO NATIVE LIST` with no partition name: the server rejects them, maybeExpectToken "partition name"; they only "parsed" before as a silent split) / throws 0 / fmtChangedMeaning 0. Suggestions: +LIST after `CONVERT PARTITION TO PARQUET `; +WHERE, WITH after the LIST. --- src/parser/ast.ts | 2 ++ src/parser/cst-types.d.ts | 2 +- src/parser/parser.ts | 10 +++++++ src/parser/toSql.ts | 13 +++++---- src/parser/visitor.ts | 8 ++++++ tests/convert-partition.test.ts | 45 ++++++++++++++++++++++++++++++ tests/fixtures/server-queries.json | 30 ++++++++++++++++++++ 7 files changed, 103 insertions(+), 7 deletions(-) create mode 100644 tests/convert-partition.test.ts diff --git a/src/parser/ast.ts b/src/parser/ast.ts index fe9e673..c6d5356 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -738,6 +738,8 @@ export interface RebaseWalAction { export interface ConvertPartitionAction { actionType: "convertPartition" partitions?: string[] + /** LIST written after `TO ` (server form) */ + listAfterTarget?: boolean target: string where?: Expression /** WITH (bloom_filter_columns = '...', bloom_filter_fpp = 0.01) */ diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 736b558..4d93e01 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -1736,7 +1736,7 @@ export interface ConvertPartitionTargetCstNode extends CstNode { } export type ConvertPartitionTargetCstChildren = { - List?: IToken[]; + List?: (IToken)[]; StringLiteral?: (IToken)[]; Comma?: (IToken)[]; To: IToken[]; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 69bd941..59990dc 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -3654,6 +3654,16 @@ class QuestDBParser extends CstParser { { ALT: () => this.CONSUME(Table) }, { ALT: () => this.SUBRULE(this.identifier) }, // Parquet, NATIVE, etc. ]) + // Server form: TO PARQUET|NATIVE LIST '

' [, ...] + // (SqlCompilerImpl.alterTableDropConvertDetachOrAttachPartition) + this.OPTION3(() => { + this.CONSUME1(List) + this.CONSUME2(StringLiteral) + this.MANY2(() => { + this.CONSUME1(Comma) + this.CONSUME3(StringLiteral) + }) + }) // Optional WHERE clause after target this.OPTION1(() => { this.CONSUME(Where) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 5fe41ec..e76cc9d 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -1095,14 +1095,15 @@ function alterTableToSql(stmt: AST.AlterTableStatement): string { } case "convertPartition": { parts.push("CONVERT PARTITION") - if (action.partitions && action.partitions.length > 0) { - parts.push("LIST") - parts.push( - action.partitions.map((p: string) => escapeString(p)).join(", "), - ) - } + const list = + action.partitions && action.partitions.length > 0 + ? "LIST " + + action.partitions.map((p: string) => escapeString(p)).join(", ") + : undefined + if (list && !action.listAfterTarget) parts.push(list) parts.push("TO") parts.push(action.target) + if (list && action.listAfterTarget) parts.push(list) if (action.where) { parts.push("WHERE") parts.push(expressionToSql(action.where)) diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index d7fc2b7..4314211 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -228,6 +228,7 @@ type ConvertPartitionTargetResult = { target: string where?: AST.Expression withParams?: AST.TableParam[] + listAfterTarget?: boolean } type GroupByClauseResult = { groupBy?: AST.Expression[] @@ -3669,6 +3670,13 @@ class QuestDBVisitor extends BaseVisitor { result.partitions = ctx.StringLiteral.map((s: IToken) => unquoteSingle(s.image), ) + if ( + ctx.List && + ctx.To && + ctx.List[0].startOffset > ctx.To[0].startOffset + ) { + result.listAfterTarget = true + } } // Target: TABLE or identifier (e.g., Parquet, NATIVE) diff --git a/tests/convert-partition.test.ts b/tests/convert-partition.test.ts new file mode 100644 index 0000000..ccd745e --- /dev/null +++ b/tests/convert-partition.test.ts @@ -0,0 +1,45 @@ +/** + * ALTER TABLE .. CONVERT PARTITION TO PARQUET|NATIVE LIST .. [WITH (..)] + * (SqlCompilerImpl: target first, then LIST | WHERE). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" + +const ok = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +describe("convert partition", () => { + it("LIST after the target", () => { + const stmt = ok( + "ALTER TABLE t CONVERT PARTITION TO PARQUET LIST '2024-01-01', '2024-01-02' WITH (bloom_filter_columns = 'id', fpp = 0.01)", + ) + expect(stmt).toMatchObject({ + action: { + target: "PARQUET", + partitions: ["2024-01-01", "2024-01-02"], + }, + }) + expect(toSql(stmt)).toBe( + "ALTER TABLE t CONVERT PARTITION TO PARQUET LIST '2024-01-01', '2024-01-02' WITH (bloom_filter_columns = 'id', fpp = 0.01)", + ) + }) + + it("existing forms still parse", () => { + expect( + toSql(ok("ALTER TABLE t CONVERT PARTITION LIST '2024-01-01' TO PARQUET")), + ).toBe("ALTER TABLE t CONVERT PARTITION LIST '2024-01-01' TO PARQUET") + ok("ALTER TABLE t CONVERT PARTITION TO NATIVE WHERE ts > 0") + }) + + it("autocomplete", () => { + expect( + getNextValidTokens("ALTER TABLE t CONVERT PARTITION TO PARQUET "), + ).toEqual(expect.arrayContaining(["List", "Where", "With"])) + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 53ebf89..50a75ed 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1789,5 +1789,35 @@ "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/integration/acl/HttpCallbackTest.java:33", "feature": "alter-table-columns", "toSql": "ALTER TABLE test RENAME COLUMN col1 TO colA, col2 TO colB" + }, + { + "query": "ALTER TABLE x CONVERT PARTITION TO PARQUET LIST '2024-01-01'", + "source": "questdb:core/src/test/java/io/questdb/test/AdaptiveParquetColumnConversionRestartTest.java:75", + "feature": "convert-partition", + "toSql": "ALTER TABLE x CONVERT PARTITION TO PARQUET LIST '2024-01-01'" + }, + { + "query": "ALTER TABLE tango CONVERT PARTITION TO PARQUET LIST '2025-01-01'", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/ArrayTest.java:2399", + "feature": "convert-partition", + "toSql": "ALTER TABLE tango CONVERT PARTITION TO PARQUET LIST '2025-01-01'" + }, + { + "query": "ALTER TABLE x CONVERT PARTITION TO PARQUET LIST '2020-01-01' WITH (bloom_filter_columns = 'x')", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/parquet/ParquetWriteTest.java:2774", + "feature": "convert-partition", + "toSql": "ALTER TABLE x CONVERT PARTITION TO PARQUET LIST '2020-01-01' WITH (bloom_filter_columns = 'x')" + }, + { + "query": "ALTER TABLE t CONVERT PARTITION TO NATIVE LIST '2024-01-01'", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/DataSyncFailureCleanupTest.java:62", + "feature": "convert-partition", + "toSql": "ALTER TABLE t CONVERT PARTITION TO NATIVE LIST '2024-01-01'" + }, + { + "query": "ALTER TABLE t_cov CONVERT PARTITION TO PARQUET LIST '2024-01-01', '2024-01-02'", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/covering/CoveringIndexParquetNativeRoundTripTest.java:273", + "feature": "convert-partition", + "toSql": "ALTER TABLE t_cov CONVERT PARTITION TO PARQUET LIST '2024-01-01', '2024-01-02'" } ] From 322b5031c9c48fc0ef8d9f873ed81826ff5bc33e Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 16:58:20 +0300 Subject: [PATCH 27/36] fix: ALTER MATERIALIZED VIEW SUSPEND WAL WITH , '' Server: compileAlterMatView's SUSPEND branch calls parseSuspendWal, the same parser tables and live views use (optional `WITH , ''`). Before: `ALTER MATERIALIZED VIEW mv SUSPEND WAL WITH 24, 'msg'` errored (the table and live view rules already accepted the WITH form). Now: new rule suspendWalWith used by the mat-view SUSPEND WAL alternative; AST AlterMaterializedViewSuspendWal gets optional code / message (printed by toSql). Tests: no server test uses the WITH form for mat views (only plain SUSPEND WAL, which already parsed); tests/alter-mat-view-wal.test.ts. Red 2 -> green; full suite 17075 passed (one run of the RSI per-position <100ms autocomplete timing test flaked under parallel load; a direct bench shows no slowdown vs HEAD and it passes on rerun). Corpus: fixed 0 / rtChanged 0 / regressed 0 / throws 0 / fmtChangedMeaning 0. Suggestions: +WITH after `ALTER MATERIALIZED VIEW mv SUSPEND WAL `. --- src/parser/ast.ts | 2 ++ src/parser/cst-types.d.ts | 14 +++++++++++ src/parser/parser.ts | 14 ++++++++++- src/parser/toSql.ts | 7 ++++++ src/parser/visitor.ts | 31 ++++++++++++++++++++++-- tests/alter-mat-view-wal.test.ts | 41 ++++++++++++++++++++++++++++++++ 6 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 tests/alter-mat-view-wal.test.ts diff --git a/src/parser/ast.ts b/src/parser/ast.ts index c6d5356..8e41aec 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -476,6 +476,8 @@ export interface AlterMaterializedViewResumeWal { export interface AlterMaterializedViewSuspendWal { actionType: "suspendWal" + code?: number | string + message?: string } export interface AlterMaterializedViewRebaseWal { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 4d93e01..b37fdcd 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -1762,6 +1762,18 @@ export type AlterMaterializedViewStatementCstChildren = { alterMaterializedViewAction: AlterMaterializedViewActionCstNode[]; }; +export interface SuspendWalWithCstNode extends CstNode { + name: "suspendWalWith"; + children: SuspendWalWithCstChildren; +} + +export type SuspendWalWithCstChildren = { + With: IToken[]; + NumberLiteral?: IToken[]; + StringLiteral: (IToken)[]; + Comma: IToken[]; +}; + export interface AlterMaterializedViewActionCstNode extends CstNode { name: "alterMaterializedViewAction"; children: AlterMaterializedViewActionCstChildren; @@ -1794,6 +1806,7 @@ export type AlterMaterializedViewActionCstChildren = { Transaction?: IToken[]; Txn?: IToken[]; Suspend?: IToken[]; + suspendWalWith?: SuspendWalWithCstNode[]; Rebase?: IToken[]; Into?: IToken[]; StringLiteral?: IToken[]; @@ -3305,6 +3318,7 @@ export interface ICstNodeVisitor extends ICstVisitor { alterTableForceAction(children: AlterTableForceActionCstChildren, param?: IN): OUT; convertPartitionTarget(children: ConvertPartitionTargetCstChildren, param?: IN): OUT; alterMaterializedViewStatement(children: AlterMaterializedViewStatementCstChildren, param?: IN): OUT; + suspendWalWith(children: SuspendWalWithCstChildren, param?: IN): OUT; alterMaterializedViewAction(children: AlterMaterializedViewActionCstChildren, param?: IN): OUT; dropStatement(children: DropStatementCstChildren, param?: IN): OUT; dropViewStatement(children: DropViewStatementCstChildren, param?: IN): OUT; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 59990dc..84a2bfd 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -3696,6 +3696,17 @@ class QuestDBParser extends CstParser { }, ) + // WITH , '' after SUSPEND WAL (materialized views) + private suspendWalWith = this.RULE("suspendWalWith", () => { + this.CONSUME(With) + this.OR([ + { ALT: () => this.CONSUME(NumberLiteral) }, + { ALT: () => this.CONSUME(StringLiteral) }, + ]) + this.CONSUME(Comma) + this.CONSUME1(StringLiteral) + }) + private alterMaterializedViewAction = this.RULE( "alterMaterializedViewAction", () => { @@ -3805,11 +3816,12 @@ class QuestDBParser extends CstParser { }) }, }, - // SUSPEND WAL + // SUSPEND WAL [WITH , ''] (parseSuspendWal) { ALT: () => { this.CONSUME(Suspend) this.CONSUME1(Wal) + this.OPTION6(() => this.SUBRULE(this.suspendWalWith)) }, }, // REBASE WAL [INTO ''] diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index e76cc9d..b6e0685 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -1426,6 +1426,13 @@ function alterMaterializedViewToSql( } case "suspendWal": parts.push("SUSPEND WAL") + if (action.code != null && action.message != null) { + const code = + typeof action.code === "number" + ? String(action.code) + : escapeString(action.code) + parts.push(`WITH ${code}, ${escapeString(action.message)}`) + } break case "rebaseWal": { parts.push("REBASE WAL") diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index 4314211..d51ebb0 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -174,6 +174,7 @@ import type { StoragePolicyTimeUnitCstChildren, StoragePolicyTtlCstChildren, TtlClauseCstChildren, + SuspendWalWithCstChildren, AlterTableAddColumnsCstChildren, AddColumnItemCstChildren, AlterColumnTypeIndexCstChildren, @@ -1836,6 +1837,20 @@ class QuestDBVisitor extends BaseVisitor { : {} } + suspendWalWith(ctx: SuspendWalWithCstChildren): { + code: number | string + message: string + } { + return { + code: ctx.NumberLiteral + ? tokenInt(ctx.NumberLiteral[0].image) + : unquoteSingle(ctx.StringLiteral[0].image), + message: unquoteSingle( + ctx.StringLiteral[ctx.StringLiteral.length - 1].image, + ), + } + } + // `[PRECISION] [NOT] NULL` is ignored (as on the server) columnNullModifier(): undefined { return undefined @@ -2570,9 +2585,21 @@ class QuestDBVisitor extends BaseVisitor { return result } - // SUSPEND WAL + // SUSPEND WAL [WITH code, 'message'] if (ctx.Suspend) { - return { actionType: "suspendWal" } + const result: AST.AlterMaterializedViewSuspendWal = { + actionType: "suspendWal", + } + if (ctx.suspendWalWith) { + Object.assign( + result, + this.visit(ctx.suspendWalWith) as { + code: number | string + message: string + }, + ) + } + return result } // REBASE WAL [INTO ''] diff --git a/tests/alter-mat-view-wal.test.ts b/tests/alter-mat-view-wal.test.ts new file mode 100644 index 0000000..051d34b --- /dev/null +++ b/tests/alter-mat-view-wal.test.ts @@ -0,0 +1,41 @@ +/** + * ALTER MATERIALIZED VIEW .. SUSPEND WAL [WITH , ''] + * (SqlCompilerImpl.parseSuspendWal, shared with tables). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" + +const ok = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +describe("ALTER MATERIALIZED VIEW SUSPEND WAL WITH", () => { + it("parses and round-trips", () => { + const stmt = ok("ALTER MATERIALIZED VIEW mv SUSPEND WAL WITH 24, 'msg'") + expect(stmt).toMatchObject({ + action: { actionType: "suspendWal", code: 24, message: "msg" }, + }) + expect(toSql(stmt)).toBe( + "ALTER MATERIALIZED VIEW mv SUSPEND WAL WITH 24, 'msg'", + ) + expect( + toSql( + ok("ALTER MATERIALIZED VIEW mv SUSPEND WAL WITH 'DISK FULL', 'oops'"), + ), + ).toBe("ALTER MATERIALIZED VIEW mv SUSPEND WAL WITH 'DISK FULL', 'oops'") + expect(toSql(ok("ALTER MATERIALIZED VIEW mv SUSPEND WAL"))).toBe( + "ALTER MATERIALIZED VIEW mv SUSPEND WAL", + ) + }) + + it("autocomplete", () => { + expect( + getNextValidTokens("ALTER MATERIALIZED VIEW mv SUSPEND WAL "), + ).toContain("With") + }) +}) From 9f19dadc8f06a55122c16401cc58554c9e0ce2a9 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 17:04:51 +0300 Subject: [PATCH 28/36] feat: PG session statements BEGIN/COMMIT/ROLLBACK, DEALLOCATE, SET, DISCARD/CLOSE/UNLISTEN/RESET The server dispatches these statement-start keywords to PG-compatibility executors (SqlCompilerImpl.registerKeywordBasedExecutors): compileBegin / compileCommit / compileRollback (optional TRANSACTION), compileDeallocate (statement name), compileSet (SET [SESSION|LOCAL] name {=|TO} v[, v]* or any tokens up to `;`, e.g. SET TIME ZONE 'UTC'), compileNoOp for DISCARD/CLOSE/UNLISTEN/RESET (one argument token). Before: BEGIN/COMMIT/ROLLBACK [TRANSACTION] were silently parsed as implicit selects (`COMMIT AS TRANSACTION`); SET ..., DEALLOCATE, DISCARD, CLOSE, UNLISTEN, RESET were parse errors. Now: new rules transactionStatement, deallocateStatement, pgNoOpStatement and setStatement (SET followed by a loose run of identifiers/literals/=/TO/,/...; SET TYPE [BYPASS] WAL keeps its own rule, gated on TYPE BYPASS|WAL). AST nodes transaction / deallocate / pgNoOp / set (raw args). New keywords are non-reserved (SELECT begin FROM t still works) and highlight as keywords. Autocomplete: inside SET only TO is offered after the name. Tests: 9 server fixture entries + tests/pg-session-statements.test.ts. Red 21 -> green; full suite 17116 passed. Corpus: fixed 47 / rtChanged 26 (implicit-select misparses of BEGIN/COMMIT/ ROLLBACK now transaction statements) / regressed 1 (prose "commit() must build ..." - server runs compileCommit and rejects `(`) / throws 0 / formatterChangedMeaning 0. --- src/autocomplete/content-assist.ts | 4 ++ src/grammar/keywords.ts | 8 +++ src/parser/ast.ts | 31 +++++++++++ src/parser/cst-types.d.ts | 72 ++++++++++++++++++++++++ src/parser/lexer.ts | 16 ++++++ src/parser/parser.ts | 79 ++++++++++++++++++++++++++- src/parser/toSql.ts | 10 ++++ src/parser/tokens.ts | 18 ++++++ src/parser/visitor.ts | 78 ++++++++++++++++++++++++++ tests/fixtures/server-queries.json | 56 +++++++++++++++++++ tests/pg-session-statements.test.ts | 85 +++++++++++++++++++++++++++++ 11 files changed, 456 insertions(+), 1 deletion(-) create mode 100644 tests/pg-session-statements.test.ts diff --git a/src/autocomplete/content-assist.ts b/src/autocomplete/content-assist.ts index 38d073e..42d6ef9 100644 --- a/src/autocomplete/content-assist.ts +++ b/src/autocomplete/content-assist.ts @@ -1043,6 +1043,10 @@ function computeSuggestions(tokens: IToken[]): ComputeResult { (s.nextTokenType.name === "Precision" && (s.ruleStack.includes("alterTableAction") || s.ruleStack[s.ruleStack.length - 1] === "columnNullModifier")) || + // SET ... takes free-form tokens; only offer TO after the name + (s.ruleStack[s.ruleStack.length - 1] === "setStatement" && + (s.nextTokenType.name !== "To" || + tokens[tokens.length - 1]?.tokenType.name === "Set")) || // INCREMENTAL is an obsolete synonym of IMMEDIATE in REFRESH modes (s.nextTokenType.name === "Incremental" && s.ruleStack[s.ruleStack.length - 1] === "materializedViewRefresh") diff --git a/src/grammar/keywords.ts b/src/grammar/keywords.ts index 0654d1e..8132211 100644 --- a/src/grammar/keywords.ts +++ b/src/grammar/keywords.ts @@ -20,6 +20,7 @@ export const keywords: string[] = [ "backup", "base", "batch", + "begin", "between", "bloom_filter", "bloom_filter_columns", @@ -35,9 +36,11 @@ export const keywords: string[] = [ "cast", "checkpoint", "cleanup", + "close", "cold", "column", "columns", + "commit", "commitLag", "compile", "compression_codec", @@ -51,6 +54,7 @@ export const keywords: string[] = [ "current", "data_page_size", "database", + "deallocate", "declare", "dedup", "deduplicate", @@ -62,6 +66,7 @@ export const keywords: string[] = [ "detach", "details", "disable", + "discard", "distinct", "drop", "else", @@ -176,12 +181,14 @@ export const keywords: string[] = [ "repair", "replace", "replica", + "reset", "resource", "respect", "resume", "revoke", "right", "role", + "rollback", "rollup", "row", "row_group_size", @@ -222,6 +229,7 @@ export const keywords: string[] = [ "type", "unbounded", "union", + "unlisten", "unlock", "unnest", "unpivot", diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 8e41aec..36a5a75 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -54,6 +54,10 @@ export type Statement = | VacuumTableStatement | ResumeWalStatement | SetTypeStatement + | SetStatement + | TransactionStatement + | DeallocateStatement + | PgNoOpStatement | ReindexTableStatement | RefreshMaterializedViewStatement | PivotStatement @@ -997,6 +1001,33 @@ export interface SetTypeStatement extends AstNode { wal: boolean } +/** PG-compatible `SET ...` (a no-op on the server); `args` are the raw tokens after SET */ +export interface SetStatement extends AstNode { + type: "set" + args: string[] +} + +/** BEGIN | COMMIT | ROLLBACK [TRANSACTION] */ +export interface TransactionStatement extends AstNode { + type: "transaction" + action: "begin" | "commit" | "rollback" + transaction?: boolean +} + +export interface DeallocateStatement extends AstNode { + type: "deallocate" + /** Statement name as written (quotes kept) */ + name: string +} + +/** DISCARD | CLOSE | UNLISTEN | RESET (no-ops on the server) */ +export interface PgNoOpStatement extends AstNode { + type: "pgNoOp" + keyword: "discard" | "close" | "unlisten" | "reset" + /** Argument as written: ALL, *, or a name */ + argument: string +} + export interface ReindexTableStatement extends AstNode { type: "reindexTable" table: QualifiedName diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index b37fdcd..6405351 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -42,6 +42,10 @@ export type StatementCstChildren = { vacuumTableStatement?: VacuumTableStatementCstNode[]; resumeWalStatement?: ResumeWalStatementCstNode[]; setTypeStatement?: SetTypeStatementCstNode[]; + setStatement?: SetStatementCstNode[]; + transactionStatement?: TransactionStatementCstNode[]; + deallocateStatement?: DeallocateStatementCstNode[]; + pgNoOpStatement?: PgNoOpStatementCstNode[]; reindexTableStatement?: ReindexTableStatementCstNode[]; refreshMaterializedViewStatement?: RefreshMaterializedViewStatementCstNode[]; pivotStatement?: PivotStatementCstNode[]; @@ -2425,6 +2429,70 @@ export type SetTypeStatementCstChildren = { Wal: IToken[]; }; +export interface SetStatementCstNode extends CstNode { + name: "setStatement"; + children: SetStatementCstChildren; +} + +export type SetStatementCstChildren = { + Set: IToken[]; + identifier?: IdentifierCstNode[]; + NumberLiteral?: IToken[]; + LongLiteral?: IToken[]; + FloatLiteral?: IToken[]; + DecimalLiteral?: IToken[]; + DurationLiteral?: IToken[]; + Equals?: IToken[]; + To?: IToken[]; + Comma?: IToken[]; + Minus?: IToken[]; + Plus?: IToken[]; + Dot?: IToken[]; + On?: IToken[]; + As?: IToken[]; + All?: IToken[]; + True?: IToken[]; + False?: IToken[]; + Null?: IToken[]; +}; + +export interface TransactionStatementCstNode extends CstNode { + name: "transactionStatement"; + children: TransactionStatementCstChildren; +} + +export type TransactionStatementCstChildren = { + Begin?: IToken[]; + Commit?: IToken[]; + Rollback?: IToken[]; + Transaction?: IToken[]; +}; + +export interface DeallocateStatementCstNode extends CstNode { + name: "deallocateStatement"; + children: DeallocateStatementCstChildren; +} + +export type DeallocateStatementCstChildren = { + Deallocate: IToken[]; + identifier: IdentifierCstNode[]; +}; + +export interface PgNoOpStatementCstNode extends CstNode { + name: "pgNoOpStatement"; + children: PgNoOpStatementCstChildren; +} + +export type PgNoOpStatementCstChildren = { + Discard?: IToken[]; + Close?: IToken[]; + Unlisten?: IToken[]; + Reset?: IToken[]; + All?: IToken[]; + Star?: IToken[]; + identifier?: IdentifierCstNode[]; +}; + export interface ReindexTableStatementCstNode extends CstNode { name: "reindexTableStatement"; children: ReindexTableStatementCstChildren; @@ -3359,6 +3427,10 @@ export interface ICstNodeVisitor extends ICstVisitor { vacuumTableStatement(children: VacuumTableStatementCstChildren, param?: IN): OUT; resumeWalStatement(children: ResumeWalStatementCstChildren, param?: IN): OUT; setTypeStatement(children: SetTypeStatementCstChildren, param?: IN): OUT; + setStatement(children: SetStatementCstChildren, param?: IN): OUT; + transactionStatement(children: TransactionStatementCstChildren, param?: IN): OUT; + deallocateStatement(children: DeallocateStatementCstChildren, param?: IN): OUT; + pgNoOpStatement(children: PgNoOpStatementCstChildren, param?: IN): OUT; reindexTableStatement(children: ReindexTableStatementCstChildren, param?: IN): OUT; refreshMaterializedViewStatement(children: RefreshMaterializedViewStatementCstChildren, param?: IN): OUT; pivotStatement(children: PivotStatementCstChildren, param?: IN): OUT; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index dfeb09c..5e8ffb2 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -6,6 +6,14 @@ import { IdentifierKeyword, keywordTokenArray, Abort, + Begin, + Close, + Commit, + Deallocate, + Discard, + Reset, + Rollback, + Unlisten, Account, Accounts, Add, @@ -349,6 +357,14 @@ import { export { IdentifierKeyword } export { Abort, + Begin, + Close, + Commit, + Deallocate, + Discard, + Reset, + Rollback, + Unlisten, Account, Accounts, Add, diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 84a2bfd..63d74f4 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -210,6 +210,14 @@ import { Native, Error, Abort, + Begin, + Close, + Commit, + Deallocate, + Discard, + Reset, + Rollback, + Unlisten, StatisticsEnabled, Stats, Primary, @@ -670,7 +678,16 @@ class QuestDBParser extends CstParser { { ALT: () => this.SUBRULE(this.revokeStatement) }, { ALT: () => this.SUBRULE(this.vacuumTableStatement) }, { ALT: () => this.SUBRULE(this.resumeWalStatement) }, - { ALT: () => this.SUBRULE(this.setTypeStatement) }, + { + GATE: () => + this.LA(2).tokenType === Type && + (this.LA(3).tokenType === Bypass || this.LA(3).tokenType === Wal), + ALT: () => this.SUBRULE(this.setTypeStatement), + }, + { ALT: () => this.SUBRULE(this.setStatement) }, + { ALT: () => this.SUBRULE(this.transactionStatement) }, + { ALT: () => this.SUBRULE(this.deallocateStatement) }, + { ALT: () => this.SUBRULE(this.pgNoOpStatement) }, { ALT: () => this.SUBRULE(this.reindexTableStatement) }, { ALT: () => this.SUBRULE(this.refreshMaterializedViewStatement) }, { @@ -4789,6 +4806,66 @@ class QuestDBParser extends CstParser { this.CONSUME(Wal) }) + // PG compatibility no-ops (SqlCompilerImpl.compileSet / compileBegin / + // compileCommit / compileRollback / compileDeallocate / compileNoOp). + // SET [SESSION | LOCAL] name {= | TO} value [, ...] or any other tokens up + // to `;` (SET TIME ZONE 'UTC'); accepted loosely as a run of plain tokens. + private setStatement = this.RULE("setStatement", () => { + this.CONSUME(Set) + this.AT_LEAST_ONE(() => { + this.OR([ + { ALT: () => this.SUBRULE(this.identifier) }, + { ALT: () => this.CONSUME(NumberLiteral) }, + { ALT: () => this.CONSUME(LongLiteral) }, + { ALT: () => this.CONSUME(FloatLiteral) }, + { ALT: () => this.CONSUME(DecimalLiteral) }, + { ALT: () => this.CONSUME(DurationLiteral) }, + { ALT: () => this.CONSUME(Equals) }, + { ALT: () => this.CONSUME(To) }, + { ALT: () => this.CONSUME(Comma) }, + { ALT: () => this.CONSUME(Minus) }, + { ALT: () => this.CONSUME(Plus) }, + { ALT: () => this.CONSUME(Dot) }, + { ALT: () => this.CONSUME(On) }, + { ALT: () => this.CONSUME(As) }, + { ALT: () => this.CONSUME(All) }, + { ALT: () => this.CONSUME(True) }, + { ALT: () => this.CONSUME(False) }, + { ALT: () => this.CONSUME(Null) }, + ]) + }) + }) + + // BEGIN | COMMIT | ROLLBACK [TRANSACTION] + private transactionStatement = this.RULE("transactionStatement", () => { + this.OR([ + { ALT: () => this.CONSUME(Begin) }, + { ALT: () => this.CONSUME(Commit) }, + { ALT: () => this.CONSUME(Rollback) }, + ]) + this.OPTION(() => this.CONSUME(Transaction)) + }) + + private deallocateStatement = this.RULE("deallocateStatement", () => { + this.CONSUME(Deallocate) + this.SUBRULE(this.identifier) + }) + + // DISCARD | CLOSE | UNLISTEN | RESET , e.g. ALL, *, a name + private pgNoOpStatement = this.RULE("pgNoOpStatement", () => { + this.OR([ + { ALT: () => this.CONSUME(Discard) }, + { ALT: () => this.CONSUME(Close) }, + { ALT: () => this.CONSUME(Unlisten) }, + { ALT: () => this.CONSUME(Reset) }, + ]) + this.OR1([ + { ALT: () => this.CONSUME(All) }, + { ALT: () => this.CONSUME(Star) }, + { ALT: () => this.SUBRULE(this.identifier) }, + ]) + }) + private reindexTableStatement = this.RULE("reindexTableStatement", () => { this.CONSUME(Reindex) this.CONSUME(Table) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index b6e0685..b34142c 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -125,6 +125,16 @@ function statementToSql(stmt: AST.Statement): string { return resumeWalToSql(stmt) case "setType": return setTypeToSql(stmt) + case "set": + return ["SET", ...stmt.args].join(" ").replace(/ ,/g, ",") + case "transaction": + return ( + stmt.action.toUpperCase() + (stmt.transaction ? " TRANSACTION" : "") + ) + case "deallocate": + return `DEALLOCATE ${stmt.name}` + case "pgNoOp": + return `${stmt.keyword.toUpperCase()} ${stmt.argument}` case "reindexTable": return reindexTableToSql(stmt) case "copyCancel": diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index f2e6b33..225f298 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -432,6 +432,16 @@ export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ // reserved on the server. "At", "Precision", + // PG session statements (BEGIN/COMMIT/ROLLBACK, DEALLOCATE, DISCARD/CLOSE/ + // UNLISTEN/RESET): statement-start words on the server, not reserved. + "Begin", + "Close", + "Commit", + "Deallocate", + "Discard", + "Reset", + "Rollback", + "Unlisten", ]) for (const name of IDENTIFIER_KEYWORD_NAMES) { @@ -480,6 +490,14 @@ function getToken(name: string): TokenType { // Keywords (from grammar/keywords.ts) export const Abort = getToken("Abort") +export const Begin = getToken("Begin") +export const Close = getToken("Close") +export const Commit = getToken("Commit") +export const Deallocate = getToken("Deallocate") +export const Discard = getToken("Discard") +export const Reset = getToken("Reset") +export const Rollback = getToken("Rollback") +export const Unlisten = getToken("Unlisten") export const Account = getToken("Account") export const Accounts = getToken("Accounts") export const Add = getToken("Add") diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index d51ebb0..d15c6f2 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -164,6 +164,10 @@ import type { SetExpressionCstChildren, SetOperationCstChildren, SetTypeStatementCstChildren, + SetStatementCstChildren, + TransactionStatementCstChildren, + DeallocateStatementCstChildren, + PgNoOpStatementCstChildren, ShowCreateDatabaseCategoryCstChildren, ShowStatementCstChildren, SimpleSelectCstChildren, @@ -400,6 +404,18 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.setTypeStatement) { return this.visit(ctx.setTypeStatement) as AST.SetTypeStatement } + if (ctx.setStatement) { + return this.visit(ctx.setStatement) as AST.SetStatement + } + if (ctx.transactionStatement) { + return this.visit(ctx.transactionStatement) as AST.TransactionStatement + } + if (ctx.deallocateStatement) { + return this.visit(ctx.deallocateStatement) as AST.DeallocateStatement + } + if (ctx.pgNoOpStatement) { + return this.visit(ctx.pgNoOpStatement) as AST.PgNoOpStatement + } if (ctx.reindexTableStatement) { return this.visit(ctx.reindexTableStatement) as AST.ReindexTableStatement } @@ -3890,6 +3906,68 @@ class QuestDBVisitor extends BaseVisitor { } } + setStatement(ctx: SetStatementCstChildren): AST.SetStatement { + const tokens: IToken[] = [] + const collect = (children: Record) => { + for (const [key, value] of Object.entries(children)) { + if (key === "Set") continue + for (const el of value as (IToken | CstNode)[]) { + if ("image" in el) tokens.push(el) + else collect(el.children) + } + } + } + collect(ctx as unknown as Record) + tokens.sort((a, b) => a.startOffset - b.startOffset) + return { type: "set", args: tokens.map((t) => t.image) } + } + + transactionStatement( + ctx: TransactionStatementCstChildren, + ): AST.TransactionStatement { + const result: AST.TransactionStatement = { + type: "transaction", + action: ctx.Begin ? "begin" : ctx.Commit ? "commit" : "rollback", + } + if (ctx.Transaction) result.transaction = true + return result + } + + deallocateStatement( + ctx: DeallocateStatementCstChildren, + ): AST.DeallocateStatement { + const id = ctx.identifier[0].children + const token = [ + ...(id.Identifier ?? []), + ...(id.QuotedIdentifier ?? []), + ...(id.StringLiteral ?? []), + ...(id.IdentifierKeyword ?? []), + ][0] + return { type: "deallocate", name: token.image } + } + + pgNoOpStatement(ctx: PgNoOpStatementCstChildren): AST.PgNoOpStatement { + const keyword = ctx.Discard + ? "discard" + : ctx.Close + ? "close" + : ctx.Unlisten + ? "unlisten" + : "reset" + let argument = "ALL" + if (ctx.Star) argument = "*" + else if (ctx.identifier) { + const id = ctx.identifier[0].children + argument = [ + ...(id.Identifier ?? []), + ...(id.QuotedIdentifier ?? []), + ...(id.StringLiteral ?? []), + ...(id.IdentifierKeyword ?? []), + ][0].image + } + return { type: "pgNoOp", keyword, argument } + } + reindexTableStatement( ctx: ReindexTableStatementCstChildren, ): AST.ReindexTableStatement { diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 50a75ed..b5297cc 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1819,5 +1819,61 @@ "source": "questdb:core/src/test/java/io/questdb/test/cairo/covering/CoveringIndexParquetNativeRoundTripTest.java:273", "feature": "convert-partition", "toSql": "ALTER TABLE t_cov CONVERT PARTITION TO PARQUET LIST '2024-01-01', '2024-01-02'" + }, + { + "query": "BEGIN TRANSACTION", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/SqlCompilerImplTest.java:3241", + "feature": "pg-session-statements", + "toSql": "BEGIN TRANSACTION" + }, + { + "query": "COMMIT TRANSACTION;", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGMultiStatementMessageTest.java:125", + "feature": "pg-session-statements", + "toSql": "COMMIT TRANSACTION" + }, + { + "query": "BEGIN; select 2", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGMultiStatementMessageTest.java:324", + "feature": "pg-session-statements", + "toSql": "BEGIN;\nSELECT 2", + "statements": 2 + }, + { + "query": "DEALLOCATE qwp_stmt", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/qwp/QwpEgressDdlExecTest.java:296", + "feature": "pg-session-statements", + "toSql": "DEALLOCATE qwp_stmt" + }, + { + "query": "SET TIME ZONE 'UTC'", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/SqlCompilerImplTest.java:3357", + "feature": "pg-session-statements", + "toSql": "SET TIME ZONE 'UTC'" + }, + { + "query": "SET x TO y, z", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/SqlCompilerImplTest.java:3330", + "feature": "pg-session-statements", + "toSql": "SET x TO y, z" + }, + { + "query": "SET application_name = 'qwp'", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/qwp/QwpEgressDdlExecTest.java:291", + "feature": "pg-session-statements", + "toSql": "SET application_name = 'qwp'" + }, + { + "query": "DISCARD ALL", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGMultiStatementMessageTest.java:860", + "feature": "pg-session-statements", + "toSql": "DISCARD ALL" + }, + { + "query": "CLOSE ALL;UNLISTEN *;RESET ALL;", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/pgwire/PGMultiStatementMessageTest.java:103", + "feature": "pg-session-statements", + "toSql": "CLOSE ALL;\nUNLISTEN *;\nRESET ALL", + "statements": 3 } ] diff --git a/tests/pg-session-statements.test.ts b/tests/pg-session-statements.test.ts new file mode 100644 index 0000000..b48c1cd --- /dev/null +++ b/tests/pg-session-statements.test.ts @@ -0,0 +1,85 @@ +/** + * PostgreSQL session statements the server accepts as (no-op) statements: + * BEGIN/COMMIT/ROLLBACK [TRANSACTION], DEALLOCATE name, SET ..., and + * DISCARD/CLOSE/UNLISTEN/RESET (SqlCompilerImpl.registerKeywordBasedExecutors). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" + +const ok = (sql: string, n = 1) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(n) + return result.ast +} + +describe("PG session statements", () => { + it("transaction statements", () => { + expect(ok("BEGIN")[0]).toMatchObject({ + type: "transaction", + action: "begin", + }) + expect(ok("rollback transaction")[0]).toMatchObject({ + type: "transaction", + action: "rollback", + transaction: true, + }) + expect(toSql(ok("commit"))).toBe("COMMIT") + }) + + it("SET forms", () => { + expect(ok("SET datestyle TO ISO, MDY")[0]).toMatchObject({ type: "set" }) + expect(toSql(ok("SET search_path TO public"))).toBe( + "SET search_path TO public", + ) + ok("SET SESSION x = y") + ok("SET LOCAL x = y") + ok("SET extra_float_digits = 3") + ok("SET x GARBAGE y") + ok("SET x = -1; SELECT 1", 2) + expect(ok("SET TYPE BYPASS WAL")[0]).toMatchObject({ + type: "setType", + bypass: true, + }) + expect(ok("set type wal")[0]).toMatchObject({ type: "setType" }) + expect(ok("SET type TO wal")[0]).toMatchObject({ type: "set" }) + }) + + it("no-op statements", () => { + expect(ok("UNLISTEN *")[0]).toMatchObject({ + type: "pgNoOp", + keyword: "unlisten", + }) + expect(toSql(ok("DISCARD TEMP"))).toBe("DISCARD TEMP") + expect(toSql(ok("DEALLOCATE 'stmt1'"))).toBe("DEALLOCATE 'stmt1'") + }) + + it("words stay usable as identifiers", () => { + for (const w of [ + "begin", + "commit", + "rollback", + "deallocate", + "discard", + "close", + "unlisten", + "reset", + ]) { + const ast = ok(`SELECT ${w} FROM ${w} WHERE ${w} > 0`) + expect(ast[0]).toMatchObject({ type: "select" }) + } + expect(ok("SELECT close, reset AS begin FROM t")[0]).toMatchObject({ + type: "select", + }) + }) + + it("autocomplete", () => { + const start = getNextValidTokens("") + for (const t of ["Begin", "Commit", "Rollback", "Set"]) { + expect(start).toContain(t) + } + expect(getNextValidTokens("BEGIN ")).toContain("Transaction") + }) +}) From 9ad7294ba2d1c3983fdabf3e7198d4f52a128226 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 17:13:07 +0300 Subject: [PATCH 29/36] feat: DROP ALL without TABLES, UPDATE t AS alias, COPY TO quoted codec/partition_by, VACUUM PARTITIONS Server: - DROP ALL[;] or legacy DROP ALL TABLES (SqlCompilerImpl.compileDrop). - UPDATE tbl [AS] alias SET ... (SqlParser.parseUpdate; SqlParserUpdateTest). - COPY .. TO .. WITH compression_codec / partition_by use expectLiteral, so quoted values ('zstd', 'DAY') work (SqlParser.parseCopy). - VACUUM PARTITIONS t is the legacy spelling of VACUUM TABLE t (SqlCompilerImpl.compileVacuum). Before: all four were parse errors. Now: TABLES optional after DROP ALL; optional AS before the UPDATE alias; StringLiteral accepted for compression_codec and partition_by in copyOption; VACUUM TABLE|PARTITIONS. AST unchanged (DROP ALL prints as DROP ALL TABLES, VACUUM PARTITIONS as VACUUM TABLE). Formatter grammar-drift: UPDATE alias AS is inline by design. Tests: 6 server fixture entries + tests/misc-statements.test.ts. Red 18 -> green; full suite 17144 passed (RSI autocomplete timing test flaked under parallel load once; passes alone). Corpus: fixed 16 / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0. Suggestions: +PARTITIONS after VACUUM, +AS after UPDATE t. --- src/parser/cst-types.d.ts | 6 ++-- src/parser/parser.ts | 24 ++++++++++++--- tests/fixtures/server-queries.json | 36 ++++++++++++++++++++++ tests/formatter/grammar-drift.test.ts | 2 +- tests/misc-statements.test.ts | 44 +++++++++++++++++++++++++++ 5 files changed, 104 insertions(+), 8 deletions(-) create mode 100644 tests/misc-statements.test.ts diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 6405351..a47b7f7 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -766,6 +766,7 @@ export interface UpdateStatementCstNode extends CstNode { export type UpdateStatementCstChildren = { Update: IToken[]; tableName: TableNameCstNode[]; + As?: IToken[]; identifier?: IdentifierCstNode[]; Set: IToken[]; setClause: (SetClauseCstNode)[]; @@ -2167,7 +2168,7 @@ export type CopyOptionCstChildren = { stringOrIdentifier?: (StringOrIdentifierCstNode)[]; Delimiter?: IToken[]; Format?: IToken[]; - StringLiteral?: IToken[]; + StringLiteral?: (IToken)[]; identifier?: IdentifierCstNode[]; Partition?: IToken[]; By?: IToken[]; @@ -2398,7 +2399,8 @@ export interface VacuumTableStatementCstNode extends CstNode { export type VacuumTableStatementCstChildren = { Vacuum: IToken[]; - Table: IToken[]; + Table?: IToken[]; + Partitions?: IToken[]; tableName: TableNameCstNode[]; }; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 63d74f4..d5015ed 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -1880,8 +1880,11 @@ class QuestDBParser extends CstParser { private updateStatement = this.RULE("updateStatement", () => { this.CONSUME(Update) this.SUBRULE(this.tableName) - // Optional alias - this.OPTION2(() => this.SUBRULE(this.identifier)) + // Optional alias, with optional AS (SqlParser.parseUpdate) + this.OPTION2(() => { + this.OPTION3(() => this.CONSUME(As)) + this.SUBRULE(this.identifier) + }) this.CONSUME(Set) this.SUBRULE(this.setClause) this.MANY(() => { @@ -3918,8 +3921,9 @@ class QuestDBParser extends CstParser { this.OR([ { ALT: () => { + // DROP ALL [TABLES] (SqlCompilerImpl.compileDrop) this.CONSUME(All) - this.CONSUME(Tables) + this.OPTION1(() => this.CONSUME(Tables)) }, }, { @@ -4374,7 +4378,11 @@ class QuestDBParser extends CstParser { }, { ALT: () => this.CONSUME(PartitionBy) }, ]) - this.SUBRULE(this.partitionPeriod) + this.OR6([ + { ALT: () => this.SUBRULE(this.partitionPeriod) }, + // expectLiteral unquotes: partition_by 'DAY' + { ALT: () => this.CONSUME3(StringLiteral) }, + ]) }, }, { @@ -4400,6 +4408,8 @@ class QuestDBParser extends CstParser { { ALT: () => this.CONSUME(Lz4Raw) }, { ALT: () => this.CONSUME(Brotli) }, { ALT: () => this.CONSUME(Lzo) }, + // expectLiteral unquotes: compression_codec 'zstd' + { ALT: () => this.CONSUME2(StringLiteral) }, ]) }, }, @@ -4779,7 +4789,11 @@ class QuestDBParser extends CstParser { private vacuumTableStatement = this.RULE("vacuumTableStatement", () => { this.CONSUME(Vacuum) - this.CONSUME(Table) + // VACUUM PARTITIONS is the legacy spelling (SqlCompilerImpl.compileVacuum) + this.OR([ + { ALT: () => this.CONSUME(Table) }, + { ALT: () => this.CONSUME(Partitions) }, + ]) this.SUBRULE(this.tableName) }) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index b5297cc..20e84d4 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1875,5 +1875,41 @@ "feature": "pg-session-statements", "toSql": "CLOSE ALL;\nUNLISTEN *;\nRESET ALL", "statements": 3 + }, + { + "query": "DROP ALL", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/DdlListenerTest.java:173", + "feature": "misc-statements", + "toSql": "DROP ALL TABLES" + }, + { + "query": "drop all;", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/mv/MatViewTest.java:3561", + "feature": "misc-statements", + "toSql": "DROP ALL TABLES" + }, + { + "query": "update tblx as xx set tt = 1 from tbly y where xx.x = y and x > 10", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserUpdateTest.java:392", + "feature": "misc-statements", + "toSql": "UPDATE tblx xx SET tt = 1 FROM tbly AS y WHERE xx.x = y AND x > 10" + }, + { + "query": "update tblx as xx set tt = 1 from (tbly LIMIT 10) y where xx.x = y and x > 10", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/SqlParserUpdateTest.java:446", + "feature": "misc-statements", + "toSql": "UPDATE tblx xx SET tt = 1 FROM (tbly LIMIT 10) AS y WHERE xx.x = y AND x > 10" + }, + { + "query": "vacuum partitions tbl", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/o3/O3PartitionPurgeTest.java:1293", + "feature": "misc-statements", + "toSql": "VACUUM TABLE tbl" + }, + { + "query": "VACUUM partitions \"таблица\";", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/VacuumTablePartitionTest.java:43", + "feature": "misc-statements", + "toSql": "VACUUM TABLE таблица" } ] diff --git a/tests/formatter/grammar-drift.test.ts b/tests/formatter/grammar-drift.test.ts index fbc9944..6a7693a 100644 --- a/tests/formatter/grammar-drift.test.ts +++ b/tests/formatter/grammar-drift.test.ts @@ -68,7 +68,7 @@ const NON_CLAUSE_RULES = new Set([ const INLINE_BY_DESIGN: Record = { select: ["Distinct"], insert: ["Into", "Atomic", "Batch"], - update: [], + update: ["As"], createTable: [ "Table", "Atomic", diff --git a/tests/misc-statements.test.ts b/tests/misc-statements.test.ts new file mode 100644 index 0000000..78dafbf --- /dev/null +++ b/tests/misc-statements.test.ts @@ -0,0 +1,44 @@ +/** + * DROP ALL [TABLES], UPDATE t AS alias, COPY TO with quoted + * compression_codec / partition_by values, VACUUM PARTITIONS t. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" + +const ok = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +describe("misc statements", () => { + it("DROP ALL", () => { + expect(toSql(ok("DROP ALL"))).toBe("DROP ALL TABLES") + expect(toSql(ok("DROP ALL TABLES"))).toBe("DROP ALL TABLES") + }) + + it("UPDATE t AS alias", () => { + expect(toSql(ok("UPDATE tblx AS xx SET tt = 1 WHERE x > 10"))).toBe( + "UPDATE tblx xx SET tt = 1 WHERE x > 10", + ) + expect(toSql(ok("UPDATE tblx xx SET tt = 1"))).toBe( + "UPDATE tblx xx SET tt = 1", + ) + }) + + it("COPY TO quoted options", () => { + ok("copy t to 'o' with format parquet compression_codec 'zstd'") + ok("copy t to 'o' with format parquet partition_by 'DAY'") + ok( + "copy t to 'o' with format parquet compression_codec 'zstd' partition_by 'DAY' row_group_size 1000", + ) + }) + + it("VACUUM PARTITIONS", () => { + expect(ok("VACUUM PARTITIONS t")).toMatchObject({ type: "vacuumTable" }) + expect(getNextValidTokens("VACUUM ")).toContain("Partitions") + }) +}) From c0cb207a82c1f0e6bea56aa0e9dbe9700aa744a6 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 17:18:41 +0300 Subject: [PATCH 30/36] feat: POSTING_DELTA/POSTING_EF, PARQUET byte_stream_split/default codec, PG type aliases as data types Server: - Index type names go through IndexType.valueOf, which also takes the single words POSTING_DELTA / POSTING_EF (besides POSTING [DELTA|EF]). - ParquetEncoding names include byte_stream_split; ParquetCompression names include default (PARQUET(enc, default), COPY compression_codec default). - ColumnType.nameTypeMap aliases: text smallint bigint real bytea regclass regprocedure array. Before: TYPE POSTING_DELTA, PARQUET(BYTE_STREAM_SPLIT), PARQUET(plain, default) and compression_codec default were parse errors; the type aliases were plain identifiers (not highlighted / suggested as types). Now: new non-reserved tokens PostingDelta, PostingEf, ByteStreamSplit (constants) and Text, Smallint, Bigint, Real, Bytea, Regclass, Regprocedure, Array (dataTypes, exported as ArrayType to avoid shadowing the global). All are IdentifierKeyword, so they stay usable as names. dataType accepts the aliases explicitly; typed literals (text 'x') use the new tokens instead of an image check; ARRAY[...] literals consume the Array token; EXPLAIN (FORMAT TEXT) accepts the Text token. Autocomplete: ARRAY is not offered at every expression start (only as a type name). Tests: 4 server fixture entries (2 regression guards for ::real / ::text[] with a `text` column) + tests/option-names.test.ts. Red 5 -> green; full suite 17163 passed. Corpus: fixed 2 / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0. Suggestions: + alias types after :: / CAST AS / column types; +POSTING_DELTA,POSTING_EF after INDEX TYPE; +BYTE_STREAM_SPLIT, DEFAULT in PARQUET(...); +TEXT after EXPLAIN (FORMAT. --- src/autocomplete/content-assist.ts | 4 ++ src/grammar/constants.ts | 3 ++ src/grammar/dataTypes.ts | 8 ++++ src/parser/cst-types.d.ts | 16 +++++++- src/parser/lexer.ts | 22 ++++++++++ src/parser/parser.ts | 61 +++++++++++++++++---------- src/parser/tokens.ts | 23 +++++++++++ src/parser/visitor.ts | 15 +++++-- tests/fixtures/server-queries.json | 24 +++++++++++ tests/option-names.test.ts | 66 ++++++++++++++++++++++++++++++ 10 files changed, 215 insertions(+), 27 deletions(-) create mode 100644 tests/option-names.test.ts diff --git a/src/autocomplete/content-assist.ts b/src/autocomplete/content-assist.ts index 42d6ef9..9fc965d 100644 --- a/src/autocomplete/content-assist.ts +++ b/src/autocomplete/content-assist.ts @@ -1047,6 +1047,10 @@ function computeSuggestions(tokens: IToken[]): ComputeResult { (s.ruleStack[s.ruleStack.length - 1] === "setStatement" && (s.nextTokenType.name !== "To" || tokens[tokens.length - 1]?.tokenType.name === "Set")) || + // ARRAY[...] literals: not offered at every expression start (it was an + // identifier before); ARRAY stays suggested as a type name + (s.nextTokenType.name === "Array" && + s.ruleStack[s.ruleStack.length - 1] === "arrayLiteral") || // INCREMENTAL is an obsolete synonym of IMMEDIATE in REFRESH modes (s.nextTokenType.name === "Incremental" && s.ruleStack[s.ruleStack.length - 1] === "materializedViewRefresh") diff --git a/src/grammar/constants.ts b/src/grammar/constants.ts index f65760e..01708e1 100644 --- a/src/grammar/constants.ts +++ b/src/grammar/constants.ts @@ -4,6 +4,7 @@ export const constants: string[] = [ "beginning", "bitmap", "brotli", + "byte_stream_split", "cadence", "century", "complete", @@ -70,6 +71,8 @@ export const constants: string[] = [ "pgwire", "plain", "posting", + "posting_delta", + "posting_ef", "prepare", "prev", "quarter", diff --git a/src/grammar/dataTypes.ts b/src/grammar/dataTypes.ts index 1d81357..d00129e 100644 --- a/src/grammar/dataTypes.ts +++ b/src/grammar/dataTypes.ts @@ -1,7 +1,10 @@ export const dataTypes: string[] = [ + "array", + "bigint", "binary", "boolean", "byte", + "bytea", "char", "date", "decimal", @@ -15,9 +18,14 @@ export const dataTypes: string[] = [ "long", "long128", "long256", + "real", + "regclass", + "regprocedure", "short", + "smallint", "string", "symbol", + "text", "timestamp", "timestamp_ns", "uuid", diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index a47b7f7..1e7e620 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -1213,6 +1213,7 @@ export type ParquetEncodingCstChildren = { RleDictionary?: IToken[]; DeltaBinaryPacked?: IToken[]; DeltaLengthByteArray?: IToken[]; + ByteStreamSplit?: IToken[]; Default?: IToken[]; }; @@ -1228,6 +1229,7 @@ export type ParquetCompressionCstChildren = { Brotli?: IToken[]; Zstd?: IToken[]; Lz4Raw?: IToken[]; + Default?: IToken[]; }; export interface OptionalStoragePolicyCstNode extends CstNode { @@ -1325,6 +1327,8 @@ export type IndexTypeOptionsCstChildren = { Posting?: IToken[]; Delta?: IToken[]; Ef?: IToken[]; + PostingDelta?: IToken[]; + PostingEf?: IToken[]; Bitmap?: IToken[]; None?: IToken[]; Include?: IToken[]; @@ -2077,6 +2081,7 @@ export type ExplainStatementCstChildren = { LParen?: IToken[]; Format?: IToken[]; Identifier?: IToken[]; + Text?: IToken[]; RParen?: IToken[]; statement: StatementCstNode[]; }; @@ -2188,6 +2193,7 @@ export type CopyOptionCstChildren = { Lz4Raw?: IToken[]; Brotli?: IToken[]; Lzo?: IToken[]; + Default?: IToken[]; CompressionLevel?: IToken[]; expression?: (ExpressionCstNode)[]; RowGroupSize?: IToken[]; @@ -2850,7 +2856,7 @@ export interface ArrayLiteralCstNode extends CstNode { } export type ArrayLiteralCstChildren = { - Identifier: IToken[]; + Array: IToken[]; arrayBracketBody: ArrayBracketBodyCstNode[]; }; @@ -2960,6 +2966,14 @@ export type DataTypeCstChildren = { Comma?: IToken[]; Interval?: IToken[]; TimestampNs?: IToken[]; + Text?: IToken[]; + Smallint?: IToken[]; + Bigint?: IToken[]; + Real?: IToken[]; + Bytea?: IToken[]; + Regclass?: IToken[]; + Regprocedure?: IToken[]; + Array?: IToken[]; Identifier?: IToken[]; LBracket?: IToken[]; RBracket?: IToken[]; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index 5e8ffb2..3d8b056 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -348,6 +348,17 @@ import { Daily, Expression, Posting, + ByteStreamSplit, + PostingDelta, + PostingEf, + ArrayType, + Bigint, + Bytea, + Real, + Regclass, + Regprocedure, + Smallint, + Text, Delta, Ef, Bitmap, @@ -699,6 +710,17 @@ export { Daily, Expression, Posting, + ByteStreamSplit, + PostingDelta, + PostingEf, + ArrayType, + Bigint, + Bytea, + Real, + Regclass, + Regprocedure, + Smallint, + Text, Delta, Ef, Bitmap, diff --git a/src/parser/parser.ts b/src/parser/parser.ts index d5015ed..520d508 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -243,6 +243,17 @@ import { Daily, Expression, Posting, + ByteStreamSplit, + PostingDelta, + PostingEf, + ArrayType, + Bigint, + Bytea, + Real, + Regclass, + Regprocedure, + Smallint, + Text, Delta, Ef, Bitmap, @@ -423,13 +434,11 @@ const TYPED_LITERAL_TYPE_TOKENS: ReadonlySet = new globalThis.Set([ Ipv4, Varchar, Decimal, -]) -const TYPED_LITERAL_TYPE_ALIASES: ReadonlySet = new globalThis.Set([ - "text", - "smallint", - "bigint", - "real", - "bytea", + Text, + Smallint, + Bigint, + Real, + Bytea, ]) class QuestDBParser extends CstParser { @@ -2712,6 +2721,7 @@ class QuestDBParser extends CstParser { { ALT: () => this.CONSUME(RleDictionary) }, { ALT: () => this.CONSUME(DeltaBinaryPacked) }, { ALT: () => this.CONSUME(DeltaLengthByteArray) }, + { ALT: () => this.CONSUME(ByteStreamSplit) }, { ALT: () => this.CONSUME(Default) }, ]) }) @@ -2724,6 +2734,7 @@ class QuestDBParser extends CstParser { { ALT: () => this.CONSUME(Brotli) }, { ALT: () => this.CONSUME(Zstd) }, { ALT: () => this.CONSUME(Lz4Raw) }, + { ALT: () => this.CONSUME(Default) }, ]) }) @@ -2852,6 +2863,9 @@ class QuestDBParser extends CstParser { ) }, }, + // IndexType.valueOf also takes POSTING_DELTA / POSTING_EF + { ALT: () => this.CONSUME(PostingDelta) }, + { ALT: () => this.CONSUME(PostingEf) }, { ALT: () => this.CONSUME(Bitmap) }, { ALT: () => this.CONSUME(None) }, ]) @@ -4249,7 +4263,8 @@ class QuestDBParser extends CstParser { this.CONSUME(LParen) this.CONSUME(Format) this.OR1([ - { ALT: () => this.CONSUME(Identifier) }, // TEXT or JSON + { ALT: () => this.CONSUME(Identifier) }, // JSON + { ALT: () => this.CONSUME(Text) }, ]) this.CONSUME(RParen) }) @@ -4408,6 +4423,7 @@ class QuestDBParser extends CstParser { { ALT: () => this.CONSUME(Lz4Raw) }, { ALT: () => this.CONSUME(Brotli) }, { ALT: () => this.CONSUME(Lzo) }, + { ALT: () => this.CONSUME(Default) }, // expectLiteral unquotes: compression_codec 'zstd' { ALT: () => this.CONSUME2(StringLiteral) }, ]) @@ -5398,15 +5414,10 @@ class QuestDBParser extends CstParser { }, { // ARRAY[...] literal — must come before functionCall/columnRef - // since ARRAY is tokenized as Identifier - GATE: () => { - const la1 = this.LA(1) - return ( - la1.tokenType === Identifier && - la1.image.toUpperCase() === "ARRAY" && - this.LA(2).tokenType === LBracket - ) - }, + // since ARRAY is a non-reserved keyword (usable as a name) + GATE: () => + this.LA(1).tokenType === ArrayType && + this.LA(2).tokenType === LBracket, ALT: () => this.SUBRULE(this.arrayLiteral), }, { @@ -5464,7 +5475,7 @@ class QuestDBParser extends CstParser { // ARRAY[...] literal: ARRAY[expr, expr, ...] or ARRAY[[...], [...], ...] private arrayLiteral = this.RULE("arrayLiteral", () => { - this.CONSUME(Identifier) // 'ARRAY' + this.CONSUME(ArrayType) this.SUBRULE(this.arrayBracketBody) }) @@ -5607,6 +5618,15 @@ class QuestDBParser extends CstParser { }, { ALT: () => this.CONSUME(Interval) }, { ALT: () => this.CONSUME(TimestampNs) }, + // PG aliases (ColumnType.nameTypeMap) + { ALT: () => this.CONSUME(Text) }, + { ALT: () => this.CONSUME(Smallint) }, + { ALT: () => this.CONSUME(Bigint) }, + { ALT: () => this.CONSUME(Real) }, + { ALT: () => this.CONSUME(Bytea) }, + { ALT: () => this.CONSUME(Regclass) }, + { ALT: () => this.CONSUME(Regprocedure) }, + { ALT: () => this.CONSUME(ArrayType) }, { ALT: () => this.CONSUME(Identifier) }, // For custom types ]) // Optional array dimensions: DOUBLE[], DOUBLE[][] @@ -5718,10 +5738,7 @@ class QuestDBParser extends CstParser { input: IToken[] } if (internals.input[internals.currIdx] !== first) return false - const isType = - TYPED_LITERAL_TYPE_TOKENS.has(first.tokenType) || - (first.tokenType === Identifier && - TYPED_LITERAL_TYPE_ALIASES.has(first.image.toLowerCase())) + const isType = TYPED_LITERAL_TYPE_TOKENS.has(first.tokenType) const next = this.LA(1).tokenType return isType && (next === StringLiteral || next === EscapeStringLiteral) } diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index 225f298..b988e0a 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -411,8 +411,20 @@ export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ "Daily", "Expression", "Posting", + "PostingDelta", + "PostingEf", "Delta", "Ef", + "ByteStreamSplit", + // PG type aliases (ColumnType.nameTypeMap); not reserved on the server + "Array", + "Bigint", + "Bytea", + "Real", + "Regclass", + "Regprocedure", + "Smallint", + "Text", "Bitmap", "Schema", "Acl", @@ -786,6 +798,14 @@ export const Timestamp = getToken("Timestamp") export const TimestampNs = getToken("TimestampNs") export const Uuid = getToken("Uuid") export const Varchar = getToken("Varchar") +export const ArrayType = getToken("Array") +export const Bigint = getToken("Bigint") +export const Bytea = getToken("Bytea") +export const Real = getToken("Real") +export const Regclass = getToken("Regclass") +export const Regprocedure = getToken("Regprocedure") +export const Smallint = getToken("Smallint") +export const Text = getToken("Text") // Constants (from grammar/constants.ts) export const True = getToken("True") @@ -854,6 +874,9 @@ export const Memory = getToken("Memory") export const Daily = getToken("Daily") export const Expression = getToken("Expression") export const Posting = getToken("Posting") +export const ByteStreamSplit = getToken("ByteStreamSplit") +export const PostingDelta = getToken("PostingDelta") +export const PostingEf = getToken("PostingEf") export const Delta = getToken("Delta") export const Ef = getToken("Ef") export const Bitmap = getToken("Bitmap") diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index d15c6f2..e92ff62 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -1908,6 +1908,10 @@ class QuestDBVisitor extends BaseVisitor { : ctx.Ef ? "posting_ef" : "posting" + } else if (ctx.PostingDelta) { + r.indexType = "posting_delta" + } else if (ctx.PostingEf) { + r.indexType = "posting_ef" } else if (ctx.Bitmap) { r.indexType = "bitmap" } else if (ctx.None) { @@ -3447,8 +3451,9 @@ class QuestDBVisitor extends BaseVisitor { type: "explain", statement: this.visit(ctx.statement) as AST.Statement, } - if (ctx.Format && ctx.Identifier) { - result.format = ctx.Identifier[0].image.toUpperCase() + const formatToken = ctx.Identifier?.[0] ?? ctx.Text?.[0] + if (ctx.Format && formatToken) { + result.format = formatToken.image.toUpperCase() } return result } @@ -3599,7 +3604,8 @@ class QuestDBVisitor extends BaseVisitor { ctx.Zstd || ctx.Lz4Raw || ctx.Brotli || - ctx.Lzo + ctx.Lzo || + ctx.Default ) { const codecToken = ctx.Uncompressed?.[0] ?? @@ -3609,7 +3615,8 @@ class QuestDBVisitor extends BaseVisitor { ctx.Zstd?.[0] ?? ctx.Lz4Raw?.[0] ?? ctx.Brotli?.[0] ?? - ctx.Lzo?.[0] + ctx.Lzo?.[0] ?? + ctx.Default?.[0] result.value = codecToken?.image } else if (ctx.partitionPeriod && (ctx.Partition || ctx.PartitionBy)) { result.value = this.visit(ctx.partitionPeriod) as diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 20e84d4..e2a57e2 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1911,5 +1911,29 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/VacuumTablePartitionTest.java:43", "feature": "misc-statements", "toSql": "VACUUM TABLE таблица" + }, + { + "query": "ALTER TABLE x ALTER COLUMN e SET PARQUET(BYTE_STREAM_SPLIT)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/AlterTableAlterColumnTest.java:881", + "feature": "option-names", + "toSql": "ALTER TABLE x ALTER COLUMN e SET PARQUET(byte_stream_split)" + }, + { + "query": "SELECT count() c FROM j WHERE json_extract(text,'$.a')::real = 0.1", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/json/JsonExtractCastScenariosTest.java:393", + "feature": "option-names", + "toSql": "SELECT count() AS c FROM j WHERE json_extract(text, '$.a')::REAL = 0.1" + }, + { + "query": "select cast(a as string)::regclass b, a from tab", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/cast/CastTest.java:5588", + "feature": "option-names", + "toSql": "SELECT CAST(a AS STRING)::REGCLASS AS b, a FROM tab" + }, + { + "query": "select * from tab where a <> all('{abc,xyz}'::text[])", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/bool/AllNotEqStrFunctionFactoryTest.java:60", + "feature": "option-names", + "toSql": "SELECT * FROM tab WHERE a <> ALL ('{abc,xyz}'::TEXT[])" } ] diff --git a/tests/option-names.test.ts b/tests/option-names.test.ts new file mode 100644 index 0000000..774f233 --- /dev/null +++ b/tests/option-names.test.ts @@ -0,0 +1,66 @@ +/** + * Index type / parquet option names accepted by the server as single words + * (IndexType.valueOf, ParquetEncoding, ParquetCompression) and the PG type + * aliases of ColumnType.nameTypeMap (highlighting / autocomplete). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" +import { dataTypes } from "../src/grammar/dataTypes" + +const ok = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +const ALIASES = [ + "text", + "smallint", + "bigint", + "real", + "bytea", + "regclass", + "regprocedure", + "array", +] + +describe("option names", () => { + it("POSTING_DELTA / POSTING_EF index types", () => { + ok("ALTER TABLE t ALTER COLUMN s ADD INDEX TYPE POSTING_DELTA") + ok("ALTER TABLE t ALTER COLUMN s ADD INDEX TYPE posting_ef") + ok("CREATE TABLE t (s SYMBOL INDEX TYPE POSTING_DELTA, ts TIMESTAMP)") + ok("ALTER TABLE t ALTER COLUMN s ADD INDEX TYPE POSTING DELTA") + }) + + it("parquet encodings and codecs", () => { + ok("ALTER TABLE x ALTER COLUMN e SET PARQUET(plain, default)") + ok("ALTER TABLE x ALTER COLUMN e SET PARQUET(byte_stream_split, zstd(3))") + ok("copy t to 'o' with format parquet compression_codec default") + expect( + getNextValidTokens("ALTER TABLE x ALTER COLUMN e SET PARQUET("), + ).toContain("ByteStreamSplit") + }) + + it("type aliases are data types and identifiers", () => { + for (const t of ALIASES) expect(dataTypes).toContain(t) + for (const t of ALIASES.filter((a) => a !== "array")) { + expect(toSql(ok(`SELECT x::${t} FROM t`))).toBe( + `SELECT x::${t.toUpperCase()} FROM t`, + ) + ok(`SELECT CAST(x AS ${t}) FROM t`) + ok(`CREATE TABLE t (a ${t}, ts TIMESTAMP)`) + // still usable as names + ok(`SELECT ${t}, ${t}.${t} FROM ${t} WHERE ${t} > 0`) + ok(`CREATE TABLE ${t} (${t} INT)`) + } + expect(toSql(ok("SELECT ARRAY[1, 2] AS array FROM t"))).toBe( + "SELECT ARRAY[1, 2] AS array FROM t", + ) + ok("SELECT array, ARRAY[[1.0], [2.0]] FROM array") + ok("SELECT x::text[] FROM t") + expect(getNextValidTokens("SELECT CAST(x AS ")).toContain("Bigint") + }) +}) From 0afa0160b9397d642672e55d2169fc9cfccb0f5a Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 17:20:58 +0300 Subject: [PATCH 31/36] =?UTF-8?q?feat:=20functions=20list=20=E2=80=94=20mi?= =?UTF-8?q?ssing=20table-valued/scalar=20names,=20files()=20table-valued?= =?UTF-8?q?=20only,=20drop=20storage=5Fpolicies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checked against server FunctionFactory signatures (OSS core griffin/engine/functions/** and Enterprise com/questdb/griffin/engine/functions): - table-valued: checkpoint_status(), export_activity(), pg_enum(), pg_catalog.pg_extension(), information_schema.{tables, columns, key_column_usage, table_constraints, referential_constraints, character_sets, questdb_columns}(); Enterprise backup_status(), table_cold_partitions(s), tick_calendars(). - scalar: build(), current_data_id(), from_base64(S), interval(NN), cancel_query, timestamp_floor_utc; Enterprise backup_instance_name(), node_role(), reload_tick_calendars(). - files(s) is a cursor function only (moved from scalar to table-valued); glob has both a scalar glob(Ss) and a cursor glob(s) (now in both lists). - storage_policies is an Enterprise view, not a function: removed from tableValuedFunctions (still parses as a table name); parser test renamed to the view form. Word lists only; no grammar change. Tests: 2 server fixture entries + tests/functions-list.test.ts. Red 3 -> green; full suite 17174 passed. Corpus: fixed 0 / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0. Suggestions: new names offered in FROM (table-valued) and expression (scalar) positions; files no longer offered as scalar; storage_policies no longer offered as a function. --- src/grammar/functions.ts | 27 ++++++++++++- tests/fixtures/server-queries.json | 12 ++++++ tests/functions-list.test.ts | 65 ++++++++++++++++++++++++++++++ tests/parser.test.ts | 6 +-- 4 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 tests/functions-list.test.ts diff --git a/src/grammar/functions.ts b/src/grammar/functions.ts index fd856d7..73f6bc8 100644 --- a/src/grammar/functions.ts +++ b/src/grammar/functions.ts @@ -42,8 +42,11 @@ export const scalarFunctions: string[] = [ "asin", "atan", "atan2", + "backup_instance_name", "bar", "base64", + "build", + "cancel_query", "ceil", "ceiling", "coalesce", @@ -51,6 +54,7 @@ export const scalarFunctions: string[] = [ "cos", "cot", "current_catalog", + "current_data_id", "current_database", "current_resource_group", "current_schema", @@ -71,11 +75,11 @@ export const scalarFunctions: string[] = [ "dump_thread_stacks", "exp", "extract", - "files", "flatten", "floor", "flush_query_cache", "format_type", + "from_base64", "geo_distance_meters", "geo_within_radius_latlon", "glob", @@ -83,6 +87,7 @@ export const scalarFunctions: string[] = [ "hour", "hydrate_table_metadata", "insertion_point", + "interval", "interval_end", "interval_start", "is_end_of_month", @@ -108,6 +113,7 @@ export const scalarFunctions: string[] = [ "month", "nanos", "netmask", + "node_role", "now", "now_ns", "nullif", @@ -134,6 +140,7 @@ export const scalarFunctions: string[] = [ "radians", "regexp_replace", "reload_config", + "reload_tick_calendars", "reload_tls", "replace", "right", @@ -191,6 +198,7 @@ export const scalarFunctions: string[] = [ "tan", "timestamp_ceil", "timestamp_floor", + "timestamp_floor_utc", "timestamp_sequence", "timestamp_sequence_ns", "timestamp_shuffle", @@ -316,14 +324,27 @@ export const tableValuedFunctions: string[] = [ "active_grants", "active_permissions", "all_permissions", + "backup_status", "backups", + "checkpoint_status", + "export_activity", + "files", "generate_series", + "glob", "information_schema._pg_expandarray", + "information_schema.character_sets", + "information_schema.columns", + "information_schema.key_column_usage", + "information_schema.questdb_columns", + "information_schema.referential_constraints", + "information_schema.table_constraints", + "information_schema.tables", "long_sequence", "parquet_scan", "pg_catalog.pg_class", "pg_catalog.pg_database", "pg_catalog.pg_description", + "pg_catalog.pg_extension", "pg_catalog.pg_index", "pg_catalog.pg_inherits", "pg_catalog.pg_locks", @@ -333,6 +354,7 @@ export const tableValuedFunctions: string[] = [ "pg_class", "pg_database", "pg_description", + "pg_enum", "pg_index", "pg_inherits", "pg_locks", @@ -341,7 +363,9 @@ export const tableValuedFunctions: string[] = [ "pg_type", "read_parquet", "sleep", + "table_cold_partitions", "table_partitions", + "tick_calendars", // STANDARD type but succeed as `SELECT * FROM ()` — meta pseudo-tables "all_tables", "export_files", @@ -365,7 +389,6 @@ export const tableValuedFunctions: string[] = [ "query_activity", "reader_pool", "resource_groups", - "storage_policies", "table_columns", "table_storage", "table_writer_metrics", diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index e2a57e2..749ae9d 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1935,5 +1935,17 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/bool/AllNotEqStrFunctionFactoryTest.java:60", "feature": "option-names", "toSql": "SELECT * FROM tab WHERE a <> ALL ('{abc,xyz}'::TEXT[])" + }, + { + "query": "SELECT * FROM checkpoint_status();", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/http/SqlValidationTest.java:117", + "feature": "functions-list", + "toSql": "SELECT * FROM checkpoint_status()" + }, + { + "query": "select worker_id, username, start_time, phase, export_path, request_source, export_sql, message from export_activity()", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/activity/ExportActivityFunctionFactoryTest.java:68", + "feature": "functions-list", + "toSql": "SELECT worker_id, username, start_time, phase, export_path, request_source, export_sql, message FROM export_activity()" } ] diff --git a/tests/functions-list.test.ts b/tests/functions-list.test.ts new file mode 100644 index 0000000..d29230d --- /dev/null +++ b/tests/functions-list.test.ts @@ -0,0 +1,65 @@ +/** + * Function name lists (highlighting / autocomplete) checked against the + * server FunctionFactory signatures (OSS + Enterprise). + */ + +import { describe, it, expect } from "vitest" +import { createAutocompleteProvider, SuggestionKind } from "../src/index" +import { scalarFunctions, tableValuedFunctions } from "../src/grammar/functions" + +const provider = createAutocompleteProvider({ tables: [], columns: {} }) +const functionsAt = (sql: string) => + provider + .getSuggestions(sql, sql.length) + .filter((s) => s.kind === SuggestionKind.Function) + .map((s) => s.label) + +describe("function lists", () => { + it("table-valued functions", () => { + for (const f of [ + "checkpoint_status", + "export_activity", + "pg_enum", + "pg_catalog.pg_extension", + "information_schema.tables", + "information_schema.columns", + "information_schema.key_column_usage", + "information_schema.table_constraints", + "information_schema.referential_constraints", + "information_schema.character_sets", + "information_schema.questdb_columns", + "backup_status", + "table_cold_partitions", + "tick_calendars", + "files", + "glob", + ]) { + expect(tableValuedFunctions).toContain(f) + } + expect(tableValuedFunctions).not.toContain("storage_policies") + }) + + it("scalar functions", () => { + for (const f of [ + "build", + "current_data_id", + "from_base64", + "interval", + "cancel_query", + "timestamp_floor_utc", + "backup_instance_name", + "node_role", + "reload_tick_calendars", + "glob", + ]) { + expect(scalarFunctions).toContain(f) + } + expect(scalarFunctions).not.toContain("files") + }) + + it("autocomplete", () => { + expect(functionsAt("SELECT * FROM checkp")).toContain("checkpoint_status") + expect(functionsAt("SELECT from_b")).toContain("from_base64") + expect(functionsAt("SELECT fil")).not.toContain("files") + }) +}) diff --git a/tests/parser.test.ts b/tests/parser.test.ts index f89f0bf..bb0b7ce 100644 --- a/tests/parser.test.ts +++ b/tests/parser.test.ts @@ -2939,9 +2939,9 @@ describe("QuestDB Parser", () => { }) }) - describe("storage_policies() meta function", () => { - it("parses SELECT * FROM storage_policies()", () => { - const result = parseToAst("SELECT * FROM storage_policies()") + describe("storage_policies (Enterprise view)", () => { + it("parses SELECT * FROM storage_policies", () => { + const result = parseToAst("SELECT * FROM storage_policies") expect(result.errors).toHaveLength(0) expect(result.ast[0].type).toBe("select") }) From 590b9b627626b3e9a1a093e21d0d132537191542 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 17:38:55 +0300 Subject: [PATCH 32/36] feat: join/clause/statement words usable as identifiers (server non-reserved) The server reserves only SqlKeywords.KEYWORDS (+ Enterprise grant/group/ revoke/service/user in EntSqlKeywords); every other word is a valid name: CREATE TABLE tango (left DOUBLE[][], right DOUBLE[][]) (ArrayTest), SELECT typeOf(l) lt FROM a4b (CompiledFilterRegressionTest), CREATE TABLE payload (k INT, by BYTE, ...) (HashJoinBuildFramesTest), create table tab (value int, when long, ...) (IODispatcherTest), SELECT sum(x) OVER window FROM t WINDOW window AS (...) (WindowFunctionTest), t.left / t.window, CREATE USER left, CREATE GROUP group. Before: all of these were parse errors (the words were reserved here). Now: - Left Right Full Inner Cross Asof Lt Splice Horizon Window By Group When Then Else Details Repair Unpivot Pivot Show Copy Truncate Vacuum Grant Revoke If Include Respect For are non-reserved (tokens.ts CLAUSE_IDENTIFIER_KEYWORD_NAMES, spread into IDENTIFIER_KEYWORD_NAMES). Grant/Revoke/Group are reserved in Enterprise; accepted anyway (never stricter than the server). - IS / ILIKE / DECLARE stay reserved in expressions but are accepted as column names in columnDefinition (AST name from the keyword token). - Alias gates: implicit table aliases stop at the server's tableAliasStop words (LEFT RIGHT FULL INNER CROSS ASOF LT SPLICE HORIZON WINDOW GROUP PIVOT) and statement keywords (SHOW COPY TRUNCATE VACUUM GRANT REVOKE DECLARE); implicit column aliases stop at FOR (columnAliasStop) and the statement keywords; pivot aggregation aliases stop at FOR; PIVOT FOR-list stops at GROUP BY. SHOW/COPY/GRANT/... at statement start never start an implicit SELECT (server dispatches them by keyword). - Content assist ignores GATEs: budgeted-content-assist refuses the same implicit-alias matches (IMPLICIT_ALIAS_STOP_TOKENS by rule), WHEN as a simple-CASE operand right after CASE, and a statement keyword as the table of an implicit SELECT / PIVOT statement. The token-based table fallback no longer takes `GROUP`/`LEFT`... as an alias without AS. - toSql keeps quoting these words; the formatter does not treat them as operands (THEN -1 stays unary) and never starts a clause after a `.` (t.window); bind-variable lexing after THEN/BY etc. unchanged. - Not done: reserved keywords after a dot (a.from, KeywordAsTableNameTest): accepting them made `SELECT t.| FROM trades t` parse `t.FROM` and broke alias-qualified column autocomplete. ANY stays reserved (= ANY(...)). Tests: 6 server fixture entries + tests/reserved-words.test.ts. Red 21 -> green; full suite 17203 passed. Corpus: fixed 75 / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0. Some fixed entries are prose or server-invalid SQL that now splits (e.g. CopyImportTest "copy x from 'f' partition by HOUR", which the server rejects with "'with' expected"). Suggestions: unchanged at all 55 checked positions. --- src/autocomplete/budgeted-content-assist.ts | 48 +++++++++++- src/autocomplete/content-assist.ts | 18 ++++- src/formatter/classification.ts | 13 +++- src/formatter/layout.ts | 4 +- src/parser/cst-types.d.ts | 3 +- src/parser/lexer.ts | 4 +- src/parser/parser.ts | 76 +++++++++++++++++-- src/parser/toSql.ts | 14 +++- src/parser/tokens.ts | 57 ++++++++++++-- src/parser/visitor.ts | 4 +- tests/fixtures/server-queries.json | 36 +++++++++ tests/reserved-words.test.ts | 84 +++++++++++++++++++++ 12 files changed, 334 insertions(+), 27 deletions(-) create mode 100644 tests/reserved-words.test.ts diff --git a/src/autocomplete/budgeted-content-assist.ts b/src/autocomplete/budgeted-content-assist.ts index 225cd45..4f6e6ec 100644 --- a/src/autocomplete/budgeted-content-assist.ts +++ b/src/autocomplete/budgeted-content-assist.ts @@ -29,7 +29,43 @@ import { Rule, Terminal, } from "chevrotain" -import { parser } from "../parser/parser" +import { + ALIAS_STOP_STATEMENT_TOKENS, + IMPLICIT_ALIAS_STOP_TOKENS, + parser, +} from "../parser/parser" +import { As, Case, IdentifierKeyword, Semicolon, When } from "../parser/lexer" + +// The parser's implicit-alias GATEs (a join / clause / statement word is not +// an alias without AS), which content assist would otherwise ignore: the +// alias path would consume e.g. `ASOF` in `FROM t ASOF JOIN u` and prune the +// join path. +function isStoppedIdentifier( + terminalType: TokenType, + ruleStack: string[], + tokenVector: IToken[], + idx: number, +): boolean { + if (terminalType !== IdentifierKeyword) return false + // `CASE WHEN`: never a simple-CASE operand named `when` + const tokenType = tokenVector[idx].tokenType + if (tokenType === When && idx > 0 && tokenVector[idx - 1].tokenType === Case) + return true + // `SHOW `, `COPY `, `GRANT ` ...: a statement keyword, not the table of an + // implicit SELECT (`show WHERE ...`) at statement start + if ( + ALIAS_STOP_STATEMENT_TOKENS.has(tokenType) && + (idx === 0 || tokenVector[idx - 1].tokenType === Semicolon) && + (ruleStack.includes("implicitSelectStatement") || + ruleStack.includes("pivotStatement")) + ) + return true + const n = ruleStack.length + if (n < 2 || ruleStack[n - 1] !== "identifier") return false + const stops = IMPLICIT_ALIAS_STOP_TOKENS.get(ruleStack[n - 2]) + if (!stops || !stops.has(tokenVector[idx].tokenType)) return false + return idx === 0 || tokenVector[idx - 1].tokenType !== As +} export interface ContentAssistSuggestion { nextTokenType: TokenType @@ -147,7 +183,15 @@ function nextPossibleTokensAfterBudgeted( if (currIdx < tokenVectorLength - 1) { const nextIdx = currIdx + 1 const actualToken = tokenVector[nextIdx] - if (tokMatcher(actualToken, prod.terminalType)) { + if ( + tokMatcher(actualToken, prod.terminalType) && + !isStoppedIdentifier( + prod.terminalType, + currRuleStack, + tokenVector, + nextIdx, + ) + ) { possiblePaths.push({ idx: nextIdx, def: currDef.slice(1), diff --git a/src/autocomplete/content-assist.ts b/src/autocomplete/content-assist.ts index 9fc965d..7b90c1f 100644 --- a/src/autocomplete/content-assist.ts +++ b/src/autocomplete/content-assist.ts @@ -2,6 +2,7 @@ import { type ILexingError, IToken, TokenType } from "chevrotain" import { parse as parseRaw } from "../parser/parser" import { visitor } from "../parser/visitor" import { QuestDBLexer } from "../parser/lexer" +import { CLAUSE_IDENTIFIER_KEYWORD_NAMES } from "../parser/tokens" import type { Statement } from "../parser/ast" import { IDENTIFIER_KEYWORD_TOKENS, @@ -581,10 +582,18 @@ function extractTables(fullSql: string, tokens: IToken[]): ExtractResult { let alias: string | undefined let aliasStart = tableNameResult.nextIndex - if (tokens[aliasStart]?.tokenType.name === "As") { + const explicitAs = tokens[aliasStart]?.tokenType.name === "As" + if (explicitAs) { aliasStart++ } - if (isIdentifierLike(tokens[aliasStart])) { + if ( + isIdentifierLike(tokens[aliasStart]) && + // `FROM t GROUP BY`, `FROM t LEFT JOIN`: clause words are no alias + (explicitAs || + !CLAUSE_IDENTIFIER_KEYWORD_NAMES.has( + tokens[aliasStart].tokenType.name, + )) + ) { alias = tokenToNamePart(tokens[aliasStart]) } @@ -1051,6 +1060,11 @@ function computeSuggestions(tokens: IToken[]): ComputeResult { // identifier before); ARRAY stays suggested as a type name (s.nextTokenType.name === "Array" && s.ruleStack[s.ruleStack.length - 1] === "arrayLiteral") || + // IS / ILIKE / DECLARE are accepted as column names, not suggested + (s.ruleStack[s.ruleStack.length - 1] === "columnDefinition" && + (s.nextTokenType.name === "Is" || + s.nextTokenType.name === "Ilike" || + s.nextTokenType.name === "Declare")) || // INCREMENTAL is an obsolete synonym of IMMEDIATE in REFRESH modes (s.nextTokenType.name === "Incremental" && s.ruleStack[s.ruleStack.length - 1] === "materializedViewRefresh") diff --git a/src/formatter/classification.ts b/src/formatter/classification.ts index 306e4b8..853e455 100644 --- a/src/formatter/classification.ts +++ b/src/formatter/classification.ts @@ -1,4 +1,8 @@ -import { IdentifierKeyword, keywordTokenArray } from "../parser/tokens" +import { + CLAUSE_IDENTIFIER_KEYWORD_NAMES, + IdentifierKeyword, + keywordTokenArray, +} from "../parser/tokens" export type SignificantKind = "word" | "operator" | "delimiter" @@ -65,7 +69,12 @@ export const wordTokenNames: ReadonlySet = new Set([ ]) const nonReservedKeywordNames = keywordTokenArray - .filter((token) => token.CATEGORIES?.includes(IdentifierKeyword)) + .filter( + (token) => + token.CATEGORIES?.includes(IdentifierKeyword) && + // `THEN -1`, `BY -x`: a sign after a clause word stays unary + !CLAUSE_IDENTIFIER_KEYWORD_NAMES.has(token.name), + ) .map((token) => token.name) export const operandTokenNames: ReadonlySet = new Set([ diff --git a/src/formatter/layout.ts b/src/formatter/layout.ts index 2aefb75..5ab4a60 100644 --- a/src/formatter/layout.ts +++ b/src/formatter/layout.ts @@ -217,7 +217,9 @@ class StatementBuilder { if ( piece !== undefined && this.previous !== null && - isGluedWords(this.previous, piece) + (isGluedWords(this.previous, piece) || + // `t.window`, `t.left`: a qualified name part, not a clause word + this.previous.token.tokenName === "Dot") ) { return null } diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 1e7e620..f9f3587 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -1163,7 +1163,8 @@ export interface ColumnDefinitionCstNode extends CstNode { } export type ColumnDefinitionCstChildren = { - identifier: IdentifierCstNode[]; + identifier?: IdentifierCstNode[]; + keywordName?: (IToken)[]; doublePrecisionType?: DoublePrecisionTypeCstNode[]; dataType?: DataTypeCstNode[]; Capacity?: IToken[]; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index 3d8b056..50852f6 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -3,6 +3,7 @@ import { createToken, IToken, Lexer, TokenType } from "chevrotain" // Import all keyword tokens from tokens.ts (auto-generated from grammar arrays) // Re-export all named tokens for parser usage import { + CLAUSE_IDENTIFIER_KEYWORD_NAMES, IdentifierKeyword, keywordTokenArray, Abort, @@ -868,7 +869,8 @@ function endsOperand(prev: IToken, colonOffset: number): boolean { return ( prev.endOffset !== undefined && prev.endOffset + 1 === colonOffset && - (type.CATEGORIES ?? []).includes(IdentifierKeyword) + (type.CATEGORIES ?? []).includes(IdentifierKeyword) && + !CLAUSE_IDENTIFIER_KEYWORD_NAMES.has(type.name) ) } diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 520d508..403b576 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -414,6 +414,45 @@ import { // Types the server accepts in a typed literal ` ''` // (ExpressionParser.processPgTimestampCast + cannotCastTo): BOOLEAN..LONG256, // UUID, IPv4, VARCHAR, DECIMAL, plus nameTypeMap aliases. +// Non-reserved words (server SqlKeywords.KEYWORDS does not list them) that +// start a join, a clause or a statement: never taken as an implicit (no AS) +// alias. Join/clause words mirror the server's tableAliasStop; the statement +// starters keep `... FROM t\nSHOW TABLES` a statement boundary as before. +export const ALIAS_STOP_STATEMENT_TOKENS: ReadonlySet = + new globalThis.Set([Show, Copy, Truncate, Vacuum, Grant, Revoke, Declare]) +const TABLE_ALIAS_STOP_TOKENS: ReadonlySet = new globalThis.Set([ + ...ALIAS_STOP_STATEMENT_TOKENS, + Left, + Right, + Full, + Inner, + Cross, + Asof, + Lt, + Splice, + Horizon, + Window, + Group, + Pivot, +]) +// Server columnAliasStop: FROM , OVER UNION EXCEPT INTERSECT ) ; FOR +const COLUMN_ALIAS_STOP_TOKENS: ReadonlySet = new globalThis.Set([ + ...ALIAS_STOP_STATEMENT_TOKENS, + For, +]) + +// Content assist ignores GATEs: the implicit-alias stops above, keyed by the +// rule whose `identifier` subrule is the alias (tableRef / selectItem / +// pivotAggregation only call `identifier` for aliases). +export const IMPLICIT_ALIAS_STOP_TOKENS: ReadonlyMap< + string, + ReadonlySet +> = new Map([ + ["tableRef", TABLE_ALIAS_STOP_TOKENS], + ["selectItem", COLUMN_ALIAS_STOP_TOKENS], + ["pivotAggregation", COLUMN_ALIAS_STOP_TOKENS], +]) + const TYPED_LITERAL_TYPE_TOKENS: ReadonlySet = new globalThis.Set([ Boolean, Byte, @@ -503,6 +542,7 @@ class QuestDBParser extends CstParser { if (la1 === Owned) return this.LA(2).tokenType !== By // If next token is AS, always allow (explicit alias) if (la1 === As) return true + if (TABLE_ALIAS_STOP_TOKENS.has(la1)) return false // Join clause words (server tableAliasStop): `ASOF JOIN t TOLERANCE 2s`, // `WINDOW JOIN t RANGE BETWEEN ...`, `... EXCLUDE PREVAILING` if (la1 === Tolerance || la1 === Range) return false @@ -710,7 +750,12 @@ class QuestDBParser extends CstParser { this.LA(1).tokenType === Compile && this.LA(2).tokenType === View, ALT: () => this.SUBRULE(this.compileViewStatement), }, - { ALT: () => this.SUBRULE(this.implicitSelectStatement) }, + { + // SHOW / COPY / GRANT ... at statement start are statement keywords + // on the server, never the table of an implicit SELECT + GATE: () => !ALIAS_STOP_STATEMENT_TOKENS.has(this.LA(1).tokenType), + ALT: () => this.SUBRULE(this.implicitSelectStatement), + }, ]) }) @@ -1025,6 +1070,7 @@ class QuestDBParser extends CstParser { GATE: () => { // If next token is AS, always allow (explicit alias) if (this.LA(1).tokenType === As) return true + if (COLUMN_ALIAS_STOP_TOKENS.has(this.LA(1).tokenType)) return false // Don't consume identifier as alias if followed by LParen — // it's a function call starting a new statement, not an alias return this.LA(2).tokenType !== LParen @@ -2618,7 +2664,14 @@ class QuestDBParser extends CstParser { private columnDefinition = this.RULE( "columnDefinition", (allowDoublePrecision?: boolean) => { - this.SUBRULE(this.identifier) + // IS / ILIKE / DECLARE are reserved in expressions here but valid + // column names on the server (not in SqlKeywords.KEYWORDS) + this.OR6([ + { ALT: () => this.SUBRULE(this.identifier) }, + { ALT: () => this.CONSUME(Is, { LABEL: "keywordName" }) }, + { ALT: () => this.CONSUME(Ilike, { LABEL: "keywordName" }) }, + { ALT: () => this.CONSUME(Declare, { LABEL: "keywordName" }) }, + ]) this.OR5({ IGNORE_AMBIGUITIES: true, DEF: [ @@ -4995,9 +5048,13 @@ class QuestDBParser extends CstParser { this.CONSUME(For) this.SUBRULE(this.pivotForClause) // Additional FOR clauses (FOR keyword optional per QuestDB syntax) - this.MANY1(() => { - this.OPTION1(() => this.CONSUME1(For)) - this.SUBRULE1(this.pivotForClause) + this.MANY1({ + GATE: () => + !(this.LA(1).tokenType === Group && this.LA(2).tokenType === By), + DEF: () => { + this.OPTION1(() => this.CONSUME1(For)) + this.SUBRULE1(this.pivotForClause) + }, }) this.OPTION(() => { this.CONSUME(Group) @@ -5012,9 +5069,12 @@ class QuestDBParser extends CstParser { private pivotAggregation = this.RULE("pivotAggregation", () => { this.SUBRULE(this.expression) - this.OPTION(() => { - this.OPTION1(() => this.CONSUME(As)) - this.SUBRULE(this.identifier) + this.OPTION({ + GATE: () => this.LA(1).tokenType !== For, + DEF: () => { + this.OPTION1(() => this.CONSUME(As)) + this.SUBRULE(this.identifier) + }, }) }) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index b34142c..69b2f5f 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -5,7 +5,10 @@ import * as AST from "./ast" import { keywords } from "../grammar/keywords" import { constants } from "../grammar/constants" -import { IDENTIFIER_KEYWORD_NAMES } from "./tokens" +import { + CLAUSE_IDENTIFIER_KEYWORD_NAMES, + IDENTIFIER_KEYWORD_NAMES, +} from "./tokens" function toPascalCase(str: string): string { if (str.includes("_")) { @@ -19,7 +22,14 @@ function toPascalCase(str: string): string { const RESERVED_KEYWORDS = new Set( [...keywords, ...constants] - .filter((k) => !IDENTIFIER_KEYWORD_NAMES.has(toPascalCase(k))) + .filter((k) => { + const name = toPascalCase(k) + // Clause words stay quoted: usable unquoted, but quoting is clearer. + return ( + !IDENTIFIER_KEYWORD_NAMES.has(name) || + CLAUSE_IDENTIFIER_KEYWORD_NAMES.has(name) + ) + }) .map((k) => k.toLowerCase()), ) diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index b988e0a..9979026 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -110,15 +110,57 @@ const allGeneratedTokens = new Map([ // only ~61 keywords are reserved and everything else is non-reserved. // // Reserved keywords NOT included (would cause parsing ambiguity): -// Structural: Select, From, Where, As, By, With, If, Exists, Into, -// Values, Set, For, Table, To, Declare, Rename -// Joins: Join, Inner, Left, Right, Full, Outer, Cross, -// Asof, Lt, Splice, On +// Structural: Select, From, Where, As, With, Exists, Into, +// Values, Set, Table, To, Declare, Rename +// Joins: Join, Outer, On // Operators: And, Or, Not, In, Between, Like, Ilike, Is -// Expressions: Case, When, Then, Else, End, Cast, True, False, Null, NaN, +// Expressions: Case, End, Cast, True, False, Null, NaN, // All, Any, Distinct, Over -// Clauses: Group, Order, Asc, Desc, Limit, Sample, Window, -// Union, Intersect, Except, Pivot, Unpivot, Lock, Truncate +// Clauses: Order, Asc, Desc, Limit, Sample, +// Union, Intersect, Except, Lock +// (IS / ILIKE / DECLARE are still accepted as column names in columnDefinition; +// join / clause words are in CLAUSE_IDENTIFIER_KEYWORD_NAMES.) + +/** + * Join / clause / statement words that the server does not reserve + * (SqlKeywords.KEYWORDS): usable as identifiers (`CREATE TABLE t (left INT, + * by BYTE, when LONG)`, `t.window`, `SELECT x lt FROM t`). Parser alias gates + * keep them from being taken as implicit aliases where they start a join / + * clause / statement. toSql still quotes them and the formatter does not treat + * them as operands (`THEN -1` keeps its unary minus). + */ +export const CLAUSE_IDENTIFIER_KEYWORD_NAMES: ReadonlySet = + new globalThis.Set([ + "Left", + "Right", + "Full", + "Inner", + "Cross", + "Asof", + "Lt", + "Splice", + "Horizon", + "Window", + "By", + "Group", + "When", + "Then", + "Else", + "Details", + "Repair", + "Unpivot", + "Pivot", + "Show", + "Copy", + "Truncate", + "Vacuum", + "Grant", + "Revoke", + "If", + "Include", + "Respect", + "For", + ]) export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ // Data type keywords @@ -454,6 +496,7 @@ export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ "Reset", "Rollback", "Unlisten", + ...CLAUSE_IDENTIFIER_KEYWORD_NAMES, ]) for (const name of IDENTIFIER_KEYWORD_NAMES) { diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index e92ff62..ab8f332 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -1643,7 +1643,9 @@ class QuestDBVisitor extends BaseVisitor { columnDefinition(ctx: ColumnDefinitionCstChildren): AST.ColumnDefinition { const result: AST.ColumnDefinition = { type: "columnDefinition", - name: this.extractIdentifierName(ctx.identifier[0].children), + name: ctx.identifier + ? this.extractIdentifierName(ctx.identifier[0].children) + : (ctx.keywordName?.[0].image ?? ""), dataType: this.visit(ctx.doublePrecisionType ?? ctx.dataType!) as string, } diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 749ae9d..8377086 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1947,5 +1947,41 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/functions/activity/ExportActivityFunctionFactoryTest.java:68", "feature": "functions-list", "toSql": "SELECT worker_id, username, start_time, phase, export_path, request_source, export_sql, message FROM export_activity()" + }, + { + "query": "CREATE TABLE tango (left DOUBLE[][], right DOUBLE[][])", + "source": "questdb:core/src/test/java/io/questdb/test/cairo/ArrayTest.java:1020", + "feature": "reserved-words", + "toSql": "CREATE TABLE tango (\"left\" DOUBLE[][], \"right\" DOUBLE[][])" + }, + { + "query": "SELECT typeOf(f) ft, typeOf(l) lt FROM a4b LIMIT 1", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/CompiledFilterRegressionTest.java:8501", + "feature": "reserved-words", + "toSql": "SELECT typeOf(f) AS ft, typeOf(l) AS \"lt\" FROM a4b LIMIT 1" + }, + { + "query": "CREATE TABLE payload (k INT, b BOOLEAN, by BYTE, sh SHORT, ch CHAR, i INT, l LONG, d DATE, ts TIMESTAMP, ns TIMESTAMP_NS, f FLOAT, dbl DOUBLE, s SYMBOL, unused STRING)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/join/HashJoinBuildFramesTest.java:75", + "feature": "reserved-words", + "toSql": "CREATE TABLE payload (k INT, b BOOLEAN, \"by\" BYTE, sh SHORT, ch CHAR, i INT, l LONG, d DATE, ts TIMESTAMP, ns TIMESTAMP_NS, f FLOAT, dbl DOUBLE, s SYMBOL, unused STRING)" + }, + { + "query": "create table tab (value int, when long, ts timestamp) timestamp(ts) partition by day bypass wal;", + "source": "questdb:core/src/test/java/io/questdb/test/cutlass/http/IODispatcherTest.java:248", + "feature": "reserved-words", + "toSql": "CREATE TABLE tab (value INT, \"when\" LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY BYPASS WAL" + }, + { + "query": "select when, version, temperature from forecasts latest on version partition by when", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/LatestByTest.java:2261", + "feature": "reserved-words", + "toSql": "SELECT \"when\", version, temperature FROM forecasts LATEST ON version PARTITION BY \"when\"" + }, + { + "query": "SELECT sum(x) OVER window FROM t WINDOW window AS (ORDER BY ts)", + "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/window/WindowFunctionTest.java:10011", + "feature": "reserved-words", + "toSql": "SELECT sum(x) OVER window FROM t WINDOW window AS (ORDER BY ts)" } ] diff --git a/tests/reserved-words.test.ts b/tests/reserved-words.test.ts new file mode 100644 index 0000000..d8fbf6e --- /dev/null +++ b/tests/reserved-words.test.ts @@ -0,0 +1,84 @@ +/** + * Words the server does not reserve (not in SqlKeywords.KEYWORDS) are usable + * as identifiers: join words, BY / GROUP / WHEN / THEN / ELSE / FOR / IF / + * PIVOT / SHOW / COPY / ... and IS / ILIKE / DECLARE as column names. They + * still start joins / clauses / statements where they did before. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens, format } from "../src/index" +import { toSql } from "../src/parser/toSql" + +const ok = (sql: string, n = 1) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(n) + return result.ast +} + +describe("non-reserved clause words as identifiers", () => { + it("column names", () => { + ok( + "SELECT left, right, full, inner, cross, asof, lt, splice, horizon, window, by, group, when, then, else FROM t", + ) + ok( + "SELECT details, repair, unpivot, show, copy, truncate, vacuum, grant, revoke, if, include, respect, pivot, for FROM t", + ) + ok("CREATE TABLE x (is INT, ilike INT, declare INT, group INT, if INT)") + ok("ALTER TABLE x ADD COLUMN is INT") + }) + + it("after a dot and as aliases / entity names", () => { + expect(toSql(ok("SELECT t.left, t.window FROM t"))).toBe( + 'SELECT t."left", t."window" FROM t', + ) + ok("SELECT a AS left, b group FROM t") + ok("CREATE USER left") + ok("CREATE TABLE group (a INT)") + }) + + it("still start joins, clauses and statements", () => { + expect(toSql(ok("SELECT * FROM t LEFT JOIN u ON t.a = u.a"))).toBe( + "SELECT * FROM t LEFT JOIN u ON t.a = u.a", + ) + expect(toSql(ok("SELECT * FROM t ASOF JOIN u TOLERANCE 1s"))).toBe( + "SELECT * FROM t ASOF JOIN u TOLERANCE 1s", + ) + expect(toSql(ok("SELECT a, count() FROM t GROUP BY a"))).toBe( + "SELECT a, count() FROM t GROUP BY a", + ) + ok("SELECT avg(x) OVER w FROM t WINDOW w AS (ORDER BY ts)") + ok("SELECT CASE WHEN a THEN b ELSE c END FROM t") + ok("SELECT * FROM t PIVOT (sum(v) FOR k IN ('a') GROUP BY s)") + ok("SELECT * FROM t\nSHOW TABLES", 2) + ok("SELECT 1\nSHOW TABLES", 2) + ok("DROP TABLE IF EXISTS t") + // never an implicit SELECT of a table named copy / show / vacuum + expect(parseToAst("copy").errors.length).toBeGreaterThan(0) + expect(parseToAst("VACUUM must remove").errors.length).toBeGreaterThan(0) + }) + + it("autocomplete keeps the clause keywords", () => { + const afterTable = getNextValidTokens("SELECT * FROM t ") + for (const k of ["Left", "Asof", "Join", "Group", "Window", "Where"]) { + expect(afterTable).toContain(k) + } + expect( + getNextValidTokens("SELECT * FROM t ASOF JOIN u ON t.ts = u.ts "), + ).toContain("Tolerance") + expect(getNextValidTokens("SELECT CASE WHEN (v > 1) ")).toContain("Then") + // a statement keyword at statement start, not an implicit-select table + expect(getNextValidTokens("SHOW ")).not.toContain("Where") + expect(getNextValidTokens("SELECT 1; COPY ")).not.toContain("Pivot") + const colName = getNextValidTokens("CREATE TABLE x (") + expect(colName).not.toContain("Is") + expect(colName).not.toContain("Declare") + }) + + it("formatter", () => { + expect(format("SELECT t.window FROM t")).toBe("SELECT t.window\nFROM t") + expect(format("SELECT CASE WHEN a THEN -1 ELSE -2 END FROM t")).toBe( + "SELECT CASE WHEN a THEN -1 ELSE -2 END\nFROM t", + ) + }) +}) From e0509dc43e507794b33e85dc6bcdef12be6d0355 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 17:43:56 +0300 Subject: [PATCH 33/36] feat: Enterprise resource groups (CREATE/ALTER/DROP RESOURCE GROUP, SET/UNSET RESOURCE GROUP) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server (EntSqlCompilerImpl): createResourceGroup — CREATE RESOURCE GROUP [IF NOT EXISTS] name [WITH (key = value, ...)]; compileAlterResourceGroup — ALTER RESOURCE GROUP name SET (...) | RESET (key, ...) | RENAME TO name; dropResourceGroup — DROP RESOURCE GROUP [IF EXISTS] name; compilePrincipalResourceGroup — ALTER USER|SERVICE ACCOUNT|GROUP x SET RESOURCE GROUP g [MAPPING PRIORITY n] | UNSET RESOURCE GROUP. Policy keys (parseResourceGroupPolicy): memory_limit cpu_weight max_active_queries max_queued_queries queue_timeout. Before: all of these were parse errors. Now: rules createResourceGroupStatement / alterResourceGroupStatement / dropResourceGroupStatement / resourceGroupPolicy / resourceGroupOption / principalResourceGroupAction (shared by alterUserAction and alterGroupStatement). Policy values: string, [-]number[unit], duration, UNLIMITED; value optional per key (RESET form). MAPPING PRIORITY accepted for users too (server rejects it there; permissive). AST: createResourceGroup / alterResourceGroup / dropResourceGroup statements, AlterUserResourceGroupAction, alterGroup actions setResourceGroup/unsetResourceGroup. All words were already non-reserved tokens. Tests: 9 server fixture entries + tests/ent-resource-groups.test.ts. Red 30 -> green; full suite 17243 passed. Corpus: fixed 155 / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0. Suggestions: +RESOURCE after CREATE/ALTER/DROP and ALTER USER|GROUP x SET; +UNSET after ALTER USER x; policy keys inside WITH (. --- src/parser/ast.ts | 51 ++++++++++- src/parser/cst-types.d.ts | 101 +++++++++++++++++++++ src/parser/lexer.ts | 18 ++++ src/parser/parser.ts | 140 +++++++++++++++++++++++++++++ src/parser/toSql.ts | 59 +++++++++++- src/parser/tokens.ts | 9 ++ src/parser/visitor.ts | 127 ++++++++++++++++++++++++++ tests/ent-resource-groups.test.ts | 75 ++++++++++++++++ tests/fixtures/server-queries.json | 54 +++++++++++ 9 files changed, 632 insertions(+), 2 deletions(-) create mode 100644 tests/ent-resource-groups.test.ts diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 36a5a75..76de83c 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -65,6 +65,9 @@ export type Statement = | SwitchStatement | AlterGroupStatement | CompileViewStatement + | CreateResourceGroupStatement + | AlterResourceGroupStatement + | DropResourceGroupStatement export interface SelectStatement extends AstNode { type: "select" @@ -528,6 +531,14 @@ export type AlterUserAction = | AlterUserCreateTokenAction | AlterUserDropTokenAction | AlterUserMemoryLimitAction + | AlterUserResourceGroupAction + +/** SET RESOURCE GROUP g [MAPPING PRIORITY n] | UNSET RESOURCE GROUP */ +export interface AlterUserResourceGroupAction { + actionType: "setResourceGroup" | "unsetResourceGroup" + resourceGroup?: QualifiedName + mappingPriority?: string +} export interface AlterUserEnableAction { actionType: "enable" @@ -1062,10 +1073,48 @@ export interface SwitchStatement extends AstNode { export interface AlterGroupStatement extends AstNode { type: "alterGroup" group: QualifiedName - action: "setAlias" | "dropAlias" | "setMemoryLimit" + action: + | "setAlias" + | "dropAlias" + | "setMemoryLimit" + | "setResourceGroup" + | "unsetResourceGroup" externalAlias?: string /** SET MEMORY LIMIT value, e.g. "2G", "512M", "0", "UNLIMITED" */ memoryLimit?: string + /** SET RESOURCE GROUP name */ + resourceGroup?: QualifiedName + /** SET RESOURCE GROUP ... MAPPING PRIORITY n */ + mappingPriority?: string +} + +/** Resource group policy entry: `key = value` (value omitted in RESET) */ +export interface ResourceGroupOption { + name: string + /** Raw value text, e.g. "'8M'", "200", "UNLIMITED" */ + value?: string +} + +export interface CreateResourceGroupStatement extends AstNode { + type: "createResourceGroup" + group: QualifiedName + ifNotExists?: boolean + /** WITH (...) policy; empty array for WITH () */ + options?: ResourceGroupOption[] +} + +export interface AlterResourceGroupStatement extends AstNode { + type: "alterResourceGroup" + group: QualifiedName + action: "set" | "reset" | "rename" + options?: ResourceGroupOption[] + newName?: QualifiedName +} + +export interface DropResourceGroupStatement extends AstNode { + type: "dropResourceGroup" + group: QualifiedName + ifExists?: boolean } export interface CompileViewStatement extends AstNode { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index f9f3587..cb4a8bd 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -826,6 +826,7 @@ export type CreateStatementCstChildren = { createUserStatement?: CreateUserStatementCstNode[]; createGroupStatement?: CreateGroupStatementCstNode[]; createServiceAccountStatement?: CreateServiceAccountStatementCstNode[]; + createResourceGroupStatement?: CreateResourceGroupStatementCstNode[]; }; export interface CreateViewBodyCstNode extends CstNode { @@ -1510,6 +1511,7 @@ export type AlterStatementCstChildren = { alterUserStatement?: AlterUserStatementCstNode[]; alterServiceAccountStatement?: AlterServiceAccountStatementCstNode[]; alterGroupStatement?: AlterGroupStatementCstNode[]; + alterResourceGroupStatement?: AlterResourceGroupStatementCstNode[]; }; export interface AlterGroupStatementCstNode extends CstNode { @@ -1529,6 +1531,7 @@ export type AlterGroupStatementCstChildren = { Memory?: IToken[]; Limit?: IToken[]; memoryLimit?: MemoryLimitCstNode[]; + principalResourceGroupAction?: PrincipalResourceGroupActionCstNode[]; }; export interface AlterViewStatementCstNode extends CstNode { @@ -1575,6 +1578,7 @@ export interface AlterUserActionCstNode extends CstNode { } export type AlterUserActionCstChildren = { + principalResourceGroupAction?: PrincipalResourceGroupActionCstNode[]; Enable?: IToken[]; Disable?: IToken[]; Set?: IToken[]; @@ -1842,6 +1846,97 @@ export type DropStatementCstChildren = { dropUserStatement?: DropUserStatementCstNode[]; dropGroupStatement?: DropGroupStatementCstNode[]; dropServiceAccountStatement?: DropServiceAccountStatementCstNode[]; + dropResourceGroupStatement?: DropResourceGroupStatementCstNode[]; +}; + +export interface CreateResourceGroupStatementCstNode extends CstNode { + name: "createResourceGroupStatement"; + children: CreateResourceGroupStatementCstChildren; +} + +export type CreateResourceGroupStatementCstChildren = { + Resource: IToken[]; + Group: IToken[]; + If?: IToken[]; + Not?: IToken[]; + Exists?: IToken[]; + qualifiedName: QualifiedNameCstNode[]; + With?: IToken[]; + resourceGroupPolicy?: ResourceGroupPolicyCstNode[]; +}; + +export interface AlterResourceGroupStatementCstNode extends CstNode { + name: "alterResourceGroupStatement"; + children: AlterResourceGroupStatementCstChildren; +} + +export type AlterResourceGroupStatementCstChildren = { + Resource: IToken[]; + Group: IToken[]; + qualifiedName: (QualifiedNameCstNode)[]; + Set?: IToken[]; + Reset?: IToken[]; + resourceGroupPolicy?: ResourceGroupPolicyCstNode[]; + Rename?: IToken[]; + To?: IToken[]; +}; + +export interface DropResourceGroupStatementCstNode extends CstNode { + name: "dropResourceGroupStatement"; + children: DropResourceGroupStatementCstChildren; +} + +export type DropResourceGroupStatementCstChildren = { + Resource: IToken[]; + Group: IToken[]; + If?: IToken[]; + Exists?: IToken[]; + qualifiedName: QualifiedNameCstNode[]; +}; + +export interface ResourceGroupPolicyCstNode extends CstNode { + name: "resourceGroupPolicy"; + children: ResourceGroupPolicyCstChildren; +} + +export type ResourceGroupPolicyCstChildren = { + LParen: IToken[]; + resourceGroupOption?: ResourceGroupOptionCstNode[]; + Comma?: IToken[]; + RParen: IToken[]; +}; + +export interface ResourceGroupOptionCstNode extends CstNode { + name: "resourceGroupOption"; + children: ResourceGroupOptionCstChildren; +} + +export type ResourceGroupOptionCstChildren = { + MemoryLimit?: IToken[]; + CpuWeight?: IToken[]; + MaxActiveQueries?: IToken[]; + MaxQueuedQueries?: IToken[]; + QueueTimeout?: IToken[]; + Equals?: IToken[]; + StringLiteral?: IToken[]; + Minus?: IToken[]; + memoryLimit?: MemoryLimitCstNode[]; +}; + +export interface PrincipalResourceGroupActionCstNode extends CstNode { + name: "principalResourceGroupAction"; + children: PrincipalResourceGroupActionCstChildren; +} + +export type PrincipalResourceGroupActionCstChildren = { + Set?: IToken[]; + Resource?: (IToken)[]; + Group?: (IToken)[]; + qualifiedName?: QualifiedNameCstNode[]; + Mapping?: IToken[]; + Priority?: IToken[]; + NumberLiteral?: IToken[]; + Unset?: IToken[]; }; export interface DropViewStatementCstNode extends CstNode { @@ -3406,6 +3501,12 @@ export interface ICstNodeVisitor extends ICstVisitor { suspendWalWith(children: SuspendWalWithCstChildren, param?: IN): OUT; alterMaterializedViewAction(children: AlterMaterializedViewActionCstChildren, param?: IN): OUT; dropStatement(children: DropStatementCstChildren, param?: IN): OUT; + createResourceGroupStatement(children: CreateResourceGroupStatementCstChildren, param?: IN): OUT; + alterResourceGroupStatement(children: AlterResourceGroupStatementCstChildren, param?: IN): OUT; + dropResourceGroupStatement(children: DropResourceGroupStatementCstChildren, param?: IN): OUT; + resourceGroupPolicy(children: ResourceGroupPolicyCstChildren, param?: IN): OUT; + resourceGroupOption(children: ResourceGroupOptionCstChildren, param?: IN): OUT; + principalResourceGroupAction(children: PrincipalResourceGroupActionCstChildren, param?: IN): OUT; dropViewStatement(children: DropViewStatementCstChildren, param?: IN): OUT; dropTableStatement(children: DropTableStatementCstChildren, param?: IN): OUT; dropMaterializedViewStatement(children: DropMaterializedViewStatementCstChildren, param?: IN): OUT; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index 50852f6..02d913b 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -346,6 +346,15 @@ import { Beginning, Now, Memory, + Resource, + Unset, + Mapping, + Priority, + MemoryLimit, + CpuWeight, + MaxActiveQueries, + MaxQueuedQueries, + QueueTimeout, Daily, Expression, Posting, @@ -708,6 +717,15 @@ export { Beginning, Now, Memory, + Resource, + Unset, + Mapping, + Priority, + MemoryLimit, + CpuWeight, + MaxActiveQueries, + MaxQueuedQueries, + QueueTimeout, Daily, Expression, Posting, diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 403b576..b055ac0 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -240,6 +240,15 @@ import { Beginning, Now, Memory, + Resource, + Unset, + Mapping, + Priority, + MemoryLimit, + CpuWeight, + MaxActiveQueries, + MaxQueuedQueries, + QueueTimeout, Daily, Expression, Posting, @@ -2021,6 +2030,7 @@ class QuestDBParser extends CstParser { { ALT: () => this.SUBRULE(this.createUserStatement) }, { ALT: () => this.SUBRULE(this.createGroupStatement) }, { ALT: () => this.SUBRULE(this.createServiceAccountStatement) }, + { ALT: () => this.SUBRULE(this.createResourceGroupStatement) }, ]) }) @@ -3151,6 +3161,7 @@ class QuestDBParser extends CstParser { { ALT: () => this.SUBRULE(this.alterUserStatement) }, { ALT: () => this.SUBRULE(this.alterServiceAccountStatement) }, { ALT: () => this.SUBRULE(this.alterGroupStatement) }, + { ALT: () => this.SUBRULE(this.alterResourceGroupStatement) }, ]) }) @@ -3182,6 +3193,7 @@ class QuestDBParser extends CstParser { this.SUBRULE(this.memoryLimit) }, }, + { ALT: () => this.SUBRULE(this.principalResourceGroupAction) }, ]) }) @@ -3235,6 +3247,7 @@ class QuestDBParser extends CstParser { private alterUserAction = this.RULE("alterUserAction", () => { this.OR([ + { ALT: () => this.SUBRULE(this.principalResourceGroupAction) }, { ALT: () => this.CONSUME(Enable) }, { ALT: () => this.CONSUME(Disable) }, { @@ -3972,9 +3985,136 @@ class QuestDBParser extends CstParser { { ALT: () => this.SUBRULE(this.dropUserStatement) }, { ALT: () => this.SUBRULE(this.dropGroupStatement) }, { ALT: () => this.SUBRULE(this.dropServiceAccountStatement) }, + { ALT: () => this.SUBRULE(this.dropResourceGroupStatement) }, ]) }) + // ========================================================================== + // Enterprise resource groups (EntSqlCompilerImpl.createResourceGroup / + // compileAlterResourceGroup / dropResourceGroup) + // ========================================================================== + + private createResourceGroupStatement = this.RULE( + "createResourceGroupStatement", + () => { + this.CONSUME(Resource) + this.CONSUME(Group) + this.OPTION(() => { + this.CONSUME(If) + this.CONSUME(Not) + this.CONSUME(Exists) + }) + this.SUBRULE(this.qualifiedName) + this.OPTION1(() => { + this.CONSUME(With) + this.SUBRULE(this.resourceGroupPolicy) + }) + }, + ) + + private alterResourceGroupStatement = this.RULE( + "alterResourceGroupStatement", + () => { + this.CONSUME(Resource) + this.CONSUME(Group) + this.SUBRULE(this.qualifiedName) + this.OR([ + { + ALT: () => { + this.OR1([ + { ALT: () => this.CONSUME(Set) }, + { ALT: () => this.CONSUME(Reset) }, + ]) + this.SUBRULE(this.resourceGroupPolicy) + }, + }, + { + ALT: () => { + this.CONSUME(Rename) + this.CONSUME(To) + this.SUBRULE1(this.qualifiedName) + }, + }, + ]) + }, + ) + + private dropResourceGroupStatement = this.RULE( + "dropResourceGroupStatement", + () => { + this.CONSUME(Resource) + this.CONSUME(Group) + this.OPTION(() => { + this.CONSUME(If) + this.CONSUME(Exists) + }) + this.SUBRULE(this.qualifiedName) + }, + ) + + // ( [key [= value] {, key [= value]}] ) — values omitted for RESET + private resourceGroupPolicy = this.RULE("resourceGroupPolicy", () => { + this.CONSUME(LParen) + this.MANY_SEP({ + SEP: Comma, + DEF: () => this.SUBRULE(this.resourceGroupOption), + }) + this.CONSUME(RParen) + }) + + private resourceGroupOption = this.RULE("resourceGroupOption", () => { + this.OR([ + { ALT: () => this.CONSUME(MemoryLimit) }, + { ALT: () => this.CONSUME(CpuWeight) }, + { ALT: () => this.CONSUME(MaxActiveQueries) }, + { ALT: () => this.CONSUME(MaxQueuedQueries) }, + { ALT: () => this.CONSUME(QueueTimeout) }, + ]) + this.OPTION(() => { + this.CONSUME(Equals) + this.OR1([ + { ALT: () => this.CONSUME(StringLiteral) }, + { + ALT: () => { + this.OPTION1(() => this.CONSUME(Minus)) + this.SUBRULE(this.memoryLimit) + }, + }, + ]) + }) + }) + + // SET RESOURCE GROUP g [MAPPING PRIORITY n] | UNSET RESOURCE GROUP + // (EntSqlCompilerImpl.compilePrincipalResourceGroup; MAPPING PRIORITY is + // server-valid only for ALTER GROUP, accepted everywhere here) + private principalResourceGroupAction = this.RULE( + "principalResourceGroupAction", + () => { + this.OR([ + { + ALT: () => { + this.CONSUME(Set) + this.CONSUME(Resource) + this.CONSUME(Group) + this.SUBRULE(this.qualifiedName) + this.OPTION(() => { + this.CONSUME(Mapping) + this.CONSUME(Priority) + this.CONSUME(NumberLiteral) + }) + }, + }, + { + ALT: () => { + this.CONSUME(Unset) + this.CONSUME1(Resource) + this.CONSUME1(Group) + }, + }, + ]) + }, + ) + private dropViewStatement = this.RULE("dropViewStatement", () => { this.CONSUME(View) this.OPTION(() => { diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 69b2f5f..b458676 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -163,6 +163,10 @@ function statementToSql(stmt: AST.Statement): string { return alterGroupToSql(stmt) case "compileView": return `COMPILE VIEW ${qualifiedNameToSql(stmt.view)}` + case "createResourceGroup": + case "alterResourceGroup": + case "dropResourceGroup": + return resourceGroupToSql(stmt) default: throw new Error( `Unknown statement type: ${(stmt as { type: string }).type}`, @@ -1541,9 +1545,56 @@ function createGroupToSql(stmt: AST.CreateGroupStatement): string { return parts.join(" ") } +function resourceGroupPolicyToSql(options: AST.ResourceGroupOption[]): string { + return `(${options + .map((o) => (o.value !== undefined ? `${o.name} = ${o.value}` : o.name)) + .join(", ")})` +} + +function resourceGroupToSql( + stmt: + | AST.CreateResourceGroupStatement + | AST.AlterResourceGroupStatement + | AST.DropResourceGroupStatement, +): string { + const name = qualifiedNameToSql(stmt.group) + if (stmt.type === "createResourceGroup") { + const parts = ["CREATE RESOURCE GROUP"] + if (stmt.ifNotExists) parts.push("IF NOT EXISTS") + parts.push(name) + if (stmt.options) + parts.push(`WITH ${resourceGroupPolicyToSql(stmt.options)}`) + return parts.join(" ") + } + if (stmt.type === "dropResourceGroup") { + return `DROP RESOURCE GROUP ${stmt.ifExists ? "IF EXISTS " : ""}${name}` + } + if (stmt.action === "rename") { + return `ALTER RESOURCE GROUP ${name} RENAME TO ${qualifiedNameToSql(stmt.newName!)}` + } + return `ALTER RESOURCE GROUP ${name} ${stmt.action.toUpperCase()} ${resourceGroupPolicyToSql(stmt.options ?? [])}` +} + +function principalResourceGroupToSql( + group: AST.QualifiedName | undefined, + mappingPriority: string | undefined, +): string { + if (!group) return "UNSET RESOURCE GROUP" + let s = `SET RESOURCE GROUP ${qualifiedNameToSql(group)}` + if (mappingPriority !== undefined) s += ` MAPPING PRIORITY ${mappingPriority}` + return s +} + function alterGroupToSql(stmt: AST.AlterGroupStatement): string { const parts: string[] = ["ALTER GROUP", qualifiedNameToSql(stmt.group)] - if (stmt.action === "setMemoryLimit") { + if ( + stmt.action === "setResourceGroup" || + stmt.action === "unsetResourceGroup" + ) { + parts.push( + principalResourceGroupToSql(stmt.resourceGroup, stmt.mappingPriority), + ) + } else if (stmt.action === "setMemoryLimit") { parts.push(`SET MEMORY LIMIT ${stmt.memoryLimit}`) } else if (stmt.action === "setAlias") { parts.push(`WITH EXTERNAL ALIAS ${escapeString(stmt.externalAlias!)}`) @@ -1595,6 +1646,12 @@ function alterUserActionToSql(action: AST.AlterUserAction): string { return `WITH PASSWORD ${escapeString(action.password!)}` case "setMemoryLimit": return `SET MEMORY LIMIT ${action.limit}` + case "setResourceGroup": + case "unsetResourceGroup": + return principalResourceGroupToSql( + action.resourceGroup, + action.mappingPriority, + ) case "createToken": { const parts = [`CREATE TOKEN TYPE ${action.tokenType}`] if (action.publicKeyX != null && action.publicKeyY != null) { diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index 9979026..f8144d9 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -914,6 +914,15 @@ export const Anchor = getToken("Anchor") export const Beginning = getToken("Beginning") export const Now = getToken("Now") export const Memory = getToken("Memory") +export const Resource = getToken("Resource") +export const Unset = getToken("Unset") +export const Mapping = getToken("Mapping") +export const Priority = getToken("Priority") +export const MemoryLimit = getToken("MemoryLimit") +export const CpuWeight = getToken("CpuWeight") +export const MaxActiveQueries = getToken("MaxActiveQueries") +export const MaxQueuedQueries = getToken("MaxQueuedQueries") +export const QueueTimeout = getToken("QueueTimeout") export const Daily = getToken("Daily") export const Expression = getToken("Expression") export const Posting = getToken("Posting") diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index ab8f332..ecd0421 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -23,6 +23,12 @@ import type { AlterTableForceActionCstChildren, AlterTableStatementCstChildren, AlterUserActionCstChildren, + PrincipalResourceGroupActionCstChildren, + CreateResourceGroupStatementCstChildren, + AlterResourceGroupStatementCstChildren, + DropResourceGroupStatementCstChildren, + ResourceGroupPolicyCstChildren, + ResourceGroupOptionCstChildren, AlterUserStatementCstChildren, AlterLiveViewStatementCstChildren, AlterViewStatementCstChildren, @@ -1491,6 +1497,11 @@ class QuestDBVisitor extends BaseVisitor { ctx.createServiceAccountStatement, ) as AST.CreateServiceAccountStatement } + if (ctx.createResourceGroupStatement) { + return this.visit( + ctx.createResourceGroupStatement, + ) as AST.CreateResourceGroupStatement + } throw new Error("Unknown create statement type") } @@ -2454,6 +2465,11 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.alterGroupStatement) { return this.visit(ctx.alterGroupStatement) as AST.AlterGroupStatement } + if (ctx.alterResourceGroupStatement) { + return this.visit( + ctx.alterResourceGroupStatement, + ) as AST.AlterResourceGroupStatement + } throw new Error("Unknown alter statement type") } @@ -2473,6 +2489,20 @@ class QuestDBVisitor extends BaseVisitor { ctx: AlterGroupStatementCstChildren, ): AST.AlterGroupStatement { const group = this.visit(ctx.qualifiedName) as AST.QualifiedName + if (ctx.principalResourceGroupAction) { + const a = this.visit( + ctx.principalResourceGroupAction, + ) as AST.AlterUserResourceGroupAction + return { + type: "alterGroup", + group, + action: a.actionType, + ...(a.resourceGroup && { resourceGroup: a.resourceGroup }), + ...(a.mappingPriority !== undefined && { + mappingPriority: a.mappingPriority, + }), + } + } if (ctx.memoryLimit) { return { type: "alterGroup", @@ -2668,7 +2698,99 @@ class QuestDBVisitor extends BaseVisitor { } } + principalResourceGroupAction( + ctx: PrincipalResourceGroupActionCstChildren, + ): AST.AlterUserResourceGroupAction { + if (ctx.Unset) return { actionType: "unsetResourceGroup" } + const result: AST.AlterUserResourceGroupAction = { + actionType: "setResourceGroup", + resourceGroup: this.visit(ctx.qualifiedName!) as AST.QualifiedName, + } + if (ctx.NumberLiteral) result.mappingPriority = ctx.NumberLiteral[0].image + return result + } + + createResourceGroupStatement( + ctx: CreateResourceGroupStatementCstChildren, + ): AST.CreateResourceGroupStatement { + const result: AST.CreateResourceGroupStatement = { + type: "createResourceGroup", + group: this.visit(ctx.qualifiedName) as AST.QualifiedName, + } + if (ctx.If) result.ifNotExists = true + if (ctx.resourceGroupPolicy) { + result.options = this.visit( + ctx.resourceGroupPolicy, + ) as AST.ResourceGroupOption[] + } + return result + } + + alterResourceGroupStatement( + ctx: AlterResourceGroupStatementCstChildren, + ): AST.AlterResourceGroupStatement { + const group = this.visit(ctx.qualifiedName[0]) as AST.QualifiedName + if (ctx.Rename) { + return { + type: "alterResourceGroup", + group, + action: "rename", + newName: this.visit(ctx.qualifiedName[1]) as AST.QualifiedName, + } + } + return { + type: "alterResourceGroup", + group, + action: ctx.Reset ? "reset" : "set", + options: this.visit( + ctx.resourceGroupPolicy!, + ) as AST.ResourceGroupOption[], + } + } + + dropResourceGroupStatement( + ctx: DropResourceGroupStatementCstChildren, + ): AST.DropResourceGroupStatement { + const result: AST.DropResourceGroupStatement = { + type: "dropResourceGroup", + group: this.visit(ctx.qualifiedName) as AST.QualifiedName, + } + if (ctx.If) result.ifExists = true + return result + } + + resourceGroupPolicy( + ctx: ResourceGroupPolicyCstChildren, + ): AST.ResourceGroupOption[] { + return (ctx.resourceGroupOption ?? []).map( + (o) => this.visit(o) as AST.ResourceGroupOption, + ) + } + + resourceGroupOption( + ctx: ResourceGroupOptionCstChildren, + ): AST.ResourceGroupOption { + const key = (ctx.MemoryLimit ?? + ctx.CpuWeight ?? + ctx.MaxActiveQueries ?? + ctx.MaxQueuedQueries ?? + ctx.QueueTimeout)![0] + const result: AST.ResourceGroupOption = { name: key.image.toLowerCase() } + if (ctx.StringLiteral) { + result.value = ctx.StringLiteral[0].image + } else if (ctx.memoryLimit) { + result.value = + (ctx.Minus ? "-" : "") + (this.visit(ctx.memoryLimit) as string) + } + return result + } + alterUserAction(ctx: AlterUserActionCstChildren): AST.AlterUserAction { + if (ctx.principalResourceGroupAction) { + return this.visit( + ctx.principalResourceGroupAction, + ) as AST.AlterUserResourceGroupAction + } if (ctx.memoryLimit) { return { actionType: "setMemoryLimit", @@ -3083,6 +3205,11 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.dropGroupStatement) { return this.visit(ctx.dropGroupStatement) as AST.DropGroupStatement } + if (ctx.dropResourceGroupStatement) { + return this.visit( + ctx.dropResourceGroupStatement, + ) as AST.DropResourceGroupStatement + } if (ctx.dropServiceAccountStatement) { return this.visit( ctx.dropServiceAccountStatement, diff --git a/tests/ent-resource-groups.test.ts b/tests/ent-resource-groups.test.ts new file mode 100644 index 0000000..bfba796 --- /dev/null +++ b/tests/ent-resource-groups.test.ts @@ -0,0 +1,75 @@ +/** + * Enterprise resource groups: CREATE / ALTER / DROP RESOURCE GROUP and + * ALTER USER | SERVICE ACCOUNT | GROUP ... SET | UNSET RESOURCE GROUP. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" + +const ok = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(1) + return result.ast[0] +} + +describe("resource groups", () => { + it("CREATE / DROP", () => { + expect(ok("CREATE RESOURCE GROUP batch")).toMatchObject({ + type: "createResourceGroup", + }) + ok("CREATE RESOURCE GROUP memory_guard WITH (memory_limit = 1_024K)") + ok("CREATE RESOURCE GROUP g WITH (memory_limit = UNLIMITED)") + ok("CREATE RESOURCE GROUP g WITH (memory_limit = 64M)") + ok("CREATE RESOURCE GROUP g WITH ()") + ok("CREATE RESOURCE GROUP g WITH (queue_timeout = 30000)") + expect(toSql(ok("drop resource group reporting"))).toBe( + "DROP RESOURCE GROUP reporting", + ) + }) + + it("ALTER USER SET / UNSET", () => { + expect(ok("ALTER USER alice SET RESOURCE GROUP DEFAULT")).toMatchObject({ + type: "alterUser", + action: { actionType: "setResourceGroup" }, + }) + expect(toSql(ok("ALTER USER john UNSET RESOURCE GROUP"))).toBe( + "ALTER USER john UNSET RESOURCE GROUP", + ) + expect(toSql(ok("ALTER GROUP acl_high UNSET RESOURCE GROUP"))).toBe( + "ALTER GROUP acl_high UNSET RESOURCE GROUP", + ) + // existing forms unchanged + ok("ALTER USER u SET MEMORY LIMIT 1G") + }) + + it("resource words stay identifiers", () => { + ok("SELECT resource, unset, memory_limit, cpu_weight FROM t") + ok("CREATE TABLE resource (mapping INT, priority INT)") + }) + + it("autocomplete", () => { + expect(getNextValidTokens("CREATE ")).toContain("Resource") + expect(getNextValidTokens("ALTER ")).toContain("Resource") + expect(getNextValidTokens("DROP ")).toContain("Resource") + expect(getNextValidTokens("ALTER RESOURCE GROUP g ")).toEqual( + expect.arrayContaining(["Set", "Reset", "Rename"]), + ) + expect(getNextValidTokens("CREATE RESOURCE GROUP g WITH (")).toEqual( + expect.arrayContaining([ + "MemoryLimit", + "CpuWeight", + "MaxActiveQueries", + "MaxQueuedQueries", + "QueueTimeout", + ]), + ) + expect(getNextValidTokens("ALTER USER u ")).toEqual( + expect.arrayContaining(["Set", "Unset"]), + ) + expect(getNextValidTokens("ALTER USER u SET ")).toEqual( + expect.arrayContaining(["Memory", "Resource"]), + ) + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 8377086..abd12c8 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -1983,5 +1983,59 @@ "source": "questdb:core/src/test/java/io/questdb/test/griffin/engine/window/WindowFunctionTest.java:10011", "feature": "reserved-words", "toSql": "SELECT sum(x) OVER window FROM t WINDOW window AS (ORDER BY ts)" + }, + { + "query": "CREATE RESOURCE GROUP analytics WITH (memory_limit = '8M', cpu_weight = 200)", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/resourcegroup/ResourceGroupSqlCompilerTest.java:220", + "feature": "ent-resource-groups", + "toSql": "CREATE RESOURCE GROUP analytics WITH (memory_limit = '8M', cpu_weight = 200)" + }, + { + "query": "CREATE RESOURCE GROUP IF NOT EXISTS analytics WITH (cpu_weight = 100)", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/resourcegroup/ResourceGroupSqlCompilerTest.java:407", + "feature": "ent-resource-groups", + "toSql": "CREATE RESOURCE GROUP IF NOT EXISTS analytics WITH (cpu_weight = 100)" + }, + { + "query": "ALTER RESOURCE GROUP DEFAULT SET (max_active_queries = 1, max_queued_queries = 2, queue_timeout = '30s', memory_limit = '8M')", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/resourcegroup/ResourceGroupHttpTest.java:931", + "feature": "ent-resource-groups", + "toSql": "ALTER RESOURCE GROUP DEFAULT SET (max_active_queries = 1, max_queued_queries = 2, queue_timeout = '30s', memory_limit = '8M')" + }, + { + "query": "ALTER RESOURCE GROUP analytics RESET (memory_limit, cpu_weight)", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/resourcegroup/ResourceGroupSqlCompilerTest.java:251", + "feature": "ent-resource-groups", + "toSql": "ALTER RESOURCE GROUP analytics RESET (memory_limit, cpu_weight)" + }, + { + "query": "ALTER RESOURCE GROUP analytics RENAME TO reporting", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/resourcegroup/ResourceGroupSqlCompilerTest.java:116", + "feature": "ent-resource-groups", + "toSql": "ALTER RESOURCE GROUP analytics RENAME TO reporting" + }, + { + "query": "DROP RESOURCE GROUP IF EXISTS batch", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/resourcegroup/ResourceGroupCatalogRestartTest.java:159", + "feature": "ent-resource-groups", + "toSql": "DROP RESOURCE GROUP IF EXISTS batch" + }, + { + "query": "ALTER GROUP a SET RESOURCE GROUP low MAPPING PRIORITY 20", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/security/ResourceGroupOwnershipTest.java:130", + "feature": "ent-resource-groups", + "toSql": "ALTER GROUP a SET RESOURCE GROUP low MAPPING PRIORITY 20" + }, + { + "query": "ALTER SERVICE ACCOUNT svc SET RESOURCE GROUP rg", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/acl/AclAuditLogTest.java:1047", + "feature": "ent-resource-groups", + "toSql": "ALTER SERVICE ACCOUNT svc SET RESOURCE GROUP rg" + }, + { + "query": "ALTER SERVICE ACCOUNT svc UNSET RESOURCE GROUP", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/acl/AclAuditLogTest.java:1048", + "feature": "ent-resource-groups", + "toSql": "ALTER SERVICE ACCOUNT svc UNSET RESOURCE GROUP" } ] From 0df1fe1b218c525392f152b53379f13d3c0bb952 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 17:46:06 +0300 Subject: [PATCH 34/36] feat: Enterprise CREATE [OR REPLACE] VIEW ... WITH AUDIT [OWNED BY x] Server (EntSqlCompilerImpl.parseCreateViewExt / parseReplaceViewExt): after the view body, WITH AUDIT and OWNED BY are independent trailing clauses in either order. Before: `... ) WITH AUDIT` started a new WITH statement and failed ("Expecting As"); `OWNED BY g WITH AUDIT` likewise. Now: createViewBody accepts a MANY of WITH AUDIT | OWNED BY x, gated so WITH not followed by AUDIT still begins the next (CTE) statement. New non-reserved keyword `audit` (highlighting; still usable as identifier). AST: CreateViewStatement.audit; toSql prints WITH AUDIT before OWNED BY. Tests: 4 server fixture entries (EntShowCreateViewTest:131/87/58, ViewAuditTest:674) + tests/ent-view-audit.test.ts. Red 15 -> green; full suite 17264 passed. Corpus: fixed 27 / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0. Suggestions: +WITH after a CREATE VIEW body, +AUDIT after it. --- src/grammar/keywords.ts | 1 + src/parser/ast.ts | 2 ++ src/parser/cst-types.d.ts | 2 ++ src/parser/lexer.ts | 2 ++ src/parser/parser.ts | 29 +++++++++++++++---- src/parser/toSql.ts | 1 + src/parser/tokens.ts | 3 ++ src/parser/visitor.ts | 3 ++ tests/ent-view-audit.test.ts | 45 ++++++++++++++++++++++++++++++ tests/fixtures/server-queries.json | 24 ++++++++++++++++ 10 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 tests/ent-view-audit.test.ts diff --git a/src/grammar/keywords.ts b/src/grammar/keywords.ts index 8132211..eb982f4 100644 --- a/src/grammar/keywords.ts +++ b/src/grammar/keywords.ts @@ -16,6 +16,7 @@ export const keywords: string[] = [ "at", "atomic", "attach", + "audit", "audited", "backup", "base", diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 76de83c..159127e 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -784,6 +784,8 @@ export interface CreateViewStatement extends AstNode { query: SelectStatement asParens?: boolean ownedBy?: string + /** Enterprise: WITH AUDIT */ + audit?: boolean } export interface AlterViewStatement extends AstNode { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index cb4a8bd..909674f 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -847,6 +847,8 @@ export type CreateViewBodyCstChildren = { selectStatement?: (SelectStatementCstNode)[]; RParen?: (IToken)[]; implicitSelectStatement?: (ImplicitSelectStatementCstNode)[]; + With?: IToken[]; + Audit?: IToken[]; Owned?: IToken[]; By?: IToken[]; stringOrIdentifier?: StringOrIdentifierCstNode[]; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index 02d913b..3ba65df 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -348,6 +348,7 @@ import { Memory, Resource, Unset, + Audit, Mapping, Priority, MemoryLimit, @@ -719,6 +720,7 @@ export { Memory, Resource, Unset, + Audit, Mapping, Priority, MemoryLimit, diff --git a/src/parser/parser.ts b/src/parser/parser.ts index b055ac0..273bfd6 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -242,6 +242,7 @@ import { Memory, Resource, Unset, + Audit, Mapping, Priority, MemoryLimit, @@ -2074,11 +2075,29 @@ class QuestDBParser extends CstParser { ALT: () => this.SUBRULE1(this.implicitSelectStatement), }, ]) - // Optional OWNED BY - this.OPTION2(() => { - this.CONSUME(Owned) - this.CONSUME(By) - this.SUBRULE(this.stringOrIdentifier) + // Trailing WITH AUDIT / OWNED BY in either order (Enterprise + // EntSqlCompilerImpl.parseCreateViewExt) + this.MANY({ + // WITH not followed by AUDIT starts a new (CTE) statement + GATE: () => + this.LA(1).tokenType !== With || this.LA(2).tokenType === Audit, + DEF: () => { + this.OR1([ + { + ALT: () => { + this.CONSUME(With) + this.CONSUME(Audit) + }, + }, + { + ALT: () => { + this.CONSUME(Owned) + this.CONSUME(By) + this.SUBRULE(this.stringOrIdentifier) + }, + }, + ]) + }, }) }) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index b458676..0b95abf 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -837,6 +837,7 @@ function createViewToSql(stmt: AST.CreateViewStatement): string { } else { parts.push(`AS ${selectToSql(stmt.query)}`) } + if (stmt.audit) parts.push("WITH AUDIT") if (stmt.ownedBy) parts.push(`OWNED BY ${escapeIdentifier(stmt.ownedBy)}`) return parts.join(" ") } diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index f8144d9..ef66fbf 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -450,6 +450,8 @@ export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ "Priority", "Resource", "Unset", + // CREATE VIEW ... WITH AUDIT (Enterprise) + "Audit", "Daily", "Expression", "Posting", @@ -916,6 +918,7 @@ export const Now = getToken("Now") export const Memory = getToken("Memory") export const Resource = getToken("Resource") export const Unset = getToken("Unset") +export const Audit = getToken("Audit") export const Mapping = getToken("Mapping") export const Priority = getToken("Priority") export const MemoryLimit = getToken("MemoryLimit") diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index ecd0421..c90494e 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -2013,6 +2013,9 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.Owned && ctx.stringOrIdentifier) { result.ownedBy = this.visit(ctx.stringOrIdentifier) as string } + if (ctx.Audit) { + result.audit = true + } return result } diff --git a/tests/ent-view-audit.test.ts b/tests/ent-view-audit.test.ts new file mode 100644 index 0000000..3d1b051 --- /dev/null +++ b/tests/ent-view-audit.test.ts @@ -0,0 +1,45 @@ +/** + * Enterprise CREATE [OR REPLACE] VIEW ... WITH AUDIT [OWNED BY x] + * (either clause order). + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, getNextValidTokens } from "../src/index" +import { toSql } from "../src/parser/toSql" + +const parse = (sql: string, n = 1) => { + const result = parseToAst(sql) + expect(result.errors).toEqual([]) + expect(result.ast).toHaveLength(n) + return result.ast +} + +describe("CREATE VIEW WITH AUDIT", () => { + it("sets audit flag", () => { + const [stmt] = parse("CREATE VIEW v AS (SELECT ts FROM t) WITH AUDIT") + expect(stmt).toMatchObject({ type: "createView", audit: true }) + expect(toSql(stmt)).toBe("CREATE VIEW v AS (SELECT ts FROM t) WITH AUDIT") + }) + + it("unparenthesised body", () => { + const [stmt] = parse("CREATE VIEW v AS SELECT ts FROM t WITH AUDIT") + expect(stmt).toMatchObject({ audit: true }) + }) + + it("WITH CTE after a view is still a separate statement", () => { + parse("CREATE VIEW v AS (SELECT 1) WITH x AS (SELECT 1) SELECT * FROM x", 2) + }) + + it("audit stays an identifier", () => { + parse("SELECT audit FROM audit") + }) + + it("autocomplete", () => { + expect( + getNextValidTokens("CREATE VIEW v AS (SELECT ts FROM t) WITH "), + ).toContain("Audit") + expect( + getNextValidTokens("CREATE VIEW v AS (SELECT ts FROM t) WITH AUDIT "), + ).toContain("Owned") + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index abd12c8..fcebdcd 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -2037,5 +2037,29 @@ "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/acl/AclAuditLogTest.java:1048", "feature": "ent-resource-groups", "toSql": "ALTER SERVICE ACCOUNT svc UNSET RESOURCE GROUP" + }, + { + "query": "CREATE VIEW v_audit_owned AS (SELECT ts, x FROM foo) WITH AUDIT OWNED BY 'group1'", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/griffin/EntShowCreateViewTest.java:131", + "feature": "ent-view-audit", + "toSql": "CREATE VIEW v_audit_owned AS (SELECT ts, x FROM foo) WITH AUDIT OWNED BY group1" + }, + { + "query": "CREATE VIEW v_owned_audit AS (SELECT ts, x FROM foo) OWNED BY 'group1' WITH AUDIT;", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/griffin/EntShowCreateViewTest.java:87", + "feature": "ent-view-audit", + "toSql": "CREATE VIEW v_owned_audit AS (SELECT ts, x FROM foo) WITH AUDIT OWNED BY group1" + }, + { + "query": "CREATE OR REPLACE VIEW v_audit AS (SELECT ts, x FROM foo WHERE x > 0) WITH AUDIT", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/griffin/EntShowCreateViewTest.java:58", + "feature": "ent-view-audit", + "toSql": "CREATE OR REPLACE VIEW v_audit AS (SELECT ts, x FROM foo WHERE x > 0) WITH AUDIT" + }, + { + "query": "CREATE VIEW view_c AS (SELECT ts, symbol, price FROM trades WHERE symbol = 'MSFT') WITH AUDIT", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/view/ViewAuditTest.java:674", + "feature": "ent-view-audit", + "toSql": "CREATE VIEW view_c AS (SELECT ts, symbol, price FROM trades WHERE symbol = 'MSFT') WITH AUDIT" } ] From 23830d3743c82d995c9af5121ee307ceb9b53b2e Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 17:51:05 +0300 Subject: [PATCH 35/36] =?UTF-8?q?feat:=20Enterprise=20GRANT/REVOKE=20?= =?UTF-8?q?=E2=80=94=20all=20permission=20names,=20WITH=20VERIFICATION=20o?= =?UTF-8?q?rder,=20ASSUME=20grantee=20lists?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server: Permission.java namePermissionMap (80 names) parsed by PermissionParser.parse; after TO/FROM grantee, GRANT takes up to two of WITH GRANT OPTION / WITH VERIFICATION in either order, REVOKE takes WITH VERIFICATION. EntSqlCompilerImpl.compileGrant/compileRevoke: ASSUME SERVICE ACCOUNT sa TO|FROM parseUserOrGroupList (comma list). Before: RENAME TABLE / RENAME COLUMN (reserved RENAME), CANCEL ANY COPY (reserved ANY), SET REFRESH LIMIT / SET MEMORY LIMIT (reserved LIMIT) were errors in GRANT/REVOKE; REVOKE ... WITH VERIFICATION, GRANT ... WITH VERIFICATION WITH GRANT OPTION and ASSUME grantee lists were errors. Now: permissionToken accepts RENAME first, ANY second, LIMIT third word; grantStatement MANY(WITH GRANT OPTION | WITH VERIFICATION); revokeStatement optional WITH VERIFICATION; ASSUME statements take grantee lists (AST toList/fromList when more than one; RevokeStatement.verification). Autocomplete: PERMISSION_NAMES (content-assist.ts) drives the permission word continuations (replaces the ad-hoc CONVERT/REMOVE/STORAGE list); ANY offered only after CANCEL. New non-reserved keywords admin, engine, settings, sql for highlighting the permission names. Tests: 5 server fixture entries (CopyThroughDemoteTest:80, AlterPermissionTest:171, AdminSecurityContextTest:139, ReplicationAclTest:260, EntSqlParserTest:1269) + tests/ent-grant-revoke.test.ts (all 80 names in GRANT/REVOKE/list forms, option order, assume lists, autocomplete). No server test runs REVOKE..WITH VERIFICATION, the VERIFICATION-first order or ASSUME lists successfully (focused tests only). Red 20 -> green; full suite 17289 passed. Corpus: fixed 17 / rtChanged 0 / regressed 0 / throws 0 / formatterChangedMeaning 0. Suggestions: more permission first words after GRANT/REVOKE; REFRESH/ PARQUET/MEMORY after GRANT SET; ANY after GRANT CANCEL; WITH after REVOKE..FROM u. --- src/autocomplete/content-assist.ts | 118 +++++++++++++++++++---- src/grammar/keywords.ts | 4 + src/parser/ast.ts | 5 + src/parser/cst-types.d.ts | 9 +- src/parser/parser.ts | 43 +++++++-- src/parser/toSql.ts | 5 +- src/parser/tokens.ts | 5 + src/parser/visitor.ts | 19 +++- tests/ent-grant-revoke.test.ts | 148 +++++++++++++++++++++++++++++ tests/fixtures/server-queries.json | 30 ++++++ 10 files changed, 355 insertions(+), 31 deletions(-) create mode 100644 tests/ent-grant-revoke.test.ts diff --git a/src/autocomplete/content-assist.ts b/src/autocomplete/content-assist.ts index 7b90c1f..c93ee5a 100644 --- a/src/autocomplete/content-assist.ts +++ b/src/autocomplete/content-assist.ts @@ -882,6 +882,90 @@ function contextFunctionSuggestions( return inSubsample ? [...SUBSAMPLE_METHODS] : [] } +// Enterprise permission names (Permission.java namePermissionMap) +export const PERMISSION_NAMES: readonly string[] = [ + "ALL", + "SELECT", + "INSERT", + "UPDATE", + "CREATE TABLE", + "DROP TABLE", + "RENAME TABLE", + "TRUNCATE TABLE", + "ADD COLUMN", + "DROP COLUMN", + "RENAME COLUMN", + "ALTER COLUMN TYPE", + "ALTER COLUMN CACHE", + "ALTER SYMBOL CAPACITY", + "ADD INDEX", + "DROP INDEX", + "REINDEX", + "ATTACH PARTITION", + "DETACH PARTITION", + "DROP PARTITION", + "SET TABLE PARAM", + "SET TABLE TYPE", + "SET TABLE FORMAT", + "RESUME WAL", + "CREATE GROUP", + "DROP GROUP", + "CREATE USER", + "DROP USER", + "CREATE SERVICE ACCOUNT", + "DROP SERVICE ACCOUNT", + "ADD USER", + "REMOVE USER", + "CANCEL ANY COPY", + "BACKUP TABLE", + "BACKUP DATABASE", + "SNAPSHOT", + "VACUUM TABLE", + "ENABLE USER", + "DISABLE USER", + "ADD PASSWORD", + "REMOVE PASSWORD", + "ADD EXTERNAL ALIAS", + "REMOVE EXTERNAL ALIAS", + "CREATE JWK", + "DROP JWK", + "CREATE REST TOKEN", + "DROP REST TOKEN", + "LIST USERS", + "USER DETAILS", + "DEDUP ENABLE", + "DEDUP DISABLE", + "HTTP", + "PGWIRE", + "ILP", + "SQL ENGINE ADMIN", + "SYSTEM ADMIN", + "CREATE MATERIALIZED VIEW", + "DROP MATERIALIZED VIEW", + "REFRESH MATERIALIZED VIEW", + "SETTINGS", + "DATABASE ADMIN", + "AUDIT VIEW", + "CREATE VIEW", + "DROP VIEW", + "CREATE LIVE VIEW", + "DROP LIVE VIEW", + "COMPILE VIEW", + "ALTER VIEW", + "SET REFRESH LIMIT", + "SET REFRESH TYPE", + "CONVERT PARTITION TO PARQUET", + "CONVERT PARTITION TO NATIVE", + "SET PARQUET SETTINGS", + "SET STORAGE POLICY", + "REMOVE STORAGE POLICY", + "ENABLE STORAGE POLICY", + "DISABLE STORAGE POLICY", + "SWITCH ROLE", + "SET MEMORY LIMIT", + "RESOURCE GROUP ADMIN", +] + // Category names valid inside SHOW CREATE DATABASE (INCLUDE|EXCLUDE) ( ... ). // The grammar consumes these via `identifier` (they are non-reserved), so only // the explicit `ALL` token surfaces on its own; re-inject the rest. @@ -922,27 +1006,21 @@ function contextKeywordSuggestions( idKeywordStacks.some((stack) => stack.includes(r)) const last = tokens[tokens.length - 1]?.tokenType.name - const prev = tokens[tokens.length - 2]?.tokenType.name const out = new Set() - // GRANT / REVOKE permission phrases whose words route through `identifier`: - // CONVERT PARTITION [TO PARQUET|NATIVE], SET TABLE FORMAT, SET TABLE TYPE, - // SET STORAGE POLICY, REMOVE STORAGE POLICY. - // (SET / TABLE / TO / PARQUET / NATIVE at their spots are explicit CONSUMEs - // and already surface; we only add the identifier-path words.) + // GRANT / REVOKE permission phrases: the grammar takes any 1-3 words (most + // via `identifier`), so offer the next word of every server permission name + // that extends the words typed so far in the current list item. if (inRule("permissionToken")) { - if (last === "Grant" || last === "Revoke") { - out.add("CONVERT") - out.add("REMOVE") - } else if (last === "Convert") { - out.add("PARTITION") - } else if (last === "Set" || last === "Remove") { - out.add("STORAGE") - } else if (last === "Table" && prev === "Set") { - out.add("FORMAT") - out.add("TYPE") - } else if (last === "Storage" && (prev === "Set" || prev === "Remove")) { - out.add("POLICY") + const words: string[] = [] + for (let i = tokens.length - 1; i >= 0; i--) { + const name = tokens[i].tokenType.name + if (name === "Grant" || name === "Revoke" || name === "Comma") break + words.unshift(tokens[i].image.toUpperCase()) + } + const prefix = words.length > 0 ? words.join(" ") + " " : "" + for (const perm of PERMISSION_NAMES) { + if (perm.startsWith(prefix)) out.add(perm.split(" ")[words.length]) } } @@ -1065,6 +1143,10 @@ function computeSuggestions(tokens: IToken[]): ComputeResult { (s.nextTokenType.name === "Is" || s.nextTokenType.name === "Ilike" || s.nextTokenType.name === "Declare")) || + // ANY only continues CANCEL in permission names (CANCEL ANY COPY) + (s.nextTokenType.name === "Any" && + s.ruleStack[s.ruleStack.length - 1] === "permissionToken" && + tokens[tokens.length - 1]?.tokenType.name !== "Cancel") || // INCREMENTAL is an obsolete synonym of IMMEDIATE in REFRESH modes (s.nextTokenType.name === "Incremental" && s.ruleStack[s.ruleStack.length - 1] === "materializedViewRefresh") diff --git a/src/grammar/keywords.ts b/src/grammar/keywords.ts index eb982f4..09f4ea8 100644 --- a/src/grammar/keywords.ts +++ b/src/grammar/keywords.ts @@ -3,6 +3,7 @@ export const keywords: string[] = [ "account", "accounts", "add", + "admin", "alias", "align", "all", @@ -73,6 +74,7 @@ export const keywords: string[] = [ "else", "enable", "end", + "engine", "error", "every", "except", @@ -199,10 +201,12 @@ export const keywords: string[] = [ "service", "set", "sets", + "settings", "show", "skip", "snapshot", "splice", + "sql", "squash", "start", "statistics_enabled", diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 159127e..d249762 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -966,6 +966,7 @@ export interface RevokeStatement extends AstNode { permissions: string[] on?: GrantOnTarget from: QualifiedName + verification?: boolean } export interface GrantOnTarget extends AstNode { @@ -988,6 +989,8 @@ export interface GrantAssumeServiceAccountStatement extends AstNode { type: "grantAssumeServiceAccount" account: QualifiedName to: QualifiedName + /** All grantees when more than one is listed (`to` is the first) */ + toList?: QualifiedName[] grantOption?: boolean } @@ -995,6 +998,8 @@ export interface RevokeAssumeServiceAccountStatement extends AstNode { type: "revokeAssumeServiceAccount" account: QualifiedName from: QualifiedName + /** All grantees when more than one is listed (`from` is the first) */ + fromList?: QualifiedName[] } export interface VacuumTableStatement extends AstNode { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 909674f..894c8e4 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -2386,7 +2386,7 @@ export type GrantStatementCstChildren = { Comma?: IToken[]; To: IToken[]; qualifiedName: QualifiedNameCstNode[]; - With?: (IToken)[]; + With?: IToken[]; Option?: IToken[]; Verification?: IToken[]; }; @@ -2406,6 +2406,8 @@ export type RevokeStatementCstChildren = { Comma?: IToken[]; From: IToken[]; qualifiedName: QualifiedNameCstNode[]; + With?: IToken[]; + Verification?: IToken[]; }; export interface PermissionListCstNode extends CstNode { @@ -2440,12 +2442,15 @@ export type PermissionTokenCstChildren = { Vacuum?: IToken[]; Lock?: IToken[]; Set?: IToken[]; + Rename?: IToken[]; Materialized?: IToken[]; View?: IToken[]; Service?: IToken[]; Account?: IToken[]; Table?: IToken[]; Group?: IToken[]; + Any?: IToken[]; + Limit?: IToken[]; To?: IToken[]; Parquet?: IToken[]; Native?: IToken[]; @@ -2478,6 +2483,7 @@ export type GrantAssumeServiceAccountStatementCstChildren = { Account: IToken[]; qualifiedName: (QualifiedNameCstNode)[]; To: IToken[]; + Comma?: IToken[]; With?: IToken[]; Option?: IToken[]; }; @@ -2494,6 +2500,7 @@ export type RevokeAssumeServiceAccountStatementCstChildren = { Account: IToken[]; qualifiedName: (QualifiedNameCstNode)[]; From: IToken[]; + Comma?: IToken[]; }; export interface VacuumTableStatementCstNode extends CstNode { diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 273bfd6..9cddb3e 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -48,6 +48,7 @@ import { On, Partition, Limit, + Any, Order, Asc, Desc, @@ -4833,14 +4834,19 @@ class QuestDBParser extends CstParser { }) this.CONSUME(To) this.SUBRULE(this.qualifiedName) - this.OPTION1(() => { + // WITH GRANT OPTION / WITH VERIFICATION in either order + // (PermissionParser.parse) + this.MANY1(() => { this.CONSUME(With) - this.CONSUME1(Grant) - this.CONSUME(Option) - }) - this.OPTION2(() => { - this.CONSUME1(With) - this.CONSUME(Verification) + this.OR1([ + { + ALT: () => { + this.CONSUME1(Grant) + this.CONSUME(Option) + }, + }, + { ALT: () => this.CONSUME(Verification) }, + ]) }) }) @@ -4869,6 +4875,10 @@ class QuestDBParser extends CstParser { }) this.CONSUME(From) this.SUBRULE(this.qualifiedName) + this.OPTION1(() => { + this.CONSUME(With) + this.CONSUME(Verification) + }) }) private permissionList = this.RULE("permissionList", () => { @@ -4899,6 +4909,7 @@ class QuestDBParser extends CstParser { { ALT: () => this.CONSUME(Vacuum) }, { ALT: () => this.CONSUME(Lock) }, { ALT: () => this.CONSUME(Set) }, + { ALT: () => this.CONSUME(Rename) }, ]) // Optional second word (for compound permissions like CREATE TABLE, ALTER MATERIALIZED VIEW) this.OPTION(() => { @@ -4918,13 +4929,21 @@ class QuestDBParser extends CstParser { // Reserved keywords that are NOT in the identifier rule { ALT: () => this.CONSUME(Table) }, { ALT: () => this.CONSUME(Group) }, + // CANCEL ANY COPY + { ALT: () => this.CONSUME(Any) }, // Any other keyword/identifier as second word (VIEW, USER, INDEX, COLUMN, // DATABASE ADMIN, etc.) { ALT: () => this.SUBRULE1(this.identifier) }, ]) // Optional third word for 3-word permissions, e.g. SET TABLE FORMAT, // SET STORAGE POLICY, SET TABLE TYPE, REMOVE STORAGE POLICY. - this.OPTION1(() => this.SUBRULE2(this.identifier)) + // SET REFRESH LIMIT / SET MEMORY LIMIT end in reserved LIMIT. + this.OPTION1(() => { + this.OR3([ + { ALT: () => this.SUBRULE2(this.identifier) }, + { ALT: () => this.CONSUME(Limit) }, + ]) + }) // Trailing "TO PARQUET|NATIVE" for the CONVERT PARTITION TO {PARQUET|NATIVE} // permission names, whose phrase embeds the TO that normally delimits // GRANT … TO / REVOKE … FROM. Gated so an ordinary "… TO " is @@ -4990,6 +5009,10 @@ class QuestDBParser extends CstParser { this.SUBRULE(this.qualifiedName) this.CONSUME(To) this.SUBRULE1(this.qualifiedName) + this.MANY(() => { + this.CONSUME(Comma) + this.SUBRULE2(this.qualifiedName) + }) this.OPTION(() => { this.CONSUME(With) this.CONSUME1(Grant) @@ -5008,6 +5031,10 @@ class QuestDBParser extends CstParser { this.SUBRULE(this.qualifiedName) this.CONSUME(From) this.SUBRULE1(this.qualifiedName) + this.MANY(() => { + this.CONSUME(Comma) + this.SUBRULE2(this.qualifiedName) + }) }, ) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index 0b95abf..e913687 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -1747,6 +1747,7 @@ function revokeToSql(stmt: AST.RevokeStatement): string { const parts: string[] = [`REVOKE ${stmt.permissions.join(", ")}`] if (stmt.on) parts.push(grantOnTargetToSql(stmt.on)) parts.push(`FROM ${qualifiedNameToSql(stmt.from)}`) + if (stmt.verification) parts.push("WITH VERIFICATION") return parts.join(" ") } @@ -1754,7 +1755,7 @@ function grantAssumeToSql( stmt: AST.GrantAssumeServiceAccountStatement, ): string { const parts: string[] = [ - `GRANT ASSUME SERVICE ACCOUNT ${qualifiedNameToSql(stmt.account)} TO ${qualifiedNameToSql(stmt.to)}`, + `GRANT ASSUME SERVICE ACCOUNT ${qualifiedNameToSql(stmt.account)} TO ${(stmt.toList ?? [stmt.to]).map(qualifiedNameToSql).join(", ")}`, ] if (stmt.grantOption) parts.push("WITH GRANT OPTION") return parts.join(" ") @@ -1763,7 +1764,7 @@ function grantAssumeToSql( function revokeAssumeToSql( stmt: AST.RevokeAssumeServiceAccountStatement, ): string { - return `REVOKE ASSUME SERVICE ACCOUNT ${qualifiedNameToSql(stmt.account)} FROM ${qualifiedNameToSql(stmt.from)}` + return `REVOKE ASSUME SERVICE ACCOUNT ${qualifiedNameToSql(stmt.account)} FROM ${(stmt.fromList ?? [stmt.from]).map(qualifiedNameToSql).join(", ")}` } // ============================================================================= diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index ef66fbf..a03b239 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -452,6 +452,11 @@ export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ "Unset", // CREATE VIEW ... WITH AUDIT (Enterprise) "Audit", + // GRANT permission-name words (SQL ENGINE ADMIN, SETTINGS, ...) + "Admin", + "Engine", + "Settings", + "Sql", "Daily", "Expression", "Posting", diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index c90494e..b1f8c51 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -3919,6 +3919,9 @@ class QuestDBVisitor extends BaseVisitor { permissions: this.visit(ctx.permissionList) as string[], from: this.visit(ctx.qualifiedName) as AST.QualifiedName, } + if (ctx.Verification) { + result.verification = true + } if (ctx.On) { if (ctx.All && ctx.Tables) { result.on = { type: "grantOn", allTables: true } @@ -3991,22 +3994,34 @@ class QuestDBVisitor extends BaseVisitor { grantAssumeServiceAccountStatement( ctx: GrantAssumeServiceAccountStatementCstChildren, ): AST.GrantAssumeServiceAccountStatement { - return { + const result: AST.GrantAssumeServiceAccountStatement = { type: "grantAssumeServiceAccount", account: this.visit(ctx.qualifiedName[0]) as AST.QualifiedName, to: this.visit(ctx.qualifiedName[1]) as AST.QualifiedName, grantOption: !!ctx.Option, } + if (ctx.qualifiedName.length > 2) { + result.toList = ctx.qualifiedName + .slice(1) + .map((n) => this.visit(n) as AST.QualifiedName) + } + return result } revokeAssumeServiceAccountStatement( ctx: RevokeAssumeServiceAccountStatementCstChildren, ): AST.RevokeAssumeServiceAccountStatement { - return { + const result: AST.RevokeAssumeServiceAccountStatement = { type: "revokeAssumeServiceAccount", account: this.visit(ctx.qualifiedName[0]) as AST.QualifiedName, from: this.visit(ctx.qualifiedName[1]) as AST.QualifiedName, } + if (ctx.qualifiedName.length > 2) { + result.fromList = ctx.qualifiedName + .slice(1) + .map((n) => this.visit(n) as AST.QualifiedName) + } + return result } // ========================================================================== diff --git a/tests/ent-grant-revoke.test.ts b/tests/ent-grant-revoke.test.ts new file mode 100644 index 0000000..718a86d --- /dev/null +++ b/tests/ent-grant-revoke.test.ts @@ -0,0 +1,148 @@ +/** + * Enterprise GRANT / REVOKE: every permission name of Permission.java + * namePermissionMap, WITH VERIFICATION / WITH GRANT OPTION in either order, + * REVOKE ... WITH VERIFICATION, ASSUME SERVICE ACCOUNT grantee lists. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst, createAutocompleteProvider } from "../src/index" +import { PERMISSION_NAMES } from "../src/autocomplete/content-assist" + +const ok = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors, sql).toEqual([]) + expect(result.ast, sql).toHaveLength(1) + return result.ast[0] +} + +// Permission.java namePermissionMap (80 names) +const SERVER_PERMISSIONS = [ + "ALL", + "SELECT", + "INSERT", + "UPDATE", + "CREATE TABLE", + "DROP TABLE", + "RENAME TABLE", + "TRUNCATE TABLE", + "ADD COLUMN", + "DROP COLUMN", + "RENAME COLUMN", + "ALTER COLUMN TYPE", + "ALTER COLUMN CACHE", + "ALTER SYMBOL CAPACITY", + "ADD INDEX", + "DROP INDEX", + "REINDEX", + "ATTACH PARTITION", + "DETACH PARTITION", + "DROP PARTITION", + "SET TABLE PARAM", + "SET TABLE TYPE", + "SET TABLE FORMAT", + "RESUME WAL", + "CREATE GROUP", + "DROP GROUP", + "CREATE USER", + "DROP USER", + "CREATE SERVICE ACCOUNT", + "DROP SERVICE ACCOUNT", + "ADD USER", + "REMOVE USER", + "CANCEL ANY COPY", + "BACKUP TABLE", + "BACKUP DATABASE", + "SNAPSHOT", + "VACUUM TABLE", + "ENABLE USER", + "DISABLE USER", + "ADD PASSWORD", + "REMOVE PASSWORD", + "ADD EXTERNAL ALIAS", + "REMOVE EXTERNAL ALIAS", + "CREATE JWK", + "DROP JWK", + "CREATE REST TOKEN", + "DROP REST TOKEN", + "LIST USERS", + "USER DETAILS", + "DEDUP ENABLE", + "DEDUP DISABLE", + "HTTP", + "PGWIRE", + "ILP", + "SQL ENGINE ADMIN", + "SYSTEM ADMIN", + "CREATE MATERIALIZED VIEW", + "DROP MATERIALIZED VIEW", + "REFRESH MATERIALIZED VIEW", + "SETTINGS", + "DATABASE ADMIN", + "AUDIT VIEW", + "CREATE VIEW", + "DROP VIEW", + "CREATE LIVE VIEW", + "DROP LIVE VIEW", + "COMPILE VIEW", + "ALTER VIEW", + "SET REFRESH LIMIT", + "SET REFRESH TYPE", + "CONVERT PARTITION TO PARQUET", + "CONVERT PARTITION TO NATIVE", + "SET PARQUET SETTINGS", + "SET STORAGE POLICY", + "REMOVE STORAGE POLICY", + "ENABLE STORAGE POLICY", + "DISABLE STORAGE POLICY", + "SWITCH ROLE", + "SET MEMORY LIMIT", + "RESOURCE GROUP ADMIN", +] + +describe("GRANT / REVOKE permissions", () => { + it("parses every server permission name", () => { + expect(SERVER_PERMISSIONS).toHaveLength(80) + for (const p of SERVER_PERMISSIONS) { + ok(`GRANT ${p} TO u`) + ok(`REVOKE ${p} FROM u`) + ok(`GRANT ${p}, SELECT ON t TO u WITH GRANT OPTION`) + ok(`revoke select, ${p.toLowerCase()} on all tables from u`) + } + }) + + it("autocomplete permission list matches the server", () => { + expect([...PERMISSION_NAMES].sort()).toEqual([...SERVER_PERMISSIONS].sort()) + }) + + it("WITH VERIFICATION / WITH GRANT OPTION in either order", () => { + ok("GRANT SELECT ON t TO u WITH GRANT OPTION WITH VERIFICATION") + ok("GRANT SELECT ON t TO u WITH VERIFICATION WITH GRANT OPTION") + ok("GRANT SELECT ON t TO u WITH VERIFICATION") + ok("REVOKE SELECT, INSERT FROM 'admin' WITH VERIFICATION") + }) + + it("ASSUME SERVICE ACCOUNT grantee lists", () => { + ok("GRANT ASSUME SERVICE ACCOUNT sa TO a, b, c WITH GRANT OPTION") + ok("REVOKE ASSUME SERVICE ACCOUNT sa FROM a, b") + }) + + it("autocomplete offers permission continuations", () => { + const provider = createAutocompleteProvider({ tables: [], columns: {} }) + const labels = (sql: string) => + provider.getSuggestions(sql, sql.length).map((s) => s.label) + expect(labels("GRANT ")).toEqual( + expect.arrayContaining(["CANCEL", "RENAME", "SWITCH", "RESOURCE", "SQL"]), + ) + expect(labels("GRANT SET ")).toEqual( + expect.arrayContaining(["REFRESH", "MEMORY", "PARQUET", "STORAGE"]), + ) + expect(labels("GRANT SET MEMORY ")).toContain("LIMIT") + expect(labels("GRANT CANCEL ")).toContain("ANY") + expect(labels("GRANT CANCEL ANY ")).toContain("COPY") + expect(labels("REVOKE SELECT, RENAME ")).toEqual( + expect.arrayContaining(["TABLE", "COLUMN"]), + ) + expect(labels("GRANT SQL ENGINE ")).toContain("ADMIN") + expect(labels("REVOKE SELECT FROM u WITH ")).toContain("VERIFICATION") + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index fcebdcd..2d4ab04 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -2061,5 +2061,35 @@ "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/view/ViewAuditTest.java:674", "feature": "ent-view-audit", "toSql": "CREATE VIEW view_c AS (SELECT ts, symbol, price FROM trades WHERE symbol = 'MSFT') WITH AUDIT" + }, + { + "query": "GRANT CANCEL ANY COPY TO canceller", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/lifecycle/CopyThroughDemoteTest.java:80", + "feature": "ent-grant-revoke", + "toSql": "GRANT CANCEL ANY COPY TO canceller" + }, + { + "query": "GRANT SET REFRESH LIMIT ON mv TO testUser", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/security/AlterPermissionTest.java:171", + "feature": "ent-grant-revoke", + "toSql": "GRANT SET REFRESH LIMIT ON mv TO testUser" + }, + { + "query": "GRANT SET MEMORY LIMIT TO granted_sa", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/security/AdminSecurityContextTest.java:139", + "feature": "ent-grant-revoke", + "toSql": "GRANT SET MEMORY LIMIT TO granted_sa" + }, + { + "query": "grant rename table on test to test_user", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/cairo/wal/transfer/ReplicationAclTest.java:260", + "feature": "ent-grant-revoke", + "toSql": "GRANT rename table ON test TO test_user" + }, + { + "query": "GRANT CANCEL ANY COPY ON my_table TO admin;", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/griffin/EntSqlParserTest.java:1269", + "feature": "ent-grant-revoke", + "toSql": "GRANT CANCEL ANY COPY ON my_table TO admin" } ] From c7bc2b95d5a22623657a5c40a387d8fa0e5d5150 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 10 Nov 2026 17:54:48 +0300 Subject: [PATCH 36/36] feat: Enterprise ADD COLUMN OWNED BY, quoted memory limits, unquoted aliases/passwords, REST token option order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server (EntSqlCompilerImpl): addColumnSuffix parses OWNED BY after the ADD COLUMN list; parseMemoryLimit unquotes its token ('64M', 'UNLIMITED'); external aliases and passwords are any (unquoted) token (parsePassword: `with password null` sets the password "null"); CREATE TOKEN TYPE REST WITH TTL v is followed by a loop of REFRESH / TRANSIENT in any order. Before: ADD COLUMN .. OWNED BY g silently split into 3 statements; SET MEMORY LIMIT '64M', unquoted EXTERNAL ALIAS, ALTER USER|SERVICE ACCOUNT .. WITH PASSWORD pw|null were errors; `.. TRANSIENT REFRESH` split into a second statement. Now: alterTableAddColumns [OWNED BY x] (AddColumnAction.ownedBy); memoryLimit accepts StringLiteral (resource group option values now go through memoryLimit only); CREATE/ALTER GROUP external alias is stringOrIdentifier ('' unescaped); alterUserAction WITH PASSWORD takes a string, NULL, number or identifier; REST token MANY(REFRESH | TRANSIENT). Tests: 6 server fixture entries (HttpExternalGroupTest:396, ResourceGroupHttpTest:1002/1012, AclAuditLogTest:717/1039, HttpBasicDynamicAuthenticationTest:221) + tests/ent-users-groups.test.ts (TRANSIENT REFRESH / service-account unquoted password: no positive server test, focused only). Red 19 -> green; full suite 17318 passed. Corpus: fixed 36 / rtChanged 1 (EntSqlParserTest "... TRANSIENT REFRESH blablabla": REFRESH now part of the token statement; server rejects the trailing word) / regressed 2 (EntSqlParserTest:42/47 "add column y int owned [blablabla]" — server syntax-error tests, "'by' expected") / throws 0 / formatterChangedMeaning 0. Suggestions: +OWNED after ADD COLUMN definitions; +NULL after WITH PASSWORD. --- src/parser/ast.ts | 2 + src/parser/cst-types.d.ts | 12 ++++-- src/parser/parser.ts | 43 +++++++++++++-------- src/parser/toSql.ts | 3 ++ src/parser/visitor.ts | 33 +++++++++++----- tests/ent-users-groups.test.ts | 62 ++++++++++++++++++++++++++++++ tests/fixtures/server-queries.json | 36 +++++++++++++++++ 7 files changed, 164 insertions(+), 27 deletions(-) create mode 100644 tests/ent-users-groups.test.ts diff --git a/src/parser/ast.ts b/src/parser/ast.ts index d249762..1c4e6c9 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -612,6 +612,8 @@ export interface AddColumnAction { actionType: "addColumn" ifNotExists?: boolean columns: ColumnDefinition[] + /** Enterprise: OWNED BY user|group */ + ownedBy?: string } export interface DropColumnAction { diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 894c8e4..af7e0fa 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -999,7 +999,7 @@ export type CreateGroupStatementCstChildren = { With?: IToken[]; External?: IToken[]; Alias?: IToken[]; - StringLiteral?: IToken[]; + stringOrIdentifier?: StringOrIdentifierCstNode[]; }; export interface CreateServiceAccountStatementCstNode extends CstNode { @@ -1527,7 +1527,7 @@ export type AlterGroupStatementCstChildren = { With?: IToken[]; External?: (IToken)[]; Alias?: (IToken)[]; - StringLiteral?: (IToken)[]; + stringOrIdentifier?: (StringOrIdentifierCstNode)[]; Drop?: IToken[]; Set?: IToken[]; Memory?: IToken[]; @@ -1591,6 +1591,9 @@ export type AlterUserActionCstChildren = { No?: IToken[]; Password?: (IToken)[]; StringLiteral?: (IToken)[]; + Null?: IToken[]; + NumberLiteral?: IToken[]; + identifier?: IdentifierCstNode[]; Create?: IToken[]; Token?: (IToken)[]; Type?: (IToken)[]; @@ -1612,6 +1615,7 @@ export interface MemoryLimitCstNode extends CstNode { } export type MemoryLimitCstChildren = { + StringLiteral?: IToken[]; Unlimited?: IToken[]; DurationLiteral?: IToken[]; NumberLiteral?: IToken[]; @@ -1704,6 +1708,9 @@ export type AlterTableAddColumnsCstChildren = { Column?: IToken[]; addColumnItem: (AddColumnItemCstNode)[]; Comma?: IToken[]; + Owned?: IToken[]; + By?: IToken[]; + stringOrIdentifier?: StringOrIdentifierCstNode[]; }; export interface AddColumnItemCstNode extends CstNode { @@ -1920,7 +1927,6 @@ export type ResourceGroupOptionCstChildren = { MaxQueuedQueries?: IToken[]; QueueTimeout?: IToken[]; Equals?: IToken[]; - StringLiteral?: IToken[]; Minus?: IToken[]; memoryLimit?: MemoryLimitCstNode[]; }; diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 9cddb3e..6071796 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -2377,7 +2377,7 @@ class QuestDBParser extends CstParser { this.CONSUME(With) this.CONSUME(External) this.CONSUME(Alias) - this.CONSUME(StringLiteral) + this.SUBRULE(this.stringOrIdentifier) }) }) @@ -3194,7 +3194,7 @@ class QuestDBParser extends CstParser { this.CONSUME(With) this.CONSUME(External) this.CONSUME(Alias) - this.CONSUME(StringLiteral) + this.SUBRULE(this.stringOrIdentifier) }, }, { @@ -3202,7 +3202,7 @@ class QuestDBParser extends CstParser { this.CONSUME(Drop) this.CONSUME1(External) this.CONSUME1(Alias) - this.CONSUME1(StringLiteral) + this.SUBRULE1(this.stringOrIdentifier) }, }, { @@ -3291,7 +3291,13 @@ class QuestDBParser extends CstParser { { ALT: () => { this.CONSUME1(Password) - this.CONSUME(StringLiteral) + // parsePassword: any token, unquoted (`null`, pw, 'pw') + this.OR6([ + { ALT: () => this.CONSUME(StringLiteral) }, + { ALT: () => this.CONSUME(Null) }, + { ALT: () => this.CONSUME(NumberLiteral) }, + { ALT: () => this.SUBRULE(this.identifier) }, + ]) }, }, ]) @@ -3327,9 +3333,15 @@ class QuestDBParser extends CstParser { { ALT: () => this.CONSUME(DurationLiteral) }, { ALT: () => this.CONSUME2(StringLiteral) }, ]) - this.OPTION1(() => this.CONSUME(Refresh)) }) - this.OPTION4(() => this.CONSUME(Transient)) + // REFRESH / TRANSIENT in any order (EntSqlCompilerImpl + // CREATE TOKEN TYPE REST loop) + this.MANY(() => { + this.OR7([ + { ALT: () => this.CONSUME(Refresh) }, + { ALT: () => this.CONSUME(Transient) }, + ]) + }) }, }, ]) @@ -3361,6 +3373,8 @@ class QuestDBParser extends CstParser { private memoryLimit = this.RULE("memoryLimit", () => { this.OR([ + // quoted form: '64M', 'UNLIMITED' (parseMemoryLimit unquotes) + { ALT: () => this.CONSUME(StringLiteral) }, { ALT: () => this.CONSUME(Unlimited) }, { ALT: () => this.CONSUME(DurationLiteral) }, { @@ -3700,6 +3714,12 @@ class QuestDBParser extends CstParser { this.CONSUME(Comma) this.SUBRULE1(this.addColumnItem) }) + // Enterprise addColumnSuffix: OWNED BY user|group + this.OPTION1(() => { + this.CONSUME(Owned) + this.CONSUME(By) + this.SUBRULE(this.stringOrIdentifier) + }) }) private addColumnItem = this.RULE("addColumnItem", () => { @@ -4092,15 +4112,8 @@ class QuestDBParser extends CstParser { ]) this.OPTION(() => { this.CONSUME(Equals) - this.OR1([ - { ALT: () => this.CONSUME(StringLiteral) }, - { - ALT: () => { - this.OPTION1(() => this.CONSUME(Minus)) - this.SUBRULE(this.memoryLimit) - }, - }, - ]) + this.OPTION1(() => this.CONSUME(Minus)) + this.SUBRULE(this.memoryLimit) }) }) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index e913687..3760e47 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -933,6 +933,9 @@ function alterTableToSql(stmt: AST.AlterTableStatement): string { ) .join(", "), ) + if (action.ownedBy) { + parts.push(`OWNED BY ${escapeIdentifier(action.ownedBy)}`) + } break } case "dropColumn": diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index b1f8c51..267acf8 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -194,6 +194,7 @@ import type { StatementCstChildren, StatementsCstChildren, StringOrIdentifierCstChildren, + StringOrIdentifierCstNode, StringOrQualifiedNameCstChildren, TableFunctionCallCstChildren, TableFunctionNameCstChildren, @@ -1835,6 +1836,9 @@ class QuestDBVisitor extends BaseVisitor { (c) => this.visit(c) as AST.ColumnDefinition, ) const result: AST.AddColumnAction = { actionType: "addColumn", columns } + if (ctx.stringOrIdentifier) { + result.ownedBy = this.visit(ctx.stringOrIdentifier) as string + } // IF NOT EXISTS on the first column is the action-level flag if (columns[0].ifNotExists) { delete columns[0].ifNotExists @@ -2050,8 +2054,8 @@ class QuestDBVisitor extends BaseVisitor { group: this.visit(ctx.qualifiedName) as AST.QualifiedName, ifNotExists: !!ctx.If, } - if (ctx.StringLiteral) { - result.externalAlias = unquoteSingle(ctx.StringLiteral[0].image) + if (ctx.stringOrIdentifier) { + result.externalAlias = this.externalAliasValue(ctx.stringOrIdentifier[0]) } return result } @@ -2514,7 +2518,7 @@ class QuestDBVisitor extends BaseVisitor { memoryLimit: this.visit(ctx.memoryLimit) as string, } } - const alias = unquoteSingle(ctx.StringLiteral![0].image) + const alias = this.externalAliasValue(ctx.stringOrIdentifier![0]) return { type: "alterGroup", group, @@ -2779,9 +2783,7 @@ class QuestDBVisitor extends BaseVisitor { ctx.MaxQueuedQueries ?? ctx.QueueTimeout)![0] const result: AST.ResourceGroupOption = { name: key.image.toLowerCase() } - if (ctx.StringLiteral) { - result.value = ctx.StringLiteral[0].image - } else if (ctx.memoryLimit) { + if (ctx.memoryLimit) { result.value = (ctx.Minus ? "-" : "") + (this.visit(ctx.memoryLimit) as string) } @@ -2807,12 +2809,17 @@ class QuestDBVisitor extends BaseVisitor { return { actionType: "disable" } } if (ctx.Password || ctx.No) { + let password: string | undefined + if (ctx.StringLiteral) + password = unquoteSingle(ctx.StringLiteral[0].image) + else if (ctx.Null) password = ctx.Null[0].image + else if (ctx.NumberLiteral) password = ctx.NumberLiteral[0].image + else if (ctx.identifier) + password = this.extractIdentifierName(ctx.identifier[0].children) return { actionType: "password", noPassword: !!ctx.No, - password: ctx.StringLiteral - ? unquoteSingle(ctx.StringLiteral[0].image) - : undefined, + password, } } if (ctx.Create && ctx.Token) { @@ -2851,6 +2858,7 @@ class QuestDBVisitor extends BaseVisitor { } memoryLimit(ctx: MemoryLimitCstChildren): string { + if (ctx.StringLiteral) return ctx.StringLiteral[0].image if (ctx.Unlimited) return "UNLIMITED" if (ctx.DurationLiteral) return ctx.DurationLiteral[0].image return ctx.NumberLiteral![0].image + (ctx.Identifier?.[0]?.image ?? "") @@ -5422,6 +5430,13 @@ class QuestDBVisitor extends BaseVisitor { // Names (OWNED BY): a single-quoted name keeps its inner text verbatim, as // the server's GenericLexer.unquote does (see unquoteSingle). + // EXTERNAL ALIAS 'cn=o''brien' | ext_g — unescapes '' (unlike stringOrIdentifier) + private externalAliasValue(node: StringOrIdentifierCstNode): string { + const str = node.children.StringLiteral + if (str) return unquoteSingle(str[0].image) + return this.visit(node) as string + } + stringOrIdentifier(ctx: StringOrIdentifierCstChildren): string { if (ctx.StringLiteral) { return ctx.StringLiteral[0].image.slice(1, -1) diff --git a/tests/ent-users-groups.test.ts b/tests/ent-users-groups.test.ts new file mode 100644 index 0000000..7f37ff3 --- /dev/null +++ b/tests/ent-users-groups.test.ts @@ -0,0 +1,62 @@ +/** + * Enterprise users/groups: ADD COLUMN ... OWNED BY, quoted SET MEMORY LIMIT, + * unquoted external aliases and passwords, REST token REFRESH/TRANSIENT order. + */ + +import { describe, it, expect } from "vitest" +import { parseToAst } from "../src/index" +import { toSql } from "../src/parser/toSql" + +const ok = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors, sql).toEqual([]) + expect(result.ast, sql).toHaveLength(1) + return result.ast[0] +} + +describe("enterprise users / groups", () => { + it("ADD COLUMN ... OWNED BY", () => { + expect( + ok("ALTER TABLE t ADD COLUMN a INT, b DOUBLE OWNED BY 'g1'"), + ).toMatchObject({ action: { actionType: "addColumn", ownedBy: "g1" } }) + }) + + it("quoted memory limits", () => { + ok("ALTER USER u SET MEMORY LIMIT 'UNLIMITED'") + ok("ALTER SERVICE ACCOUNT sa SET MEMORY LIMIT '2G'") + ok("ALTER USER u SET MEMORY LIMIT 512M") + }) + + it("unquoted external alias", () => { + expect(ok("ALTER GROUP g WITH EXTERNAL ALIAS ext_g")).toMatchObject({ + action: "setAlias", + externalAlias: "ext_g", + }) + expect(ok("CREATE GROUP g WITH EXTERNAL ALIAS 'x'")).toMatchObject({ + externalAlias: "x", + }) + }) + + it("unquoted passwords", () => { + expect(ok("ALTER SERVICE ACCOUNT sa WITH PASSWORD pw")).toMatchObject({ + action: { actionType: "password", password: "pw" }, + }) + ok("ALTER USER u WITH PASSWORD 'x'") + ok("ALTER USER u WITH NO PASSWORD") + }) + + it("REST token REFRESH / TRANSIENT in any order", () => { + const stmt = ok( + "ALTER USER u CREATE TOKEN TYPE REST WITH TTL '42m' TRANSIENT REFRESH", + ) + expect(stmt).toMatchObject({ + action: { actionType: "createToken", refresh: true, transient: true }, + }) + expect(toSql(stmt)).toBe( + "ALTER USER u CREATE TOKEN TYPE REST WITH TTL '42m' REFRESH TRANSIENT", + ) + ok("ALTER USER u CREATE TOKEN TYPE REST WITH TTL '42m' REFRESH TRANSIENT") + ok("ALTER USER u CREATE TOKEN TYPE REST WITH TTL '42m' REFRESH") + ok("ALTER USER u CREATE TOKEN TYPE REST") + }) +}) diff --git a/tests/fixtures/server-queries.json b/tests/fixtures/server-queries.json index 2d4ab04..57bfe4d 100644 --- a/tests/fixtures/server-queries.json +++ b/tests/fixtures/server-queries.json @@ -2091,5 +2091,41 @@ "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/griffin/EntSqlParserTest.java:1269", "feature": "ent-grant-revoke", "toSql": "GRANT CANCEL ANY COPY ON my_table TO admin" + }, + { + "query": "alter table trades add column value double owned by group11", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/acl/HttpExternalGroupTest.java:396", + "feature": "ent-users-groups", + "toSql": "ALTER TABLE trades ADD COLUMN value DOUBLE OWNED BY group11" + }, + { + "query": "ALTER USER memory_user SET MEMORY LIMIT '64M'", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/resourcegroup/ResourceGroupHttpTest.java:1002", + "feature": "ent-users-groups", + "toSql": "ALTER USER memory_user SET MEMORY LIMIT '64M'" + }, + { + "query": "ALTER GROUP memory_low SET MEMORY LIMIT '1M'", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/resourcegroup/ResourceGroupHttpTest.java:1012", + "feature": "ent-users-groups", + "toSql": "ALTER GROUP memory_low SET MEMORY LIMIT '1M'" + }, + { + "query": "CREATE GROUP g1 WITH EXTERNAL ALIAS ext_g", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/acl/AclAuditLogTest.java:717", + "feature": "ent-users-groups", + "toSql": "CREATE GROUP g1 WITH EXTERNAL ALIAS 'ext_g'" + }, + { + "query": "ALTER GROUP g DROP EXTERNAL ALIAS ext_g", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/acl/AclAuditLogTest.java:1039", + "feature": "ent-users-groups", + "toSql": "ALTER GROUP g DROP EXTERNAL ALIAS 'ext_g'" + }, + { + "query": "alter user minion with password null", + "source": "questdb-enterprise:questdb-ent/src/test/java/com/questdb/acl/HttpBasicDynamicAuthenticationTest.java:221", + "feature": "ent-users-groups", + "toSql": "ALTER USER minion WITH PASSWORD 'null'" } ]