From fdda675f1b3be3c16475a25e8a0ef069df1bebcc Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 17 Oct 2026 22:39:14 -0400 Subject: [PATCH 1/4] fix: start a clause after a select list's trailing comma BigQuery and Snowflake allow `SELECT a, b, FROM t`, whose FROM was read as a column after the change that keeps a column named like a clause (ex. `offset`) in its list. --- src/generation/generate.rs | 22 ++++++++++++++++++++-- tests/specs/BigQuery/BigQuery_All.txt | 16 +++++++++++++++- tests/specs/Snowflake/Snowflake_All.txt | 22 ++++++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/generation/generate.rs b/src/generation/generate.rs index cc482f2..9ea3cf1 100644 --- a/src/generation/generate.rs +++ b/src/generation/generate.rs @@ -752,8 +752,26 @@ impl ClauseScan { fn match_clause(&mut self, elements: &[Element], i: usize) -> Option<(ClauseKind, usize)> { let token = word_at(elements, i)?; let previous = prev_significant(elements, i); - // a word that's part of a qualified name, or an item of a list (ex. a column named `offset`) - if previous.is_some_and(|e| e.is_kind(TokenKind::Dot) || e.is_kind(TokenKind::Comma)) + // a word that's part of a qualified name, or an item of a list (ex. a column named `offset`), + // though a list may end in a comma before a clause no column can be named like (ex. BigQuery's + // `SELECT a, b, FROM t`) + let after_comma = previous.is_some_and(|e| e.is_kind(TokenKind::Comma)) + && ![ + "FROM", + "WHERE", + "GROUP", + "HAVING", + "ORDER", + "UNION", + "INTERSECT", + "EXCEPT", + "MINUS", + "INTO", + ] + .iter() + .any(|w| token.is_word(w)); + if after_comma + || previous.is_some_and(|e| e.is_kind(TokenKind::Dot)) || elements.get(i + 1).is_some_and(|e| e.is_kind(TokenKind::Dot)) { return None; diff --git a/tests/specs/BigQuery/BigQuery_All.txt b/tests/specs/BigQuery/BigQuery_All.txt index f06b583..edcd9d2 100644 --- a/tests/specs/BigQuery/BigQuery_All.txt +++ b/tests/specs/BigQuery/BigQuery_All.txt @@ -100,7 +100,7 @@ BEGIN END; == should format call == -CALL dataset.p(1, y); +CALL dataset.p(1 , y); [expect] CALL dataset.p(1, y); @@ -147,3 +147,17 @@ SELECT r'\'' AS a, `a\`b` FROM t; [expect] SELECT r'\'' AS a, `a\`b` FROM t; + +== should format select as struct == +SELECT AS STRUCT 1 AS a, 2 AS b; + +[expect] +SELECT AS STRUCT 1 AS a, 2 AS b; + +== should end a select list in a trailing comma == +SELECT a, b, FROM t WHERE x = 1; + +[expect] +SELECT a, b, +FROM t +WHERE x = 1; diff --git a/tests/specs/Snowflake/Snowflake_All.txt b/tests/specs/Snowflake/Snowflake_All.txt index fe80e75..f438012 100644 --- a/tests/specs/Snowflake/Snowflake_All.txt +++ b/tests/specs/Snowflake/Snowflake_All.txt @@ -127,3 +127,25 @@ FROM @%t; PUT file:///tmp/x.csv @~/staged; LIST @my_stage/data/2024/; REMOVE @%mytable/old; + +== should end a select list in a trailing comma == +SELECT a, b, FROM t; + +[expect] +SELECT a, b, +FROM t; + +== should format a match recognize == +SELECT * FROM t MATCH_RECOGNIZE (PARTITION BY a ORDER BY b MEASURES c AS d ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW PATTERN (A B*) DEFINE B AS b > 1); + +[expect] +SELECT * +FROM t MATCH_RECOGNIZE ( + PARTITION BY a + ORDER BY b + MEASURES c AS d + ONE ROW PER MATCH + AFTER MATCH SKIP PAST LAST ROW + PATTERN (A B*) + DEFINE B AS b > 1 +); From 42277322c6ad9f79c1e9485106cdf0c4dacaf3eb Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 17 Oct 2026 22:39:14 -0400 Subject: [PATCH 2/4] test: cover more syntax, options and safety checks - specs for syntax no spec reached: subquery aliases, BETWEEN within conditions, comments before END, ELSE and EXCEPTION, lateral views, LOCK IN SHARE MODE, TOP PERCENT, SELECT AS STRUCT, MATCH_RECOGNIZE, number and quoted string literals, unfinished procedural code and unformattable bodies - specs for the casing and legacy uppercase options together, explicit preserve, joins as top level within a subquery, and range formatting in SQL Server, MySQL and PostgreSQL - give the specs whose input was already formatted messy input, so they check formatting rather than only stability - unit tests for checking the line breaks between joined strings and template tags in place, and a test of CRLF output --- src/format_text.rs | 18 +++++++++ tests/specs/Comments/BlankLines.txt | 10 ++--- tests/specs/Comments/Comments_All.txt | 29 +++++++++++++-- tests/specs/Config/Casing_Preserve.txt | 3 +- tests/specs/Config/JoinsAsTopLevel_True.txt | 14 +++++++ tests/specs/Config/LinesBetweenQueries_2.txt | 4 +- tests/specs/Config/Uppercase_False.txt | 8 ++++ tests/specs/Config/Uppercase_WithCasing.txt | 8 ++++ tests/specs/Ddl/CreateTable.txt | 23 ++++-------- tests/specs/Ddl/Ddl_All.txt | 27 ++++++-------- tests/specs/Ddl/Transactions.txt | 20 ++++------ tests/specs/Dml/Delete.txt | 2 +- tests/specs/Dml/Insert.txt | 6 +-- tests/specs/Generic/Generic_All.txt | 10 ++--- tests/specs/Generic/Generic_Heuristics.txt | 27 +++++++++----- tests/specs/Generic/Templates.txt | 4 +- tests/specs/MySql/Procedures.txt | 34 +++++++++++++++-- tests/specs/MySql/Syntax.txt | 14 +++++-- tests/specs/Oracle/Syntax.txt | 7 ++++ tests/specs/PostgreSql/Functions.txt | 14 +++++++ tests/specs/PostgreSql/Procedures.txt | 39 +++++++++++++++++--- tests/specs/PostgreSql/Syntax.txt | 10 ++++- tests/specs/Range/Range_MySql.txt | 17 +++++++++ tests/specs/Range/Range_PostgreSql.txt | 21 +++++++++++ tests/specs/Range/Range_SqlServer.txt | 38 +++++++++++++++++++ tests/specs/Select/Case.txt | 13 +++++++ tests/specs/Select/Expressions.txt | 7 ++++ tests/specs/Select/Join.txt | 9 +++++ tests/specs/Select/Select_Basic.txt | 7 ++-- tests/specs/Select/Select_LineWidth.txt | 10 +++++ tests/specs/Select/Select_Subqueries.txt | 8 ++++ tests/specs/SqlServer/Batches.txt | 29 ++++++++++++--- tests/specs/SqlServer/ControlFlow.txt | 20 +++++++--- tests/specs/SqlServer/Procedures.txt | 14 +++---- tests/specs/SqlServer/Syntax.txt | 7 ++++ tests/specs/Sqlite/Sqlite_All.txt | 5 +-- tests/test.rs | 10 +++++ 37 files changed, 433 insertions(+), 113 deletions(-) create mode 100644 tests/specs/Config/Uppercase_False.txt create mode 100644 tests/specs/Config/Uppercase_WithCasing.txt create mode 100644 tests/specs/Range/Range_MySql.txt create mode 100644 tests/specs/Range/Range_PostgreSql.txt create mode 100644 tests/specs/Range/Range_SqlServer.txt diff --git a/src/format_text.rs b/src/format_text.rs index a8d9cde..ff761fa 100644 --- a/src/format_text.rs +++ b/src/format_text.rs @@ -294,6 +294,24 @@ mod test { assert!(verify(&tokens, "SELECT a\nFROM t", &config).is_err()); } + #[test] + fn verification_compares_line_breaks_between_strings() { + let config = ConfigurationBuilder::new().dialect(Dialect::PostgreSql).build(); + let tokens = lexer::lex("SELECT 'a'\n'b'", Dialect::PostgreSql).unwrap(); + assert!(verify(&tokens, "SELECT\n 'a'\n 'b'", &config).is_ok()); + assert!(verify(&tokens, "SELECT 'a' 'b'", &config).is_err()); + } + + #[test] + fn verification_compares_template_tags_in_place() { + let config = ConfigurationBuilder::new().build(); + let tokens = lexer::lex("SELECT a\n{% if x %}\n, b\n{% endif %}", Dialect::Generic).unwrap(); + // a tag may move onto or off a line of its own + assert!(verify(&tokens, "SELECT a {% if x %}, b\n{% endif %}", &config).is_ok()); + // but not across the code around it + assert!(verify(&tokens, "SELECT a,\n{% if x %}\nb\n{% endif %}", &config).is_err()); + } + #[test] fn verification_compares_routine_bodies_as_code() { let config = ConfigurationBuilder::new().dialect(Dialect::PostgreSql).build(); diff --git a/tests/specs/Comments/BlankLines.txt b/tests/specs/Comments/BlankLines.txt index 20374fc..5edec1d 100644 --- a/tests/specs/Comments/BlankLines.txt +++ b/tests/specs/Comments/BlankLines.txt @@ -1,8 +1,8 @@ == should keep a blank line between statements == -SELECT 1; +SELECT 1; -SELECT 2; -SELECT 3; +SELECT 2; +SELECT 3; [expect] SELECT 1; @@ -59,10 +59,10 @@ BEGIN END; == should keep a blank line before a comment == -SELECT 1; +SELECT 1; -- about the next one -SELECT 2; +SELECT 2; [expect] SELECT 1; diff --git a/tests/specs/Comments/Comments_All.txt b/tests/specs/Comments/Comments_All.txt index 3db1d9e..1030db5 100644 --- a/tests/specs/Comments/Comments_All.txt +++ b/tests/specs/Comments/Comments_All.txt @@ -1,8 +1,8 @@ == should keep comments between statements == -- first -SELECT 1; +SELECT 1; /* second */ -SELECT 2; +SELECT 2; [expect] -- first @@ -193,7 +193,7 @@ CREATE TABLE t ( ); == should keep a comment at the end of the file == -SELECT 1; +SELECT 1; -- the end [expect] @@ -231,3 +231,26 @@ group by a having -- why count(*) > 1; + +== should keep a comment after the last item of a list == +SELECT a, + b + -- after the last +FROM t; + +[expect] +SELECT a, b +-- after the last +FROM t; + +== should keep a comment after an opening bracket == +SELECT f( -- c + a, b) FROM t; + +[expect] +SELECT f( + -- c + a, + b +) +FROM t; diff --git a/tests/specs/Config/Casing_Preserve.txt b/tests/specs/Config/Casing_Preserve.txt index 0af22d7..7a2b000 100644 --- a/tests/specs/Config/Casing_Preserve.txt +++ b/tests/specs/Config/Casing_Preserve.txt @@ -1,4 +1,5 @@ -== should preserve casing by default == +~~ casing: preserve ~~ +== should preserve casing == SELECT a FROM t where b = 1 Order By a; [expect] diff --git a/tests/specs/Config/JoinsAsTopLevel_True.txt b/tests/specs/Config/JoinsAsTopLevel_True.txt index 545a520..c5e2fdc 100644 --- a/tests/specs/Config/JoinsAsTopLevel_True.txt +++ b/tests/specs/Config/JoinsAsTopLevel_True.txt @@ -8,3 +8,17 @@ FROM users u INNER JOIN orders o ON u.id = o.user_id LEFT JOIN payments p ON o.id = p.order_id WHERE o.total > 100; + +== should write joins in a subquery at the indentation of its from == +SELECT * FROM (SELECT a FROM t JOIN u ON t.id = u.id LEFT JOIN v ON v.id = u.id WHERE t.a > 100000) x JOIN w ON w.id = x.id; + +[expect] +SELECT * +FROM ( + SELECT a + FROM t + JOIN u ON t.id = u.id + LEFT JOIN v ON v.id = u.id + WHERE t.a > 100000 +) x +JOIN w ON w.id = x.id; diff --git a/tests/specs/Config/LinesBetweenQueries_2.txt b/tests/specs/Config/LinesBetweenQueries_2.txt index 1daf0bf..52f2cec 100644 --- a/tests/specs/Config/LinesBetweenQueries_2.txt +++ b/tests/specs/Config/LinesBetweenQueries_2.txt @@ -37,9 +37,9 @@ END; SELECT 3; == should not separate a statement from the batch separator after it == -SELECT 1 +SELECT 1 GO -SELECT 2 +SELECT 2 [expect] SELECT 1 diff --git a/tests/specs/Config/Uppercase_False.txt b/tests/specs/Config/Uppercase_False.txt new file mode 100644 index 0000000..eb609bc --- /dev/null +++ b/tests/specs/Config/Uppercase_False.txt @@ -0,0 +1,8 @@ +~~ uppercase: false ~~ +== should preserve casing when the legacy uppercase option is false == +select a FROM t Where b = 1; + +[expect] +select a +FROM t +Where b = 1; diff --git a/tests/specs/Config/Uppercase_WithCasing.txt b/tests/specs/Config/Uppercase_WithCasing.txt new file mode 100644 index 0000000..5592fe7 --- /dev/null +++ b/tests/specs/Config/Uppercase_WithCasing.txt @@ -0,0 +1,8 @@ +~~ uppercase: true, casing: lower ~~ +== should prefer casing over the legacy uppercase option == +select a FROM t Where b = 1; + +[expect] +select a +from t +where b = 1; diff --git a/tests/specs/Ddl/CreateTable.txt b/tests/specs/Ddl/CreateTable.txt index 06b4e7a..2d23852 100644 --- a/tests/specs/Ddl/CreateTable.txt +++ b/tests/specs/Ddl/CreateTable.txt @@ -1,5 +1,5 @@ == should format a create table == -CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255) NOT NULL, email VARCHAR(255) UNIQUE) +CREATE TABLE users (id INT PRIMARY KEY , name VARCHAR(255) NOT NULL , email VARCHAR(255) UNIQUE) [expect] CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255) NOT NULL, email VARCHAR(255) UNIQUE) @@ -28,14 +28,7 @@ CREATE TABLE orders ( ); == should format table constraints == -CREATE TABLE t ( - a INT, - b INT, - CONSTRAINT pk PRIMARY KEY (a, b), - CONSTRAINT fk FOREIGN KEY (b) REFERENCES u (id) ON DELETE CASCADE ON UPDATE NO ACTION, - CHECK (a > 0), - UNIQUE (b) -); +CREATE TABLE t ( a INT, b INT, CONSTRAINT pk PRIMARY KEY (a, b), CONSTRAINT fk FOREIGN KEY (b) REFERENCES u (id) ON DELETE CASCADE ON UPDATE NO ACTION, CHECK (a > 0), UNIQUE (b) ); [expect] CREATE TABLE t ( @@ -49,9 +42,9 @@ CREATE TABLE t ( == should keep blank lines between columns == CREATE TABLE t ( - a INT, +a INT, - b INT +b INT ); [expect] @@ -62,7 +55,7 @@ CREATE TABLE t ( ); == should format if not exists and temporary == -CREATE TEMPORARY TABLE IF NOT EXISTS t (a INT); +CREATE TEMPORARY TABLE IF NOT EXISTS t (a INT); [expect] CREATE TEMPORARY TABLE IF NOT EXISTS t (a INT); @@ -83,19 +76,19 @@ CREATE TABLE t (id INT) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE t (id INT) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; == should format partitioning == -CREATE TABLE t (id INT, d DATE) PARTITION BY RANGE (d); +CREATE TABLE t (id INT , d DATE) PARTITION BY RANGE (d); [expect] CREATE TABLE t (id INT, d DATE) PARTITION BY RANGE (d); == should format generated columns == -CREATE TABLE t (a INT, b INT GENERATED ALWAYS AS (a * 2) STORED, c INT GENERATED BY DEFAULT AS IDENTITY); +CREATE TABLE t (a INT , b INT GENERATED ALWAYS AS (a * 2) STORED , c INT GENERATED BY DEFAULT AS IDENTITY); [expect] CREATE TABLE t (a INT, b INT GENERATED ALWAYS AS (a * 2) STORED, c INT GENERATED BY DEFAULT AS IDENTITY); == should format like == -CREATE TABLE t (LIKE u INCLUDING ALL); +CREATE TABLE t (LIKE u INCLUDING ALL); [expect] CREATE TABLE t (LIKE u INCLUDING ALL); diff --git a/tests/specs/Ddl/Ddl_All.txt b/tests/specs/Ddl/Ddl_All.txt index 50636db..33349ee 100644 --- a/tests/specs/Ddl/Ddl_All.txt +++ b/tests/specs/Ddl/Ddl_All.txt @@ -34,19 +34,19 @@ SELECT x FROM a; == should format a create index == -CREATE UNIQUE INDEX IF NOT EXISTS idx ON t (a, b DESC) WHERE c IS NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx ON t (a , b DESC) WHERE c IS NULL; [expect] CREATE UNIQUE INDEX IF NOT EXISTS idx ON t (a, b DESC) WHERE c IS NULL; == should format an alter table with one action == -ALTER TABLE t ADD COLUMN c INT NOT NULL DEFAULT 0; +ALTER TABLE t ADD COLUMN c INT NOT NULL DEFAULT 0; [expect] ALTER TABLE t ADD COLUMN c INT NOT NULL DEFAULT 0; == should format an alter table with several actions == -ALTER TABLE t ADD COLUMN c INT, DROP COLUMN d; +ALTER TABLE t ADD COLUMN c INT , DROP COLUMN d; [expect] ALTER TABLE t ADD COLUMN c INT, DROP COLUMN d; @@ -69,21 +69,19 @@ ADD CONSTRAINT fk FOREIGN KEY (a) REFERENCES u (id) ALTER TABLE t ADD CONSTRAINT fk FOREIGN KEY (a) REFERENCES u (id) == should format alter column == -ALTER TABLE t ALTER COLUMN a SET DEFAULT 1, ALTER COLUMN b TYPE TEXT; +ALTER TABLE t ALTER COLUMN a SET DEFAULT 1 , ALTER COLUMN b TYPE TEXT; [expect] ALTER TABLE t ALTER COLUMN a SET DEFAULT 1, ALTER COLUMN b TYPE TEXT; == should format rename == -ALTER TABLE t RENAME COLUMN a TO b; +ALTER TABLE t RENAME COLUMN a TO b; [expect] ALTER TABLE t RENAME COLUMN a TO b; == should format drops == -DROP TABLE IF EXISTS a, b CASCADE; -DROP VIEW v; -DROP INDEX idx; +DROP TABLE IF EXISTS a, b CASCADE; DROP VIEW v; DROP INDEX idx; [expect] DROP TABLE IF EXISTS a, b CASCADE; @@ -91,27 +89,26 @@ DROP VIEW v; DROP INDEX idx; == should format truncate == -TRUNCATE TABLE t RESTART IDENTITY; +TRUNCATE TABLE t RESTART IDENTITY; [expect] TRUNCATE TABLE t RESTART IDENTITY; == should format grants == -GRANT SELECT, INSERT ON t TO u WITH GRANT OPTION; -REVOKE ALL ON t FROM u; +GRANT SELECT, INSERT ON t TO u WITH GRANT OPTION; REVOKE ALL ON t FROM u; [expect] GRANT SELECT, INSERT ON t TO u WITH GRANT OPTION; REVOKE ALL ON t FROM u; == should format a create schema == -CREATE SCHEMA IF NOT EXISTS app AUTHORIZATION admin; +CREATE SCHEMA IF NOT EXISTS app AUTHORIZATION admin; [expect] CREATE SCHEMA IF NOT EXISTS app AUTHORIZATION admin; == should format a create sequence == -CREATE SEQUENCE s START WITH 1 INCREMENT BY 1; +CREATE SEQUENCE s START WITH 1 INCREMENT BY 1; [expect] CREATE SEQUENCE s START WITH 1 INCREMENT BY 1; @@ -129,13 +126,13 @@ CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); == should format a create domain == -CREATE DOMAIN positive AS INT CHECK (VALUE > 0); +CREATE DOMAIN positive AS INT CHECK (VALUE > 0); [expect] CREATE DOMAIN positive AS INT CHECK (VALUE > 0); == should format a create extension == -CREATE EXTENSION IF NOT EXISTS pgcrypto; +CREATE EXTENSION IF NOT EXISTS pgcrypto; [expect] CREATE EXTENSION IF NOT EXISTS pgcrypto; diff --git a/tests/specs/Ddl/Transactions.txt b/tests/specs/Ddl/Transactions.txt index 2e5ef1a..b2cfdf7 100644 --- a/tests/specs/Ddl/Transactions.txt +++ b/tests/specs/Ddl/Transactions.txt @@ -15,25 +15,21 @@ WHERE id = 2; COMMIT; == should format begin transaction == -BEGIN TRANSACTION; -ROLLBACK TRANSACTION; +BEGIN TRANSACTION; ROLLBACK TRANSACTION; [expect] BEGIN TRANSACTION; ROLLBACK TRANSACTION; == should format start transaction == -START TRANSACTION ISOLATION LEVEL SERIALIZABLE; -COMMIT; +START TRANSACTION ISOLATION LEVEL SERIALIZABLE; COMMIT; [expect] START TRANSACTION ISOLATION LEVEL SERIALIZABLE; COMMIT; == should format savepoints == -SAVEPOINT a; -ROLLBACK TO SAVEPOINT a; -RELEASE SAVEPOINT a; +SAVEPOINT a; ROLLBACK TO SAVEPOINT a; RELEASE SAVEPOINT a; [expect] SAVEPOINT a; @@ -41,23 +37,21 @@ ROLLBACK TO SAVEPOINT a; RELEASE SAVEPOINT a; == should format begin work == -BEGIN WORK; -COMMIT WORK; +BEGIN WORK; COMMIT WORK; [expect] BEGIN WORK; COMMIT WORK; == should format sqlite's begin == -BEGIN IMMEDIATE TRANSACTION; -END TRANSACTION; +BEGIN IMMEDIATE TRANSACTION; END TRANSACTION; [expect] BEGIN IMMEDIATE TRANSACTION; END TRANSACTION; == should format set statements == -SET search_path TO app, public; +SET search_path TO app , public; SET TIME ZONE 'UTC'; [expect] @@ -65,7 +59,7 @@ SET search_path TO app, public; SET TIME ZONE 'UTC'; == should format use == -USE my_database; +USE my_database; [expect] USE my_database; diff --git a/tests/specs/Dml/Delete.txt b/tests/specs/Dml/Delete.txt index edf4b82..8ffa3c5 100644 --- a/tests/specs/Dml/Delete.txt +++ b/tests/specs/Dml/Delete.txt @@ -6,7 +6,7 @@ DELETE FROM t WHERE id = 1; == should format a delete without a where == -DELETE FROM t; +DELETE FROM t; [expect] DELETE FROM t; diff --git a/tests/specs/Dml/Insert.txt b/tests/specs/Dml/Insert.txt index 577dde1..1604d94 100644 --- a/tests/specs/Dml/Insert.txt +++ b/tests/specs/Dml/Insert.txt @@ -29,7 +29,7 @@ FROM u WHERE c = 1; == should format default values == -INSERT INTO t DEFAULT VALUES; +INSERT INTO t DEFAULT VALUES; [expect] INSERT INTO t DEFAULT VALUES; @@ -70,8 +70,8 @@ RETURNING id, a; == should format an insert written over several lines == INSERT INTO t ( - a, - b +a, +b ) VALUES (1, 2); diff --git a/tests/specs/Generic/Generic_All.txt b/tests/specs/Generic/Generic_All.txt index 75db763..05d3106 100644 --- a/tests/specs/Generic/Generic_All.txt +++ b/tests/specs/Generic/Generic_All.txt @@ -20,9 +20,9 @@ SELECT * FROM #temp; == should read go as a batch separator == -SELECT 1 +SELECT 1 GO -SELECT 2 +SELECT 2 [expect] SELECT 1 @@ -59,9 +59,9 @@ SELECT $$text$$; SELECT $$text$$; == should format t-sql without semicolons == -DECLARE @a INT -SET @a = 1 -SELECT @a +DECLARE @a INT +SET @a = 1 +SELECT @a [expect] DECLARE @a INT diff --git a/tests/specs/Generic/Generic_Heuristics.txt b/tests/specs/Generic/Generic_Heuristics.txt index df106c0..81bd7eb 100644 --- a/tests/specs/Generic/Generic_Heuristics.txt +++ b/tests/specs/Generic/Generic_Heuristics.txt @@ -6,9 +6,7 @@ select percentile_disc(array[[null, 1, 0.5], [0.75, 0.25, null]]) within group ( from t; == should not read an alias in a trigger as declarations == -CREATE TRIGGER t AFTER UPDATE ON p REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE PROCEDURE f(); -CREATE TRIGGER u AFTER UPDATE ON p REFERENCING NEW TABLE AS new_table FOR EACH STATEMENT EXECUTE PROCEDURE f(); -SELECT 1; +CREATE TRIGGER t AFTER UPDATE ON p REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE PROCEDURE f(); CREATE TRIGGER u AFTER UPDATE ON p REFERENCING NEW TABLE AS new_table FOR EACH STATEMENT EXECUTE PROCEDURE f(); SELECT 1; [expect] CREATE TRIGGER t AFTER UPDATE ON p REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE PROCEDURE f(); @@ -39,7 +37,7 @@ CREATE OPERATOR #%# (leftarg = int8, procedure = f); == should read a hash at the start of a line as a comment == #=== a section === -SELECT 1; +SELECT 1; [expect] #=== a section === @@ -47,15 +45,15 @@ SELECT 1; == should read a hash after a statement as a comment == USE db; #inline -SELECT 1; +SELECT 1; [expect] USE db; #inline SELECT 1; == should not split a grant at its privileges == -GRANT SELECT, UPDATE ON t TO u -ALTER TABLE t DROP COLUMN c +GRANT SELECT , UPDATE ON t TO u +ALTER TABLE t DROP COLUMN c [expect] GRANT SELECT, UPDATE ON t TO u @@ -96,8 +94,8 @@ INSERT INTO t (a) VALUES (2) SELECT 3 == should split a second query that starts a line == -SELECT 1 -SELECT 2 +SELECT 1 +SELECT 2 [expect] SELECT 1 @@ -140,3 +138,14 @@ create trigger tr after insert on t for each row insert into log values (1); create view v as select event, name from t; + +== should not split at words that continue a statement on the next line == +CREATE TEMPORARY TABLE t (a int) +ON COMMIT PRESERVE ROWS +SELECT * FROM t +USE INDEX (i) + +[expect] +CREATE TEMPORARY TABLE t (a int) ON COMMIT PRESERVE ROWS +SELECT * +FROM t USE INDEX (i) diff --git a/tests/specs/Generic/Templates.txt b/tests/specs/Generic/Templates.txt index 2328dcf..2ca23b3 100644 --- a/tests/specs/Generic/Templates.txt +++ b/tests/specs/Generic/Templates.txt @@ -30,7 +30,7 @@ where a = 1 {% if x %} and b = 2 {% endif %}; == should keep a config block before a query == {{ config(materialized='table') }} -select 1 as id +select 1 as id [expect] {{ config(materialized='table') }} @@ -38,7 +38,7 @@ select 1 as id == should keep template comments == {# a comment #} -select 1; +select 1; [expect] {# a comment #} diff --git a/tests/specs/MySql/Procedures.txt b/tests/specs/MySql/Procedures.txt index 2783c5d..d584dff 100644 --- a/tests/specs/MySql/Procedures.txt +++ b/tests/specs/MySql/Procedures.txt @@ -146,7 +146,7 @@ END // DELIMITER ; == should format a procedure with a single statement body == -CREATE PROCEDURE p() SELECT 1; +CREATE PROCEDURE p() SELECT 1; [expect] CREATE PROCEDURE p() SELECT 1; @@ -181,7 +181,7 @@ END // DELIMITER ; == should format a trigger with a single statement == -CREATE TRIGGER t_ai AFTER INSERT ON t FOR EACH ROW INSERT INTO log (id) VALUES (NEW.id); +CREATE TRIGGER t_ai AFTER INSERT ON t FOR EACH ROW INSERT INTO log (id) VALUES (NEW.id); [expect] CREATE TRIGGER t_ai AFTER INSERT ON t FOR EACH ROW INSERT INTO log (id) VALUES (NEW.id); @@ -215,9 +215,35 @@ END // DELIMITER ; == should format call == -CALL get_user(1, @name); -SELECT @name; +CALL get_user(1, @name); SELECT @name; [expect] CALL get_user(1, @name); SELECT @name; + +== should leave an if without an end as statements == +CREATE PROCEDURE p() BEGIN IF a THEN SET b = 1; END; + +[expect] +CREATE PROCEDURE p() +BEGIN + IF a THEN SET b = 1; +END; + +== should leave a repeat without an until as statements == +CREATE PROCEDURE p() BEGIN REPEAT SET a = a + 1; END; + +[expect] +CREATE PROCEDURE p() +BEGIN + REPEAT SET a = a + 1; +END; + +== should leave a while without an end as statements == +CREATE PROCEDURE p() BEGIN WHILE a < 1 DO SET a = a + 1; END; + +[expect] +CREATE PROCEDURE p() +BEGIN + WHILE a < 1 DO SET a = a + 1; +END; diff --git a/tests/specs/MySql/Syntax.txt b/tests/specs/MySql/Syntax.txt index 7a302fa..0485133 100644 --- a/tests/specs/MySql/Syntax.txt +++ b/tests/specs/MySql/Syntax.txt @@ -40,8 +40,7 @@ SELECT "text" AS a; SELECT "text" AS a; == should format user variables == -SET @a := 1, @b = 2; -SELECT @a + @b, @@session.sql_mode, @@global.max_connections; +SET @a := 1, @b = 2; SELECT @a + @b, @@session.sql_mode, @@global.max_connections; [expect] SET @a := 1, @b = 2; @@ -102,7 +101,7 @@ CREATE TABLE `t` ( ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; == should format alter table == -ALTER TABLE t ADD COLUMN c INT AFTER b, DROP INDEX idx, MODIFY COLUMN d TEXT; +ALTER TABLE t ADD COLUMN c INT AFTER b , DROP INDEX idx , MODIFY COLUMN d TEXT; [expect] ALTER TABLE t ADD COLUMN c INT AFTER b, DROP INDEX idx, MODIFY COLUMN d TEXT; @@ -145,3 +144,12 @@ SELECT * FROM t WHERE x = @v # note AND y = 1; + +== should format a lock in share mode == +SELECT * FROM t WHERE a = 1 LOCK IN SHARE MODE; + +[expect] +SELECT * +FROM t +WHERE a = 1 +LOCK IN SHARE MODE; diff --git a/tests/specs/Oracle/Syntax.txt b/tests/specs/Oracle/Syntax.txt index 2db3ef1..f93fe2e 100644 --- a/tests/specs/Oracle/Syntax.txt +++ b/tests/specs/Oracle/Syntax.txt @@ -147,3 +147,10 @@ EXECUTE dbms_stats.gather_schema_stats( - cascade => TRUE); SELECT 1 FROM dual; + +== should read quoted strings with bracket delimiters == +SELECT q'b>', q'{x}y}', q'(z)' FROM dual; + +[expect] +SELECT q'b>', q'{x}y}', q'(z)' +FROM dual; diff --git a/tests/specs/PostgreSql/Functions.txt b/tests/specs/PostgreSql/Functions.txt index 1f5c87e..224afc1 100644 --- a/tests/specs/PostgreSql/Functions.txt +++ b/tests/specs/PostgreSql/Functions.txt @@ -379,3 +379,17 @@ BEGIN END; $$ LANGUAGE plpgsql; + +== should keep a body that's empty or can't be formatted as it is == +CREATE FUNCTION f() RETURNS int AS $$ $$ LANGUAGE sql; +CREATE FUNCTION g() RETURNS int AS $$ SELECT ( $$ LANGUAGE sql; + +[expect] +CREATE FUNCTION f() +RETURNS int +AS $$ $$ +LANGUAGE sql; +CREATE FUNCTION g() +RETURNS int +AS $$ SELECT ( $$ +LANGUAGE sql; diff --git a/tests/specs/PostgreSql/Procedures.txt b/tests/specs/PostgreSql/Procedures.txt index 03edead..e0ce8c9 100644 --- a/tests/specs/PostgreSql/Procedures.txt +++ b/tests/specs/PostgreSql/Procedures.txt @@ -24,7 +24,7 @@ END; $$; == should format call == -CALL transfer(1, 2, 100.00); +CALL transfer(1 , 2 , 100.00); [expect] CALL transfer(1, 2, 100.00); @@ -62,23 +62,50 @@ print( 1 ) $$; == should format a trigger == -CREATE TRIGGER set_updated_at BEFORE UPDATE ON users FOR EACH ROW EXECUTE FUNCTION set_updated_at(); +CREATE TRIGGER set_updated_at BEFORE UPDATE ON users FOR EACH ROW EXECUTE FUNCTION set_updated_at(); [expect] CREATE TRIGGER set_updated_at BEFORE UPDATE ON users FOR EACH ROW EXECUTE FUNCTION set_updated_at(); == should format a rule == -CREATE RULE r AS ON INSERT TO t DO INSTEAD (INSERT INTO u VALUES (NEW.a); INSERT INTO v VALUES (NEW.a)); +CREATE RULE r AS ON INSERT TO t DO INSTEAD (INSERT INTO u VALUES (NEW.a); INSERT INTO v VALUES (NEW.a)); [expect] CREATE RULE r AS ON INSERT TO t DO INSTEAD (INSERT INTO u VALUES (NEW.a); INSERT INTO v VALUES (NEW.a)); == should format a cursor == -DECLARE c CURSOR FOR SELECT * FROM t WHERE a = 1; -FETCH NEXT FROM c; -CLOSE c; +DECLARE c CURSOR FOR SELECT * FROM t WHERE a = 1; FETCH NEXT FROM c; CLOSE c; [expect] DECLARE c CURSOR FOR SELECT * FROM t WHERE a = 1; FETCH NEXT FROM c; CLOSE c; + +== should keep a comment at the start of an exception section == +DO $$ BEGIN + NULL; +EXCEPTION + -- handle it + WHEN others THEN NULL; +END $$; + +[expect] +DO $$ +BEGIN + NULL; +EXCEPTION + -- handle it + WHEN others THEN + NULL; +END +$$; + +== should leave a case statement without an end as statements == +DO $$ BEGIN CASE x WHEN 1 THEN NULL; END; $$; + +[expect] +DO $$ +BEGIN + CASE x WHEN 1 THEN NULL; +END; +$$; diff --git a/tests/specs/PostgreSql/Syntax.txt b/tests/specs/PostgreSql/Syntax.txt index 89d55d3..7ae5c1d 100644 --- a/tests/specs/PostgreSql/Syntax.txt +++ b/tests/specs/PostgreSql/Syntax.txt @@ -68,11 +68,11 @@ FROM t WHERE id = :id AND name = :'name' AND col = :"column"; == should format copy data == -COPY t (a, b) FROM stdin; +COPY t (a , b) FROM stdin; 1 foo 2 bar \. -SELECT 1; +SELECT 1; [expect] COPY t (a, b) FROM stdin; @@ -148,3 +148,9 @@ select 'bar' as x, 'a' 'b' as y from t; + +== should format an alter table if exists only == +ALTER TABLE IF EXISTS ONLY t ADD COLUMN a int , ADD COLUMN b int; + +[expect] +ALTER TABLE IF EXISTS ONLY t ADD COLUMN a int, ADD COLUMN b int; diff --git a/tests/specs/Range/Range_MySql.txt b/tests/specs/Range/Range_MySql.txt new file mode 100644 index 0000000..cf8da72 --- /dev/null +++ b/tests/specs/Range/Range_MySql.txt @@ -0,0 +1,17 @@ +~~ dialect: mysql ~~ +== should format within a delimiter block == +select 1; +DELIMITER // +CREATE PROCEDURE p() BEGIN select [|2|]; END// +DELIMITER ; +select 3; + +[expect] +select 1; +DELIMITER // +CREATE PROCEDURE p() +BEGIN + select 2; +END// +DELIMITER ; +select 3; diff --git a/tests/specs/Range/Range_PostgreSql.txt b/tests/specs/Range/Range_PostgreSql.txt new file mode 100644 index 0000000..fce2ff6 --- /dev/null +++ b/tests/specs/Range/Range_PostgreSql.txt @@ -0,0 +1,21 @@ +~~ dialect: postgresql ~~ +== should format the whole function the range is in == +select 1; +CREATE FUNCTION f() RETURNS int AS $$ +BEGIN +RETURN [|2|]; +END +$$ LANGUAGE plpgsql; +select 3; + +[expect] +select 1; +CREATE FUNCTION f() +RETURNS int +AS $$ +BEGIN + RETURN 2; +END +$$ +LANGUAGE plpgsql; +select 3; diff --git a/tests/specs/Range/Range_SqlServer.txt b/tests/specs/Range/Range_SqlServer.txt new file mode 100644 index 0000000..c7b17b9 --- /dev/null +++ b/tests/specs/Range/Range_SqlServer.txt @@ -0,0 +1,38 @@ +~~ dialect: sqlserver ~~ +== should format only the batch statement the range is in == +select 1 +GO +select [|2|] +GO +select 3 + +[expect] +select 1 +GO +select 2 +GO +select 3 + +== should format the whole procedure the range is in == +select 1 +GO +CREATE PROCEDURE p AS +BEGIN +select [|2|] +IF @a = 1 PRINT 3 +END +GO +select 4 + +[expect] +select 1 +GO +CREATE PROCEDURE p +AS +BEGIN + select 2 + IF @a = 1 + PRINT 3 +END +GO +select 4 diff --git a/tests/specs/Select/Case.txt b/tests/specs/Select/Case.txt index 8237f13..b5a09b8 100644 --- a/tests/specs/Select/Case.txt +++ b/tests/specs/Select/Case.txt @@ -85,3 +85,16 @@ SELECT CASE WHEN a THEN 1 END FROM t; [expect] SELECT CASE WHEN a THEN 1 END FROM t; + +== should keep a comment before the end of a case == +SELECT CASE WHEN a THEN 1 + -- before end +END FROM t; + +[expect] +SELECT + CASE + WHEN a THEN 1 + -- before end + END +FROM t; diff --git a/tests/specs/Select/Expressions.txt b/tests/specs/Select/Expressions.txt index f37bfe5..03b0c00 100644 --- a/tests/specs/Select/Expressions.txt +++ b/tests/specs/Select/Expressions.txt @@ -193,3 +193,10 @@ SELECT "Column Name", `other`, [bracketed] FROM "My Table"; [expect] SELECT "Column Name", `other`, [bracketed] FROM "My Table"; + +== should keep number literals as they are == +SELECT 0b101, 0o17, 0x1F, 1e5, 1.5e-3, 1_000 FROM t; + +[expect] +SELECT 0b101, 0o17, 0x1F, 1e5, 1.5e-3, 1_000 +FROM t; diff --git a/tests/specs/Select/Join.txt b/tests/specs/Select/Join.txt index 2dc033e..afc3ebe 100644 --- a/tests/specs/Select/Join.txt +++ b/tests/specs/Select/Join.txt @@ -107,3 +107,12 @@ SELECT LEFT(name, 3) FROM a LEFT JOIN b ON a.id = b.id; SELECT LEFT(name, 3) FROM a LEFT JOIN b ON a.id = b.id; + +== should format lateral views == +SELECT * FROM t LATERAL VIEW OUTER explode(a) x AS y LATERAL VIEW explode(b) z; + +[expect] +SELECT * +FROM t + LATERAL VIEW OUTER explode(a) x AS y + LATERAL VIEW explode(b) z; diff --git a/tests/specs/Select/Select_Basic.txt b/tests/specs/Select/Select_Basic.txt index b6b0ef1..994a069 100644 --- a/tests/specs/Select/Select_Basic.txt +++ b/tests/specs/Select/Select_Basic.txt @@ -20,7 +20,7 @@ limit 10 offset 5; == should keep a select without a from on one line == -SELECT 1; +SELECT 1; [expect] SELECT 1; @@ -85,8 +85,7 @@ SELECT 2; SELECT 3 == should keep statements without semicolons apart == -SELECT 1; -SELECT 2 +SELECT 1; SELECT 2 [expect] SELECT 1; @@ -194,7 +193,7 @@ FROM t QUALIFY rn = 1; == should format a table statement == -TABLE users; +TABLE users; [expect] TABLE users; diff --git a/tests/specs/Select/Select_LineWidth.txt b/tests/specs/Select/Select_LineWidth.txt index d05ba02..380b938 100644 --- a/tests/specs/Select/Select_LineWidth.txt +++ b/tests/specs/Select/Select_LineWidth.txt @@ -173,3 +173,13 @@ WITH active_users AS ( ) SELECT * FROM active_users; + +== should break conditions around a between == +SELECT * FROM t WHERE a BETWEEN 1 AND 2 AND b = 1 OR c NOT BETWEEN x AND y; + +[expect] +SELECT * +FROM t +WHERE a BETWEEN 1 AND 2 + AND b = 1 + OR c NOT BETWEEN x AND y; diff --git a/tests/specs/Select/Select_Subqueries.txt b/tests/specs/Select/Select_Subqueries.txt index c686ab0..2f41640 100644 --- a/tests/specs/Select/Select_Subqueries.txt +++ b/tests/specs/Select/Select_Subqueries.txt @@ -102,3 +102,11 @@ SELECT * FROM t WHERE a = ANY(SELECT a FROM u) OR b > ALL (SELECT b FROM u); SELECT * FROM t WHERE a = ANY(SELECT a FROM u) OR b > ALL (SELECT b FROM u); + +== should keep the alias of a subquery after its brackets == +select * from (select a from t) as x join (select b from u) y on x.a = y.b; + +[expect] +select * +from (select a from t) as x + join (select b from u) y on x.a = y.b; diff --git a/tests/specs/SqlServer/Batches.txt b/tests/specs/SqlServer/Batches.txt index 31e0dff..d300620 100644 --- a/tests/specs/SqlServer/Batches.txt +++ b/tests/specs/SqlServer/Batches.txt @@ -1,8 +1,8 @@ ~~ dialect: sqlserver ~~ == should keep go on its own line == -SELECT 1 +SELECT 1 GO -SELECT 2 +SELECT 2 GO [expect] @@ -20,7 +20,7 @@ INSERT INTO t DEFAULT VALUES GO 10 == should keep a comment after go == -SELECT 1 +SELECT 1 GO -- end of batch [expect] @@ -28,11 +28,11 @@ SELECT 1 GO -- end of batch == should keep blank lines around go == -SELECT 1 +SELECT 1 GO -SELECT 2 +SELECT 2 [expect] SELECT 1 @@ -141,3 +141,22 @@ INSERT INTO #t VALUES (1) UPDATE #t SET a = 2 + +== should start a statement at a common table expression with columns == +SELECT 1 +WITH c (a) AS (SELECT 1) SELECT a FROM c + +[expect] +SELECT 1 +WITH c (a) AS (SELECT 1) +SELECT a +FROM c + +== should keep the set of an alter in it == +ALTER DATABASE d +SET RECOVERY SIMPLE +SELECT 1 + +[expect] +ALTER DATABASE d SET RECOVERY SIMPLE +SELECT 1 diff --git a/tests/specs/SqlServer/ControlFlow.txt b/tests/specs/SqlServer/ControlFlow.txt index ac2081e..5f3224b 100644 --- a/tests/specs/SqlServer/ControlFlow.txt +++ b/tests/specs/SqlServer/ControlFlow.txt @@ -134,11 +134,7 @@ WAITFOR DELAY '00:00:05'; RAISERROR('failed: %s', 16, 1, @msg) WITH NOWAIT; == should format transactions == -BEGIN TRAN; -BEGIN TRANSACTION t1; -COMMIT TRAN; -ROLLBACK TRANSACTION t1; -SAVE TRANSACTION s; +BEGIN TRAN; BEGIN TRANSACTION t1; COMMIT TRAN; ROLLBACK TRANSACTION t1; SAVE TRANSACTION s; [expect] BEGIN TRAN; @@ -204,3 +200,17 @@ IF @x IS NULL RETURN SELECT * FROM t + +== should keep a comment before an else == +IF @a = 1 +PRINT 1 +-- otherwise +ELSE +PRINT 2 + +[expect] +IF @a = 1 + PRINT 1 +-- otherwise +ELSE + PRINT 2 diff --git a/tests/specs/SqlServer/Procedures.txt b/tests/specs/SqlServer/Procedures.txt index 6c3f61f..a03ca5a 100644 --- a/tests/specs/SqlServer/Procedures.txt +++ b/tests/specs/SqlServer/Procedures.txt @@ -46,11 +46,11 @@ SELECT @a + @b; GO == should format a procedure without a block == -CREATE PROCEDURE dbo.p +CREATE PROCEDURE dbo.p AS -SET NOCOUNT ON -SELECT 1 -SELECT 2 +SET NOCOUNT ON +SELECT 1 +SELECT 2 GO [expect] @@ -144,10 +144,10 @@ END GO == should format exec == -EXEC dbo.GetUser @Id = 1; +EXEC dbo.GetUser @Id = 1; EXEC dbo.GetOrders 1, '2020-01-01'; -EXECUTE @result = dbo.p @a = 1, @b = @x OUTPUT; -EXEC (@sql); +EXECUTE @result = dbo.p @a = 1 , @b = @x OUTPUT; +EXEC (@sql); EXEC sp_executesql @sql, N'@id INT', @id = 1; [expect] diff --git a/tests/specs/SqlServer/Syntax.txt b/tests/specs/SqlServer/Syntax.txt index 111b36b..e4c753a 100644 --- a/tests/specs/SqlServer/Syntax.txt +++ b/tests/specs/SqlServer/Syntax.txt @@ -128,3 +128,10 @@ SELECT id, offset, filter, identity FROM t; [expect] SELECT id, offset, filter, identity FROM t; + +== should format a top percent == +SELECT TOP (10) PERCENT a FROM t; + +[expect] +SELECT TOP (10) PERCENT a +FROM t; diff --git a/tests/specs/Sqlite/Sqlite_All.txt b/tests/specs/Sqlite/Sqlite_All.txt index c1ee9d9..fd032da 100644 --- a/tests/specs/Sqlite/Sqlite_All.txt +++ b/tests/specs/Sqlite/Sqlite_All.txt @@ -58,14 +58,13 @@ INSERT OR IGNORE INTO t (a) VALUES (1); == should format a create table == -CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL COLLATE NOCASE, data BLOB) WITHOUT ROWID; +CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT , name TEXT NOT NULL COLLATE NOCASE , data BLOB) WITHOUT ROWID; [expect] CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL COLLATE NOCASE, data BLOB) WITHOUT ROWID; == should format pragmas == -PRAGMA foreign_keys = ON; -PRAGMA table_info(t); +PRAGMA foreign_keys = ON; PRAGMA table_info(t); [expect] PRAGMA foreign_keys = ON; diff --git a/tests/test.rs b/tests/test.rs index 3d97e44..1a50ef4 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -24,6 +24,16 @@ fn should_keep_windows_newlines_in_auto_mode() { assert_eq!(file_text.unwrap(), "SELECT *\r\nFROM dbo.Test\r\n"); } +#[test] +fn should_write_crlf_newlines() { + let config = ConfigurationBuilder::new() + .new_line_kind(dprint_core::configuration::NewLineKind::CarriageReturnLineFeed) + .build(); + let file_text = format_text(&PathBuf::from("file.sql"), "SELECT * FROM dbo.Test\n", &config).unwrap(); + + assert_eq!(file_text.unwrap(), "SELECT *\r\nFROM dbo.Test\r\n"); +} + #[test] fn should_format_empty_file() { let config = ConfigurationBuilder::new().build(); From fe33661c49ba1582d3d5f050cb4a102ef87cdbbb Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 17 Oct 2026 23:33:16 -0400 Subject: [PATCH 3/4] fix: keep output stable for comments against values, percents and CASE operands Found by formatting the corpus of real projects: - keep a block comment against the value after it (ex. MySQL's `/*!50705 x,*/'a'`), which once gained a space the second time - keep a `%` against the word after it (ex. `SELECT %x`, a Databricks `%python` magic), which once gained a space the second time - don't read a SET statement as a SQL*Plus command once its `=` is spaced (ex. `SET AUTOCOMMIT=0`), which refused to format it - don't break the branches of a CASE onto lines because its operand broke --- src/generation/generate.rs | 12 +++++++++--- src/lexer.rs | 4 +++- tests/specs/MySql/Syntax.txt | 12 ++++++++++++ tests/specs/Oracle/Syntax.txt | 11 +++++++++++ tests/specs/Select/Expressions.txt | 7 +++++++ tests/specs/Select/Select_LineWidth.txt | 11 +++++++++++ 6 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/generation/generate.rs b/src/generation/generate.rs index 9ea3cf1..2aafd1a 100644 --- a/src/generation/generate.rs +++ b/src/generation/generate.rs @@ -2023,9 +2023,10 @@ fn case_forces_multi_line(case: &Case) -> bool { .branches .first() .map(|b| b.leading_comments.first().copied().unwrap_or(b.when)); + // not the operand, which comes before the branches whatever they do, and which the line width + // may have broken (ex. `CASE f(` then `a` then `) WHEN 1 ..`) first_branch_token.is_some_and(|t| t.newlines_before > 0) || !case.end_comments.is_empty() - || is_known_multi_line(&case.operand) || case .branches .iter() @@ -2317,7 +2318,9 @@ fn needs_space(elements: &[Element], i: usize) -> bool { if let Some(operator) = previous.as_token().filter(|t| t.kind == TokenKind::Operator) { return match operator.text { "::" => false, - "%" if is_attribute_operator(elements, i - 1) => false, + // a `%` against the word after it stays against it, wherever it is, since the space before + // it may be one the formatter wrote (ex. after `SELECT`) + "%" if next.is_kind(TokenKind::Word) && !next_first.space_before => false, ":" | ".." => next_first.space_before, // a `*` where a unary operator would be is a wildcard (ex. `SELECT *`) "*" => true, @@ -2448,7 +2451,10 @@ fn gen_leading_comments(comments: &[Tok], next: Option) -> PrintItems { items.extend(gen_comment(comment)); let following = comments.get(i + 1).copied().or(next); if !comment.is_line_comment() && following.is_some_and(|t| t.newlines_before == 0) { - items.push_space(); + // a block comment against what follows stays against it (ex. MySQL's `/*!50705 x,*/'a'`) + if following.is_some_and(|t| t.space_before) { + items.push_space(); + } } else { items.push_signal(Signal::NewLine); } diff --git a/src/lexer.rs b/src/lexer.rs index 6de94ce..03848ca 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1308,7 +1308,9 @@ fn is_sql_plus_command(first_word: &str, rest: &str) -> bool { return true; } if first_word.eq_ignore_ascii_case("set") { - let option = rest.split_whitespace().next().unwrap_or(""); + // the option's name ends wherever a name can't continue, since the formatter may put a space + // there (ex. `SET AUTOCOMMIT=0` becoming `SET AUTOCOMMIT = 0`) + let option = &rest[..rest.find(|c: char| !is_ident_char(c)).unwrap_or(rest.len())]; return SET_OPTIONS.iter().any(|o| o.eq_ignore_ascii_case(option)); } // `EXEC p` runs a procedure, while `EXECUTE IMMEDIATE` is only in PL/SQL diff --git a/tests/specs/MySql/Syntax.txt b/tests/specs/MySql/Syntax.txt index 0485133..aa0d0cc 100644 --- a/tests/specs/MySql/Syntax.txt +++ b/tests/specs/MySql/Syntax.txt @@ -153,3 +153,15 @@ SELECT * FROM t WHERE a = 1 LOCK IN SHARE MODE; + +== should keep an executable comment against the value after it == +INSERT INTO t VALUES (1,'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',/*!50705 0x0000000001010000003E0A325D63345CC0761FDB8D99D94840,*/'2014-09-25'); + +[expect] +INSERT INTO t +VALUES ( + 1, + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + /*!50705 0x0000000001010000003E0A325D63345CC0761FDB8D99D94840,*/'2014-09-25' +); diff --git a/tests/specs/Oracle/Syntax.txt b/tests/specs/Oracle/Syntax.txt index f93fe2e..1ae67f7 100644 --- a/tests/specs/Oracle/Syntax.txt +++ b/tests/specs/Oracle/Syntax.txt @@ -154,3 +154,14 @@ SELECT q'b>', q'{x}y}', q'(z)' FROM dual; [expect] SELECT q'b>', q'{x}y}', q'(z)' FROM dual; + +== should not read a set statement as a sql plus command once it's formatted == +SET AUTOCOMMIT=0; +SET define=off +UPDATE t SET a=1; + +[expect] +SET AUTOCOMMIT=0; +SET define=off +UPDATE t +SET a = 1; diff --git a/tests/specs/Select/Expressions.txt b/tests/specs/Select/Expressions.txt index 03b0c00..338e966 100644 --- a/tests/specs/Select/Expressions.txt +++ b/tests/specs/Select/Expressions.txt @@ -200,3 +200,10 @@ SELECT 0b101, 0o17, 0x1F, 1e5, 1.5e-3, 1_000 FROM t; [expect] SELECT 0b101, 0o17, 0x1F, 1e5, 1.5e-3, 1_000 FROM t; + +== should keep a percent against the word after it == +SELECT%x, a % b, a%b FROM t; + +[expect] +SELECT %x, a % b, a%b +FROM t; diff --git a/tests/specs/Select/Select_LineWidth.txt b/tests/specs/Select/Select_LineWidth.txt index 380b938..a0e60de 100644 --- a/tests/specs/Select/Select_LineWidth.txt +++ b/tests/specs/Select/Select_LineWidth.txt @@ -183,3 +183,14 @@ FROM t WHERE a BETWEEN 1 AND 2 AND b = 1 OR c NOT BETWEEN x AND y; + +== should keep the branches of a case after an operand that breaks == +SELECT COUNT(CASE DATALENGTH(chr.[UTF8_value_column]) WHEN 1 THEN 'x' END) AS b FROM t; + +[expect] +SELECT COUNT( + CASE DATALENGTH( + chr.[UTF8_value_column] + ) WHEN 1 THEN 'x' END +) AS b +FROM t; From 0bb76717342f1b9b73e7f03472c524f8e53d04e6 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 17 Oct 2026 23:33:16 -0400 Subject: [PATCH 4/4] test: add a runner that formats the SQL files of real projects `deno run -A scripts/corpus.ts` fetches only the .sql files of 14 projects pinned to commits (sqlfluff's fixtures, PostgreSQL's regression tests, pgTAP, SQL Server samples, the First Responder Kit, Ola Hallengren's maintenance solution, Oracle's sample schemas, test_db, sakila, chinook, bigquery-utils, Snowflake quickstarts, jaffle_shop and dbt-utils) and formats each in its own dialect, or in all of them with --all-dialects. It reports panics, refusals, unstable output, timeouts and slow files, and --compare writes how the output differs from the formatter at another ref. It isn't run in CI. --- .gitignore | 1 + CLAUDE.md | 13 ++ dprint.json | 3 +- examples/corpus.rs | 373 +++++++++++++++++++++++++++++++++++++++++++++ scripts/corpus.ts | 305 ++++++++++++++++++++++++++++++++++++ 5 files changed, 694 insertions(+), 1 deletion(-) create mode 100644 examples/corpus.rs create mode 100644 scripts/corpus.ts diff --git a/.gitignore b/.gitignore index 9254077..bf5c7e3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /target deployment/npm/plugin.wasm deployment/npm/node_modules +/.corpus diff --git a/CLAUDE.md b/CLAUDE.md index 37cbcea..46953b8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,6 +9,7 @@ SQL formatting plugin for dprint, with its own lexer, parser and formatter that - `cargo clippy --all-targets --all-features -- -D warnings` — lint - `cargo build --target wasm32-unknown-unknown --features wasm --release` — build the wasm plugin - `cd deployment/npm && node setup.js && npm test` — test the npm package (requires wasm build first) +- `deno run -A scripts/corpus.ts` — format the SQL files of real projects to find bugs (see [Corpus](#corpus)) ## Architecture @@ -26,6 +27,7 @@ The pipeline is lex → parse → generate IR → print → verify. - `deployment/npm/` — npm package that ships `plugin.wasm` - `deployment/schema.json` — JSON schema for plugin configuration - `scripts/generateReleaseNotes.ts` — generates changelog for GitHub releases +- `scripts/corpus.ts` and `examples/corpus.rs` — the corpus of real projects' SQL files and its runner ## Principles @@ -53,6 +55,17 @@ Spec files live in `tests/specs/` and use the dprint-development format: - `[|` and `|]` mark the range to format in a spec's text - A spec file can't start with a SQL comment, since a first line starting with `--` names the spec's file +## Corpus + +`deno run -A scripts/corpus.ts` fetches the `.sql` files of the projects pinned in `scripts/corpus.ts` into `.corpus/`, and formats each in its own dialect. It reports any file that panics, is refused (formatting would have changed more than whitespace), is unstable or times out, and exits with an error if any do. It also lists slow files. Files that don't parse in their dialect aren't problems, and `--verbose` lists them. + +- `--all-dialects` formats every file in every dialect, which finds the bugs only a dialect a file isn't written in shows +- `--line-width 30,120` formats at several widths, since a narrow width breaks far more +- `--compare main` also formats with the formatter at a git ref and writes the differences in output to `.corpus/compare.diff`, to review how a change formats real code +- project names (ex. `pgtap sakila`) limit it to those projects + +It isn't run in CI. Run it after changing the lexer, parser or layout, and before a release. When it finds a bug, add a spec reproducing it along with the fix. To add a project, add it to the list with a pinned commit and the dialect of its files. + ## Releases - Manual: trigger the `release` workflow with patch/minor diff --git a/dprint.json b/dprint.json index 57509f0..a1a85f8 100644 --- a/dprint.json +++ b/dprint.json @@ -9,7 +9,8 @@ "excludes": [ "**/node_modules", "**/*-lock.json", - "**/target" + "**/target", + ".corpus" ], "plugins": [ "npm:@dprint/typescript@0.96.1", diff --git a/examples/corpus.rs b/examples/corpus.rs new file mode 100644 index 0000000..5536159 --- /dev/null +++ b/examples/corpus.rs @@ -0,0 +1,373 @@ +//! Formats SQL files from real projects to find the formatter's bugs: a panic, a refusal to +//! format (output that would change more than whitespace), output that changes when it's +//! formatted again, or a file that takes too long. `scripts/corpus.ts` fetches a pinned set of +//! projects and runs this over them. +//! +//! ```shellsession +//! cargo run --release --example corpus -- [options] [path...] +//! ``` +//! +//! - `--manifest ` reads the files to format from lines of `\t` +//! - `--dialect ` is the dialect of the paths given directly (default `generic`) +//! - `--all-dialects` formats every file in every dialect rather than only its own +//! - `--line-width ` is a line width to format at, which may be given more than once (default +//! 120) +//! - `--out ` writes what each file is formatted to beneath ``, so that two versions of +//! the formatter can be compared +//! - `--verbose` also lists the files that don't parse, which is expected of a file in a dialect +//! it isn't written in +//! +//! A panic is only caught when the release profile unwinds, which the script sets up. + +use std::collections::BTreeMap; +use std::path::Component; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::mpsc; +use std::time::Duration; +use std::time::Instant; + +use dprint_core::configuration::ConfigKeyMap; +use dprint_core::configuration::GlobalConfiguration; +use dprint_plugin_sql::FormatError; +use dprint_plugin_sql::configuration::Configuration; +use dprint_plugin_sql::configuration::resolve_config; + +const DIALECTS: &[&str] = &[ + "generic", + "postgresql", + "mysql", + "sqlserver", + "sqlite", + "oracle", + "bigquery", + "snowflake", +]; +/// How long a file may take before it's reported as slow. +const SLOW: Duration = Duration::from_secs(1); +/// How long a file may take before it's given up on. +const TIMEOUT: Duration = Duration::from_secs(30); + +fn main() { + let options = Options::parse(); + let mut jobs = Vec::new(); + for (dialect, path) in &options.files { + let dialects: Vec<&str> = if options.all_dialects { + DIALECTS.to_vec() + } else { + vec![dialect.as_str()] + }; + for dialect in dialects { + for &line_width in &options.line_widths { + jobs.push(Job { + path: path.clone(), + dialect: dialect.to_string(), + line_width, + }); + } + } + } + + let outcomes = run_jobs(jobs, options.out.as_deref()); + + let mut totals: BTreeMap = BTreeMap::new(); + let mut problems = 0; + for (job, outcome) in &outcomes { + let totals = totals.entry(job.config_name()).or_default(); + totals.files += 1; + let label = match outcome { + Outcome::Ok => { + totals.ok += 1; + None + } + Outcome::NotUtf8 => { + totals.skipped += 1; + options.verbose.then(|| "SKIPPED as it isn't UTF-8".to_string()) + } + Outcome::Slow(elapsed) => { + totals.ok += 1; + totals.slow += 1; + Some(format!("SLOW {}ms", elapsed.as_millis())) + } + Outcome::ParseError(message) => { + totals.parse_errors += 1; + options.verbose.then(|| format!("PARSE {message}")) + } + Outcome::Refused => { + totals.refused += 1; + Some("REFUSED".to_string()) + } + Outcome::Unstable(first_difference) => { + totals.unstable += 1; + Some(format!("UNSTABLE {first_difference}")) + } + Outcome::Panic(message) => { + totals.panics += 1; + Some(format!("PANIC {message}")) + } + Outcome::Timeout => { + totals.timeouts += 1; + Some(format!("TIMEOUT after {}s", TIMEOUT.as_secs())) + } + }; + if let Some(label) = label { + if !matches!(outcome, Outcome::ParseError(_) | Outcome::Slow(_) | Outcome::NotUtf8) { + problems += 1; + } + println!("[{}] {} {}", job.config_name(), job.path.display(), label); + } + } + for (config, totals) in &totals { + println!( + "[{config}] files: {}, ok: {}, skipped: {}, parse errors: {}, refused: {}, unstable: {}, panics: {}, timeouts: {}, slow: {}", + totals.files, + totals.ok, + totals.skipped, + totals.parse_errors, + totals.refused, + totals.unstable, + totals.panics, + totals.timeouts, + totals.slow + ); + } + if problems > 0 { + println!("{problems} problems"); + std::process::exit(1); + } +} + +struct Options { + files: Vec<(String, PathBuf)>, + all_dialects: bool, + line_widths: Vec, + out: Option, + verbose: bool, +} + +impl Options { + fn parse() -> Self { + let mut args = std::env::args().skip(1); + let mut manifest = None; + let mut dialect = "generic".to_string(); + let mut paths = Vec::new(); + let mut options = Options { + files: Vec::new(), + all_dialects: false, + line_widths: Vec::new(), + out: None, + verbose: false, + }; + while let Some(arg) = args.next() { + let mut value = || args.next().unwrap_or_else(|| panic!("expected a value after {arg}")); + match arg.as_str() { + "--manifest" => manifest = Some(PathBuf::from(value())), + "--dialect" => dialect = value(), + "--all-dialects" => options.all_dialects = true, + "--line-width" => options.line_widths.push(value().parse().expect("a line width")), + "--out" => options.out = Some(PathBuf::from(value())), + "--verbose" => options.verbose = true, + _ if arg.starts_with("--") => panic!("unknown option {arg}"), + _ => paths.push(PathBuf::from(arg)), + } + } + if options.line_widths.is_empty() { + options.line_widths.push(120); + } + if let Some(manifest) = manifest { + let text = std::fs::read_to_string(&manifest).expect("the manifest to be readable"); + for line in text.lines().filter(|line| !line.is_empty()) { + let (dialect, path) = line.split_once('\t').expect("a line of `\\t`"); + options.files.push((dialect.to_string(), PathBuf::from(path))); + } + } + for path in paths { + let mut files = Vec::new(); + collect_sql_files(&path, &mut files); + options + .files + .extend(files.into_iter().map(|file| (dialect.clone(), file))); + } + options + } +} + +#[derive(Clone)] +struct Job { + path: PathBuf, + dialect: String, + line_width: u32, +} + +impl Job { + fn config_name(&self) -> String { + format!("{} {}", self.dialect, self.line_width) + } + + fn config(&self) -> Configuration { + let mut map = ConfigKeyMap::new(); + map.insert("dialect".to_string(), self.dialect.clone().into()); + map.insert("lineWidth".to_string(), (self.line_width as i32).into()); + let result = resolve_config(map, &GlobalConfiguration::default()); + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + result.config + } +} + +enum Outcome { + Ok, + /// dprint only formats UTF-8 files (ex. not a UTF-16 one, which SQL Server tools often write). + NotUtf8, + Slow(Duration), + ParseError(String), + Refused, + Unstable(String), + Panic(String), + Timeout, +} + +#[derive(Default)] +struct Totals { + files: usize, + ok: usize, + skipped: usize, + parse_errors: usize, + refused: usize, + unstable: usize, + panics: usize, + timeouts: usize, + slow: usize, +} + +/// Runs the jobs on a thread per core, returning their outcomes in the order of the jobs. +fn run_jobs(jobs: Vec, out: Option<&Path>) -> Vec<(Job, Outcome)> { + // a panic is an outcome to report rather than something to print + std::panic::set_hook(Box::new(|_| {})); + let queue = Arc::new(Mutex::new(jobs.into_iter().enumerate())); + let results = Arc::new(Mutex::new(Vec::new())); + let threads = std::thread::available_parallelism().map_or(4, |n| n.get()); + let workers: Vec<_> = (0..threads) + .map(|_| { + let queue = queue.clone(); + let results = results.clone(); + let out = out.map(Path::to_path_buf); + std::thread::spawn(move || { + loop { + let Some((index, job)) = queue.lock().unwrap().next() else { + return; + }; + let outcome = run_job(&job, out.as_deref()); + results.lock().unwrap().push((index, job, outcome)); + } + }) + }) + .collect(); + for worker in workers { + worker.join().unwrap(); + } + let mut results = std::mem::take(&mut *results.lock().unwrap()); + results.sort_by_key(|(index, _, _)| *index); + results.into_iter().map(|(_, job, outcome)| (job, outcome)).collect() +} + +fn run_job(job: &Job, out: Option<&Path>) -> Outcome { + let Ok(text) = std::fs::read_to_string(&job.path) else { + return Outcome::NotUtf8; + }; + // on a thread of its own so that a file that never finishes can be given up on + let (sender, receiver) = mpsc::channel(); + let config = job.config(); + let path = job.path.clone(); + let start = Instant::now(); + std::thread::spawn(move || { + let first = std::panic::catch_unwind(|| dprint_plugin_sql::format_text(&path, &text, &config)); + let second = match &first { + Ok(Ok(Some(formatted))) => Some(std::panic::catch_unwind(|| { + dprint_plugin_sql::format_text(&path, formatted, &config) + })), + _ => None, + }; + let _ = sender.send((text, first, second)); + }); + let Ok((text, first, second)) = receiver.recv_timeout(TIMEOUT) else { + return Outcome::Timeout; + }; + let elapsed = start.elapsed(); + let formatted = match first { + Err(panic) => return Outcome::Panic(panic_message(panic)), + Ok(Err(FormatError::Parse(err))) => { + return Outcome::ParseError(err.to_string().lines().next().unwrap_or("").to_string()); + } + Ok(Err(err)) => { + write_output(out, job, &format!("{err}\n"), true); + return Outcome::Refused; + } + Ok(Ok(formatted)) => formatted.unwrap_or(text), + }; + write_output(out, job, &formatted, false); + match second { + None | Some(Ok(Ok(None))) => {} + Some(Ok(Ok(Some(again)))) => { + let difference = formatted + .lines() + .zip(again.lines()) + .enumerate() + .find(|(_, (a, b))| a != b) + .map(|(i, (a, b))| format!("at line {}: {a:?} became {b:?}", i + 1)) + .unwrap_or_default(); + return Outcome::Unstable(difference); + } + Some(Ok(Err(_))) => return Outcome::Unstable("formatting the output failed".to_string()), + Some(Err(panic)) => return Outcome::Panic(format!("formatting the output: {}", panic_message(panic))), + } + if elapsed > SLOW { + Outcome::Slow(elapsed) + } else { + Outcome::Ok + } +} + +/// Writes what a file was formatted to, or why it couldn't be, beneath `out`. +fn write_output(out: Option<&Path>, job: &Job, text: &str, is_error: bool) { + let Some(out) = out else { + return; + }; + let mut path = out.join(job.config_name().replace(' ', "-")); + // the components of the file's path that are names, whether it's relative or absolute + for component in job.path.components() { + if let Component::Normal(name) = component { + path.push(name); + } + } + if is_error { + path.as_mut_os_string().push(".error"); + } + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, text).unwrap(); +} + +fn panic_message(panic: Box) -> String { + panic + .downcast_ref::<&str>() + .map(|s| s.to_string()) + .or_else(|| panic.downcast_ref::().cloned()) + .unwrap_or_default() +} + +fn collect_sql_files(path: &Path, files: &mut Vec) { + if path.is_dir() { + let mut entries: Vec<_> = std::fs::read_dir(path) + .unwrap() + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .collect(); + entries.sort(); + for entry in entries { + collect_sql_files(&entry, files); + } + } else if path.extension().is_some_and(|extension| extension == "sql") { + files.push(path.to_path_buf()); + } +} diff --git a/scripts/corpus.ts b/scripts/corpus.ts new file mode 100644 index 0000000..54c7cb3 --- /dev/null +++ b/scripts/corpus.ts @@ -0,0 +1,305 @@ +// Formats the SQL files of a pinned set of open source projects to find the formatter's bugs (see +// `examples/corpus.rs`), and optionally compares what they're formatted to with another version +// of the formatter. +// +// deno run -A scripts/corpus.ts [options] [project...] +// +// --all-dialects format every file in every dialect rather than only its own +// --line-width the line widths to format at (default 120) +// --compare also format with the formatter at a git ref (ex. `main`), and write the +// differences to .corpus/compare.diff +// --verbose also list the files that don't parse +// +// The projects are fetched into .corpus/projects, only their .sql files, at the commits below. + +interface Project { + name: string; + url: string; + commit: string; + /** The directories to take .sql files from, or the whole project when there are none. */ + paths?: string[]; + /** The dialect of a file, from its path within the project. */ + dialect: (path: string) => string; +} + +const projects: Project[] = [ + { + name: "sqlfluff", + url: "https://github.com/sqlfluff/sqlfluff", + commit: "cf59ad38d4d1435dbad9118bd791ef03795ace0a", + paths: ["test/fixtures/dialects"], + dialect: (path) => { + const folder = path.split("/")[3]; + return ({ + ansi: "generic", + bigquery: "bigquery", + greenplum: "postgresql", + mariadb: "mysql", + mysql: "mysql", + oracle: "oracle", + postgres: "postgresql", + redshift: "postgresql", + snowflake: "snowflake", + sqlite: "sqlite", + tsql: "sqlserver", + } as Record)[folder] ?? "generic"; + }, + }, + { + name: "postgres", + url: "https://github.com/postgres/postgres", + commit: "45fc51fcbf4534ea2f6f85e36f8155dbd5ba9851", + paths: ["src/test/regress/sql", "src/pl/plpgsql/src/sql"], + dialect: () => "postgresql", + }, + { + name: "pgtap", + url: "https://github.com/theory/pgtap", + commit: "21ceed193bf370892d530c5115680839ee322851", + dialect: () => "postgresql", + }, + { + name: "sql-server-samples", + url: "https://github.com/microsoft/sql-server-samples", + commit: "e5d7bb90ca0b695ac14f61458047bfb5a498459e", + dialect: () => "sqlserver", + }, + { + name: "first-responder-kit", + url: "https://github.com/BrentOzarULTD/SQL-Server-First-Responder-Kit", + commit: "53c380edf86e93fa07bcb2f2caf82c3e0d20fb68", + dialect: () => "sqlserver", + }, + { + name: "sql-server-maintenance-solution", + url: "https://github.com/olahallengren/sql-server-maintenance-solution", + commit: "41617578643e722b790ea4d57ef80f8de38ae747", + dialect: () => "sqlserver", + }, + { + name: "db-sample-schemas", + url: "https://github.com/oracle-samples/db-sample-schemas", + commit: "6660bad68c07bd143430ace58565b3f727e17263", + dialect: () => "oracle", + }, + { + name: "test_db", + url: "https://github.com/datacharmer/test_db", + commit: "e324b56193ca506ab7cc1ab143a9153d8c4535d7", + dialect: () => "mysql", + }, + { + name: "sakila", + url: "https://github.com/jOOQ/sakila", + commit: "e089a5b1ec9af0df7a9c6a5d47d49fa1736a4e84", + dialect: dialectFromName, + }, + { + name: "chinook", + url: "https://github.com/lerocha/chinook-database", + commit: "7f67772503d71ba90f19283c38e93923addb43fa", + dialect: dialectFromName, + }, + { + name: "bigquery-utils", + url: "https://github.com/GoogleCloudPlatform/bigquery-utils", + commit: "d5bde4fb0d2dbfa0477f1f2a6d87edda65bb7907", + dialect: () => "bigquery", + }, + { + name: "sfquickstarts", + url: "https://github.com/Snowflake-Labs/sfquickstarts", + commit: "b186c148d09b971fcf99e693ffa70312b196e0ec", + dialect: () => "snowflake", + }, + { + name: "jaffle_shop", + url: "https://github.com/dbt-labs/jaffle_shop", + commit: "fd7bfacae4f497ff044a6a0275268676bf1b64c3", + dialect: () => "generic", + }, + { + name: "dbt-utils", + url: "https://github.com/dbt-labs/dbt-utils", + commit: "2a8f07bcbd02fb7a8c011298a1bdaf3e92759c14", + dialect: () => "generic", + }, +]; + +const rootDir = new URL("../", import.meta.url).pathname.replace(/^\/(\w:)/, "$1"); +const corpusDir = `${rootDir}.corpus`; +const projectsDir = `${corpusDir}/projects`; + +const options = parseArgs(Deno.args); +const selected = options.projects.length === 0 + ? projects + : options.projects.map((name) => { + const project = projects.find((p) => p.name === name); + if (project == null) { + throw new Error(`Unknown project ${name}. Expected one of: ${projects.map((p) => p.name).join(", ")}`); + } + return project; + }); + +for (const project of selected) { + await fetchProject(project); +} +const manifest = `${corpusDir}/manifest.tsv`; +await Deno.writeTextFile(manifest, (await manifestLines(selected)).join("\n") + "\n"); + +const runnerArgs = ["--manifest", manifest]; +if (options.allDialects) { + runnerArgs.push("--all-dialects"); +} +for (const width of options.lineWidths) { + runnerArgs.push("--line-width", width); +} +if (options.verbose) { + runnerArgs.push("--verbose"); +} + +if (options.compare == null) { + Deno.exit(await runCorpus(rootDir, `${rootDir}target/corpus`, runnerArgs)); +} + +// compare with the formatter at another ref, running this version of the runner against it +const baseDir = `${corpusDir}/base`; +await removeWorktree(baseDir); +await run("git", ["worktree", "add", "--quiet", "--detach", baseDir, options.compare], rootDir); +let code; +try { + await Deno.mkdir(`${baseDir}/examples`, { recursive: true }); + await Deno.copyFile(`${rootDir}examples/corpus.rs`, `${baseDir}/examples/corpus.rs`); + const outDir = `${corpusDir}/out`; + await Deno.remove(outDir, { recursive: true }).catch(() => {}); + console.log(`\n=== ${options.compare}`); + await runCorpus(baseDir, `${rootDir}target/corpus-base`, [...runnerArgs, "--out", `${outDir}/base`]); + console.log("\n=== this version"); + code = await runCorpus(rootDir, `${rootDir}target/corpus`, [...runnerArgs, "--out", `${outDir}/head`]); + const diff = await output("git", ["diff", "--no-index", "--", "base", "head"], outDir); + await Deno.writeTextFile(`${corpusDir}/compare.diff`, diff); + const stat = await output("git", ["diff", "--no-index", "--shortstat", "--", "base", "head"], outDir); + console.log(`\n${stat.trim() || "no differences"} (see .corpus/compare.diff)`); +} finally { + await removeWorktree(baseDir); +} +Deno.exit(code); + +/** Fetches the project's .sql files at its commit, unless they're there already. */ +async function fetchProject(project: Project) { + const dir = `${projectsDir}/${project.name}`; + const head = await output("git", ["rev-parse", "HEAD"], dir).catch(() => ""); + if (head.trim() === project.commit) { + return; + } + console.log(`Fetching ${project.name}...`); + await Deno.mkdir(dir, { recursive: true }); + if (head === "") { + await run("git", ["init", "--quiet"], dir); + await run("git", ["remote", "add", "origin", project.url], dir); + } + // only the blobs of the .sql files are fetched, which keeps large projects small + await run("git", ["sparse-checkout", "set", "--no-cone", "*.sql"], dir); + await run("git", ["fetch", "--quiet", "--depth", "1", "--filter=blob:none", "origin", project.commit], dir); + await run("git", ["checkout", "--quiet", "--detach", "FETCH_HEAD"], dir); +} + +async function manifestLines(projects: Project[]) { + const lines = []; + for (const project of projects) { + const dir = `${projectsDir}/${project.name}`; + for (const path of project.paths ?? [""]) { + for await (const file of sqlFiles(`${dir}/${path}`.replace(/\/$/, ""))) { + const relative = file.slice(dir.length + 1); + lines.push(`${project.dialect(relative)}\tprojects/${project.name}/${relative}`); + } + } + } + return lines.sort(); +} + +async function* sqlFiles(dir: string): AsyncGenerator { + for await (const entry of Deno.readDir(dir)) { + const path = `${dir}/${entry.name}`; + if (entry.isDirectory && entry.name !== ".git") { + yield* sqlFiles(path); + } else if (entry.isFile && entry.name.endsWith(".sql")) { + yield path; + } + } +} + +/** Builds and runs the runner with the formatter in `crateDir`, returning its exit code. */ +async function runCorpus(crateDir: string, targetDir: string, args: string[]) { + const env = { + // a panic is only caught when it unwinds + CARGO_PROFILE_RELEASE_PANIC: "unwind", + CARGO_TARGET_DIR: targetDir, + }; + await run("cargo", ["build", "--quiet", "--release", "--example", "corpus"], crateDir, env); + const exe = `${targetDir}/release/examples/corpus${Deno.build.os === "windows" ? ".exe" : ""}`; + // the paths in the manifest are relative to the corpus directory + const { code } = await new Deno.Command(exe, { args, cwd: corpusDir }).spawn().status; + return code; +} + +/** The dialect of a file named after its database (ex. `Chinook_PostgreSql.sql`). */ +function dialectFromName(path: string) { + const lower = path.toLowerCase(); + const names: [string, string][] = [ + ["postgres", "postgresql"], + ["mysql", "mysql"], + ["sql-server", "sqlserver"], + ["sqlserver", "sqlserver"], + ["sqlite", "sqlite"], + ["oracle", "oracle"], + ]; + return names.find(([name]) => lower.includes(name))?.[1] ?? "generic"; +} + +async function removeWorktree(dir: string) { + await output("git", ["worktree", "remove", "--force", dir], rootDir).catch(() => {}); +} + +async function run(command: string, args: string[], cwd: string, env?: Record) { + const { code } = await new Deno.Command(command, { args, cwd, env }).spawn().status; + if (code !== 0) { + throw new Error(`${command} ${args.join(" ")} failed with exit code ${code}`); + } +} + +async function output(command: string, args: string[], cwd: string) { + const result = await new Deno.Command(command, { args, cwd, stderr: "null" }).output(); + // `git diff --no-index` exits with 1 when there are differences + if (result.code > 1) { + throw new Error(`${command} ${args.join(" ")} failed with exit code ${result.code}`); + } + return new TextDecoder().decode(result.stdout); +} + +function parseArgs(args: string[]) { + const options = { + allDialects: false, + lineWidths: [] as string[], + compare: undefined as string | undefined, + verbose: false, + projects: [] as string[], + }; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === "--all-dialects") { + options.allDialects = true; + } else if (arg === "--line-width") { + options.lineWidths.push(...args[++i].split(",")); + } else if (arg === "--compare") { + options.compare = args[++i]; + } else if (arg === "--verbose") { + options.verbose = true; + } else if (arg.startsWith("--")) { + throw new Error(`Unknown option ${arg}`); + } else { + options.projects.push(arg); + } + } + return options; +}