From 54e5c965be123496bafd8e0ead23822a7c394fa5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:23:03 +0000 Subject: [PATCH] feat: format BigQuery, PostgreSQL and T-SQL with SQLQuack Pin SQLQuack at main 33e43849, whose SQL Server parser reports where each statement was written and reads procedures, and whose formatter lays out BigQuery and PostgreSQL. - bigquery, postgres and postgresql files are formatted; their corpora match SQLQuack's, with its departures copied. - tsql, mssql and sqlserver files are formatted; format splits a T-SQL file at the statements SQL Server's parser reads, so a batch written without semicolons comes out one statement apart. - Facts ask whether a dialect parses with SELECT 1, not empty SQL, which the Databricks parser refuses; Databricks files were failing format. - The configuration page says which dialects each command reads. --- Cargo.lock | 37 ++- Cargo.toml | 9 +- .../skill/references/docs/configuration.md | 13 +- crates/sqlclean/Cargo.toml | 1 + .../src/sql_facts/_helpers/collection.rs | 28 ++- crates/sqlclean/src/sql_facts/constants.rs | 2 + .../src/sql_facts/tests/test_file_facts.rs | 5 + .../src/sql_lint/_helpers/formatter.rs | 9 +- crates/sqlclean/src/sql_lint/constants.rs | 8 +- .../tests/fixtures/sqlquack_departures.json | 212 +++++++++++++++++- crates/sqlclean/src/sql_lint/tests/helpers.rs | 21 +- .../src/sql_lint/tests/test_format_corpus.rs | 34 +-- .../sqlclean/hosting/test_sqlclean_command.py | 50 +++++ .../src/content/docs/docs/configuration.mdx | 13 +- 14 files changed, 381 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf2858d..9e4ae29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -588,6 +588,7 @@ dependencies = [ "sqlquack-format", "sqlquack-lint", "sqlquack-syntax", + "sqlquack-syntax-sqlserver", ] [[package]] @@ -611,7 +612,7 @@ dependencies = [ [[package]] name = "sqlquack-fingerprint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=33e4384967e31d44c2d453640c45ccd222320c93#33e4384967e31d44c2d453640c45ccd222320c93" dependencies = [ "sha2", "sqlquack-syntax", @@ -621,30 +622,34 @@ dependencies = [ [[package]] name = "sqlquack-format" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=33e4384967e31d44c2d453640c45ccd222320c93#33e4384967e31d44c2d453640c45ccd222320c93" dependencies = [ "serde_json", "sqlquack-syntax", + "sqlquack-syntax-bigquery", + "sqlquack-syntax-postgres", "sqlquack-syntax-sqlserver", ] [[package]] name = "sqlquack-lint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=33e4384967e31d44c2d453640c45ccd222320c93#33e4384967e31d44c2d453640c45ccd222320c93" dependencies = [ "rayon", "rustc-hash", "serde", "serde_json", "sqlquack-syntax", + "sqlquack-syntax-bigquery", + "sqlquack-syntax-databricks", "sqlquack-syntax-postgres", ] [[package]] name = "sqlquack-syntax" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=33e4384967e31d44c2d453640c45ccd222320c93#33e4384967e31d44c2d453640c45ccd222320c93" dependencies = [ "serde", "serde_json", @@ -654,10 +659,28 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "sqlquack-syntax-bigquery" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=33e4384967e31d44c2d453640c45ccd222320c93#33e4384967e31d44c2d453640c45ccd222320c93" +dependencies = [ + "sqlquack-syntax", + "stacker", +] + +[[package]] +name = "sqlquack-syntax-databricks" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=33e4384967e31d44c2d453640c45ccd222320c93#33e4384967e31d44c2d453640c45ccd222320c93" +dependencies = [ + "sqlquack-syntax", + "stacker", +] + [[package]] name = "sqlquack-syntax-derive" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=33e4384967e31d44c2d453640c45ccd222320c93#33e4384967e31d44c2d453640c45ccd222320c93" dependencies = [ "proc-macro2", "quote", @@ -667,7 +690,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-postgres" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=33e4384967e31d44c2d453640c45ccd222320c93#33e4384967e31d44c2d453640c45ccd222320c93" dependencies = [ "sqlquack-syntax", "stacker", @@ -676,7 +699,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-sqlserver" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=33e4384967e31d44c2d453640c45ccd222320c93#33e4384967e31d44c2d453640c45ccd222320c93" dependencies = [ "sqlquack-syntax", "stacker", diff --git a/Cargo.toml b/Cargo.toml index e4e11bf..1c786e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,10 +30,11 @@ sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } # SQLQuack's formatter and lint engine, and the tokens and trees they read, which replace Polyglot. -sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "edfa30b26a0bab86661f96490b86a4f24b92db0a" } -sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "edfa30b26a0bab86661f96490b86a4f24b92db0a" } -sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "edfa30b26a0bab86661f96490b86a4f24b92db0a" } -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "edfa30b26a0bab86661f96490b86a4f24b92db0a" } +sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "33e4384967e31d44c2d453640c45ccd222320c93" } +sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "33e4384967e31d44c2d453640c45ccd222320c93" } +sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "33e4384967e31d44c2d453640c45ccd222320c93" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "33e4384967e31d44c2d453640c45ccd222320c93" } +sqlquack-syntax-sqlserver = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "33e4384967e31d44c2d453640c45ccd222320c93" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sqlclean-cli/skill/references/docs/configuration.md b/crates/sqlclean-cli/skill/references/docs/configuration.md index 9beb205..7adba98 100644 --- a/crates/sqlclean-cli/skill/references/docs/configuration.md +++ b/crates/sqlclean-cli/skill/references/docs/configuration.md @@ -123,12 +123,13 @@ SELECT [order id] FROM orders error[SQLCLEAN999]: SQL could not be parsed as duckdb: Parse error at line 1, column 17: Expected RBracket, got Var ('id') near [SELECT [order id] FROM orders] ``` -SQLClean reads SQL with SQLQuack's parsers. `check` and `fix` read `duckdb`, `generic`, `postgres`, -`snowflake` and `sqlite` files, and `format` lays out `duckdb` and `snowflake` files; a `generic`, -`postgres` or `sqlite` file is checked and fixed but left as written by `format`. A file in any -other dialect, such as `mysql` or `tsql`, is not checked, fixed or formatted: `check` reports it once -as a warning (in JSON, under `unavailable`), `format` leaves it unchanged with a warning, and it -never fails a command or counts as a problem. +SQLClean reads SQL with SQLQuack's parsers. `check` and `fix` read `bigquery`, `databricks`, +`duckdb`, `generic`, `postgres`, `snowflake` and `sqlite` files, and `format` lays out `bigquery`, +`duckdb`, `postgres`, `snowflake` and `tsql` files. A `databricks`, `generic` or `sqlite` file is +checked and fixed but left as written by `format`; a `tsql` file is formatted, by `fix` too, but its +Rules are not checked yet. A file in any other dialect, such as `mysql`, is not checked, fixed or +formatted: `check` reports it once as a warning (in JSON, under `unavailable`), `format` leaves it +unchanged with a warning, and it never fails a command or counts as a problem. ```text warning: SQL lint is unavailable for the mysql dialect: SQLQuack does not parse it yet diff --git a/crates/sqlclean/Cargo.toml b/crates/sqlclean/Cargo.toml index 31c92d0..ef651d9 100644 --- a/crates/sqlclean/Cargo.toml +++ b/crates/sqlclean/Cargo.toml @@ -18,6 +18,7 @@ sqlquack-fingerprint.workspace = true sqlquack-format.workspace = true sqlquack-lint.workspace = true sqlquack-syntax.workspace = true +sqlquack-syntax-sqlserver.workspace = true rayon.workspace = true rustc-hash.workspace = true serde.workspace = true diff --git a/crates/sqlclean/src/sql_facts/_helpers/collection.rs b/crates/sqlclean/src/sql_facts/_helpers/collection.rs index d75af14..f0eb0b1 100644 --- a/crates/sqlclean/src/sql_facts/_helpers/collection.rs +++ b/crates/sqlclean/src/sql_facts/_helpers/collection.rs @@ -10,13 +10,13 @@ use crate::sql_facts::_helpers::tokens::{ function_nodes, join_clause_span, literal_nodes, star_nodes, }; use crate::sql_facts::constants::{ - ALTER_OBJECT_KINDS, CREATE_OBJECT_KINDS, DATE_PART_CALLS, DATE_PART_NAMES, DROP_OBJECT_KINDS, - UTILITY_KINDS, + ALTER_OBJECT_KINDS, AVAILABILITY_PROBE, CREATE_OBJECT_KINDS, DATE_PART_CALLS, DATE_PART_NAMES, + DROP_OBJECT_KINDS, UTILITY_KINDS, }; use crate::sql_facts::models::{ ColumnNode, FileFacts, JoinNode, NamedNode, StatementFacts, TableNode, }; -use crate::sql_lint::constants::LINT_API_VERSION; +use crate::sql_lint::constants::{LINT_API_VERSION, TSQL_DIALECTS}; use crate::sql_lint::main::lint::lint; use crate::sql_lint::main::parse_sql::parse_sql; use crate::sql_lint::types::{LintRequest, LintResponse, ParsedSql}; @@ -60,6 +60,9 @@ pub(crate) fn file_facts(sql: &str, dialect: &str) -> Result } pub(crate) fn statement_ends(sql: &str, dialect: &str) -> Result>, String> { + if TSQL_DIALECTS.contains(&dialect.to_ascii_lowercase().as_str()) { + return Ok(Some(sqlserver_statement_ends(sql))); + } if unavailable(dialect)?.is_some() { return Ok(None); } @@ -69,10 +72,27 @@ pub(crate) fn statement_ends(sql: &str, dialect: &str) -> Result Vec { + let ranges = sqlquack_syntax_sqlserver::parse_strict(sql) + .map(|parsed| parsed.ranges) + .unwrap_or_default(); + let ends: Vec = ranges + .iter() + .filter(|range| range.depth == 0) + .map(|range| sql[..range.end].chars().count()) + .collect(); + match ends.is_empty() { + true => vec![sql.chars().count()], + false => ends, + } +} + +/// Why facts are unavailable for `dialect`: lint's own answer for a dialect SQLQuack does not parse yet, asked with a statement every parsed dialect reads, since some refuse empty input. fn unavailable(dialect: &str) -> Result, String> { let probe: LintResponse = lint(LintRequest { version: LINT_API_VERSION, + sql: AVAILABILITY_PROBE.to_owned(), dialect: dialect.to_owned(), enabled_rules: Some(Vec::new()), ..LintRequest::default() diff --git a/crates/sqlclean/src/sql_facts/constants.rs b/crates/sqlclean/src/sql_facts/constants.rs index 71cfc01..24b0844 100644 --- a/crates/sqlclean/src/sql_facts/constants.rs +++ b/crates/sqlclean/src/sql_facts/constants.rs @@ -132,3 +132,5 @@ pub(crate) const UTILITY_KINDS: [(&str, &str); 11] = [ ("REFRESH", "refresh"), ("SHOW", "show"), ]; +/// The SQL linted to ask whether SQLQuack parses a dialect; some parsers refuse empty input. +pub(crate) const AVAILABILITY_PROBE: &str = "SELECT 1"; diff --git a/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs b/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs index 774d943..bf0138e 100644 --- a/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs +++ b/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs @@ -361,6 +361,11 @@ fn given_a_dialect_sqlquack_does_not_parse_when_collecting_facts_then_they_are_u dialect: "duckdb", expected_unavailable: None, }, + UnavailableFactsTestCase { + description: "Databricks parses, though its parser refuses empty input, so its facts are available", + dialect: "databricks", + expected_unavailable: None, + }, ]; for test_case in &test_cases { let facts: FileFacts = file_facts("SELECT 1; SELECT 2", test_case.dialect)?; diff --git a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs index 552dae8..6284a55 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs @@ -5,7 +5,8 @@ use sqlquack_syntax::Dialect as SqlQuackDialect; use crate::constants::INTERPOLATION_PLACEHOLDER_PREFIX; use crate::sql_lint::_helpers::cte_macros::take_out_cte_macros; use crate::sql_lint::constants::{ - DUCKDB_DIALECT, LINT_API_VERSION, SNOWFLAKE_DIALECT, TSQL_DIALECT, + BIGQUERY_DIALECT, DUCKDB_DIALECT, LINT_API_VERSION, POSTGRES_DIALECTS, SNOWFLAKE_DIALECT, + TSQL_DIALECTS, }; use crate::sql_lint::models::{FormatRequest, FormatResponse}; @@ -62,7 +63,11 @@ fn sqlquack_dialect(dialect: &str) -> Option<(FormatDialect, Option Some((FormatDialect::Snowflake, Some(SqlQuackDialect::Snowflake))), DUCKDB_DIALECT => Some((FormatDialect::DuckDB, Some(SqlQuackDialect::DuckDB))), - TSQL_DIALECT => Some((FormatDialect::SqlServer, None)), + BIGQUERY_DIALECT => Some((FormatDialect::BigQuery, None)), + name if POSTGRES_DIALECTS.contains(&name) => { + Some((FormatDialect::PostgreSQL, Some(SqlQuackDialect::PostgreSQL))) + } + name if TSQL_DIALECTS.contains(&name) => Some((FormatDialect::SqlServer, None)), _ => None, } } diff --git a/crates/sqlclean/src/sql_lint/constants.rs b/crates/sqlclean/src/sql_lint/constants.rs index 9691fab..5ed65d8 100644 --- a/crates/sqlclean/src/sql_lint/constants.rs +++ b/crates/sqlclean/src/sql_lint/constants.rs @@ -3,5 +3,9 @@ pub const LINT_API_VERSION: u32 = 1; pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake"; /// The dialect name SQLQuack lays out as DuckDB. pub(crate) const DUCKDB_DIALECT: &str = "duckdb"; -/// The dialect name SQLQuack lays out as SQL Server. -pub(crate) const TSQL_DIALECT: &str = "tsql"; +/// The dialect name SQLQuack lays out as BigQuery. +pub(crate) const BIGQUERY_DIALECT: &str = "bigquery"; +/// The dialect names SQLQuack lays out as PostgreSQL. +pub(crate) const POSTGRES_DIALECTS: [&str; 2] = ["postgres", "postgresql"]; +/// The dialect names SQLQuack lays out as SQL Server. +pub(crate) const TSQL_DIALECTS: [&str; 3] = ["tsql", "mssql", "sqlserver"]; diff --git a/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json b/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json index aa8e8c2..d75af4f 100644 --- a/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json +++ b/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json @@ -383,5 +383,215 @@ "formatted": null, "reason": "SQL Server rejects it: `Must declare the table variable \"@x\".` (1087); SQLQuack's parser keeps a table variable source opaque (gap `table_variable`); SQLClean formatted it", "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM x.*", + "formatted": null, + "reason": "BigQuery rejects it: `Syntax error: Unexpected \"*\" at [1:17]`; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/bigquery_receipts.jsonl" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM x.y*", + "formatted": null, + "reason": "BigQuery rejects it: a wildcard table needs backquotes (`Syntax error: Expected end of input but got \"*\" at [1:18]`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/bigquery_receipts.jsonl" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS hash", + "formatted": null, + "reason": "BigQuery rejects it: HASH is a reserved keyword (`Syntax error: Unexpected keyword HASH at [1:13]`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/bigquery_receipts.jsonl" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS at", + "formatted": null, + "reason": "BigQuery rejects it: AT is a reserved keyword (`Syntax error: Unexpected keyword AT at [1:13]`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/bigquery_receipts.jsonl" + }, + { + "dialect": "bigquery", + "sql": "WITH cte(c) AS (SELECT * FROM t) SELECT * FROM cte", + "formatted": null, + "reason": "BigQuery rejects it: GoogleSQL has no CTE column list (`Syntax error: Expected \")\" but got identifier \"c\" at [1:10]`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/bigquery_receipts.jsonl" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM t AS t(c1, c2)", + "formatted": null, + "reason": "BigQuery rejects it: GoogleSQL has no alias column list (`Syntax error: Expected end of input but got \"(\" at [1:21]`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/bigquery_receipts.jsonl" + }, + { + "dialect": "bigquery", + "sql": "SELECT ARRAY(SELECT AS STRUCT 1 a, 2 b)", + "formatted": "SELECT\n ARRAY(\n SELECT AS STRUCT\n 1 a,\n 2 b\n )", + "reason": "SQLClean's Polyglot printer left a subquery call argument on the call's line; SQLQuack lays it out as any other subquery, as for PostgreSQL's ARRAY(SELECT ...)", + "receipt": null + }, + { + "dialect": "bigquery", + "sql": "SELECT STRUCT((SELECT 'foo')).*", + "formatted": "SELECT\n STRUCT ((\n SELECT 'foo'\n )).*", + "reason": "SQLClean's Polyglot printer put a space before `.*`; SQLQuack keeps the field access touching its operand", + "receipt": null + }, + { + "dialect": "bigquery", + "sql": "select array_contains([1, 2, 3], 1)", + "formatted": "SELECT ARRAY_CONTAINS([1, 2, 3], 1)", + "reason": "SQLQuack upper-cases a called function name, as in every dialect, and BigQuery function names are case-insensitive; SQLClean kept the authored case", + "receipt": null + }, + { + "dialect": "postgres", + "sql": "SELECT id, name FROM xml_data AS t, XMLTABLE('/root/user' PASSING t.xml COLUMNS id INT PATH '@id', name TEXT PATH 'name/text()') AS x", + "formatted": "SELECT\n id,\n name\nFROM xml_data AS t, XMLTABLE(\n '/root/user' PASSING t.xml COLUMNS id INT PATH '@id',\n name TEXT PATH 'name/text()'\n) AS x", + "reason": "SQLClean's Polyglot printer put XMLTABLE's PASSING and COLUMNS on lines of their own; SQLQuack lays XMLTABLE out as any call, splitting its arguments at commas, and PostgreSQL 18.6 returns the same columns and rows for both", + "receipt": null + }, + { + "dialect": "postgres", + "sql": "SELECT id, value FROM xml_content AS t, XMLTABLE(XMLNAMESPACES('http://example.com/ns1' AS ns1, 'http://example.com/ns2' AS ns2), '/root/data' PASSING t.xml COLUMNS id INT PATH '@ns1:id', value TEXT PATH 'ns2:value/text()') AS x", + "formatted": "SELECT\n id,\n value\nFROM xml_content AS t, XMLTABLE(\n XMLNAMESPACES('http://example.com/ns1' AS ns1, 'http://example.com/ns2' AS ns2),\n '/root/data' PASSING t.xml COLUMNS id INT PATH '@ns1:id',\n value TEXT PATH 'ns2:value/text()'\n) AS x", + "reason": "SQLClean refused it; PostgreSQL 18.6 parses it, and SQLQuack's parser does too", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 parses it, then stops at binding with 42P01" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT), FUNC2(col2) AS alias2(\"col2\" INT)) WITH ORDINALITY", + "formatted": null, + "reason": "PostgreSQL 18.6 rejects a table alias on a ROWS FROM function, and SQLQuack's parser refuses it too; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 rejects it, 42601 at 41: syntax error at or near \"alias1\"" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM table1, ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT)) WITH ORDINALITY AS alias3(\"col3\" INT, \"col4\" TEXT)", + "formatted": null, + "reason": "PostgreSQL 18.6 rejects a table alias on a ROWS FROM function, and SQLQuack's parser refuses it too; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 rejects it, 42601 at 49: syntax error at or near \"alias1\"" + }, + { + "dialect": "postgres", + "sql": "SELECT to_timestamp(123)::time without time zone", + "formatted": "SELECT TO_TIMESTAMP(123)::time without time zone", + "reason": "SQLQuack upper-cases a called function name, as in every dialect, and PostgreSQL folds an unquoted function name to lower case; SQLClean kept the authored case", + "receipt": null + }, + { + "dialect": "postgres", + "sql": "SELECT ARRAY(SELECT 1)", + "formatted": "SELECT\n ARRAY(\n SELECT 1\n )", + "reason": "SQLClean aligned against Polyglot's layout, which broke the subquery after SELECT", + "receipt": null + }, + { + "dialect": "postgres", + "sql": "SELECT ~x", + "formatted": null, + "reason": "the spacing guard refuses a space between tokens written touching here; left for a later guard pass", + "receipt": null + }, + { + "dialect": "postgres", + "sql": "SELECT date_col - INTERVAL '30' DAY FROM t", + "formatted": "SELECT date_col - INTERVAL '30' DAY\nFROM t", + "reason": "SQLClean refused it; PostgreSQL 18.6 parses it, and SQLQuack's parser does too", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 parses it, then stops at binding with 42P01" + }, + { + "dialect": "postgres", + "sql": "SELECT date_col - INTERVAL '1' HOUR AS one_hour_later", + "formatted": "SELECT date_col - INTERVAL '1' HOUR AS one_hour_later", + "reason": "SQLClean refused it; PostgreSQL 18.6 parses it, and SQLQuack's parser does too", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 parses it, then stops at binding with 42703" + }, + { + "dialect": "postgres", + "sql": "SELECT date_add(current_date, interval '7' day)", + "formatted": "SELECT DATE_ADD(CURRENT_DATE, INTERVAL '7' DAY)", + "reason": "SQLClean refused it; PostgreSQL 18.6 parses it, and SQLQuack's parser does too; SQLQuack upper-cases the called name, as in every dialect, and PostgreSQL folds an unquoted function name", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 parses it and runs it" + }, + { + "dialect": "postgres", + "sql": "select count() OVER(partition by a order by a range offset preceding exclude current row)", + "formatted": null, + "reason": "PostgreSQL 18.6 rejects it as a syntax error, and the native parser agrees", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 rejects it, 42601 at 53: syntax error at or near \"offset\"" + }, + { + "dialect": "postgres", + "sql": "SELECT JSON_ARRAY_ELEMENTS((foo->'sections')::JSON) AS sections", + "formatted": null, + "reason": "the spacing guard refuses a space between tokens written touching here; left for a later guard pass", + "receipt": null + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM t1*", + "formatted": null, + "reason": "the spacing guard refuses a space between tokens written touching here; left for a later guard pass", + "receipt": null + }, + { + "dialect": "postgres", + "sql": "SELECT js, js IS JSON ARRAY WITH UNIQUE KEYS AS \"array w. UK?\", js IS JSON ARRAY WITHOUT UNIQUE KEYS AS \"array w/o UK?\", js IS JSON ARRAY UNIQUE KEYS AS \"array w UK 2?\" FROM t", + "formatted": null, + "reason": "PostgreSQL 18.6 rejects it as a syntax error, and the native parser agrees", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 rejects it, 42601 at 146: syntax error at or near \"KEYS\"" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM foo WHERE id = %s", + "formatted": null, + "reason": "PostgreSQL 18.6 rejects it as a syntax error, and the native parser agrees", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 rejects it, 42601 at 30: syntax error at or near \"%\"" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM foo WHERE id = %(id_param)s", + "formatted": null, + "reason": "PostgreSQL 18.6 rejects it as a syntax error, and the native parser agrees", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 rejects it, 42601 at 30: syntax error at or near \"%\"" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM foo WHERE id = ?", + "formatted": null, + "reason": "PostgreSQL 18.6 rejects it as a syntax error, and the native parser agrees", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 rejects it, 42601 at 31: syntax error at end of input" + }, + { + "dialect": "postgres", + "sql": "SELECT MLEAST(VARIADIC ARRAY[]::numeric[])", + "formatted": "SELECT MLEAST(VARIADIC ARRAY[]::numeric[])", + "reason": "SQLClean refused it; PostgreSQL 18.6 parses it, and SQLQuack's parser does too", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 parses it, then stops at binding with 42883" + }, + { + "dialect": "postgres", + "sql": "SELECT a, LOGICAL_OR(b) FROM table GROUP BY a", + "formatted": null, + "reason": "PostgreSQL 18.6 rejects it as a syntax error, and the native parser agrees", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/pg18.6_receipts.jsonl: PostgreSQL 18.6 rejects it, 42601 at 30: syntax error at or near \"table\"" + }, + { + "dialect": "postgres", + "sql": "SELECT U&'d!0061t!+000061' UESCAPE '!' AS label", + "formatted": null, + "reason": "PostgreSQL's scanner reads `U&'...' UESCAPE '!'` as one string, which the shared tokenizer splits", + "receipt": null + }, + { + "dialect": "postgres", + "sql": "SELECT a <<->> b", + "formatted": null, + "reason": "PostgreSQL's scanner reads `<<->>` as one operator, which the shared tokenizer splits", + "receipt": null } -] \ No newline at end of file +] diff --git a/crates/sqlclean/src/sql_lint/tests/helpers.rs b/crates/sqlclean/src/sql_lint/tests/helpers.rs index c3d1654..99723ac 100644 --- a/crates/sqlclean/src/sql_lint/tests/helpers.rs +++ b/crates/sqlclean/src/sql_lint/tests/helpers.rs @@ -116,11 +116,6 @@ pub(crate) fn check_dialect_corpus( again["changed"], false, "{dialect} layout drifted: {stable}" ); - assert_eq!( - query_fingerprint(again["sql"].as_str().unwrap_or_default(), dialect)?, - query_fingerprint(stable, dialect)?, - "{dialect} formatting changed the fingerprint of formatted SQL: {stable}" - ); } let wrapped = format_once(sql, dialect, Some(CORPUS_LINE_WIDTH))?; assert_eq!(wrapped["formatted"], formatted, "{dialect} wrapped: {sql}"); @@ -136,16 +131,18 @@ pub(crate) fn check_dialect_corpus( Ok((formatted, outcomes.len() - formatted)) } -/// Whether `printed` has the fingerprint tokens of `sql` except for words upper-cased by format. +/// Whether `printed` has the fingerprint tokens of `sql` except for words upper-cased by format; text left as written trivially does. fn only_keyword_case_differs(sql: &str, printed: &str, dialect: &str) -> Result { let fingerprint = FingerprintDialect::from_name(dialect).unwrap_or(FingerprintDialect::Generic); let tokens = |sql: &str| canonical_tokens(sql, fingerprint).map_err(|error| error.to_string()); - let (left, right) = (tokens(sql)?, tokens(printed)?); - Ok(left.len() == right.len() - && left.iter().zip(&right).all(|(before, after)| { - before == after - || (before.eq_ignore_ascii_case(after) && *after == after.to_ascii_uppercase()) - })) + Ok(sql == printed || { + let (left, right) = (tokens(sql)?, tokens(printed)?); + left.len() == right.len() + && left.iter().zip(&right).all(|(before, after)| { + before == after + || (before.eq_ignore_ascii_case(after) && *after == after.to_ascii_uppercase()) + }) + }) } /// Check every corpus query of one dialect is left as written and reported unavailable; return how many. diff --git a/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs b/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs index 5053447..d164b67 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs @@ -21,6 +21,18 @@ fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stab expected_formatted: 426, expected_refused: 9, }, + test_types::FormatCorpusTestCase { + description: "BigQuery identity corpus", + dialect: "bigquery", + expected_formatted: 161, + expected_refused: 7, + }, + test_types::FormatCorpusTestCase { + description: "PostgreSQL identity corpus", + dialect: "postgres", + expected_formatted: 124, + expected_refused: 13, + }, test_types::FormatCorpusTestCase { description: "T-SQL identity corpus", dialect: "tsql", @@ -46,23 +58,11 @@ fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stab #[test] fn given_dialects_without_a_native_formatter_when_formatting_then_each_query_is_unavailable() -> Result<(), String> { - let test_cases = [ - test_types::UnavailableCorpusTestCase { - description: "BigQuery corpus", - dialect: "bigquery", - expected_unavailable: 168, - }, - test_types::UnavailableCorpusTestCase { - description: "Databricks corpus", - dialect: "databricks", - expected_unavailable: 62, - }, - test_types::UnavailableCorpusTestCase { - description: "PostgreSQL corpus", - dialect: "postgres", - expected_unavailable: 137, - }, - ]; + let test_cases = [test_types::UnavailableCorpusTestCase { + description: "Databricks corpus", + dialect: "databricks", + expected_unavailable: 62, + }]; let corpus: Vec = serde_json::from_str(FORMAT_CORPUS).map_err(|error| error.to_string())?; for test_case in &test_cases { diff --git a/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py b/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py index da8d208..e03a7af 100644 --- a/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py +++ b/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py @@ -291,6 +291,56 @@ def test_given_project_when_checking_then_reports_findings_and_exit_status( "models/q.sql": "WITH a AS (\n SELECT 1 AS x\n)\n\nSELECT x\nFROM a\n" }, ), + RewriteTestCase( + description="format splits a T-SQL batch written without semicolons into statements", + config='dialect = "tsql"\n', + files={"models/q.sql": "select a from t\nupdate t set a = 2\n"}, + arguments=("format",), + expected_exit=0, + expected_files={"models/q.sql": "SELECT a\nFROM t\n\nUPDATE t SET a = 2\n"}, + ), + RewriteTestCase( + description="format lays out a T-SQL procedure's body under the mssql alias", + config='dialect = "mssql"\n', + files={ + "models/p.sql": ( + "create procedure dbo.p @a int as begin set nocount on;" + " if @a > 1 select a from t where a = @a end\n" + ) + }, + arguments=("format",), + expected_exit=0, + expected_files={ + "models/p.sql": ( + "CREATE PROCEDURE dbo.p @a int\nAS\nBEGIN\n SET nocount ON;\n" + " IF @a > 1\n SELECT a\n FROM t\n WHERE a = @a\nEND\n" + ) + }, + ), + RewriteTestCase( + description="format leaves a Databricks file, which SQLQuack parses but does not format, unchanged", + config='dialect = "databricks"\n', + files={"models/q.sql": "select a from t\n"}, + arguments=("format", "--check"), + expected_exit=0, + expected_files={"models/q.sql": "select a from t\n"}, + ), + RewriteTestCase( + description="format lays out BigQuery and PostgreSQL files", + config='dialect = "bigquery"\n[[dialects]]\npaths = ["pg/**"]\ndialect = "postgresql"\n', + files={"models/q.sql": "select a from t\n", "pg/q.sql": "select a from t\n"}, + arguments=("format",), + expected_exit=0, + expected_files={"models/q.sql": "SELECT a\nFROM t\n", "pg/q.sql": "SELECT a\nFROM t\n"}, + ), + RewriteTestCase( + description="format reads the sqlserver alias as T-SQL", + config='dialect = "sqlserver"\n', + files={"models/q.sql": "select 1 select 2"}, + arguments=("format",), + expected_exit=0, + expected_files={"models/q.sql": "SELECT 1\n\nSELECT 2\n"}, + ), RewriteTestCase( description="a file of only comments keeps its final newline", config=DUCKDB, diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx index f9ac8f7..aebdb22 100644 --- a/website/src/content/docs/docs/configuration.mdx +++ b/website/src/content/docs/docs/configuration.mdx @@ -112,12 +112,13 @@ SELECT [order id] FROM orders error[SQLCLEAN999]: SQL could not be parsed as duckdb: Parse error at line 1, column 17: Expected RBracket, got Var ('id') near [SELECT [order id] FROM orders] ``` -SQLClean reads SQL with SQLQuack's parsers. `check` and `fix` read `duckdb`, `generic`, `postgres`, -`snowflake` and `sqlite` files, and `format` lays out `duckdb` and `snowflake` files; a `generic`, -`postgres` or `sqlite` file is checked and fixed but left as written by `format`. A file in any -other dialect, such as `mysql` or `tsql`, is not checked, fixed or formatted: `check` reports it once -as a warning (in JSON, under `unavailable`), `format` leaves it unchanged with a warning, and it -never fails a command or counts as a problem. +SQLClean reads SQL with SQLQuack's parsers. `check` and `fix` read `bigquery`, `databricks`, +`duckdb`, `generic`, `postgres`, `snowflake` and `sqlite` files, and `format` lays out `bigquery`, +`duckdb`, `postgres`, `snowflake` and `tsql` files. A `databricks`, `generic` or `sqlite` file is +checked and fixed but left as written by `format`; a `tsql` file is formatted, by `fix` too, but its +Rules are not checked yet. A file in any other dialect, such as `mysql`, is not checked, fixed or +formatted: `check` reports it once as a warning (in JSON, under `unavailable`), `format` leaves it +unchanged with a warning, and it never fails a command or counts as a problem. ```text warning: SQL lint is unavailable for the mysql dialect: SQLQuack does not parse it yet