From 0ab5dc689f7230f3cbdf0d4e0bb4ba1dbe418959 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:21:21 +0000 Subject: [PATCH 01/21] wip(format): lay out Snowflake and DuckDB with SQLQuack's formatter Snowflake and DuckDB requests now go to sqlquack-format, pinned by git revision, which lays out SQL on SQLQuack's own tokenizer and parser and checks token identity and idempotency itself. Other dialects still use the Polyglot layout oracle. Work in progress: 6 formatter tests still expect Polyglot's layout. --- Cargo.lock | 34 ++++++++++++++ Cargo.toml | 3 ++ crates/sqlclean/Cargo.toml | 2 + .../src/sql_lint/_helpers/formatter.rs | 45 ++++++++++++++++++- crates/sqlclean/src/sql_lint/constants.rs | 4 ++ 5 files changed, 87 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 1afe2d1..60dfd71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -628,6 +628,8 @@ dependencies = [ "serde", "serde_json", "sha2", + "sqlquack-format", + "sqlquack-syntax", ] [[package]] @@ -648,6 +650,38 @@ dependencies = [ "toml", ] +[[package]] +name = "sqlquack-format" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=babbc292ee09f298ba2280f69f87e16d8fcdfbe3#babbc292ee09f298ba2280f69f87e16d8fcdfbe3" +dependencies = [ + "serde_json", + "sqlquack-syntax", +] + +[[package]] +name = "sqlquack-syntax" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=babbc292ee09f298ba2280f69f87e16d8fcdfbe3#babbc292ee09f298ba2280f69f87e16d8fcdfbe3" +dependencies = [ + "serde", + "serde_json", + "sqlquack-syntax-derive", + "stacker", + "thiserror", + "unicode-segmentation", +] + +[[package]] +name = "sqlquack-syntax-derive" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=babbc292ee09f298ba2280f69f87e16d8fcdfbe3#babbc292ee09f298ba2280f69f87e16d8fcdfbe3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "stacker" version = "0.1.25" diff --git a/Cargo.toml b/Cargo.toml index 2604256..d903c96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,9 @@ serde_json = { version = "1", features = ["preserve_order"] } sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } +# SQLQuack's formatter, which lays out the dialects it parses natively without Polyglot. +sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "babbc292ee09f298ba2280f69f87e16d8fcdfbe3" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "babbc292ee09f298ba2280f69f87e16d8fcdfbe3" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sqlclean/Cargo.toml b/crates/sqlclean/Cargo.toml index f9dc9b9..9b69421 100644 --- a/crates/sqlclean/Cargo.toml +++ b/crates/sqlclean/Cargo.toml @@ -16,6 +16,8 @@ test-support = [] [dependencies] polyglot-sql.workspace = true polyglot-sql-function-catalogs.workspace = true +sqlquack-format.workspace = true +sqlquack-syntax.workspace = true rayon.workspace = true rustc-hash.workspace = true serde.workspace = true diff --git a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs index c881702..37c55ab 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs @@ -4,6 +4,8 @@ use std::str::FromStr; use polyglot_sql::tokens::{Token, TokenType}; use polyglot_sql::{ComplexityGuardOptions, Dialect, DialectType, ParseOptions}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; +use sqlquack_format::FormatOptions; +use sqlquack_syntax::Dialect as SqlQuackDialect; use crate::constants::INTERPOLATION_PLACEHOLDER_PREFIX; use crate::sql_lint::_helpers::formatter_syntax::{cte_macro_indices, protect_syntax}; @@ -13,7 +15,8 @@ use crate::sql_lint::_helpers::token_layout::{ oracle_layout, print_authored_tokens, tokenize, verify_printed_tokens, }; use crate::sql_lint::constants::{ - CAST_TYPE_SEPARATOR_KEYWORD, CLOSE_PARENTHESIS, LINT_API_VERSION, OPEN_PARENTHESIS, + CAST_TYPE_SEPARATOR_KEYWORD, CLOSE_PARENTHESIS, DUCKDB_DIALECT, LINT_API_VERSION, + OPEN_PARENTHESIS, SNOWFLAKE_DIALECT, }; use crate::sql_lint::models::{FormatRequest, FormatResponse}; @@ -74,6 +77,9 @@ fn try_format_sql(request: FormatRequest) -> Result { request.version )); } + if let Some(dialect) = sqlquack_dialect(&request.dialect) { + return format_with_sqlquack(request, dialect); + } let original = request.sql; let dialect_type = DialectType::from_str(&request.dialect).map_err(|error| error.to_string())?; @@ -108,6 +114,43 @@ fn try_format_sql(request: FormatRequest) -> Result { }) } +/// The SQLQuack dialect that lays out `dialect`, for the dialects SQLQuack parses natively. +fn sqlquack_dialect(dialect: &str) -> Option { + match dialect.to_ascii_lowercase().as_str() { + SNOWFLAKE_DIALECT => Some(SqlQuackDialect::Snowflake), + DUCKDB_DIALECT => Some(SqlQuackDialect::DuckDB), + _ => None, + } +} + +/// Lay out `request` with SQLQuack's formatter, which checks token identity and idempotency itself. +fn format_with_sqlquack( + request: FormatRequest, + dialect: SqlQuackDialect, +) -> Result { + let options = FormatOptions { + line_width: request.line_width, + indent: request.indent.unwrap_or(DEFAULT_INDENT), + token_widths: request.token_widths, + parse: sqlquack_syntax::ParseOptions { + complexity_guard: Some(sqlquack_syntax::ComplexityGuardOptions { + max_function_call_depth: request.max_function_call_depth, + ..Default::default() + }), + }, + ..FormatOptions::new(dialect) + }; + let formatted = + sqlquack_format::format_sql(&request.sql, &options).map_err(|refusal| refusal.reason)?; + Ok(FormatResponse { + version: LINT_API_VERSION, + changed: formatted.changed, + sql: formatted.sql, + formatted: true, + reason: None, + }) +} + /// The printed output of [`format_layout`], its tokens, and the oracle it was printed against. type FormattedLayout = (String, Vec, Option); diff --git a/crates/sqlclean/src/sql_lint/constants.rs b/crates/sqlclean/src/sql_lint/constants.rs index 908275a..f1a2948 100644 --- a/crates/sqlclean/src/sql_lint/constants.rs +++ b/crates/sqlclean/src/sql_lint/constants.rs @@ -72,3 +72,7 @@ pub(crate) const SELECT_ITEM_END_TOKENS: [TokenType; 8] = [ TokenType::Identifier, TokenType::QuotedIdentifier, ]; +/// The dialect name SQLQuack lays out as Snowflake. +pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake"; +/// The dialect name SQLQuack lays out as DuckDB. +pub(crate) const DUCKDB_DIALECT: &str = "duckdb"; From 63c4b4aee8792e992cb34f61b167f10626c19600 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:01:03 +0000 Subject: [PATCH 02/21] wip(format): gate Snowflake and DuckDB on SQLQuack's layout, and drop the Polyglot AST API SQLClean's corpus check now expects SQLQuack's named departures, listed with their reasons in sqlquack_departures.json: 15 DuckDB and 13 Snowflake queries that SQLQuack lays out or refuses differently. Called names fold to upper case as SQLBuild's query fingerprint folds them, except in unaliased Snowflake select items, whose text names their column. The re-print count test runs on PostgreSQL, which still uses the Polyglot layout. RuleContext.polyglot_ast() and the ast extra are removed. Work in progress: CTE macro placeholders are not laid out yet. --- Cargo.lock | 6 +- Cargo.toml | 4 +- .../docs/custom-rules/statement-facts.md | 4 - .../tests/fixtures/FORMAT_CORPUS_NOTICE.md | 4 + .../tests/fixtures/sqlquack_departures.json | 198 ++++++++++++++++++ crates/sqlclean/src/sql_lint/tests/helpers.rs | 24 ++- .../sql_lint/tests/test_authored_tokens.rs | 4 +- .../sql_lint/tests/test_comment_formatter.rs | 2 +- .../src/sql_lint/tests/test_format_corpus.rs | 8 +- .../tests/test_idempotency_reprint.rs | 2 +- .../src/sql_lint/tests/test_sql_lint.rs | 10 +- pyproject.toml | 6 - src/sqlclean/rules/constants.py | 2 - src/sqlclean/rules/models.py | 20 -- uv.lock | 26 +-- .../docs/custom-rules/statement-facts.mdx | 4 - 16 files changed, 243 insertions(+), 81 deletions(-) create mode 100644 crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json diff --git a/Cargo.lock b/Cargo.lock index 60dfd71..6572ef3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -653,7 +653,7 @@ dependencies = [ [[package]] name = "sqlquack-format" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=babbc292ee09f298ba2280f69f87e16d8fcdfbe3#babbc292ee09f298ba2280f69f87e16d8fcdfbe3" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=f2985b374694f3da56c4f1e5b36fbc5d81e101ef#f2985b374694f3da56c4f1e5b36fbc5d81e101ef" dependencies = [ "serde_json", "sqlquack-syntax", @@ -662,7 +662,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=babbc292ee09f298ba2280f69f87e16d8fcdfbe3#babbc292ee09f298ba2280f69f87e16d8fcdfbe3" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=f2985b374694f3da56c4f1e5b36fbc5d81e101ef#f2985b374694f3da56c4f1e5b36fbc5d81e101ef" dependencies = [ "serde", "serde_json", @@ -675,7 +675,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-derive" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=babbc292ee09f298ba2280f69f87e16d8fcdfbe3#babbc292ee09f298ba2280f69f87e16d8fcdfbe3" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=f2985b374694f3da56c4f1e5b36fbc5d81e101ef#f2985b374694f3da56c4f1e5b36fbc5d81e101ef" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index d903c96..62634da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,8 +41,8 @@ sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } # SQLQuack's formatter, which lays out the dialects it parses natively without Polyglot. -sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "babbc292ee09f298ba2280f69f87e16d8fcdfbe3" } -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "babbc292ee09f298ba2280f69f87e16d8fcdfbe3" } +sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "f2985b374694f3da56c4f1e5b36fbc5d81e101ef" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "f2985b374694f3da56c4f1e5b36fbc5d81e101ef" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sqlclean-cli/skill/references/docs/custom-rules/statement-facts.md b/crates/sqlclean-cli/skill/references/docs/custom-rules/statement-facts.md index b029079..c5d0b00 100644 --- a/crates/sqlclean-cli/skill/references/docs/custom-rules/statement-facts.md +++ b/crates/sqlclean-cli/skill/references/docs/custom-rules/statement-facts.md @@ -79,10 +79,6 @@ One statement of a file, without its terminating semicolon. Each tuple is in source order and includes nodes inside CTEs and subqueries. -`statement.polyglot_ast()` returns the full Polyglot syntax tree of the statement, parsed on first -use. It needs the `ast` extra: `pip install "sqlclean[ast]"`. Without it, the call fails with -`polyglot_ast() needs the Polyglot SQL package: install the ast extra: pip install 'sqlclean[ast]'`. - ## Cte A common table expression, located at its name. diff --git a/crates/sqlclean/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md b/crates/sqlclean/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md index 39e0868..11a5ab1 100644 --- a/crates/sqlclean/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md +++ b/crates/sqlclean/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md @@ -30,3 +30,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` + +`sqlquack_departures.json` holds the corpus queries that SQLQuack's formatter lays out or refuses +differently from the layouts in `format_corpus.json`, with the reason for each. Its queries come from +the same SQLGlot fixtures, under the same licence. diff --git a/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json b/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json new file mode 100644 index 0000000..4f2cfa4 --- /dev/null +++ b/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json @@ -0,0 +1,198 @@ +[ + { + "dialect": "duckdb", + "sql": "SELECT str[0:1]", + "formatted": "SELECT str[0:1]", + "reason": "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x, y : COALESCE(x, y, 0) + 1)", + "formatted": "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x, y: COALESCE(x, y, 0) + 1)", + "reason": "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM(LIST_FILTER([0, 1, 2, 3, 4, 5], LAMBDA x : x % 2 = 0), LAMBDA y : y * y)", + "formatted": "SELECT LIST_TRANSFORM(LIST_FILTER([0, 1, 2, 3, 4, 5], LAMBDA x: x % 2 = 0), LAMBDA y: y * y)", + "reason": "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x : COALESCE(x, 0) + 1)", + "formatted": "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x: COALESCE(x, 0) + 1)", + "reason": "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM(nbr, LAMBDA x : x + 1) FROM article AS a", + "formatted": "SELECT LIST_TRANSFORM(nbr, LAMBDA x: x + 1)\nFROM article AS a", + "reason": "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT JSON_EXTRACT(c, '$[*].id')[0:2]", + "formatted": "SELECT JSON_EXTRACT(c, '$[*].id')[0:2]", + "reason": "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT JSON_EXTRACT_STRING(c, '$[*].id')[0:2]", + "formatted": "SELECT JSON_EXTRACT_STRING(c, '$[*].id')[0:2]", + "reason": "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT [1, 2, 3][1 + 1:LENGTH([1, 2, 3]) + -1]", + "formatted": "SELECT [1, 2, 3][1 + 1:LENGTH([1, 2, 3]) + -1]", + "reason": "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT x::INT[3][3]", + "formatted": "SELECT x::INT[3][3]", + "reason": "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT ARRAY[[[1]]]::INT[1][1][1]", + "formatted": "SELECT ARRAY[[[1]]]::INT[1][1][1]", + "reason": "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "WITH t1 AS (FROM (FROM t2 SELECT foo1, foo2)) FROM t1", + "formatted": "WITH t1 AS (\n FROM (\n FROM t2\n SELECT\n foo1,\n foo2\n )\n)\n\nFROM t1", + "reason": "SQLClean aligned against Polyglot's SELECT-first rewrite of the FROM-first query", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT e: 1 + 2, f: len('asdf'), s: (SELECT 42)", + "formatted": "SELECT\n e:1 + 2,\n f:len('asdf'),\n s:(\n SELECT 42\n )", + "reason": "SQLClean aligned against Polyglot's path reading of the prefix alias and lost the line break after a comma", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT (@col) + 1 FROM t", + "formatted": "SELECT\n (\n @col\n )\n + 1\nFROM t", + "reason": "SQLClean aligned against Polyglot's ABS() rewrite of `@`", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT (@-1) + 1", + "formatted": "SELECT\n (\n @-1\n )\n + 1", + "reason": "SQLClean aligned against Polyglot's ABS() rewrite of `@`", + "receipt": null + }, + { + "dialect": "duckdb", + "sql": "SELECT file[:256] FROM GLOB('*')", + "formatted": "SELECT file[:256]\nFROM GLOB('*')", + "reason": "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + "receipt": null + }, + { + "dialect": "snowflake", + "sql": "WITH t (SELECT 1 AS c) SELECT c FROM t", + "formatted": null, + "reason": "Snowflake rejects a CTE body without AS, so SQLQuack's parser refuses it", + "receipt": "sf.query.with.select.k8b29420d51a4" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_PART('month' FROM CAST('2024-04-08' AS DATE))", + "formatted": null, + "reason": "Snowflake rejects a quoted date part in the FROM form of DATE_PART, so SQLQuack's parser refuses it", + "receipt": "sf.functions.date-part.date-part-from.k10b598e047b5" + }, + { + "dialect": "snowflake", + "sql": "WITH t AS (SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data) SELECT data: level1 : level2 : level3::VARIANT FROM t", + "formatted": "WITH t AS (\n SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data\n)\n\nSELECT data: level1 : level2 : level3::VARIANT\nFROM t", + "reason": "Snowflake names an unaliased select item by its text, spaces included, so the item keeps its spacing; SQLClean respaced it", + "receipt": "cases/relational/unaliased_output_names.toml#unaliased_expressions" + }, + { + "dialect": "snowflake", + "sql": "SELECT RPAD(TO_BINARY('Hi', 'UTF8'), 10, TO_BINARY('_', 'UTF8'))", + "formatted": "SELECT\n RPAD(TO_BINARY('Hi', 'UTF8'), 10, TO_BINARY('_', 'UTF8'))", + "reason": "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + "receipt": "cases/relational/unaliased_output_names.toml#unaliased_expressions" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_FROM_PARTS(CAST('2024-05-09' AS DATE), CAST('14:30:45' AS TIME))", + "formatted": "SELECT\n TIMESTAMP_FROM_PARTS(CAST('2024-05-09' AS DATE), CAST('14:30:45' AS TIME))", + "reason": "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + "receipt": "cases/relational/unaliased_output_names.toml#unaliased_expressions" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAYS_ZIP([1, 2, 3], ['a', 'b', 'c'], [10, 20, 30])", + "formatted": "SELECT\n ARRAYS_ZIP([1, 2, 3], ['a', 'b', 'c'], [10, 20, 30])", + "reason": "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + "receipt": "cases/relational/unaliased_output_names.toml#unaliased_expressions" + }, + { + "dialect": "snowflake", + "sql": "SELECT AI_CLASSIFY('text', ['travel', 'cooking'], OBJECT_CONSTRUCT('output_mode', 'multi'))", + "formatted": "SELECT\n AI_CLASSIFY('text', ['travel', 'cooking'], OBJECT_CONSTRUCT('output_mode', 'multi'))", + "reason": "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + "receipt": "cases/relational/unaliased_output_names.toml#unaliased_expressions" + }, + { + "dialect": "snowflake", + "sql": "SELECT {fn DAYNAME('2022-5-13')}", + "formatted": "SELECT {fn DAYNAME('2022-5-13')}", + "reason": "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean broke a line inside its `{fn ...}` escape", + "receipt": "cases/relational/unaliased_output_names.toml#unaliased_expressions" + }, + { + "dialect": "snowflake", + "sql": "SELECT {fn LOG(5)}", + "formatted": "SELECT {fn LOG(5)}", + "reason": "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean broke a line inside its `{fn ...}` escape", + "receipt": "cases/relational/unaliased_output_names.toml#unaliased_expressions" + }, + { + "dialect": "snowflake", + "sql": "SELECT {fn CEILING(5.3)}", + "formatted": "SELECT {fn CEILING(5.3)}", + "reason": "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean broke a line inside its `{fn ...}` escape", + "receipt": "cases/relational/unaliased_output_names.toml#unaliased_expressions" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 1, 'HOUR', 'start')", + "formatted": "SELECT\n TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 1, 'HOUR', 'start')", + "reason": "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + "receipt": "cases/relational/unaliased_output_names.toml#unaliased_expressions" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER', SEARCH_MODE => 'OR')", + "formatted": "SELECT\n SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER', SEARCH_MODE => 'OR')", + "reason": "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + "receipt": "cases/relational/unaliased_output_names.toml#unaliased_expressions" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(line, 'king', SEARCH_MODE => 'AND', ANALYZER => 'PATTERN_ANALYZER')", + "formatted": "SELECT\n SEARCH(line, 'king', SEARCH_MODE => 'AND', ANALYZER => 'PATTERN_ANALYZER')", + "reason": "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + "receipt": "cases/relational/unaliased_output_names.toml#unaliased_expressions" + } +] \ No newline at end of file diff --git a/crates/sqlclean/src/sql_lint/tests/helpers.rs b/crates/sqlclean/src/sql_lint/tests/helpers.rs index d3c11d2..9c2f683 100644 --- a/crates/sqlclean/src/sql_lint/tests/helpers.rs +++ b/crates/sqlclean/src/sql_lint/tests/helpers.rs @@ -66,6 +66,23 @@ fn diagnostic_values(response: &str) -> Result, String> { .ok_or_else(|| "diagnostics should be an array".to_string()) } +/// Corpus queries SQLQuack's formatter lays out or refuses differently from SQLClean's Polyglot layout, each with its reason. +const SQLQUACK_DEPARTURES: &str = include_str!("fixtures/sqlquack_departures.json"); + +/// The expected layout of a corpus `entry`: SQLQuack's named departure when it has one, else the recorded layout. +fn expected_layout(entry: &Value) -> Value { + let departures: Vec = serde_json::from_str(SQLQUACK_DEPARTURES).unwrap_or_default(); + departures + .into_iter() + .find(|departure| { + departure["dialect"] == entry["dialect"] && departure["sql"] == entry["sql"] + }) + .map_or_else( + || entry["formatted"].clone(), + |departure| departure["formatted"].clone(), + ) +} + /// A narrow width so that wrapping is exercised on most corpus queries. const CORPUS_LINE_WIDTH: usize = 40; @@ -84,7 +101,7 @@ pub(crate) fn check_dialect_corpus( .filter(|_| formatted) .cloned() .unwrap_or(Value::Null); - let expected = entry["formatted"].as_str().map_or(Value::Null, Value::from); + let expected = expected_layout(entry); assert_eq!(actual, expected, "{dialect}: {sql}"); let printed = actual.as_str().unwrap_or(sql); assert!( @@ -95,10 +112,7 @@ pub(crate) fn check_dialect_corpus( .as_str() .map(|previous| relaid_previous_layout(previous, dialect)) .transpose()?; - let layouts = [ - entry["formatted"].as_str().map(str::to_string), - previous_layout, - ]; + let layouts = [expected.as_str().map(str::to_string), previous_layout]; for stable in layouts.iter().flatten() { let stable = stable.as_str(); let again = format_once(stable, dialect, None)?; diff --git a/crates/sqlclean/src/sql_lint/tests/test_authored_tokens.rs b/crates/sqlclean/src/sql_lint/tests/test_authored_tokens.rs index cb29e9c..dcc8d0a 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_authored_tokens.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_authored_tokens.rs @@ -13,9 +13,9 @@ fn given_generator_respellings_when_formatting_then_authored_spellings_are_kept( -> Result<(), String> { let test_cases = [ test_types::FormatTestCase { - description: "Snowflake STARTSWITH keeps its authored name", + description: "Snowflake STARTSWITH keeps its authored spelling and folds to upper case like every called name", sql: "select startswith(order_code, 'EU') as is_eu_order from orders", - expected_sql: "SELECT startswith(order_code, 'EU') AS is_eu_order\nFROM orders", + expected_sql: "SELECT STARTSWITH(order_code, 'EU') AS is_eu_order\nFROM orders", expected_changed: true, }, test_types::FormatTestCase { diff --git a/crates/sqlclean/src/sql_lint/tests/test_comment_formatter.rs b/crates/sqlclean/src/sql_lint/tests/test_comment_formatter.rs index a7496a2..579ebec 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_comment_formatter.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_comment_formatter.rs @@ -65,7 +65,7 @@ fn given_trailing_comments_after_the_last_token_when_formatting_then_no_newline_ test_types::FormatTestCase { description: "line comments after the last token keep no trailing newline", sql: "select 1 as order_id from orders\n\n-- first note\n\n-- second note", - expected_sql: "SELECT 1 AS order_id\nFROM orders -- first note\n -- second note", + expected_sql: "SELECT 1 AS order_id\nFROM orders -- first note\n-- second note", expected_changed: true, }, test_types::FormatTestCase { diff --git a/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs b/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs index c40d2ff..ebbd2bb 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs @@ -24,8 +24,8 @@ fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stab test_types::FormatCorpusTestCase { description: "DuckDB identity corpus", dialect: "duckdb", - expected_formatted: 178, - expected_refused: 11, + expected_formatted: 189, + expected_refused: 0, }, test_types::FormatCorpusTestCase { description: "PostgreSQL identity corpus", @@ -36,8 +36,8 @@ fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stab test_types::FormatCorpusTestCase { description: "Snowflake identity corpus", dialect: "snowflake", - expected_formatted: 429, - expected_refused: 6, + expected_formatted: 427, + expected_refused: 8, }, test_types::FormatCorpusTestCase { description: "T-SQL identity corpus", diff --git a/crates/sqlclean/src/sql_lint/tests/test_idempotency_reprint.rs b/crates/sqlclean/src/sql_lint/tests/test_idempotency_reprint.rs index edf70b5..c3176dd 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_idempotency_reprint.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_idempotency_reprint.rs @@ -24,7 +24,7 @@ fn given_formatted_and_unformatted_bodies_when_formatting_then_only_changed_bodi for test_case in &test_cases { RELAYOUTS.with(|relayouts| relayouts.set(0)); let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": "duckdb"}).to_string(), + &json!({"version": 1, "sql": test_case.sql, "dialect": "postgres"}).to_string(), )?; let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; assert_eq!( diff --git a/crates/sqlclean/src/sql_lint/tests/test_sql_lint.rs b/crates/sqlclean/src/sql_lint/tests/test_sql_lint.rs index 3b72db4..b4f8b4e 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_sql_lint.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_sql_lint.rs @@ -1825,9 +1825,15 @@ fn given_format_cases_when_formatting_then_output_matches() -> Result<(), String expected_changed: true, }, test_types::FormatTestCase { - description: "lowercase function canonicalization preserves structure", + description: "an unaliased Snowflake select item keeps its authored text, which names its column", sql: "select count(1) from items", - expected_sql: "SELECT COUNT(1)\nFROM items", + expected_sql: "SELECT count(1)\nFROM items", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "an aliased call folds to upper case", + sql: "select count(1) as n from items", + expected_sql: "SELECT COUNT(1) AS n\nFROM items", expected_changed: true, }, test_types::FormatTestCase { diff --git a/pyproject.toml b/pyproject.toml index 954000e..189abf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,11 +25,6 @@ dependencies = [ "sqlclean-cli==0.4.3", # x-release-please-version ] -[project.optional-dependencies] -ast = [ - "polyglot-sql-chio==0.13.10", -] - [project.urls] Homepage = "https://github.com/sqlstack-io/sqlclean" Issues = "https://github.com/sqlstack-io/sqlclean/issues" @@ -42,7 +37,6 @@ packages = ["src/sqlclean"] dev = [ "commitizen>=4.9.1", "fensu>=0.28.0", - "polyglot-sql-chio==0.13.10", # optional AST API "pre-commit>=4.3.0", "pytest>=8.4.2", "pytest-xdist>=3.8.0", diff --git a/src/sqlclean/rules/constants.py b/src/sqlclean/rules/constants.py index c1bd270..41cc266 100644 --- a/src/sqlclean/rules/constants.py +++ b/src/sqlclean/rules/constants.py @@ -8,5 +8,3 @@ "must accept exactly one typed Statement, SqlFile or Repository subject and one typed " "RuleContext, both keyword-only" ) -POLYGLOT_MODULE: str = "polyglot_sql" -POLYGLOT_INSTALL_HINT: str = "install the ast extra: pip install 'sqlclean[ast]'" diff --git a/src/sqlclean/rules/models.py b/src/sqlclean/rules/models.py index ab8c093..2ef7c62 100644 --- a/src/sqlclean/rules/models.py +++ b/src/sqlclean/rules/models.py @@ -1,13 +1,8 @@ """Structured values Rules receive and return.""" from dataclasses import dataclass, field -from functools import cached_property -from importlib import import_module -from types import ModuleType from sqlclean.rules.classes.rule_option import RuleOption -from sqlclean.rules.constants import POLYGLOT_INSTALL_HINT, POLYGLOT_MODULE -from sqlclean.rules.exceptions import RuleUsageError from sqlclean.rules.types import FixSafety, RuleCheck, RuleSubjectKind @@ -143,21 +138,6 @@ class Statement: columns: tuple[ColumnReference, ...] literals: tuple[Literal, ...] - def polyglot_ast(self) -> object: - """Return Polyglot's syntax tree of this statement, parsed on first use.""" - - return self._polyglot_ast - - @cached_property - def _polyglot_ast(self) -> object: - try: - polyglot: ModuleType = import_module(POLYGLOT_MODULE) - except ImportError as error: - raise RuleUsageError( - f"polyglot_ast() needs the Polyglot SQL package: {POLYGLOT_INSTALL_HINT}" - ) from error - return polyglot.parse_one(self.text, dialect=self.dialect) - @dataclass(frozen=True) class SqlFile: diff --git a/uv.lock b/uv.lock index f60a8d1..1aaacbd 100644 --- a/uv.lock +++ b/uv.lock @@ -439,19 +439,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, ] -[[package]] -name = "polyglot-sql-chio" -version = "0.13.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/8c/b48d0fcca36c2e6df0a2ab8aa24e029373264b6dfa9db683eb533f6b6249/polyglot_sql_chio-0.13.10.tar.gz", hash = "sha256:4365617889c3779fd1a3ebdd15ab03c92c517cb1be36dedf82e5853b31d39781", size = 2279296, upload-time = "2026-10-28T23:07:03.083Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/7a/c66b36a40c0931d9647215f315f90e2e0d83328165dce7275d624f694381/polyglot_sql_chio-0.13.10-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:a1abe78cde4b54d145f42356ca4c3a7585775aa4efee954209084ea993633f90", size = 13701198, upload-time = "2026-10-28T23:06:50.006Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e2/b2ba58e445163b9e29a715d9e3eab5fd9e276e8bbe58f5710bd77aae70d7/polyglot_sql_chio-0.13.10-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:a7d01559916bff555fd0c258f4a52a98e3f2fa25b1f4c482ef35cb868704c3b6", size = 12941723, upload-time = "2026-10-28T23:06:52.738Z" }, - { url = "https://files.pythonhosted.org/packages/c6/08/a9e5cb57c9938551edeb0884cc1b57ef54bbb38eb5f371bb146b663881ab/polyglot_sql_chio-0.13.10-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e5ee843dccffe87501113238ba333c17dce55b31867fcdc1b81e245b9c3eb4", size = 12895624, upload-time = "2026-10-28T23:06:55.52Z" }, - { url = "https://files.pythonhosted.org/packages/a0/9e/0879b3a267899906c231270d0167d8c32c14dcc2ae346b86834c4f7bb8b2/polyglot_sql_chio-0.13.10-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd2ca982a19c7fbeb460914eceae60d9c5f7e6e631473f0141585b778772c1ce", size = 12661001, upload-time = "2026-10-28T23:06:58.287Z" }, - { url = "https://files.pythonhosted.org/packages/e7/13/dbdbce022153042edb59abe07e64c51a0e0a9a8f4c86cf2043f1c42a1da7/polyglot_sql_chio-0.13.10-cp312-abi3-win_amd64.whl", hash = "sha256:f366391d63fbf455f71c07fb03e4ac1cf860609de7fd875759647ffe98659320", size = 12301030, upload-time = "2026-10-28T23:07:00.945Z" }, -] - [[package]] name = "pre-commit" version = "4.7.0" @@ -621,16 +608,10 @@ dependencies = [ { name = "sqlclean-cli" }, ] -[package.optional-dependencies] -ast = [ - { name = "polyglot-sql-chio" }, -] - [package.dev-dependencies] dev = [ { name = "commitizen" }, { name = "fensu" }, - { name = "polyglot-sql-chio" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-xdist" }, @@ -639,17 +620,12 @@ dev = [ ] [package.metadata] -requires-dist = [ - { name = "polyglot-sql-chio", marker = "extra == 'ast'", specifier = "==0.13.10" }, - { name = "sqlclean-cli", directory = "packages/sqlclean-cli" }, -] -provides-extras = ["ast"] +requires-dist = [{ name = "sqlclean-cli", directory = "packages/sqlclean-cli" }] [package.metadata.requires-dev] dev = [ { name = "commitizen", specifier = ">=4.9.1" }, { name = "fensu", specifier = ">=0.28.0" }, - { name = "polyglot-sql-chio", specifier = "==0.13.10" }, { name = "pre-commit", specifier = ">=4.3.0" }, { name = "pytest", specifier = ">=8.4.2" }, { name = "pytest-xdist", specifier = ">=3.8.0" }, diff --git a/website/src/content/docs/docs/custom-rules/statement-facts.mdx b/website/src/content/docs/docs/custom-rules/statement-facts.mdx index 50138a1..4d9e983 100644 --- a/website/src/content/docs/docs/custom-rules/statement-facts.mdx +++ b/website/src/content/docs/docs/custom-rules/statement-facts.mdx @@ -62,10 +62,6 @@ One statement of a file, without its terminating semicolon. Each tuple is in source order and includes nodes inside CTEs and subqueries. -`statement.polyglot_ast()` returns the full Polyglot syntax tree of the statement, parsed on first -use. It needs the `ast` extra: `pip install "sqlclean[ast]"`. Without it, the call fails with -`polyglot_ast() needs the Polyglot SQL package: install the ast extra: pip install 'sqlclean[ast]'`. - ## Cte A common table expression, located at its name. From 7c7abfc9239d2d2efbcf30dbed05acab07d5d1d7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:40:43 +0000 Subject: [PATCH 03/21] wip(lint): lint and collect statement facts with SQLQuack instead of Polyglot The lint entry points (lint, lint_json, lint_batch_json, parse_sql, catalogue, default_rules, cross_join_subject, fill_finding_path and validate_dialect) now call sqlquack-lint, the port of this lint engine and its quality Rules onto SQLQuack's tokens and trees. The lint request, response and parse types are SQLQuack's, named in sql_lint::types; the format types stay in sql_lint::models. The placeholder constants take sqlquack-lint's values. sql_quality is removed: its Rules run inside sqlquack-lint. sql_facts reads SQLQuack's tokens and trees through parse_sql, with the same collection code. Lint for a dialect SQLQuack does not parse yet, such as T-SQL or MySQL, answers that lint is unavailable rather than reporting findings, so the T-SQL bare UNION test is ignored until that parser lands. The formatter is unchanged and still reads Polyglot; the one lint helper it used, token_depths, stays in sql_lint/_helpers/engine.rs. SQLQuack is pinned at wip/lint-port 4f9ac36d8. --- Cargo.lock | 37 + Cargo.toml | 3 + .../src/checking/_helpers/builtin.rs | 2 +- crates/sqlclean/Cargo.toml | 2 + crates/sqlclean/src/constants.rs | 27 +- crates/sqlclean/src/lib.rs | 3 +- .../src/sql_facts/_helpers/collection.rs | 9 +- .../src/sql_facts/_helpers/statements.rs | 2 +- .../sqlclean/src/sql_facts/_helpers/tokens.rs | 2 +- .../src/sql_lint/_helpers/additional.rs | 1886 ---------------- .../sqlclean/src/sql_lint/_helpers/engine.rs | 1936 +---------------- .../src/sql_lint/_helpers/inline_relations.rs | 223 -- .../src/sql_lint/_helpers/join_predicates.rs | 734 ------- crates/sqlclean/src/sql_lint/_helpers/mod.rs | 5 - .../sqlclean/src/sql_lint/_helpers/parsing.rs | 44 - .../src/sql_lint/_helpers/terminal_shape.rs | 373 ---- crates/sqlclean/src/sql_lint/constants.rs | 67 - .../src/sql_lint/main/batch_engine.rs | 19 +- .../sqlclean/src/sql_lint/main/catalogue.rs | 6 +- .../src/sql_lint/main/cross_join_subject.rs | 10 +- .../src/sql_lint/main/default_rules.rs | 2 +- crates/sqlclean/src/sql_lint/main/engine.rs | 5 +- .../src/sql_lint/main/fill_finding_path.rs | 4 +- crates/sqlclean/src/sql_lint/main/lint.rs | 4 +- .../sqlclean/src/sql_lint/main/parse_sql.rs | 6 +- .../src/sql_lint/main/validate_dialect.rs | 13 +- crates/sqlclean/src/sql_lint/mod.rs | 3 +- crates/sqlclean/src/sql_lint/models.rs | 180 +- .../src/sql_lint/tests/test_sql_lint.rs | 1 + crates/sqlclean/src/sql_lint/types.rs | 20 + .../src/sql_quality/_helpers/cte_columns.rs | 457 ---- .../src/sql_quality/_helpers/json_parses.rs | 201 -- .../sqlclean/src/sql_quality/_helpers/keys.rs | 437 ---- .../src/sql_quality/_helpers/literals.rs | 69 - .../src/sql_quality/_helpers/ranking.rs | 218 -- .../sql_quality/_helpers/removal_safety.rs | 102 - .../src/sql_quality/_helpers/safety.rs | 709 ------ .../src/sql_quality/_helpers/setting_help.rs | 26 - .../src/sql_quality/_helpers/syntax.rs | 124 -- crates/sqlclean/src/sql_quality/constants.rs | 228 -- .../src/sql_quality/main/diagnostics.rs | 33 - crates/sqlclean/src/sql_quality/main/mod.rs | 1 - crates/sqlclean/src/sql_quality/mod.rs | 27 - crates/sqlclean/src/sql_quality/models.rs | 22 - crates/sqlclean/src/sql_quality/tests.rs | 12 - .../sqlclean/src/sql_quality/tests/helpers.rs | 135 -- .../src/sql_quality/tests/test_json_parses.rs | 282 --- .../sql_quality/tests/test_quality_rules.rs | 435 ---- .../sql_quality/tests/test_quality_scale.rs | 43 - .../sql_quality/tests/test_safety_rules.rs | 207 -- .../src/sql_quality/tests/test_types.rs | 49 - crates/sqlclean/src/sql_quality/types.rs | 6 - fensu.toml | 5 + 53 files changed, 110 insertions(+), 9346 deletions(-) delete mode 100644 crates/sqlclean/src/sql_lint/_helpers/additional.rs delete mode 100644 crates/sqlclean/src/sql_lint/_helpers/inline_relations.rs delete mode 100644 crates/sqlclean/src/sql_lint/_helpers/join_predicates.rs delete mode 100644 crates/sqlclean/src/sql_lint/_helpers/parsing.rs delete mode 100644 crates/sqlclean/src/sql_lint/_helpers/terminal_shape.rs create mode 100644 crates/sqlclean/src/sql_lint/types.rs delete mode 100644 crates/sqlclean/src/sql_quality/_helpers/cte_columns.rs delete mode 100644 crates/sqlclean/src/sql_quality/_helpers/json_parses.rs delete mode 100644 crates/sqlclean/src/sql_quality/_helpers/keys.rs delete mode 100644 crates/sqlclean/src/sql_quality/_helpers/literals.rs delete mode 100644 crates/sqlclean/src/sql_quality/_helpers/ranking.rs delete mode 100644 crates/sqlclean/src/sql_quality/_helpers/removal_safety.rs delete mode 100644 crates/sqlclean/src/sql_quality/_helpers/safety.rs delete mode 100644 crates/sqlclean/src/sql_quality/_helpers/setting_help.rs delete mode 100644 crates/sqlclean/src/sql_quality/_helpers/syntax.rs delete mode 100644 crates/sqlclean/src/sql_quality/constants.rs delete mode 100644 crates/sqlclean/src/sql_quality/main/diagnostics.rs delete mode 100644 crates/sqlclean/src/sql_quality/main/mod.rs delete mode 100644 crates/sqlclean/src/sql_quality/mod.rs delete mode 100644 crates/sqlclean/src/sql_quality/models.rs delete mode 100644 crates/sqlclean/src/sql_quality/tests.rs delete mode 100644 crates/sqlclean/src/sql_quality/tests/helpers.rs delete mode 100644 crates/sqlclean/src/sql_quality/tests/test_json_parses.rs delete mode 100644 crates/sqlclean/src/sql_quality/tests/test_quality_rules.rs delete mode 100644 crates/sqlclean/src/sql_quality/tests/test_quality_scale.rs delete mode 100644 crates/sqlclean/src/sql_quality/tests/test_safety_rules.rs delete mode 100644 crates/sqlclean/src/sql_quality/tests/test_types.rs delete mode 100644 crates/sqlclean/src/sql_quality/types.rs diff --git a/Cargo.lock b/Cargo.lock index 1afe2d1..e2f644a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -628,6 +628,8 @@ dependencies = [ "serde", "serde_json", "sha2", + "sqlquack-lint", + "sqlquack-syntax", ] [[package]] @@ -648,6 +650,41 @@ dependencies = [ "toml", ] +[[package]] +name = "sqlquack-lint" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db#4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db" +dependencies = [ + "rayon", + "rustc-hash", + "serde", + "serde_json", + "sqlquack-syntax", +] + +[[package]] +name = "sqlquack-syntax" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db#4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db" +dependencies = [ + "serde", + "serde_json", + "sqlquack-syntax-derive", + "stacker", + "thiserror", + "unicode-segmentation", +] + +[[package]] +name = "sqlquack-syntax-derive" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db#4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "stacker" version = "0.1.25" diff --git a/Cargo.toml b/Cargo.toml index 2604256..ca72390 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,9 @@ serde_json = { version = "1", features = ["preserve_order"] } sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } +# SQLQuack's lint engine and the tokens and trees it reads, which replace Polyglot for lint and facts. +sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sqlclean-cli/src/checking/_helpers/builtin.rs b/crates/sqlclean-cli/src/checking/_helpers/builtin.rs index fa51aae..3ca621f 100644 --- a/crates/sqlclean-cli/src/checking/_helpers/builtin.rs +++ b/crates/sqlclean-cli/src/checking/_helpers/builtin.rs @@ -5,7 +5,7 @@ use std::collections::BTreeSet; use sqlclean::sql_lint::constants::LINT_API_VERSION; use sqlclean::sql_lint::main::fill_finding_path::fill_finding_path; use sqlclean::sql_lint::main::lint::lint; -use sqlclean::sql_lint::models::{LintDiagnostic, LintRequest}; +use sqlclean::sql_lint::types::{LintDiagnostic, LintRequest}; use crate::checking::constants::{ PARSE_ERROR_CODE, PARSE_ERROR_MESSAGE, PARSE_ERROR_REMEDIATION, UNSAFE_FIX_BUILTIN_RULES, diff --git a/crates/sqlclean/Cargo.toml b/crates/sqlclean/Cargo.toml index f9dc9b9..9a0a2b3 100644 --- a/crates/sqlclean/Cargo.toml +++ b/crates/sqlclean/Cargo.toml @@ -16,6 +16,8 @@ test-support = [] [dependencies] polyglot-sql.workspace = true polyglot-sql-function-catalogs.workspace = true +sqlquack-lint.workspace = true +sqlquack-syntax.workspace = true rayon.workspace = true rustc-hash.workspace = true serde.workspace = true diff --git a/crates/sqlclean/src/constants.rs b/crates/sqlclean/src/constants.rs index 068c43d..edbe621 100644 --- a/crates/sqlclean/src/constants.rs +++ b/crates/sqlclean/src/constants.rs @@ -1,25 +1,14 @@ -//! Constants shared by the engine modules, and the placeholders hosts write for template code. +//! The placeholders hosts write for template code, the same values SQLQuack's lint engine reads. + +use sqlquack_lint::constants as lint; -pub(crate) const SQL_WILDCARD: &str = "*"; -pub(crate) const OPEN_PAREN: &str = "("; -pub(crate) const CLOSE_PAREN: &str = ")"; -pub(crate) const EMPTY_FIXTURE_QUERY_TOKEN_COUNT: usize = 6; -pub(crate) const ENCLOSING_CTE_TOKEN_COUNT: usize = 3; -/// The indent of each line of a configuration snippet printed in remediation text. -pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// The configuration file remediation text names when the request does not name one. -pub const DEFAULT_CONFIG_FILE: &str = "sqlclean.toml"; +pub const DEFAULT_CONFIG_FILE: &str = lint::DEFAULT_CONFIG_FILE; /// The identifier prefix a host writes in place of each template interpolation. -pub const INTERPOLATION_PLACEHOLDER_PREFIX: &str = "__sqlclean_interpolation_"; +pub const INTERPOLATION_PLACEHOLDER_PREFIX: &str = lint::INTERPOLATION_PLACEHOLDER_PREFIX; /// The identifier prefix a host writes in place of a SQL macro parameter. -pub const MACRO_PARAMETER_PLACEHOLDER_PREFIX: &str = "__sqlclean_macro_parameter_"; +pub const MACRO_PARAMETER_PLACEHOLDER_PREFIX: &str = lint::MACRO_PARAMETER_PLACEHOLDER_PREFIX; /// The prefix of a relation parameter placeholder, written `__`. -pub const RELATION_PARAMETER_PLACEHOLDER_PREFIX: &str = "__sqlclean_relation_parameter_"; +pub const RELATION_PARAMETER_PLACEHOLDER_PREFIX: &str = lint::RELATION_PARAMETER_PLACEHOLDER_PREFIX; /// The identifier prefix a host writes in place of a value parameter. -pub const VALUE_PARAMETER_PLACEHOLDER_PREFIX: &str = "__sqlclean_value_parameter_"; -/// Placeholders standing for names the host generates, which Rules never judge as authored names. -pub(crate) const GENERATED_PLACEHOLDER_PREFIXES: [&str; 3] = [ - INTERPOLATION_PLACEHOLDER_PREFIX, - RELATION_PARAMETER_PLACEHOLDER_PREFIX, - VALUE_PARAMETER_PLACEHOLDER_PREFIX, -]; +pub const VALUE_PARAMETER_PLACEHOLDER_PREFIX: &str = lint::VALUE_PARAMETER_PLACEHOLDER_PREFIX; diff --git a/crates/sqlclean/src/lib.rs b/crates/sqlclean/src/lib.rs index 78fd494..cff9944 100644 --- a/crates/sqlclean/src/lib.rs +++ b/crates/sqlclean/src/lib.rs @@ -1,9 +1,8 @@ -//! SQLClean's engine: lint and quality Rules, statement facts and the formatter, over Polyglot. +//! SQLClean's engine: lint and quality Rules, statement facts and the formatter, over SQLQuack for lint and facts; the formatter still reads Polyglot. #![forbid(unsafe_code)] pub mod constants; pub mod sql_facts; pub mod sql_lint; -pub mod sql_quality; pub mod sql_tokens; diff --git a/crates/sqlclean/src/sql_facts/_helpers/collection.rs b/crates/sqlclean/src/sql_facts/_helpers/collection.rs index e331e03..e38aa97 100644 --- a/crates/sqlclean/src/sql_facts/_helpers/collection.rs +++ b/crates/sqlclean/src/sql_facts/_helpers/collection.rs @@ -1,8 +1,9 @@ //! Walk each parsed statement once and collect its structural facts. -use polyglot_sql::expressions::{Column, Cte, Join, TableRef, With}; -use polyglot_sql::tokens::{Span, Token, TokenType}; -use polyglot_sql::{Expression, ExpressionWalk}; +use sqlquack_syntax::expressions::names::models::{Column, TableRef}; +use sqlquack_syntax::expressions::queries::models::{Cte, Join, With}; +use sqlquack_syntax::tokens::models::{Span, Token, TokenType}; +use sqlquack_syntax::{Expression, ExpressionWalk}; use crate::sql_facts::_helpers::statements::{StatementTokens, statement_token_ranges}; use crate::sql_facts::_helpers::tokens::{ @@ -12,7 +13,7 @@ use crate::sql_facts::models::{ ColumnNode, FileFacts, JoinNode, NamedNode, StatementFacts, TableNode, }; use crate::sql_lint::main::parse_sql::parse_sql; -use crate::sql_lint::models::ParsedSql; +use crate::sql_lint::types::ParsedSql; pub(crate) fn file_facts(sql: &str, dialect: &str) -> Result { let ( diff --git a/crates/sqlclean/src/sql_facts/_helpers/statements.rs b/crates/sqlclean/src/sql_facts/_helpers/statements.rs index fbc66d8..1c04020 100644 --- a/crates/sqlclean/src/sql_facts/_helpers/statements.rs +++ b/crates/sqlclean/src/sql_facts/_helpers/statements.rs @@ -1,6 +1,6 @@ //! Statement source ranges from the token stream. -use polyglot_sql::tokens::{Token, TokenType}; +use sqlquack_syntax::tokens::models::{Token, TokenType}; /// The first and last token index of one statement, and the end of its terminator. pub(crate) struct StatementTokens { diff --git a/crates/sqlclean/src/sql_facts/_helpers/tokens.rs b/crates/sqlclean/src/sql_facts/_helpers/tokens.rs index 933a1c1..9828304 100644 --- a/crates/sqlclean/src/sql_facts/_helpers/tokens.rs +++ b/crates/sqlclean/src/sql_facts/_helpers/tokens.rs @@ -1,6 +1,6 @@ //! Function calls and literals, which the syntax tree does not locate, read from tokens. -use polyglot_sql::tokens::{Token, TokenType}; +use sqlquack_syntax::tokens::models::{Token, TokenType}; use crate::sql_facts::models::{LiteralNode, NamedNode, StarNode}; diff --git a/crates/sqlclean/src/sql_lint/_helpers/additional.rs b/crates/sqlclean/src/sql_lint/_helpers/additional.rs deleted file mode 100644 index f18e557..0000000 --- a/crates/sqlclean/src/sql_lint/_helpers/additional.rs +++ /dev/null @@ -1,1886 +0,0 @@ -use std::collections::HashSet; - -use polyglot_sql::tokens::{Span, Token, TokenType}; - -use crate::constants::{CLOSE_PAREN, OPEN_PAREN, SQL_WILDCARD}; -use crate::constants::{EMPTY_FIXTURE_QUERY_TOKEN_COUNT, ENCLOSING_CTE_TOKEN_COUNT}; -use crate::constants::{ - GENERATED_PLACEHOLDER_PREFIXES, MACRO_PARAMETER_PLACEHOLDER_PREFIX, - RELATION_PARAMETER_PLACEHOLDER_PREFIX, -}; -use crate::sql_lint::_helpers::engine::{ - direct_indices, is_comment, is_layout, is_query_from, query_end, significant_after, - significant_before, token_depths, -}; -use crate::sql_lint::constants::RELATION_MODIFIER_KEYWORDS; -use crate::sql_lint::models::{ - AdditionalFactOptions, AdditionalQueryFacts, FixtureHarness, SqlMacroOutput, -}; -use rustc_hash::{FxHashMap, FxHashSet}; - -const COUNT_ONE_LITERAL: &str = "1"; -const QUALIFIED_REFERENCE_LENGTH: usize = 3; -const QUALIFIER_PAIR_LENGTH: usize = 2; -const IMPLICIT_ALIAS_MINIMUM_LENGTH: usize = 2; - -struct QuerySlice<'a> { - tokens: &'a [Token], - depths: &'a [usize], - direct: &'a [usize], - start: usize, - end: usize, - depth: usize, -} - -struct ReferenceFactContext<'a> { - query: &'a QuerySlice<'a>, - outer_relations: &'a FxHashSet, - external_identifiers: &'a HashSet, - sql_macro_outputs: &'a [SqlMacroOutput], - query_start: usize, - query_end: usize, -} - -struct ReferenceCandidateContext<'a> { - tokens: &'a [Token], - references: &'a [usize], - external_identifiers: &'a HashSet, - projection_aliases: &'a FxHashMap, -} - -struct SelectFactContext<'a> { - tokens: &'a [Token], - depths: &'a [usize], - external_identifiers: &'a HashSet, - sql_macro_outputs: &'a [SqlMacroOutput], - dependency_identifiers: &'a HashSet, - allows_ceremonial_select: bool, - allows_dynamic_output_star: bool, - fixture_harness: Option<&'a FixtureHarness>, -} - -struct ProjectedStarContext<'a> { - tokens: &'a [Token], - depths: &'a [usize], - direct: &'a [usize], - query_end: usize, -} - -pub(super) fn collect_additional_facts( - tokens: &[Token], - options: &AdditionalFactOptions<'_>, -) -> AdditionalQueryFacts { - let mut facts = AdditionalQueryFacts::default(); - let depths = token_depths(tokens); - let significant: Vec = (0..tokens.len()) - .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) - .collect(); - for (position, &index) in significant.iter().enumerate() { - let token = &tokens[index]; - if token.token_type == TokenType::Union - && significant.get(position + 1).is_none_or(|&next| { - !matches!( - tokens[next].token_type, - TokenType::All | TokenType::Distinct | TokenType::By - ) - }) - { - facts.bare_unions.push(token.span); - } - if token.token_type == TokenType::Else - && significant - .get(position + 1) - .is_some_and(|&next| tokens[next].token_type == TokenType::Null) - { - let next = significant[position + 1]; - facts.redundant_else_nulls.push(Span { - start: token.span.start, - end: tokens[next].span.end, - line: token.span.line, - column: token.span.column, - }); - } - if token.token_type == TokenType::Distinct - && significant - .get(position + 1) - .is_some_and(|&next| tokens[next].token_type == TokenType::LParen) - && let Some(span) = - parenthesized_distinct_span(tokens, &depths, significant[position + 1]) - { - facts.parenthesized_distincts.push(span); - } - if token.token_type == TokenType::Semicolon - && significant - .get(position.wrapping_sub(1)) - .is_some_and(|&previous| tokens[previous].token_type == TokenType::Semicolon) - { - facts.consecutive_semicolons.push(token.span); - } - if is_identifier(token) - && let Some((&as_index, &alias_index)) = significant - .get(position + 1) - .zip(significant.get(position + 2)) - && tokens[as_index].token_type == TokenType::As - && is_identifier(&tokens[alias_index]) - && token.token_type == tokens[alias_index].token_type - && token.text == tokens[alias_index].text - && is_simple_projection_reference(tokens, &depths, &significant, position) - { - facts.redundant_self_aliases.push(Span { - start: token.span.end, - end: tokens[alias_index].span.end, - line: tokens[as_index].span.line, - column: tokens[as_index].span.column, - }); - } - if token.text.eq_ignore_ascii_case("COUNT") - && let Some(span) = count_one_span(tokens, &significant, position) - { - facts.count_one_literals.push(span); - } - if token.text.eq_ignore_ascii_case("ROW_NUMBER") - && let Some(span) = unstable_row_number_span(tokens, &depths, index) - { - facts.unstable_row_numbers.push(span); - } - if token.token_type == TokenType::Not - && let Some(span) = null_not_in_span(tokens, &depths, &significant, position) - { - facts.null_not_in_predicates.push(span); - } - if token.token_type == TokenType::Case - && let Some(span) = simple_boolean_case_span(tokens, &significant, position) - { - facts.simple_boolean_cases.push(span); - } - if token.token_type == TokenType::Else - && significant - .get(position + 1) - .is_some_and(|&next| tokens[next].token_type == TokenType::Case) - { - facts - .nested_else_cases - .push(tokens[significant[position + 1]].span); - } - if position + 2 < significant.len() - && constant_predicate(tokens, &significant[position..position + 3]) - { - facts - .constant_predicates - .push(tokens[significant[position + 1]].span); - } - } - let select_facts = collect_select_additional_facts(&SelectFactContext { - tokens, - depths: &depths, - external_identifiers: options.external_identifiers, - sql_macro_outputs: options.sql_macro_outputs, - dependency_identifiers: options.dependency_identifiers, - allows_ceremonial_select: options.allows_ceremonial_select, - allows_dynamic_output_star: options.allows_dynamic_output_star, - fixture_harness: options.fixture_harness, - }); - facts.duplicate_output_aliases = select_facts.duplicate_output_aliases; - facts.unaliased_calculations = select_facts.unaliased_calculations; - facts.projected_stars = select_facts.projected_stars; - facts.duplicate_table_aliases = select_facts.duplicate_table_aliases; - facts.unused_table_aliases = select_facts.unused_table_aliases; - facts.null_rejected_left_joins = select_facts.null_rejected_left_joins; - facts.implicit_inner_joins = select_facts.implicit_inner_joins; - facts.unqualified_multi_source_columns = select_facts.unqualified_multi_source_columns; - facts.inconsistent_single_source_qualification = - select_facts.inconsistent_single_source_qualification; - facts.unknown_relation_qualifiers = select_facts.unknown_relation_qualifiers; - facts.unused_joined_relations = select_facts.unused_joined_relations; - facts.mixed_group_order_references = select_facts.mixed_group_order_references; - facts.ambiguous_order_directions = select_facts.ambiguous_order_directions; - facts.set_arity_mismatches = set_arity_mismatch_spans(tokens, &depths); - facts -} - -fn is_simple_projection_reference( - tokens: &[Token], - depths: &[usize], - significant: &[usize], - position: usize, -) -> bool { - let index = significant[position]; - let Some(boundary) = (0..index).rev().find(|&candidate| { - depths[candidate] == depths[index] - && (tokens[candidate].token_type == TokenType::Select - || is_query_from(tokens, candidate) - || tokens[candidate].token_type == TokenType::Join) - }) else { - return false; - }; - if tokens[boundary].token_type != TokenType::Select { - return false; - } - let item_boundary = significant[..position] - .iter() - .rposition(|&candidate| { - candidate > boundary - && depths[candidate] == depths[index] - && tokens[candidate].token_type == TokenType::Comma - }) - .map_or(boundary, |boundary_position| significant[boundary_position]); - let prefix: Vec = significant[..position] - .iter() - .copied() - .filter(|&candidate| candidate > item_boundary && depths[candidate] == depths[index]) - .collect(); - if is_bare_star_prefix(tokens, &prefix) { - return true; - } - let mut qualifier_start = prefix.len(); - while qualifier_start >= QUALIFIER_PAIR_LENGTH - && tokens[prefix[qualifier_start - 1]].token_type == TokenType::Dot - && is_identifier(&tokens[prefix[qualifier_start - 2]]) - { - qualifier_start -= 2; - } - qualifier_start < prefix.len() && is_bare_star_prefix(tokens, &prefix[..qualifier_start]) -} - -fn collect_select_additional_facts(context: &SelectFactContext<'_>) -> AdditionalQueryFacts { - let tokens = context.tokens; - let depths = context.depths; - let mut facts = AdditionalQueryFacts::default(); - for (select_index, token) in tokens.iter().enumerate() { - if token.token_type != TokenType::Select { - continue; - } - let depth = depths[select_index]; - let end = query_end(tokens, depths, select_index, depth); - let direct: Vec = direct_indices(depths, select_index + 1, end, depth) - .into_iter() - .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) - .collect(); - let from_position = direct - .iter() - .position(|&index| is_query_from(tokens, index)) - .unwrap_or(direct.len()); - facts - .duplicate_output_aliases - .extend(duplicate_alias_spans(tokens, &direct[..from_position])); - if !is_exists_subquery(tokens, select_index) - && !is_set_continuation_select(tokens, select_index) - && !is_scalar_subquery(tokens, depths, select_index) - && !has_enclosing_cte_column_list(tokens, depths, select_index) - && !is_ceremonial_select( - tokens, - &direct[..from_position], - context.allows_ceremonial_select, - ) - { - facts - .unaliased_calculations - .extend(unaliased_calculation_spans( - tokens, - &direct[..from_position], - )); - } - if !is_dependency_import_select(context, select_index, &direct, from_position) - && !is_compiler_proven_dynamic_output_select( - context, - select_index, - &direct, - from_position, - ) - && !is_canonical_empty_input_fixture(context, select_index, end) - && !has_irreducible_projected_star( - &ProjectedStarContext { - tokens, - depths, - direct: &direct, - query_end: end, - }, - from_position, - ) - { - facts - .projected_stars - .extend(projected_star_spans(tokens, &direct[..from_position])); - } - if from_position < direct.len() { - let clause_end = direct[from_position + 1..] - .iter() - .position(|&index| is_after_relation_clause(tokens[index].token_type)) - .map_or(direct.len(), |offset| from_position + 1 + offset); - facts.duplicate_table_aliases.extend(duplicate_alias_spans( - tokens, - &direct[from_position + 1..clause_end], - )); - let query_slice = QuerySlice { - tokens, - depths, - direct: &direct[from_position + 1..clause_end], - start: select_index, - end, - depth, - }; - facts - .unused_table_aliases - .extend(unused_alias_spans(&query_slice)); - facts - .null_rejected_left_joins - .extend(null_rejected_left_join_spans(tokens, &direct, clause_end)); - facts.implicit_inner_joins.extend(implicit_inner_join_spans( - tokens, - &direct[from_position + 1..clause_end], - )); - let reference_slice = QuerySlice { - tokens, - depths, - direct: &direct, - start: from_position, - end: clause_end, - depth, - }; - let outer_relations = outer_relation_names(tokens, depths, select_index, depth); - let reference_context = ReferenceFactContext { - query: &reference_slice, - outer_relations: &outer_relations, - external_identifiers: context.external_identifiers, - sql_macro_outputs: context.sql_macro_outputs, - query_start: select_index, - query_end: end, - }; - let reference_facts = collect_reference_facts(&reference_context); - facts - .unqualified_multi_source_columns - .extend(reference_facts.unqualified_multi_source_columns); - facts - .inconsistent_single_source_qualification - .extend(reference_facts.inconsistent_single_source_qualification); - facts - .unknown_relation_qualifiers - .extend(reference_facts.unknown_relation_qualifiers); - facts - .unused_joined_relations - .extend(reference_facts.unused_joined_relations); - } - facts - .mixed_group_order_references - .extend(mixed_reference_clause_spans(tokens, &direct)); - facts - .ambiguous_order_directions - .extend(ambiguous_order_direction_spans(tokens, &direct)); - } - facts -} - -fn is_canonical_empty_input_fixture( - context: &SelectFactContext<'_>, - select_index: usize, - query_end: usize, -) -> bool { - let Some(harness) = context.fixture_harness else { - return false; - }; - let Some(empty_relation) = harness.empty_relation.as_deref() else { - return false; - }; - let significant_query: Vec<&str> = context.tokens[select_index..query_end] - .iter() - .filter(|token| !is_layout(token) && !is_comment(token)) - .map(|token| token.text.as_str()) - .collect(); - if significant_query.len() != EMPTY_FIXTURE_QUERY_TOKEN_COUNT - || !significant_query[0].eq_ignore_ascii_case("select") - || significant_query[1] != SQL_WILDCARD - || !significant_query[2].eq_ignore_ascii_case("from") - || !significant_query[3].eq_ignore_ascii_case(empty_relation) - || significant_query[4] != OPEN_PAREN - || significant_query[5] != CLOSE_PAREN - { - return false; - } - let previous: Vec<&Token> = context.tokens[..select_index] - .iter() - .filter(|token| !is_layout(token) && !is_comment(token)) - .rev() - .take(ENCLOSING_CTE_TOKEN_COUNT) - .collect(); - if previous.len() != ENCLOSING_CTE_TOKEN_COUNT - || previous[0].token_type != TokenType::LParen - || !previous[1].text.eq_ignore_ascii_case("as") - { - return false; - } - let cte_name = previous[2].text.trim_matches(['"', '`']); - harness - .input_cte_prefixes - .iter() - .any(|prefix| cte_name.starts_with(prefix.as_str())) -} - -fn is_compiler_proven_dynamic_output_select( - context: &SelectFactContext<'_>, - select_index: usize, - direct: &[usize], - from_position: usize, -) -> bool { - if !context.allows_dynamic_output_star - || context.depths[select_index] != 0 - || direct[..from_position].len() != 1 - || context.tokens[direct[0]].token_type != TokenType::Star - { - return false; - } - !context.tokens[select_index + 1..] - .iter() - .zip(&context.depths[select_index + 1..]) - .any(|(token, depth)| token.token_type == TokenType::Select && *depth == 0) -} - -fn has_irreducible_projected_star( - context: &ProjectedStarContext<'_>, - from_position: usize, -) -> bool { - let ProjectedStarContext { - tokens, - depths, - direct, - query_end, - } = context; - let projection = &direct[..from_position]; - if projection.len() != 1 || tokens[projection[0]].token_type != TokenType::Star { - return false; - } - - let Some(relation_start) = from_position - .checked_add(1) - .filter(|&start| start < direct.len()) - else { - return false; - }; - let relation_end = direct[relation_start..] - .iter() - .position(|&index| is_after_relation_clause(tokens[index].token_type)) - .map_or(direct.len(), |offset| relation_start + offset); - let relation = &direct[relation_start..relation_end]; - if relation - .iter() - .any(|&index| matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma)) - { - return false; - } - if is_lone_audit_parameter_relation(tokens, relation) { - return true; - } - - let mut pivots = relation - .iter() - .copied() - .filter(|&index| tokens[index].text.eq_ignore_ascii_case("PIVOT")); - let Some(pivot_index) = pivots.next() else { - return false; - }; - if pivots.next().is_some() { - return false; - } - dynamic_pivot_uses_any(tokens, depths, pivot_index, *query_end) -} - -fn is_lone_audit_parameter_relation(tokens: &[Token], relation: &[usize]) -> bool { - let Some(&relation_index) = relation.first() else { - return false; - }; - let normalized = tokens[relation_index].text.to_ascii_lowercase(); - let Some(index) = normalized - .strip_prefix(RELATION_PARAMETER_PLACEHOLDER_PREFIX) - .and_then(|value| value.strip_suffix("__")) - else { - return false; - }; - if index.is_empty() || !index.chars().all(|character| character.is_ascii_digit()) { - return false; - } - match relation.get(1..) { - Some([]) => true, - Some([alias]) => is_identifier(&tokens[*alias]), - Some([as_index, alias]) => { - tokens[*as_index].token_type == TokenType::As && is_identifier(&tokens[*alias]) - } - _ => false, - } -} - -fn dynamic_pivot_uses_any( - tokens: &[Token], - depths: &[usize], - pivot_index: usize, - query_end: usize, -) -> bool { - let Some(pivot_open) = significant_after(tokens, pivot_index) else { - return false; - }; - if tokens[pivot_open].token_type != TokenType::LParen { - return false; - } - let pivot_depth = depths[pivot_open]; - let Some(pivot_close) = (pivot_open + 1..query_end).find(|&index| { - depths[index] == pivot_depth + 1 && tokens[index].token_type == TokenType::RParen - }) else { - return false; - }; - (pivot_open + 1..pivot_close).any(|in_index| { - depths[in_index] == pivot_depth + 1 - && tokens[in_index].token_type == TokenType::In - && significant_after(tokens, in_index).is_some_and(|any_open| { - any_open < pivot_close - && tokens[any_open].token_type == TokenType::LParen - && significant_after(tokens, any_open).is_some_and(|any_index| { - any_index < pivot_close && tokens[any_index].token_type == TokenType::Any - }) - }) - }) -} - -fn has_enclosing_cte_column_list(tokens: &[Token], depths: &[usize], select_index: usize) -> bool { - let select_depth = depths[select_index]; - let Some(body_open) = (0..select_index).rev().find(|&index| { - tokens[index].token_type == TokenType::LParen - && depths[index].saturating_add(1) == select_depth - }) else { - return false; - }; - let Some(as_index) = significant_before(tokens, body_open) else { - return false; - }; - if tokens[as_index].token_type != TokenType::As { - return false; - } - significant_before(tokens, as_index) - .is_some_and(|index| tokens[index].token_type == TokenType::RParen) -} - -fn is_dependency_import_select( - context: &SelectFactContext<'_>, - select_index: usize, - direct: &[usize], - from_position: usize, -) -> bool { - let tokens = context.tokens; - let depths = context.depths; - if from_position != 1 - || tokens[direct[0]].token_type != TokenType::Star - || !is_top_level_cte_body(tokens, depths, select_index) - { - return false; - } - if direct.len() <= from_position + 1 || has_cte_body_set_operation(tokens, depths, select_index) - { - return false; - } - let relation = &direct[from_position + 1..]; - let Some(&relation_index) = relation.first() else { - return false; - }; - if !context - .dependency_identifiers - .contains(&tokens[relation_index].text.to_ascii_lowercase()) - { - return false; - } - match relation { - [_] => true, - [_, alias] => is_identifier(&tokens[*alias]), - [_, explicit_as, alias] => { - tokens[*explicit_as].token_type == TokenType::As && is_identifier(&tokens[*alias]) - } - _ => false, - } -} - -fn has_cte_body_set_operation(tokens: &[Token], depths: &[usize], select_index: usize) -> bool { - let select_depth = depths[select_index]; - let body_end = (select_index + 1..tokens.len()) - .find(|&index| { - depths[index] == select_depth && tokens[index].token_type == TokenType::RParen - }) - .unwrap_or(tokens.len()); - (select_index + 1..body_end).any(|index| { - depths[index] == select_depth - && matches!( - tokens[index].token_type, - TokenType::Union | TokenType::Intersect | TokenType::Except - ) - }) -} - -fn is_top_level_cte_body(tokens: &[Token], depths: &[usize], select_index: usize) -> bool { - let Some(open_index) = significant_before(tokens, select_index) else { - return false; - }; - if tokens[open_index].token_type != TokenType::LParen || depths[open_index] != 0 { - return false; - } - significant_before(tokens, open_index) - .is_some_and(|index| tokens[index].token_type == TokenType::As) -} - -fn collect_reference_facts(context: &ReferenceFactContext<'_>) -> AdditionalQueryFacts { - let query = context.query; - let QuerySlice { - tokens, - direct, - start: from_position, - end: relation_end, - .. - } = query; - let from_position = *from_position; - let relation_end = *relation_end; - let mut facts = AdditionalQueryFacts::default(); - let relations = &direct[from_position + 1..relation_end]; - let relation_count = if is_values_relation(tokens, relations) { - 1 - } else { - 1 + relations - .iter() - .filter(|&&index| { - matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma) - }) - .count() - }; - let mut known = relation_names(tokens, relations); - known.extend(context.outer_relations.iter().cloned()); - let references: Vec = direct[..from_position] - .iter() - .chain(direct[relation_end..].iter()) - .copied() - .collect(); - let projection_aliases = projection_alias_positions(tokens, &direct[..from_position]); - let candidate_context = ReferenceCandidateContext { - tokens, - references: &references, - external_identifiers: context.external_identifiers, - projection_aliases: &projection_aliases, - }; - let qualified: Vec = references - .iter() - .enumerate() - .filter_map(|(position, &index)| simple_qualifier(tokens, &references, position, index)) - .collect(); - let unqualified: Vec = references - .iter() - .enumerate() - .filter_map(|(position, &index)| unqualified_reference(&candidate_context, position, index)) - .collect(); - if relation_count > 1 { - facts - .unqualified_multi_source_columns - .extend(unqualified.iter().map(|&index| tokens[index].span)); - } else { - facts - .inconsistent_single_source_qualification - .extend(inconsistent_qualification( - tokens, - (&qualified, &unqualified), - context.sql_macro_outputs, - )); - } - facts.unknown_relation_qualifiers.extend( - qualified - .iter() - .filter(|&&index| !known.contains(&tokens[index].text.to_ascii_lowercase())) - .map(|&index| tokens[index].span), - ); - let projects_bare_star = has_bare_projected_star(tokens, &direct[..from_position]); - if !projects_bare_star { - facts.unused_joined_relations.extend(unused_join_spans( - query, - context.query_start, - context.query_end, - )); - } - facts -} - -/// The unqualified references of a single-relation scope that also qualifies some, each judged as it was written. In a macro body a plain slot read lints as a pair of macro parameter placeholders and counts as unqualified. In a call's output, a read qualified by a relation the call passes takes neither style: the slot decides how it expands, not the caller or the macro author. A scope whose every reference a macro call generated is judged once, in the macro file. -fn inconsistent_qualification( - tokens: &[Token], - (qualified, unqualified): (&[usize], &[usize]), - outputs: &[SqlMacroOutput], -) -> Vec { - let output_of = |index: usize| -> Option { - let start: usize = tokens[index].span.start; - outputs - .iter() - .position(|output| output.start <= start && start < output.end) - }; - let mut references = qualified.iter().chain(unqualified).copied(); - if let Some(first) = references.next() - && let Some(output) = output_of(first) - && references.all(|index| output_of(index) == Some(output)) - { - return Vec::new(); - } - let passed_relation_read = |index: &usize| { - output_of(*index).is_some_and(|output| { - let qualifier: String = unquoted_name(&tokens[*index].text); - outputs[output] - .relations - .iter() - .any(|relation| relation.eq_ignore_ascii_case(&qualifier)) - }) - }; - let (plain_reads, qualified): (Vec, Vec) = qualified - .iter() - .copied() - .filter(|index| !passed_relation_read(index)) - .partition(|&index| is_plain_slot_read(tokens, index)); - if qualified.is_empty() || (unqualified.is_empty() && plain_reads.is_empty()) { - return Vec::new(); - } - let mut unqualified: Vec = unqualified.to_vec(); - unqualified.extend(plain_reads); - unqualified.sort_unstable(); - unqualified - .iter() - .map(|&index| tokens[index].span) - .collect() -} - -/// An identifier's name without its quotes. -fn unquoted_name(text: &str) -> String { - text.trim_matches(|character| matches!(character, '"' | '`' | '[' | ']')) - .to_owned() -} - -/// Whether the qualified reference at `index` is a plain SQL macro slot read, both its relation and its slot a macro parameter. -fn is_plain_slot_read(tokens: &[Token], index: usize) -> bool { - let is_parameter = |token: &Token| { - token - .text - .to_ascii_lowercase() - .starts_with(MACRO_PARAMETER_PLACEHOLDER_PREFIX) - }; - is_parameter(&tokens[index]) && tokens.get(index + 2).is_some_and(is_parameter) -} - -fn simple_qualifier( - tokens: &[Token], - references: &[usize], - position: usize, - index: usize, -) -> Option { - let previous = position.checked_sub(1).map(|value| references[value]); - let dot = references.get(position + 1).copied(); - let column = references.get(position + 2).copied(); - let following = references.get(position + 3).copied(); - (is_identifier(&tokens[index]) - && !is_generated_identifier(&tokens[index]) - && previous.is_none_or(|candidate| { - !matches!( - tokens[candidate].token_type, - TokenType::Dot | TokenType::Colon | TokenType::DColon | TokenType::DotColon - ) - }) - && dot.is_some_and(|candidate| tokens[candidate].token_type == TokenType::Dot) - && column.is_some_and(|candidate| is_identifier(&tokens[candidate])) - && following.is_none_or(|candidate| tokens[candidate].token_type != TokenType::Dot)) - .then_some(index) -} - -fn unqualified_reference( - context: &ReferenceCandidateContext<'_>, - position: usize, - index: usize, -) -> Option { - let ReferenceCandidateContext { - tokens, - references, - external_identifiers, - projection_aliases, - } = context; - let previous = position.checked_sub(1).map(|value| references[value]); - let next = references.get(position + 1).copied(); - (is_identifier(&tokens[index]) - && !is_generated_identifier(&tokens[index]) - && !is_context_value(&tokens[index]) - && !external_identifiers.contains(&tokens[index].text.to_ascii_lowercase()) - && projection_aliases - .get(&tokens[index].text.to_ascii_lowercase()) - .is_none_or(|alias_index| *alias_index >= index) - && previous.is_none_or(|candidate| { - !matches!( - tokens[candidate].token_type, - TokenType::Dot - | TokenType::As - | TokenType::Colon - | TokenType::DColon - | TokenType::DotColon - ) - }) - && next.is_none_or(|candidate| { - !matches!( - tokens[candidate].token_type, - TokenType::Dot - | TokenType::LParen - | TokenType::Colon - | TokenType::DColon - | TokenType::DotColon - ) - })) - .then_some(index) -} - -fn relation_names(tokens: &[Token], relation_clause: &[usize]) -> FxHashSet { - let mut names: FxHashSet = FxHashSet::default(); - for (position, &index) in relation_clause.iter().enumerate() { - let starts_relation = position == 0 - || matches!( - tokens[relation_clause[position - 1]].token_type, - TokenType::Join | TokenType::Comma - ); - if starts_relation && is_identifier(&tokens[index]) { - names.insert(tokens[index].text.to_ascii_lowercase()); - } - if tokens[index].token_type == TokenType::As - && let Some(&alias) = relation_clause.get(position + 1) - && is_identifier(&tokens[alias]) - { - names.insert(tokens[alias].text.to_ascii_lowercase()); - } - } - let mut segment_start = 0_usize; - for segment_end in relation_clause - .iter() - .enumerate() - .filter_map(|(position, &index)| { - matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma) - .then_some(position) - }) - .chain(std::iter::once(relation_clause.len())) - { - let segment = &relation_clause[segment_start..segment_end]; - segment_start = segment_end + 1; - let relation_end = segment - .iter() - .position(|&index| is_relation_condition_start(&tokens[index])) - .unwrap_or(segment.len()); - if let Some(&implicit_name) = segment[..relation_end] - .iter() - .rev() - .find(|&&index| is_identifier(&tokens[index])) - { - names.insert(tokens[implicit_name].text.to_ascii_lowercase()); - } - } - names -} - -fn outer_relation_names( - tokens: &[Token], - depths: &[usize], - select_index: usize, - depth: usize, -) -> FxHashSet { - let mut names: FxHashSet = FxHashSet::default(); - for outer_select in (0..select_index).filter(|&index| { - tokens[index].token_type == TokenType::Select - && depths[index] < depth - && query_end(tokens, depths, index, depths[index]) > select_index - }) { - let outer_depth = depths[outer_select]; - let outer_end = query_end(tokens, depths, outer_select, outer_depth); - let direct: Vec = direct_indices(depths, outer_select + 1, outer_end, outer_depth) - .into_iter() - .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) - .collect(); - let Some(from_position) = direct - .iter() - .position(|&index| is_query_from(tokens, index)) - else { - continue; - }; - let clause_end = direct[from_position + 1..] - .iter() - .position(|&index| is_after_relation_clause(tokens[index].token_type)) - .map_or(direct.len(), |offset| from_position + 1 + offset); - names.extend(relation_names( - tokens, - &direct[from_position + 1..clause_end], - )); - } - names -} - -fn projected_star_spans(tokens: &[Token], projection: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - for (position, &index) in projection.iter().enumerate() { - if tokens[index].token_type != TokenType::Star { - continue; - } - let previous = position.checked_sub(1).map(|value| projection[value]); - let qualified = - previous.is_some_and(|candidate| tokens[candidate].token_type == TokenType::Dot); - let mut item_start = 0_usize; - for prior_position in (0..position).rev() { - if tokens[projection[prior_position]].token_type == TokenType::Comma { - item_start = prior_position + 1; - break; - } - } - let bare = is_bare_star_prefix(tokens, &projection[item_start..position]); - if qualified || bare { - spans.push(tokens[index].span); - } - } - spans -} - -fn has_bare_projected_star(tokens: &[Token], projection: &[usize]) -> bool { - projection.iter().enumerate().any(|(position, &index)| { - if tokens[index].token_type != TokenType::Star { - return false; - } - let mut item_start = 0_usize; - for prior_position in (0..position).rev() { - if tokens[projection[prior_position]].token_type == TokenType::Comma { - item_start = prior_position + 1; - break; - } - } - is_bare_star_prefix(tokens, &projection[item_start..position]) - }) -} - -fn is_exists_subquery(tokens: &[Token], select_index: usize) -> bool { - let Some(open_index) = significant_before(tokens, select_index) else { - return false; - }; - tokens[open_index].token_type == TokenType::LParen - && significant_before(tokens, open_index) - .is_some_and(|index| tokens[index].token_type == TokenType::Exists) -} - -fn is_set_continuation_select(tokens: &[Token], select_index: usize) -> bool { - let mut previous = significant_before(tokens, select_index); - if previous.is_some_and(|index| { - matches!( - tokens[index].token_type, - TokenType::All | TokenType::Distinct - ) - }) { - previous = previous.and_then(|index| significant_before(tokens, index)); - } - previous.is_some_and(|index| { - matches!( - tokens[index].token_type, - TokenType::Union | TokenType::Intersect | TokenType::Except - ) - }) -} - -fn is_bare_star_prefix(tokens: &[Token], prefix: &[usize]) -> bool { - let mut position = 0_usize; - while prefix.get(position).is_some_and(|&index| { - matches!( - tokens[index].token_type, - TokenType::Distinct | TokenType::All - ) - }) { - position += 1; - } - if position == prefix.len() { - return true; - } - if tokens[prefix[position]].token_type != TokenType::Top { - return false; - } - position += 1; - if !prefix - .get(position) - .is_some_and(|&index| tokens[index].token_type == TokenType::Number) - { - return false; - } - position += 1; - if prefix - .get(position) - .is_some_and(|&index| tokens[index].token_type == TokenType::Percent) - { - position += 1; - } - if prefix - .get(position) - .is_some_and(|&index| tokens[index].token_type == TokenType::With) - && prefix - .get(position + 1) - .is_some_and(|&index| tokens[index].token_type == TokenType::Ties) - { - position += 2; - } - position == prefix.len() -} - -fn is_scalar_subquery(tokens: &[Token], depths: &[usize], select_index: usize) -> bool { - let Some(open_index) = significant_before(tokens, select_index) else { - return false; - }; - if tokens[open_index].token_type != TokenType::LParen { - return false; - } - let Some(previous) = significant_before(tokens, open_index) else { - return false; - }; - if matches!( - tokens[previous].token_type, - TokenType::From | TokenType::Join | TokenType::As | TokenType::Exists | TokenType::Lateral - ) { - return false; - } - if tokens[previous].token_type != TokenType::Comma { - return true; - } - let parent_depth = depths[previous]; - let query_start = (0..previous) - .rev() - .find(|&index| { - tokens[index].token_type == TokenType::Select && depths[index] < parent_depth - }) - .map_or(0, |index| index + 1); - let relation_boundary = (query_start..previous).rev().find(|&index| { - depths[index] == parent_depth - && (tokens[index].token_type == TokenType::Select - || is_query_from(tokens, index) - || matches!( - tokens[index].token_type, - TokenType::Join | TokenType::Semicolon - )) - }); - !relation_boundary.is_some_and(|index| { - is_query_from(tokens, index) || tokens[index].token_type == TokenType::Join - }) -} - -fn is_ceremonial_select(tokens: &[Token], projection: &[usize], allowed: bool) -> bool { - allowed - && projection.len() == 1 - && tokens[projection[0]].token_type == TokenType::Number - && tokens[projection[0]].text == COUNT_ONE_LITERAL -} - -fn is_values_relation(tokens: &[Token], relation_clause: &[usize]) -> bool { - relation_clause - .first() - .is_some_and(|&index| tokens[index].token_type == TokenType::Values) -} - -fn is_generated_identifier(token: &Token) -> bool { - let normalized = token.text.to_ascii_lowercase(); - GENERATED_PLACEHOLDER_PREFIXES - .iter() - .any(|prefix| normalized.starts_with(prefix)) -} - -fn is_relation_condition_start(token: &Token) -> bool { - matches!( - token.token_type, - TokenType::On | TokenType::Using | TokenType::MatchCondition - ) || token.text.eq_ignore_ascii_case("MATCH_CONDITION") -} - -fn is_context_value(token: &Token) -> bool { - matches!( - token.text.to_ascii_uppercase().as_str(), - "CURRENT_CATALOG" - | "CURRENT_DATE" - | "CURRENT_DATETIME" - | "CURRENT_ROLE" - | "CURRENT_SCHEMA" - | "CURRENT_TIME" - | "CURRENT_TIMESTAMP" - | "CURRENT_USER" - ) -} - -fn projection_alias_positions(tokens: &[Token], projection: &[usize]) -> FxHashMap { - let mut aliases: FxHashMap = FxHashMap::default(); - for window in projection.windows(2) { - if tokens[window[0]].token_type == TokenType::As && is_identifier(&tokens[window[1]]) { - aliases.insert(tokens[window[1]].text.to_ascii_lowercase(), window[1]); - } - } - aliases -} - -fn unused_join_spans(query: &QuerySlice<'_>, query_start: usize, query_end: usize) -> Vec { - let QuerySlice { - tokens, - direct, - start: from_position, - end: relation_end, - .. - } = query; - let from_position = *from_position; - let relation_end = *relation_end; - let relation_start_index = direct.get(from_position + 1).copied().unwrap_or(query_end); - let relation_end_index = direct.get(relation_end).copied().unwrap_or(query_end); - let references: Vec = (query_start..relation_start_index) - .chain(relation_end_index..query_end) - .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) - .collect(); - if contains_possible_unqualified_reference(tokens, &references) { - return Vec::new(); - } - let mut spans: Vec = Vec::new(); - for position in from_position + 1..relation_end { - let index = direct[position]; - if tokens[index].token_type != TokenType::Join { - continue; - } - let Some(alias) = relation_alias_after_join(tokens, direct, position) else { - continue; - }; - let used_outside_relation_clause = - contains_qualified_reference(tokens, &references, &alias); - let downstream_relation_start = direct[position + 1..relation_end] - .iter() - .position(|&candidate| { - matches!( - tokens[candidate].token_type, - TokenType::Join | TokenType::Comma - ) - }) - .map(|offset| position + 1 + offset); - let used_by_downstream_relation = downstream_relation_start.is_some_and(|start| { - let downstream_start_index = direct[start]; - let downstream_end_index = direct.get(relation_end).copied().unwrap_or(query_end); - let downstream_references: Vec = (downstream_start_index..downstream_end_index) - .filter(|&candidate| { - !is_layout(&tokens[candidate]) && !is_comment(&tokens[candidate]) - }) - .collect(); - contains_qualified_reference(tokens, &downstream_references, &alias) - }); - let used = used_outside_relation_clause || used_by_downstream_relation; - if !used { - spans.push(tokens[index].span); - } - } - spans -} - -fn contains_possible_unqualified_reference(tokens: &[Token], references: &[usize]) -> bool { - references.iter().enumerate().any(|(position, &index)| { - if !is_identifier(&tokens[index]) - || is_generated_identifier(&tokens[index]) - || is_context_value(&tokens[index]) - { - return false; - } - let previous = if position == 0 { - None - } else { - Some(references[position - 1]) - }; - let next = references.get(position + 1).copied(); - previous.is_none_or(|candidate| { - !matches!(tokens[candidate].token_type, TokenType::Dot | TokenType::As) - }) && next.is_none_or(|candidate| { - !matches!( - tokens[candidate].token_type, - TokenType::Dot | TokenType::LParen - ) - }) - }) -} - -fn contains_qualified_reference(tokens: &[Token], references: &[usize], alias: &str) -> bool { - references.windows(2).any(|window| { - tokens[window[0]].text.eq_ignore_ascii_case(alias) - && tokens[window[1]].token_type == TokenType::Dot - }) -} - -fn is_after_relation_clause(token_type: TokenType) -> bool { - matches!( - token_type, - TokenType::Where - | TokenType::Group - | TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Offset - | TokenType::Fetch - | TokenType::Union - | TokenType::Intersect - | TokenType::Except - | TokenType::Semicolon - ) -} - -fn unaliased_calculation_spans(tokens: &[Token], projection: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - let mut start = 0_usize; - let ends: Vec = projection - .iter() - .enumerate() - .filter_map(|(position, &index)| { - (tokens[index].token_type == TokenType::Comma).then_some(position) - }) - .chain(std::iter::once(projection.len())) - .collect(); - for end in ends { - let item = &projection[start..end]; - start = end + 1; - if item.is_empty() - || item - .iter() - .any(|&index| tokens[index].token_type == TokenType::As) - || item - .iter() - .any(|&index| is_generated_identifier(&tokens[index])) - || simple_projection(tokens, item) - || implicit_alias_projection(tokens, item) - { - continue; - } - spans.push(tokens[item[0]].span); - } - spans -} - -fn simple_projection(tokens: &[Token], item: &[usize]) -> bool { - let trimmed: Vec = item - .iter() - .copied() - .skip_while(|&index| { - matches!( - tokens[index].token_type, - TokenType::Distinct | TokenType::All - ) - }) - .collect(); - let wildcard = trimmed.first().is_some_and(|&index| { - tokens[index].token_type == TokenType::Star - || (trimmed.len() >= QUALIFIED_REFERENCE_LENGTH - && is_identifier(&tokens[index]) - && tokens[trimmed[1]].token_type == TokenType::Dot - && tokens[trimmed[2]].token_type == TokenType::Star) - }); - wildcard - || trimmed.len() == 1 - && (is_bare_column_token(&tokens[trimmed[0]]) - || tokens[trimmed[0]].token_type == TokenType::Star) - || trimmed.len() == QUALIFIED_REFERENCE_LENGTH - && is_identifier(&tokens[trimmed[0]]) - && tokens[trimmed[1]].token_type == TokenType::Dot - && (is_bare_column_token(&tokens[trimmed[2]]) - || tokens[trimmed[2]].token_type == TokenType::Star) -} - -fn is_bare_column_token(token: &Token) -> bool { - is_identifier(token) - || (!matches!( - token.token_type, - TokenType::Null - | TokenType::True - | TokenType::False - | TokenType::Number - | TokenType::String - ) && token - .text - .chars() - .next() - .is_some_and(|character| character.is_alphabetic() || character == '_') - && token - .text - .chars() - .all(|character| character.is_alphanumeric() || character == '_')) -} - -fn implicit_alias_projection(tokens: &[Token], item: &[usize]) -> bool { - item.len() >= IMPLICIT_ALIAS_MINIMUM_LENGTH - && is_identifier(&tokens[*item.last().unwrap_or(&item[0])]) - && !matches!( - tokens[item[item.len() - 2]].token_type, - TokenType::Dot | TokenType::Plus | TokenType::Dash | TokenType::Star | TokenType::Slash - ) -} - -fn unused_alias_spans(query: &QuerySlice<'_>) -> Vec { - let QuerySlice { - tokens, - depths, - direct: relation_clause, - start: query_start, - end: query_end, - depth, - } = query; - let query_start = *query_start; - let query_end = *query_end; - let depth = *depth; - let mut spans: Vec = Vec::new(); - for (span_start, alias_index) in relation_alias_positions(tokens, relation_clause) { - if significant_after(tokens, alias_index) - .is_some_and(|index| tokens[index].token_type == TokenType::LParen) - { - continue; - } - let alias = &tokens[alias_index].text; - let relation_name = significant_before(tokens, span_start); - let used = (query_start..query_end).any(|index| { - index != alias_index - && Some(index) != relation_name - && depths[index] >= depth - && is_identifier(&tokens[index]) - && tokens[index].text.eq_ignore_ascii_case(alias) - && significant_before(tokens, index).is_none_or(|previous| { - !matches!(tokens[previous].token_type, TokenType::Dot | TokenType::As) - }) - && significant_after(tokens, index) - .is_none_or(|next| tokens[next].token_type != TokenType::LParen) - }); - if !used { - spans.push(Span { - start: tokens[span_start].span.start, - end: tokens[alias_index].span.end, - line: tokens[span_start].span.line, - column: tokens[span_start].span.column, - }); - } - } - spans -} - -fn relation_alias_positions(tokens: &[Token], relation_clause: &[usize]) -> Vec<(usize, usize)> { - let mut aliases: Vec<(usize, usize)> = Vec::new(); - let mut segment_start = 0_usize; - let mut in_condition = false; - for (position, &index) in relation_clause.iter().enumerate() { - if matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma) { - segment_start = position + 1; - in_condition = false; - continue; - } - if in_condition || is_relation_condition_start(&tokens[index]) { - in_condition = true; - continue; - } - if tokens[index].token_type == TokenType::As { - if let Some(&alias) = relation_clause.get(position + 1) - && is_identifier(&tokens[alias]) - { - aliases.push((index, alias)); - } - continue; - } - if is_implicit_relation_alias( - tokens, - &relation_clause[segment_start..], - position - segment_start, - ) { - aliases.push((index, index)); - } - } - aliases -} - -fn is_implicit_relation_alias(tokens: &[Token], segment: &[usize], position: usize) -> bool { - let candidate = &tokens[segment[position]]; - if position == 0 - || !is_identifier(candidate) - || candidate.token_type != TokenType::QuotedIdentifier - && RELATION_MODIFIER_KEYWORDS - .iter() - .any(|keyword| candidate.text.eq_ignore_ascii_case(keyword)) - { - return false; - } - let previous = &tokens[segment[position - 1]]; - if previous.token_type == TokenType::LParen { - return true; - } - is_identifier(previous) - && position - .checked_sub(QUALIFIER_PAIR_LENGTH) - .is_none_or(|before| tokens[segment[before]].token_type == TokenType::Dot) -} - -fn null_rejected_left_join_spans( - tokens: &[Token], - direct: &[usize], - relation_end: usize, -) -> Vec { - let Some(where_position) = direct - .iter() - .position(|&index| tokens[index].token_type == TokenType::Where) - else { - return Vec::new(); - }; - if direct[where_position + 1..] - .iter() - .any(|&index| tokens[index].token_type == TokenType::Or) - { - return Vec::new(); - } - let mut spans: Vec = Vec::new(); - for position in 0..relation_end { - let index = direct[position]; - if tokens[index].token_type != TokenType::Left { - continue; - } - let Some(join_position) = direct[position + 1..relation_end] - .iter() - .position(|&candidate| tokens[candidate].token_type == TokenType::Join) - .map(|offset| position + 1 + offset) - else { - continue; - }; - let Some(alias) = relation_alias_after_join(tokens, direct, join_position) else { - continue; - }; - if right_side_null_rejected(tokens, &direct[where_position + 1..], &alias) { - spans.push(tokens[index].span); - } - } - spans -} - -fn right_side_null_rejected(tokens: &[Token], predicate: &[usize], alias: &str) -> bool { - for window in predicate.windows(4) { - if !tokens[window[0]].text.eq_ignore_ascii_case(alias) - || tokens[window[1]].token_type != TokenType::Dot - || !is_identifier(&tokens[window[2]]) - { - continue; - } - if matches!( - tokens[window[3]].token_type, - TokenType::Eq - | TokenType::Neq - | TokenType::Lt - | TokenType::Lte - | TokenType::Gt - | TokenType::Gte - | TokenType::Like - | TokenType::In - | TokenType::Between - ) { - return true; - } - } - for window in predicate.windows(6) { - if tokens[window[0]].text.eq_ignore_ascii_case(alias) - && tokens[window[1]].token_type == TokenType::Dot - && is_identifier(&tokens[window[2]]) - && tokens[window[3]].token_type == TokenType::Is - && tokens[window[4]].token_type == TokenType::Not - && tokens[window[5]].token_type == TokenType::Null - { - return true; - } - } - false -} - -fn relation_alias_after_join( - tokens: &[Token], - direct: &[usize], - join_position: usize, -) -> Option { - let relation = &direct[join_position + 1..]; - let relation_end = relation - .iter() - .position(|&index| { - is_relation_condition_start(&tokens[index]) - || is_after_relation_clause(tokens[index].token_type) - || matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma) - }) - .unwrap_or(relation.len()); - relation[..relation_end] - .iter() - .rev() - .find(|&&index| is_identifier(&tokens[index])) - .map(|&index| tokens[index].text.to_ascii_lowercase()) -} - -fn implicit_inner_join_spans(tokens: &[Token], relation_clause: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - for (position, &index) in relation_clause.iter().enumerate() { - if is_plain_conditioned_join(tokens, relation_clause, position, index) { - spans.push(tokens[index].span); - } - } - spans -} - -fn is_plain_conditioned_join( - tokens: &[Token], - relation_clause: &[usize], - position: usize, - index: usize, -) -> bool { - if tokens[index].token_type != TokenType::Join - || position > 0 - && matches!( - tokens[relation_clause[position - 1]].token_type, - TokenType::Inner - | TokenType::Left - | TokenType::Right - | TokenType::Full - | TokenType::Outer - | TokenType::Cross - | TokenType::Natural - | TokenType::AsOf - | TokenType::Semi - | TokenType::Anti - ) - { - return false; - } - for &candidate in &relation_clause[position + 1..] { - if tokens[candidate].token_type == TokenType::Join { - break; - } - if matches!( - tokens[candidate].token_type, - TokenType::On | TokenType::Using - ) { - return true; - } - } - false -} - -fn ambiguous_order_direction_spans(tokens: &[Token], direct: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - for (position, &index) in direct.iter().enumerate() { - if tokens[index].token_type != TokenType::Order - || direct - .get(position + 1) - .is_none_or(|&by| tokens[by].token_type != TokenType::By) - { - continue; - } - let end = direct[position + 2..] - .iter() - .position(|&candidate| is_clause_boundary(tokens[candidate].token_type)) - .map_or(direct.len(), |offset| position + 2 + offset); - let clause = &direct[position + 2..end]; - let item_ends: Vec = clause - .iter() - .enumerate() - .filter_map(|(item_position, &candidate)| { - (tokens[candidate].token_type == TokenType::Comma).then_some(item_position) - }) - .chain(std::iter::once(clause.len())) - .collect(); - let mut item_start = 0_usize; - let mut explicit = false; - let mut implicit = false; - for item_end in item_ends { - let item = &clause[item_start..item_end]; - item_start = item_end + 1; - let has_direction = item.last().is_some_and(|&candidate| { - matches!( - tokens[candidate].token_type, - TokenType::Asc | TokenType::Desc - ) - }); - explicit |= has_direction; - implicit |= !has_direction && !item.is_empty(); - } - if explicit && implicit { - spans.push(tokens[index].span); - } - } - spans -} - -fn duplicate_alias_spans(tokens: &[Token], indices: &[usize]) -> Vec { - let mut seen: FxHashSet = FxHashSet::default(); - let mut duplicates: Vec = Vec::new(); - for window in indices.windows(2) { - if tokens[window[0]].token_type != TokenType::As || !is_identifier(&tokens[window[1]]) { - continue; - } - let normalized = tokens[window[1]].text.to_ascii_lowercase(); - if !seen.insert(normalized) { - duplicates.push(tokens[window[1]].span); - } - } - duplicates -} - -fn mixed_reference_clause_spans(tokens: &[Token], direct: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - for (position, &index) in direct.iter().enumerate() { - if !matches!( - tokens[index].token_type, - TokenType::Group | TokenType::Order - ) || direct - .get(position + 1) - .is_none_or(|&by| tokens[by].token_type != TokenType::By) - { - continue; - } - let end = direct[position + 2..] - .iter() - .position(|&candidate| is_clause_boundary(tokens[candidate].token_type)) - .map_or(direct.len(), |offset| position + 2 + offset); - let clause = &direct[position + 2..end]; - let mut has_number = false; - let mut has_name = false; - for (item_position, &candidate) in clause.iter().enumerate() { - if item_position == 0 - || tokens[clause[item_position - 1]].token_type == TokenType::Comma - { - has_number |= tokens[candidate].token_type == TokenType::Number; - has_name |= tokens[candidate].token_type != TokenType::Number; - } - } - if has_number && has_name { - spans.push(tokens[index].span); - } - } - spans -} - -fn is_clause_boundary(token_type: TokenType) -> bool { - matches!( - token_type, - TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Offset - | TokenType::Fetch - | TokenType::Union - | TokenType::Intersect - | TokenType::Except - | TokenType::Semicolon - ) -} - -fn parenthesized_distinct_span( - tokens: &[Token], - depths: &[usize], - open_index: usize, -) -> Option { - let close_depth = depths[open_index] + 1; - let close_index = (open_index + 1..tokens.len()).find(|&index| { - tokens[index].token_type == TokenType::RParen && depths[index] == close_depth - })?; - Some(Span { - start: tokens[significant_before(tokens, open_index)?].span.start, - end: tokens[close_index].span.end, - line: tokens[significant_before(tokens, open_index)?].span.line, - column: tokens[significant_before(tokens, open_index)?].span.column, - }) -} - -fn count_one_span(tokens: &[Token], significant: &[usize], position: usize) -> Option { - let open = *significant.get(position + 1)?; - let one = *significant.get(position + 2)?; - let close = *significant.get(position + 3)?; - (tokens[open].token_type == TokenType::LParen - && tokens[one].token_type == TokenType::Number - && tokens[one].text == COUNT_ONE_LITERAL - && tokens[close].token_type == TokenType::RParen) - .then_some(tokens[one].span) -} - -fn simple_boolean_case_span( - tokens: &[Token], - significant: &[usize], - position: usize, -) -> Option { - let tail = &significant[position + 1..]; - let end_offset = tail - .iter() - .position(|&index| tokens[index].token_type == TokenType::End)?; - let body = &tail[..=end_offset]; - if body - .iter() - .any(|&index| tokens[index].token_type == TokenType::Case) - { - return None; - } - let when_position = body - .iter() - .position(|&index| tokens[index].token_type == TokenType::When)?; - let else_position = body - .iter() - .position(|&index| tokens[index].token_type == TokenType::Else)?; - let else_value = *body.get(else_position + 1)?; - let mut thens = body - .iter() - .enumerate() - .filter(|&(_, &index)| tokens[index].token_type == TokenType::Then) - .peekable(); - thens.peek()?; - let every_branch_returns_true = thens.all(|(offset, _)| { - body.get(offset + 1) - .is_some_and(|&value| tokens[value].token_type == TokenType::True) - && body.get(offset + 2).is_some_and(|&next| { - matches!(tokens[next].token_type, TokenType::When | TokenType::Else) - }) - }); - if when_position != 0 - || !every_branch_returns_true - || tokens[else_value].token_type != TokenType::False - || else_position + 2 != body.len() - 1 - { - return None; - } - let start = tokens[significant[position]].span; - let end = tokens[*body.last()?].span; - Some(Span { - start: start.start, - end: end.end, - line: start.line, - column: start.column, - }) -} - -fn unstable_row_number_span( - tokens: &[Token], - depths: &[usize], - function_index: usize, -) -> Option { - let over_index = (function_index + 1..tokens.len()).find(|&index| { - depths[index] == depths[function_index] && tokens[index].token_type == TokenType::Over - })?; - let open_index = significant_after(tokens, over_index)?; - if tokens[open_index].token_type != TokenType::LParen { - return None; - } - let close_depth = depths[open_index] + 1; - let close_index = (open_index + 1..tokens.len()).find(|&index| { - tokens[index].token_type == TokenType::RParen && depths[index] == close_depth - })?; - (!tokens[open_index + 1..close_index] - .iter() - .any(|token| matches!(token.token_type, TokenType::Order | TokenType::OrderBy))) - .then_some(tokens[function_index].span) -} - -fn null_not_in_span( - tokens: &[Token], - depths: &[usize], - significant: &[usize], - position: usize, -) -> Option { - let in_index = *significant.get(position + 1)?; - let open_index = *significant.get(position + 2)?; - if tokens[in_index].token_type != TokenType::In - || tokens[open_index].token_type != TokenType::LParen - { - return None; - } - let close_depth = depths[open_index] + 1; - let close_index = (open_index + 1..tokens.len()).find(|&index| { - tokens[index].token_type == TokenType::RParen && depths[index] == close_depth - })?; - let list_depth = depths[open_index] + 1; - let list = &tokens[open_index + 1..close_index]; - (!list - .iter() - .any(|token| token.token_type == TokenType::Select) - && list.iter().enumerate().any(|(offset, token)| { - token.token_type == TokenType::Null && depths[open_index + 1 + offset] == list_depth - })) - .then_some(tokens[significant[position]].span) -} - -fn constant_predicate(tokens: &[Token], indices: &[usize]) -> bool { - let left = &tokens[indices[0]]; - let operator = &tokens[indices[1]]; - let right = &tokens[indices[2]]; - matches!( - left.token_type, - TokenType::Number | TokenType::String | TokenType::True | TokenType::False - ) && left.token_type == right.token_type - && left.text == right.text - && matches!(operator.token_type, TokenType::Eq | TokenType::Neq) -} - -fn set_arity_mismatch_spans(tokens: &[Token], depths: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - if !matches!( - token.token_type, - TokenType::Union | TokenType::Intersect | TokenType::Except - ) { - continue; - } - let depth = depths[index]; - let Some(left_select) = (0..index).rev().find(|&candidate| { - depths[candidate] == depth && tokens[candidate].token_type == TokenType::Select - }) else { - continue; - }; - let Some(right_select) = (index + 1..tokens.len()).find(|&candidate| { - depths[candidate] == depth && tokens[candidate].token_type == TokenType::Select - }) else { - continue; - }; - let left_query = QuerySlice { - tokens, - depths, - direct: &[], - start: left_select, - end: index, - depth, - }; - let left_count = projection_arity(&left_query); - let right_end = query_end(tokens, depths, right_select, depth); - let right_query = QuerySlice { - tokens, - depths, - direct: &[], - start: right_select, - end: right_end, - depth, - }; - let right_count = projection_arity(&right_query); - if left_count - .zip(right_count) - .is_some_and(|(left, right)| left != right) - { - spans.push(token.span); - } - } - spans -} - -fn projection_arity(query: &QuerySlice<'_>) -> Option { - let QuerySlice { - tokens, - depths, - start: select, - end, - depth, - .. - } = query; - let select = *select; - let end = *end; - let depth = *depth; - let projection_end = (select + 1..end) - .find(|&index| depths[index] == depth && is_query_from(tokens, index)) - .unwrap_or(end); - if (select + 1..projection_end) - .any(|index| depths[index] == depth && tokens[index].token_type == TokenType::Star) - { - return None; - } - Some( - 1 + (select + 1..projection_end) - .filter(|&index| depths[index] == depth && tokens[index].token_type == TokenType::Comma) - .count(), - ) -} - -fn is_identifier(token: &Token) -> bool { - matches!( - token.token_type, - TokenType::Identifier | TokenType::QuotedIdentifier | TokenType::Var - ) -} diff --git a/crates/sqlclean/src/sql_lint/_helpers/engine.rs b/crates/sqlclean/src/sql_lint/_helpers/engine.rs index b0d9ca7..caed3ec 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/engine.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/engine.rs @@ -1,657 +1,9 @@ -use std::borrow::Cow; -use std::collections::{HashMap, HashSet, VecDeque}; +//! Token nesting depths, the one lint helper the formatter still reads. -use polyglot_sql::expressions::With; -use polyglot_sql::tokens::{Span, Token, TokenType}; -use polyglot_sql::{DialectType, Expression, ExpressionWalk}; +use polyglot_sql::tokens::{Token, TokenType}; -use crate::sql_lint::constants::{LINT_API_VERSION, SELECT_ITEM_END_TOKENS}; -use crate::sql_lint::models::{ - AdditionalFactOptions, CrossJoinRequest, FixtureHarness, LintDiagnostic, LintEdit, LintRequest, - LintResponse, LintRuleMetadata, ParsedSql, QueryFacts, SqlMacroOutput, -}; - -use crate::sql_lint::_helpers::additional::collect_additional_facts; -use crate::sql_lint::_helpers::inline_relations::collect_inline_query_relation_spans; -use crate::sql_lint::_helpers::join_predicates::{ - condition_tokens, cross_join_diagnostics, trivial_on_condition, -}; -use crate::sql_lint::_helpers::parsing::parse; -use crate::sql_lint::_helpers::terminal_shape::{ - collect_terminal_shape_facts, first_row_selection_clause, limits_to_no_rows, - projects_only_constants, -}; - -const NULL_COMPARISON: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN001", - message: "Comparison with NULL is never true", - remediation: "Use IS NULL or IS NOT NULL when testing for NULL.", -}; - -const IMPLICIT_CARTESIAN_JOIN: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN002", - message: "Comma-separated sources create an implicit cartesian join", - remediation: "Replace comma-separated sources with an explicit keyed join, or use CROSS JOIN when the cartesian product is intentional.", -}; -const JOIN_WITHOUT_CONDITION: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN003", - message: "Non-cross join has no meaningful condition", - remediation: "Add a meaningful ON or USING condition, or declare an intentional cartesian product with CROSS JOIN.", -}; -const UNORDERED_LIMIT: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN004", - message: "Row selection is nondeterministic", - remediation: "Add ORDER BY with a deterministic tie-breaker before LIMIT or OFFSET.", -}; -const UNUSED_CTE: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN005", - message: "CTE is unreachable from the final query", - remediation: "Reference the CTE from the final query or another reachable CTE, or remove it.", -}; -const REDUNDANT_DISTINCT: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN006", - message: "DISTINCT is redundant with the grouped output", - remediation: "Remove DISTINCT; the equivalent GROUP BY already determines the output groups.", -}; -const POSITIONAL_SET_STAR: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN007", - message: "Positional set operation is vulnerable to column-order drift", - remediation: "Enumerate columns in the same order in every set-operation branch.", -}; -const EXPLICIT_UNION: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN008", - message: "Bare UNION leaves duplicate handling implicit", - remediation: "Use UNION DISTINCT or UNION ALL to state duplicate handling explicitly.", -}; -const DUPLICATE_TABLE_ALIAS: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN009", - message: "Table alias is duplicated in the same query scope", - remediation: "Give every relation in this query scope a unique alias.", -}; -const DUPLICATE_OUTPUT_ALIAS: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN010", - message: "Output alias is duplicated in the same select list", - remediation: "Give every projected expression a unique output alias.", -}; -const MIXED_GROUP_ORDER_REFERENCES: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN011", - message: "Clause mixes positional and named references", - remediation: "Use column names or positions consistently throughout this clause.", -}; -const REDUNDANT_ELSE_NULL: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN012", - message: "ELSE NULL is redundant", - remediation: "Remove ELSE NULL; CASE expressions return NULL when ELSE is omitted.", -}; -const PARENTHESIZED_DISTINCT: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN013", - message: "DISTINCT is written as though it were a function", - remediation: "Write DISTINCT as a SELECT modifier without function-like parentheses.", -}; -const CONSTANT_PREDICATE: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN014", - message: "Predicate compares identical constant values", - remediation: "Remove the scaffold predicate or replace it with the intended condition.", -}; -const CONSECUTIVE_SEMICOLON: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN015", - message: "Empty statement follows another terminator", - remediation: "Remove the redundant statement terminator.", -}; -const REDUNDANT_SELF_ALIAS: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN016", - message: "Expression is aliased to its existing name", - remediation: "Remove the self-alias when the inferred output identity remains stable.", -}; -const COUNT_ONE: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN017", - message: "Row count uses a constant expression", - remediation: "Use COUNT(*) to state that rows, rather than a nullable expression, are counted.", -}; -const UNSTABLE_ROW_NUMBER: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN018", - message: "ROW_NUMBER has no deterministic ordering", - remediation: "Add ORDER BY with a stable tie-breaker inside the window specification.", -}; -const NULL_NOT_IN: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN019", - message: "NOT IN list contains NULL and can reject every row", - remediation: "Remove NULL from the list or express the anti-join with a NULL-safe NOT EXISTS.", -}; -const SET_ARITY_MISMATCH: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN020", - message: "Set-operation branches project different column counts", - remediation: "Project the same number of columns in every set-operation branch.", -}; -const PROJECTED_STAR: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN021", - message: "Query output has an uncontrolled column shape", - remediation: "Enumerate the intended output columns explicitly.", -}; -const UNALIASED_CALCULATION: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN022", - message: "Calculated projection has no stable output name", - remediation: "Add an explicit AS alias that describes the projected value.", -}; -const UNUSED_TABLE_ALIAS: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN023", - message: "Table alias is never referenced", - remediation: "Use the alias to qualify references or remove it.", -}; -const NULL_REJECTED_LEFT_JOIN: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN024", - message: "WHERE predicate rejects NULL rows from a LEFT JOIN", - remediation: "Move the right-side predicate into ON or use INNER JOIN when row rejection is intentional.", -}; -const IMPLICIT_INNER_JOIN: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN025", - message: "JOIN type is implicit", - remediation: "Write INNER JOIN explicitly.", -}; -const AMBIGUOUS_ORDER_DIRECTIONS: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN026", - message: "ORDER BY mixes explicit and implicit directions", - remediation: "State ASC or DESC for every ordering expression in this clause.", -}; -const UNQUALIFIED_MULTI_SOURCE_COLUMN: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN027", - message: "Column reference is unqualified in a multi-relation query", - remediation: "Qualify the column with its relation alias.", -}; -const INCONSISTENT_SINGLE_SOURCE_QUALIFICATION: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN028", - message: "Single-relation query mixes qualified and unqualified columns", - remediation: "Use one consistent qualification style in this query scope.", -}; -const UNKNOWN_RELATION_QUALIFIER: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN029", - message: "Qualified reference uses a relation absent from this query scope", - remediation: "Use an available relation alias or add the intended relation to FROM.", -}; -const SIMPLE_BOOLEAN_CASE: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN030", - message: "CASE expression only converts a condition to TRUE or FALSE", - remediation: "Use a NULL-safe boolean expression directly.", -}; -const NESTED_ELSE_CASE: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN031", - message: "CASE expression is nested directly inside ELSE", - remediation: "Flatten the nested branch into the surrounding CASE expression.", -}; -const UNUSED_JOINED_RELATION: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN032", - message: "Joined relation contributes no referenced values", - remediation: "Use the joined relation or remove the join. When the join intentionally filters rows, record it as a reviewed rule exception with a reason in the configuration file.", -}; -const COMMENT_ATTACHMENT: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN033", - message: "Comment is not attached to the syntax node it explains", - remediation: "Use a standalone -- comment block immediately above the relevant CTE, JOIN, filter, predicate, selected expression, CASE branch, grouping, or ordering node.", -}; -const CTE_ONLY_BODY: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN034", - message: "Query keeps transformation logic outside top-level CTEs", - remediation: "Move transformation logic into named top-level CTEs before the terminal SELECT.", -}; -const TERMINAL_SELECT: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN035", - message: "Terminal SELECT does not read plainly from the final top-level CTE", - remediation: "Name the final logical CTE and leave one terminal SELECT from that CTE.", -}; -const NESTED_CTE: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN036", - message: "CTE is nested inside another query", - remediation: "Hoist the nested CTE into the statement's top-level WITH list.", -}; -const RECURSIVE_CTE: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN037", - message: "Recursive CTE is not permitted", - remediation: "Replace recursion with an explicit upstream relation or a bounded non-recursive query shape.", -}; -const CROSS_JOIN: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN038", - message: "CROSS JOIN creates an explicit cartesian product", - remediation: "Use a keyed JOIN, or make one side provably at most one row (LIMIT 1, an aggregate without GROUP BY, or a one-row VALUES). When the cartesian product is intentional, record it as a reviewed [[rules.rule_exceptions]] entry with rule, path, relation and reason in the configuration file.", -}; -const INLINE_QUERY_RELATION: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN039", - message: "Inline query relation obscures data flow", - remediation: "Extract the derived relation into a named top-level CTE; use explicit LATERAL only when the relation must remain correlated.", -}; -const JOIN_EXPRESSION: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN040", - message: "JOIN ON contains a computed expression", - remediation: "Compute keys, including variant path access, in input CTEs and join on plain columns. Literal-only predicates such as ON TRUE are allowed.", -}; -const FINAL_CTE_NAME: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN041", - message: "Only the last top-level CTE may be named final, and it must use that name", - remediation: "Name the last top-level CTE final and rename any other final CTE; SQLCLEAN035 checks the terminal SELECT.", -}; - -pub(crate) const DEFAULT_RULES: [&str; 19] = [ - NULL_COMPARISON.code, - IMPLICIT_CARTESIAN_JOIN.code, - JOIN_WITHOUT_CONDITION.code, - UNORDERED_LIMIT.code, - UNUSED_CTE.code, - REDUNDANT_DISTINCT.code, - POSITIONAL_SET_STAR.code, - DUPLICATE_TABLE_ALIAS.code, - DUPLICATE_OUTPUT_ALIAS.code, - UNSTABLE_ROW_NUMBER.code, - NULL_NOT_IN.code, - SET_ARITY_MISMATCH.code, - INLINE_QUERY_RELATION.code, - crate::sql_quality::constants::DUPLICATE_CTE_NAME.code, - crate::sql_quality::constants::NULL_MATCH.code, - crate::sql_quality::constants::LITERAL_ZERO_DIVISION.code, - crate::sql_quality::constants::NATURAL_JOIN.code, - crate::sql_quality::constants::REVERSED_BETWEEN.code, - crate::sql_quality::constants::MISPLACED_AGGREGATE.code, -]; - -const ALL_RULE_METADATA: [&LintRuleMetadata; 53] = [ - &NULL_COMPARISON, - &IMPLICIT_CARTESIAN_JOIN, - &JOIN_WITHOUT_CONDITION, - &UNORDERED_LIMIT, - &UNUSED_CTE, - &REDUNDANT_DISTINCT, - &POSITIONAL_SET_STAR, - &EXPLICIT_UNION, - &DUPLICATE_TABLE_ALIAS, - &DUPLICATE_OUTPUT_ALIAS, - &MIXED_GROUP_ORDER_REFERENCES, - &REDUNDANT_ELSE_NULL, - &PARENTHESIZED_DISTINCT, - &CONSTANT_PREDICATE, - &CONSECUTIVE_SEMICOLON, - &REDUNDANT_SELF_ALIAS, - &COUNT_ONE, - &UNSTABLE_ROW_NUMBER, - &NULL_NOT_IN, - &SET_ARITY_MISMATCH, - &PROJECTED_STAR, - &UNALIASED_CALCULATION, - &UNUSED_TABLE_ALIAS, - &NULL_REJECTED_LEFT_JOIN, - &IMPLICIT_INNER_JOIN, - &AMBIGUOUS_ORDER_DIRECTIONS, - &UNQUALIFIED_MULTI_SOURCE_COLUMN, - &INCONSISTENT_SINGLE_SOURCE_QUALIFICATION, - &UNKNOWN_RELATION_QUALIFIER, - &SIMPLE_BOOLEAN_CASE, - &NESTED_ELSE_CASE, - &UNUSED_JOINED_RELATION, - &COMMENT_ATTACHMENT, - &CTE_ONLY_BODY, - &TERMINAL_SELECT, - &NESTED_CTE, - &RECURSIVE_CTE, - &CROSS_JOIN, - &INLINE_QUERY_RELATION, - &JOIN_EXPRESSION, - &FINAL_CTE_NAME, - &crate::sql_quality::constants::UNUSED_CTE_OUTPUT, - &crate::sql_quality::constants::RANKING_SORT_CAP, - &crate::sql_quality::constants::LONG_LITERAL, - &crate::sql_quality::constants::REPEATED_JSON_PARSE, - &crate::sql_quality::constants::DUPLICATE_CTE_NAME, - &crate::sql_quality::constants::NULL_MATCH, - &crate::sql_quality::constants::LITERAL_ZERO_DIVISION, - &crate::sql_quality::constants::UNFILTERED_WRITE, - &crate::sql_quality::constants::INSERT_WITHOUT_COLUMNS, - &crate::sql_quality::constants::NATURAL_JOIN, - &crate::sql_quality::constants::REVERSED_BETWEEN, - &crate::sql_quality::constants::MISPLACED_AGGREGATE, -]; - -/// Whether the harness generated the CTE `name`. -fn is_harness_cte(harness: &FixtureHarness, name: &str) -> bool { - harness - .cte_prefixes - .iter() - .any(|prefix| name.starts_with(prefix.as_str())) -} - -struct QueryTokenContext<'a> { - tokens: &'a [Token], - depths: &'a [usize], - direct: &'a [usize], - query_end: usize, - depth: usize, -} - -struct ExpressionRange { - start: usize, - end: usize, - depth: usize, - strip_alias: bool, -} - -struct DiagnosticContext<'a> { - sql: &'a str, - dialect: DialectType, - facts: &'a QueryFacts, - tokens: &'a [Token], - enabled: &'a HashSet, -} - -struct FactBuildOptions<'a> { - external_identifiers: &'a HashSet, - sql_macro_outputs: &'a [SqlMacroOutput], - dependency_identifiers: &'a HashSet, - externally_referenced_ctes: &'a HashSet, - allows_ceremonial_select: bool, - allows_dynamic_output_star: bool, - fixture_harness: Option<&'a FixtureHarness>, -} - -pub(crate) fn lint_json_impl(request_json: &str) -> Result { - let request: LintRequest = - serde_json::from_str(request_json).map_err(|error| error.to_string())?; - let response = lint(request)?; - serde_json::to_string(&response).map_err(|error| error.to_string()) -} - -pub(crate) fn rule_metadata() -> Vec { - ALL_RULE_METADATA.iter().map(|rule| **rule).collect() -} - -pub(crate) fn lint(request: LintRequest) -> Result { - if request.version != LINT_API_VERSION { - return Err(format!( - "unsupported native lint request version {}; expected {LINT_API_VERSION}", - request.version - )); - } - let enabled = enabled_rules(request.enabled_rules, &request.ignored_rules)?; - let ParsedSql { - dialect: dialect_type, - tokens, - statements, - } = parse(&request.sql, &request.dialect)?; - let external_identifiers: HashSet = request - .external_identifiers - .iter() - .map(|identifier| identifier.to_ascii_lowercase()) - .collect(); - let dependency_identifiers: HashSet = request - .dependency_identifiers - .iter() - .map(|identifier| identifier.to_ascii_lowercase()) - .collect(); - let externally_referenced_ctes: HashSet = request - .externally_referenced_ctes - .iter() - .map(|name| name.to_ascii_lowercase()) - .collect(); - let options = FactBuildOptions { - external_identifiers: &external_identifiers, - sql_macro_outputs: &request.sql_macro_outputs, - dependency_identifiers: &dependency_identifiers, - externally_referenced_ctes: &externally_referenced_ctes, - allows_ceremonial_select: request.fixture_harness.is_some(), - allows_dynamic_output_star: request.allows_dynamic_output_star, - fixture_harness: request.fixture_harness.as_ref(), - }; - let facts = build_facts(&statements, &tokens, &request.sql, &options); - let context = DiagnosticContext { - sql: &request.sql, - dialect: dialect_type, - facts: &facts, - tokens: &tokens, - enabled: &enabled, - }; - let mut diagnostics = diagnostics(&context); - if enabled.contains(CROSS_JOIN.code) { - let cross_joins = CrossJoinRequest { - statements: &statements, - tokens: &tokens, - cross_spans: &facts.cross_joins, - relation_keys: &request.relation_keys, - relation_names: &request.relation_names, - }; - diagnostics.extend(cross_join_diagnostics(&cross_joins, &CROSS_JOIN)); - } - let depths = token_depths(&tokens); - if enabled.contains(JOIN_EXPRESSION.code) { - diagnostics.extend(diagnostics_for_spans( - &JOIN_EXPRESSION, - &crate::sql_lint::_helpers::join_predicates::join_expression_spans( - &tokens, - &depths, - dialect_type, - ), - Some("computing a join key requires an authored CTE rewrite"), - )); - } - if enabled.contains(FINAL_CTE_NAME.code) && request.fixture_harness.is_none() { - diagnostics.extend(diagnostics_for_spans( - &FINAL_CTE_NAME, - &crate::sql_lint::_helpers::terminal_shape::final_cte_name_spans(&tokens, &depths), - Some("renaming a CTE requires updating its references"), - )); - } - diagnostics.extend(crate::sql_quality::main::diagnostics::diagnostics( - &crate::sql_quality::models::QualityRequest { - statements: &statements, - tokens: &tokens, - dialect: dialect_type, - enabled: &enabled, - max_literal_length: request - .max_literal_length - .unwrap_or(crate::sql_quality::constants::DEFAULT_MAX_LITERAL_LENGTH), - max_ranking_order_by: request - .max_ranking_order_by - .unwrap_or(crate::sql_quality::constants::DEFAULT_MAX_RANKING_ORDER_BY), - relation_keys: &request.relation_keys, - externally_referenced_ctes: &externally_referenced_ctes, - fixture_body: request.fixture_harness.is_some(), - config_file: request - .config_file - .as_deref() - .unwrap_or(crate::constants::DEFAULT_CONFIG_FILE), - long_literal_guidance: request - .long_literal_guidance - .as_deref() - .unwrap_or(crate::sql_quality::constants::LONG_LITERAL_GUIDANCE), - }, - )?); - diagnostics.sort_by_key(|diagnostic| (diagnostic.start, diagnostic.end, diagnostic.code)); - Ok(LintResponse { - version: LINT_API_VERSION, - diagnostics, - }) -} - -fn enabled_rules( - requested: Option>, - ignored: &[String], -) -> Result, String> { - let selectors = - requested.unwrap_or_else(|| DEFAULT_RULES.iter().map(ToString::to_string).collect()); - let mut rules = expand_rule_selectors(&selectors)?; - for code in expand_rule_selectors(ignored)? { - rules.remove(&code); - } - Ok(rules) -} - -fn expand_rule_selectors(selectors: &[String]) -> Result, String> { - let mut selected: HashSet = HashSet::new(); - for selector in selectors { - let matches: Vec<&str> = ALL_RULE_METADATA - .iter() - .map(|rule| rule.code) - .filter(|code| code.starts_with(selector)) - .collect(); - if matches.is_empty() { - return Err(format!("unknown native lint rule '{selector}'")); - } - selected.extend(matches.into_iter().map(ToString::to_string)); - } - Ok(selected) -} - -fn build_facts( - expressions: &[Expression], - tokens: &[Token], - sql: &str, - options: &FactBuildOptions<'_>, -) -> QueryFacts { - let mut facts: QueryFacts = - collect_token_query_facts(tokens, sql, options.allows_ceremonial_select); - facts.null_comparisons = null_comparison_spans(tokens); - facts.additional = collect_additional_facts( - tokens, - &AdditionalFactOptions { - external_identifiers: options.external_identifiers, - sql_macro_outputs: options.sql_macro_outputs, - dependency_identifiers: options.dependency_identifiers, - allows_ceremonial_select: options.allows_ceremonial_select, - allows_dynamic_output_star: options.allows_dynamic_output_star, - fixture_harness: options.fixture_harness, - }, - ); - for expression in expressions { - facts - .unused_cte_names - .extend(collect_unused_ctes(expression, options)); - } - facts -} - -fn collect_token_query_facts( - tokens: &[Token], - sql: &str, - allows_ceremonial_select: bool, -) -> QueryFacts { - let depths = token_depths(tokens); - let mut facts = collect_rule_migration_facts(tokens, &depths, sql, allows_ceremonial_select); - for (select_index, token) in tokens.iter().enumerate() { - if token.token_type != TokenType::Select { - continue; - } - let depth = depths[select_index]; - let end = query_end(tokens, &depths, select_index, depth); - let direct = direct_indices(&depths, select_index + 1, end, depth); - let order = first_type(tokens, &direct, TokenType::Order); - let limit = first_row_selection_clause(tokens, &direct); - if order.is_none() - && let Some(index) = limit - && !limits_to_no_rows(tokens, index) - && !projects_only_constants(tokens, &direct, index) - { - facts.unordered_limits.push(tokens[index].span); - } - let context = QueryTokenContext { - tokens, - depths: &depths, - direct: &direct, - query_end: end, - depth, - }; - if let Some(span) = redundant_distinct_span(&context) { - facts.redundant_distincts.push(span); - } - let (implicit, missing_conditions) = collect_from_and_join_facts(tokens, &direct); - facts.implicit_cartesian_joins.extend(implicit); - facts.joins_without_condition.extend(missing_conditions); - } - facts.positional_set_stars = collect_set_operation_facts(tokens, &depths); - facts -} - -fn collect_rule_migration_facts( - tokens: &[Token], - depths: &[usize], - sql: &str, - allows_ceremonial_select: bool, -) -> QueryFacts { - let mut facts = QueryFacts::default(); - let significant: Vec = (0..tokens.len()) - .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) - .collect(); - - let root_with = crate::sql_lint::_helpers::terminal_shape::root_with_indices(tokens, depths); - - for (position, &index) in significant.iter().enumerate() { - let text = tokens[index].text.as_str(); - if text.eq_ignore_ascii_case("cross") - && significant - .get(position + 1) - .is_some_and(|&next| tokens[next].token_type == TokenType::Join) - { - facts.cross_joins.push(tokens[index].span); - } - if tokens[index].token_type == TokenType::With { - let starts_hierarchy = position.checked_sub(1).is_some_and(|previous| { - tokens[significant[previous]].token_type == TokenType::Start - }); - if !root_with.contains(&index) && !starts_hierarchy { - facts.nested_ctes.push(tokens[index].span); - } - if !starts_hierarchy - && significant - .get(position + 1) - .is_some_and(|&next| tokens[next].text.eq_ignore_ascii_case("recursive")) - { - facts.recursive_ctes.push(tokens[index].span); - } - } - } - facts - .inline_query_relations - .extend(collect_inline_query_relation_spans( - tokens, - depths, - &significant, - )); - - let (cte_only_bodies, terminal_selects) = - collect_terminal_shape_facts(tokens, depths, &significant, allows_ceremonial_select); - facts.cte_only_bodies.extend(cte_only_bodies); - facts.terminal_selects.extend(terminal_selects); - facts - .invalid_comment_attachments - .extend(collect_comment_attachment_facts(tokens, sql)); - facts -} - -fn collect_comment_attachment_facts(tokens: &[Token], sql: &str) -> Vec { - let mut invalid: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - if !token.trailing_comments.is_empty() { - invalid.push(token.span); - } - if token.comments.is_empty() { - continue; - } - let gap_start = index - .checked_sub(1) - .map_or(0, |previous| tokens[previous].span.end); - let gap = char_slice(sql, gap_start, token.span.start).unwrap_or_default(); - let block_comment = gap.contains("/*"); - let detached = gap.replace("\r\n", "\n").ends_with("\n\n"); - let unsupported_target = matches!( - token.token_type, - TokenType::Comma - | TokenType::RParen - | TokenType::Semicolon - | TokenType::Then - | TokenType::Else - | TokenType::From - ); - if block_comment || detached || unsupported_target { - invalid.push(token.span); - } - } - invalid -} - -pub(super) fn token_depths(tokens: &[Token]) -> Vec { +/// Each token's bracket nesting depth, counted before the token itself. +pub(crate) fn token_depths(tokens: &[Token]) -> Vec { let mut depth = 0_usize; tokens .iter() @@ -672,1283 +24,3 @@ pub(super) fn token_depths(tokens: &[Token]) -> Vec { }) .collect() } - -pub(super) fn query_end(tokens: &[Token], depths: &[usize], start: usize, depth: usize) -> usize { - (start + 1..tokens.len()) - .find(|&index| { - depths[index] < depth - || depths[index] == depth - && matches!( - tokens[index].token_type, - TokenType::RParen - | TokenType::Semicolon - | TokenType::Union - | TokenType::Intersect - | TokenType::Except - ) - }) - .unwrap_or(tokens.len()) -} - -pub(super) fn direct_indices( - depths: &[usize], - start: usize, - end: usize, - depth: usize, -) -> Vec { - (start..end) - .filter(|&index| depths[index] == depth) - .collect() -} - -fn first_type(tokens: &[Token], indices: &[usize], token_type: TokenType) -> Option { - indices - .iter() - .copied() - .find(|&index| tokens[index].token_type == token_type) -} - -pub(super) fn is_query_from(tokens: &[Token], index: usize) -> bool { - tokens[index].token_type == TokenType::From - && significant_before(tokens, index) - .is_none_or(|previous| tokens[previous].token_type != TokenType::Distinct) -} - -fn redundant_distinct_span(context: &QueryTokenContext<'_>) -> Option { - let QueryTokenContext { - tokens, - depths, - direct, - query_end, - depth, - } = context; - let group_position = group_by_position(tokens, direct)?; - let group_index = direct[group_position]; - let projection_end = direct - .iter() - .copied() - .find(|&index| is_query_from(tokens, index)) - .filter(|index| *index < group_index) - .unwrap_or(group_index); - let distinct_position = direct.iter().position(|&index| { - index < projection_end && tokens[index].token_type == TokenType::Distinct - })?; - if direct - .get(distinct_position + 1) - .is_some_and(|&index| tokens[index].token_type == TokenType::On) - { - return None; - } - let projection_start = direct[distinct_position] + 1; - let group_start = *direct.get(group_position + 2)?; - let group_end = direct[group_position + 2..] - .iter() - .copied() - .find(|&index| { - matches!( - tokens[index].token_type, - TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Offset - ) - }) - .unwrap_or(*query_end); - let projection_signatures = expression_signatures( - tokens, - depths, - &ExpressionRange { - start: projection_start, - end: projection_end, - depth: *depth, - strip_alias: true, - }, - ); - let group_signatures = expression_signatures( - tokens, - depths, - &ExpressionRange { - start: group_start, - end: group_end, - depth: *depth, - strip_alias: false, - }, - ); - if group_signatures.is_empty() - || !group_signatures - .iter() - .all(|group| projection_signatures.contains(group)) - { - return None; - } - Some(tokens[direct[distinct_position]].span) -} - -fn group_by_position(tokens: &[Token], direct: &[usize]) -> Option { - direct.windows(2).position(|window| { - tokens[window[0]].token_type == TokenType::Group - && tokens[window[1]].text.eq_ignore_ascii_case("BY") - }) -} - -fn expression_signatures( - tokens: &[Token], - depths: &[usize], - range: &ExpressionRange, -) -> Vec { - let mut boundaries: Vec = (range.start..range.end) - .filter(|&index| { - depths[index] == range.depth && tokens[index].token_type == TokenType::Comma - }) - .collect(); - boundaries.push(range.end); - let mut segment_start = range.start; - let mut signatures: Vec = Vec::new(); - for segment_end in boundaries { - let effective_end = alias_start(tokens, depths, range, segment_start..segment_end); - let parts: Vec = tokens[segment_start..effective_end] - .iter() - .map(|token| token.text.to_ascii_lowercase()) - .collect(); - let signature = parts.join(" "); - if !signature.is_empty() { - signatures.push(signature); - } - segment_start = segment_end + 1; - } - signatures -} - -fn alias_start( - tokens: &[Token], - depths: &[usize], - range: &ExpressionRange, - segment: std::ops::Range, -) -> usize { - if !range.strip_alias { - return segment.end; - } - segment - .clone() - .find(|&index| depths[index] == range.depth && tokens[index].token_type == TokenType::As) - .unwrap_or(segment.end) -} - -fn collect_from_and_join_facts(tokens: &[Token], direct: &[usize]) -> (Vec, Vec) { - let mut implicit: Vec = Vec::new(); - let mut missing_conditions: Vec = Vec::new(); - let Some(from_position) = direct - .iter() - .position(|&index| is_query_from(tokens, index)) - else { - return (implicit, missing_conditions); - }; - let clause_end = direct[from_position + 1..] - .iter() - .position(|&index| is_post_from_clause(tokens[index].token_type)) - .map_or(direct.len(), |offset| from_position + 1 + offset); - let relation_clause = &direct[from_position + 1..clause_end]; - let values_relation = relation_clause - .first() - .is_some_and(|&index| tokens[index].token_type == TokenType::Values); - for (position, &index) in relation_clause.iter().enumerate() { - if tokens[index].token_type == TokenType::Comma - && !values_relation - && relation_clause - .get(position + 1) - .is_none_or(|&next| tokens[next].token_type != TokenType::Lateral) - { - implicit.push(tokens[index].span); - } - } - for (position, &index) in direct.iter().enumerate() { - if tokens[index].token_type != TokenType::Join - || is_conditionless_join(tokens, direct, position) - { - continue; - } - let condition_end = direct[position + 1..] - .iter() - .position(|&candidate| { - tokens[candidate].token_type == TokenType::Join - || is_post_from_clause(tokens[candidate].token_type) - }) - .map_or(direct.len(), |offset| position + 1 + offset); - let condition = &direct[position + 1..condition_end]; - let on_position = condition - .iter() - .position(|&candidate| tokens[candidate].token_type == TokenType::On); - let has_using = condition - .iter() - .any(|&candidate| tokens[candidate].token_type == TokenType::Using); - if !has_using - && on_position.is_none_or(|offset| { - let end = direct.get(condition_end).copied().unwrap_or(tokens.len()); - trivial_on_condition(tokens, &condition_tokens(tokens, condition[offset], end)) - }) - { - missing_conditions.push(tokens[index].span); - } - } - (implicit, missing_conditions) -} - -fn is_post_from_clause(token_type: TokenType) -> bool { - matches!( - token_type, - TokenType::Join - | TokenType::Where - | TokenType::Group - | TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Offset - | TokenType::Fetch - ) -} - -fn is_conditionless_join(tokens: &[Token], direct: &[usize], join_position: usize) -> bool { - direct[..join_position] - .iter() - .rev() - .take_while(|&&index| { - !matches!( - tokens[index].token_type, - TokenType::From | TokenType::Join | TokenType::Comma - ) - }) - .any(|&index| { - matches!( - tokens[index].text.to_ascii_uppercase().as_str(), - "CROSS" | "NATURAL" | "APPLY" | "POSITIONAL" | "PASTE" - ) - }) -} - -fn collect_set_operation_facts(tokens: &[Token], depths: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - if !matches!( - token.token_type, - TokenType::Union | TokenType::Intersect | TokenType::Except - ) { - continue; - } - let depth = depths[index]; - let by_name = tokens[index + 1..] - .iter() - .take_while(|candidate| candidate.token_type != TokenType::Select) - .any(|candidate| candidate.text.eq_ignore_ascii_case("BY")) - && tokens[index + 1..] - .iter() - .take_while(|candidate| candidate.token_type != TokenType::Select) - .any(|candidate| candidate.text.eq_ignore_ascii_case("NAME")); - if !by_name - && (select_before_projects_star(tokens, depths, index, depth) - || select_after_projects_star(tokens, depths, index, depth)) - { - spans.push(token.span); - } - } - spans -} - -fn select_before_projects_star( - tokens: &[Token], - depths: &[usize], - end: usize, - depth: usize, -) -> bool { - let Some(start) = (0..end) - .rev() - .find(|&index| depths[index] == depth && tokens[index].token_type == TokenType::Select) - .or_else(|| { - (0..end) - .rev() - .filter(|&index| tokens[index].token_type == TokenType::Select) - .min_by_key(|&index| depths[index]) - }) - else { - return false; - }; - projection_projects_star( - tokens, - depths, - &ExpressionRange { - start, - end, - depth: depths[start], - strip_alias: false, - }, - ) -} - -fn select_after_projects_star( - tokens: &[Token], - depths: &[usize], - start: usize, - depth: usize, -) -> bool { - let Some(select) = (start + 1..tokens.len()) - .find(|&index| depths[index] == depth && tokens[index].token_type == TokenType::Select) - .or_else(|| { - (start + 1..tokens.len()) - .filter(|&index| tokens[index].token_type == TokenType::Select) - .min_by_key(|&index| depths[index]) - }) - else { - return false; - }; - let select_depth = depths[select]; - projection_projects_star( - tokens, - depths, - &ExpressionRange { - start: select, - end: query_end(tokens, depths, select, select_depth), - depth: select_depth, - strip_alias: false, - }, - ) -} - -fn projection_projects_star(tokens: &[Token], depths: &[usize], range: &ExpressionRange) -> bool { - (range.start + 1..range.end) - .take_while(|&index| depths[index] != range.depth || !is_query_from(tokens, index)) - .any(|index| depths[index] == range.depth && tokens[index].token_type == TokenType::Star) -} - -fn collect_unused_ctes(expression: &Expression, options: &FactBuildOptions<'_>) -> Vec { - let mut unused: Vec = Vec::new(); - for node in expression.dfs() { - let with: Option<&With> = match node { - Expression::Select(select) => select.with.as_ref(), - Expression::Union(operation) => operation.with.as_ref(), - Expression::Intersect(operation) => operation.with.as_ref(), - Expression::Except(operation) => operation.with.as_ref(), - _ => None, - }; - if let Some(with) = with { - unused.extend(collect_unused_for_with( - with, - &outside_with_table_names(node, with), - options, - )); - } - } - unused -} - -/// Table names `root` references outside its own `with`, counted so a name the WITH also uses still counts. -fn outside_with_table_names(root: &Expression, with: &With) -> HashSet { - let mut counts: HashMap = HashMap::new(); - for name in table_name_occurrences(root) { - *counts.entry(name).or_default() += 1; - } - let inside = with - .ctes - .iter() - .map(|cte| &cte.this) - .chain(with.search.as_deref()); - for expression in inside { - for name in table_name_occurrences(expression) { - if let Some(count) = counts.get_mut(&name) { - *count -= 1; - } - } - } - counts - .into_iter() - .filter_map(|(name, count)| (count > 0).then_some(name)) - .collect() -} - -fn collect_unused_for_with( - with: &With, - root_tables: &HashSet, - options: &FactBuildOptions<'_>, -) -> Vec { - let names: HashSet = with - .ctes - .iter() - .map(|cte| cte.alias.name.to_ascii_lowercase()) - .collect(); - let dependencies: HashMap> = with - .ctes - .iter() - .map(|cte| { - ( - cte.alias.name.to_ascii_lowercase(), - table_names(&cte.this) - .intersection(&names) - .cloned() - .collect(), - ) - }) - .collect(); - let mut reachable: HashSet = root_tables.intersection(&names).cloned().collect(); - reachable.extend( - options - .externally_referenced_ctes - .intersection(&names) - .cloned(), - ); - if let Some(harness) = options.fixture_harness { - reachable.extend( - names - .iter() - .filter(|name| is_harness_cte(harness, name)) - .cloned(), - ); - } - let mut queue: VecDeque = reachable.iter().cloned().collect(); - while let Some(name) = queue.pop_front() { - if let Some(required) = dependencies.get(&name) { - for dependency in required { - if reachable.insert(dependency.clone()) { - queue.push_back(dependency.clone()); - } - } - } - } - with.ctes - .iter() - .filter(|cte| !reachable.contains(&cte.alias.name.to_ascii_lowercase())) - .map(|cte| cte.alias.name.clone()) - .collect() -} - -fn table_names(expression: &Expression) -> HashSet { - table_name_occurrences(expression).collect() -} - -fn table_name_occurrences(expression: &Expression) -> impl Iterator + '_ { - expression.dfs().filter_map(|node| match node { - Expression::Table(table) => Some(table.name.name.to_ascii_lowercase()), - _ => None, - }) -} - -fn null_comparison_fix(tokens: &[Token], span: Span) -> Option { - let index = token_index_for_span(tokens, span)?; - let token = &tokens[index]; - if !adjacent_null(tokens, index, true) { - return None; - } - let replacement = match token.token_type { - TokenType::Eq => "IS", - TokenType::Neq => "IS NOT", - _ => return None, - }; - Some(LintEdit { - start: span.start, - end: span.end, - replacement: replacement.to_string(), - }) -} - -fn redundant_distinct_fix(tokens: &[Token], span: Span) -> LintEdit { - let end = token_index_for_span(tokens, span) - .and_then(|index| significant_after(tokens, index)) - .filter(|&index| !is_comment(&tokens[index])) - .map_or(span.end, |index| tokens[index].span.start); - LintEdit { - start: span.start, - end, - replacement: String::new(), - } -} - -fn implicit_cartesian_fix(sql: &str, tokens: &[Token], span: Span) -> Option { - let comma_index = token_index_for_span(tokens, span)?; - let depths = token_depths(tokens); - let depth = depths[comma_index]; - let from_index = (0..comma_index).rev().find(|&index| { - depths[index] == depth - && (is_query_from(tokens, index) || tokens[index].token_type == TokenType::Semicolon) - })?; - if !is_query_from(tokens, from_index) { - return None; - } - let clause_end = (comma_index + 1..tokens.len()) - .find(|&index| depths[index] == depth && is_post_from_clause(tokens[index].token_type)) - .unwrap_or(tokens.len()); - if tokens[from_index + 1..clause_end] - .iter() - .any(|token| token.token_type == TokenType::Join) - { - return None; - } - let next_index = significant_after(tokens, comma_index)?; - let end = tokens[next_index].span.start; - let replaced = char_slice(sql, span.start, end)?; - if contains_comment(replaced) { - return None; - } - Some(LintEdit { - start: span.start, - end, - replacement: " CROSS JOIN ".to_string(), - }) -} - -fn conditionless_join_fix(tokens: &[Token], span: Span) -> Option { - let join_index = token_index_for_span(tokens, span)?; - let depths = token_depths(tokens); - let depth = depths[join_index]; - if significant_before(tokens, join_index).is_some_and(|index| { - matches!( - tokens[index].text.to_ascii_uppercase().as_str(), - "INNER" - | "LEFT" - | "RIGHT" - | "FULL" - | "OUTER" - | "CROSS" - | "NATURAL" - | "ASOF" - | "SEMI" - | "ANTI" - ) - }) { - return None; - } - let condition_end = (join_index + 1..tokens.len()) - .find(|&index| { - depths[index] == depth - && (tokens[index].token_type == TokenType::Join - || is_post_from_clause(tokens[index].token_type)) - }) - .unwrap_or(tokens.len()); - if tokens[join_index + 1..condition_end] - .iter() - .any(|token| matches!(token.token_type, TokenType::On | TokenType::Using)) - { - return None; - } - Some(LintEdit { - start: span.start, - end: span.end, - replacement: "CROSS JOIN".to_string(), - }) -} - -fn unused_cte_fix(sql: &str, tokens: &[Token], name_span: Span) -> Option { - let name_index = token_index_for_span(tokens, name_span)?; - if tokens[name_index].text.starts_with("__") { - return None; - } - let as_index = significant_after(tokens, name_index)?; - if tokens[as_index].token_type != TokenType::As { - return None; - } - let open_index = significant_after(tokens, as_index)?; - if tokens[open_index].token_type != TokenType::LParen { - return None; - } - let depths = token_depths(tokens); - let close_depth = depths[open_index] + 1; - let close_index = (open_index + 1..tokens.len()).find(|&index| { - tokens[index].token_type == TokenType::RParen && depths[index] == close_depth - })?; - let body_tokens = &tokens[open_index + 1..close_index]; - let first_body = body_tokens.iter().find(|token| !is_layout(token))?; - if !matches!(first_body.token_type, TokenType::Select | TokenType::With) - || body_tokens.iter().any(|token| { - matches!( - token.text.to_ascii_uppercase().as_str(), - "INSERT" | "UPDATE" | "DELETE" | "MERGE" - ) - }) - { - return None; - } - let previous_index = significant_before(tokens, name_index)?; - let next_index = significant_after(tokens, close_index)?; - let (start, end) = if tokens[next_index].token_type == TokenType::Comma { - let following_index = significant_after(tokens, next_index)?; - (name_span.start, tokens[following_index].span.start) - } else if tokens[previous_index].token_type == TokenType::Comma { - ( - tokens[previous_index].span.start, - tokens[close_index].span.end, - ) - } else if tokens[previous_index].token_type == TokenType::With { - ( - tokens[previous_index].span.start, - tokens[next_index].span.start, - ) - } else { - return None; - }; - if contains_comment(char_slice(sql, start, end)?) { - return None; - } - Some(LintEdit { - start, - end, - replacement: String::new(), - }) -} - -fn token_index_for_span(tokens: &[Token], span: Span) -> Option { - tokens.iter().position(|token| token.span == span) -} - -pub(super) fn significant_before(tokens: &[Token], index: usize) -> Option { - (0..index) - .rev() - .find(|&candidate| !is_layout(&tokens[candidate])) -} - -pub(super) fn significant_after(tokens: &[Token], index: usize) -> Option { - (index + 1..tokens.len()).find(|&candidate| !is_layout(&tokens[candidate])) -} - -pub(super) fn is_layout(token: &Token) -> bool { - matches!(token.token_type, TokenType::Space | TokenType::Break) -} - -pub(super) fn is_comment(token: &Token) -> bool { - matches!( - token.token_type, - TokenType::LineComment | TokenType::BlockComment - ) -} - -fn diagnostics(context: &DiagnosticContext<'_>) -> Vec { - let mut output: Vec = core_diagnostics(context); - output.extend(additional_style_diagnostics(context)); - output.extend(additional_relation_diagnostics(context)); - output.extend(rule_migration_diagnostics(context)); - output -} - -fn core_diagnostics(context: &DiagnosticContext<'_>) -> Vec { - let DiagnosticContext { - sql, - dialect, - facts, - tokens, - enabled, - } = context; - let mut diagnostics: Vec = Vec::new(); - if enabled.contains(NULL_COMPARISON.code) { - diagnostics.extend(facts.null_comparisons.iter().map(|span| { - let fix = null_comparison_fix(tokens, *span); - diagnostic( - &NULL_COMPARISON, - Some(*span), - fix, - Some("only equality and inequality comparisons have a deterministic repair"), - ) - })); - } - if enabled.contains(IMPLICIT_CARTESIAN_JOIN.code) { - diagnostics.extend(facts.implicit_cartesian_joins.iter().map(|span| { - let fix = implicit_cartesian_fix(sql, tokens, *span); - diagnostic( - &IMPLICIT_CARTESIAN_JOIN, - Some(*span), - fix, - Some("mixed joins, comments, or generated SQL require an explicit authored repair"), - ) - })); - } - if enabled.contains(JOIN_WITHOUT_CONDITION.code) { - diagnostics.extend(facts.joins_without_condition.iter().map(|span| { - let fix = conditionless_join_fix(tokens, *span); - diagnostic( - &JOIN_WITHOUT_CONDITION, - Some(*span), - fix, - Some("qualified joins or joins with placeholder conditions require user intent"), - ) - })); - } - if enabled.contains(UNORDERED_LIMIT.code) { - diagnostics.extend(diagnostics_for_spans( - &UNORDERED_LIMIT, - &facts.unordered_limits, - Some("deterministic ordering columns and tie-breakers require user intent"), - )); - } - if enabled.contains(UNUSED_CTE.code) { - for span in unused_cte_spans(tokens, &facts.unused_cte_names) { - let fix = unused_cte_fix(sql, tokens, span); - diagnostics.push(diagnostic( - &UNUSED_CTE, - Some(span), - fix, - Some("only contiguous authored, comment-free, select-only CTEs can be removed"), - )); - } - } - if enabled.contains(REDUNDANT_DISTINCT.code) { - diagnostics.extend(facts.redundant_distincts.iter().map(|span| { - diagnostic( - &REDUNDANT_DISTINCT, - Some(*span), - Some(redundant_distinct_fix(tokens, *span)), - None, - ) - })); - } - if enabled.contains(POSITIONAL_SET_STAR.code) { - diagnostics.extend(diagnostics_for_spans( - &POSITIONAL_SET_STAR, - &facts.positional_set_stars, - Some( - "column names and their intended positional order require resolved schema evidence", - ), - )); - } - if enabled.contains(EXPLICIT_UNION.code) { - diagnostics.extend(facts.additional.bare_unions.iter().map(|span| { - let fix = explicit_union_fix(*dialect, *span); - diagnostic( - &EXPLICIT_UNION, - Some(*span), - fix, - Some("the active dialect does not accept explicit UNION DISTINCT"), - ) - })); - } - diagnostics -} - -fn additional_style_diagnostics(context: &DiagnosticContext<'_>) -> Vec { - let DiagnosticContext { - sql, - facts, - tokens, - enabled, - .. - } = context; - let mut diagnostics: Vec = Vec::new(); - if enabled.contains(DUPLICATE_TABLE_ALIAS.code) { - diagnostics.extend(diagnostics_for_spans( - &DUPLICATE_TABLE_ALIAS, - &facts.additional.duplicate_table_aliases, - Some("choosing a new relation alias requires user intent"), - )); - } - if enabled.contains(DUPLICATE_OUTPUT_ALIAS.code) { - diagnostics.extend(diagnostics_for_spans( - &DUPLICATE_OUTPUT_ALIAS, - &facts.additional.duplicate_output_aliases, - Some("choosing a stable output name requires user intent"), - )); - } - if enabled.contains(MIXED_GROUP_ORDER_REFERENCES.code) { - diagnostics.extend(diagnostics_for_spans( - &MIXED_GROUP_ORDER_REFERENCES, - &facts.additional.mixed_group_order_references, - Some("normalization requires reliable projection-name resolution"), - )); - } - if enabled.contains(REDUNDANT_ELSE_NULL.code) { - diagnostics.extend(facts.additional.redundant_else_nulls.iter().map(|span| { - diagnostic( - &REDUNDANT_ELSE_NULL, - Some(*span), - Some(deletion(*span)), - None, - ) - })); - } - if enabled.contains(PARENTHESIZED_DISTINCT.code) { - diagnostics.extend(facts.additional.parenthesized_distincts.iter().map(|span| { - diagnostic( - &PARENTHESIZED_DISTINCT, - Some(*span), - parenthesized_distinct_fix(sql, tokens, *span), - Some("the parentheses group an expression that continues after them, or hold comments"), - ) - })); - } - if enabled.contains(CONSTANT_PREDICATE.code) { - diagnostics.extend(diagnostics_for_spans( - &CONSTANT_PREDICATE, - &facts.additional.constant_predicates, - Some("removing or replacing a scaffold predicate requires user intent"), - )); - } - if enabled.contains(CONSECUTIVE_SEMICOLON.code) { - diagnostics.extend(facts.additional.consecutive_semicolons.iter().map(|span| { - diagnostic( - &CONSECUTIVE_SEMICOLON, - Some(*span), - Some(deletion(*span)), - None, - ) - })); - } - if enabled.contains(REDUNDANT_SELF_ALIAS.code) { - diagnostics.extend(facts.additional.redundant_self_aliases.iter().map(|span| { - diagnostic( - &REDUNDANT_SELF_ALIAS, - Some(*span), - None, - Some("removing the alias may change statically inferred output identity"), - ) - })); - } - if enabled.contains(COUNT_ONE.code) { - diagnostics.extend(facts.additional.count_one_literals.iter().map(|span| { - diagnostic( - &COUNT_ONE, - Some(*span), - Some(LintEdit { - start: span.start, - end: span.end, - replacement: "*".to_string(), - }), - None, - ) - })); - } - if enabled.contains(UNSTABLE_ROW_NUMBER.code) { - diagnostics.extend(diagnostics_for_spans( - &UNSTABLE_ROW_NUMBER, - &facts.additional.unstable_row_numbers, - Some("ordering columns and a stable tie-breaker require user intent"), - )); - } - diagnostics -} - -fn additional_relation_diagnostics(context: &DiagnosticContext<'_>) -> Vec { - let DiagnosticContext { - sql, - facts, - tokens, - enabled, - .. - } = context; - let mut diagnostics: Vec = Vec::new(); - if enabled.contains(NULL_NOT_IN.code) { - diagnostics.extend(diagnostics_for_spans( - &NULL_NOT_IN, - &facts.additional.null_not_in_predicates, - Some("the intended NULL and anti-join semantics require user intent"), - )); - } - if enabled.contains(SET_ARITY_MISMATCH.code) { - diagnostics.extend(diagnostics_for_spans( - &SET_ARITY_MISMATCH, - &facts.additional.set_arity_mismatches, - Some("the intended branch schema requires user intent"), - )); - } - if enabled.contains(PROJECTED_STAR.code) { - diagnostics.extend(diagnostics_for_spans( - &PROJECTED_STAR, - &facts.additional.projected_stars, - Some("column enumeration requires reliable resolved schema evidence"), - )); - } - if enabled.contains(UNALIASED_CALCULATION.code) { - diagnostics.extend(diagnostics_for_spans( - &UNALIASED_CALCULATION, - &facts.additional.unaliased_calculations, - Some("a meaningful stable output name requires user intent"), - )); - } - if enabled.contains(UNUSED_TABLE_ALIAS.code) { - diagnostics.extend(facts.additional.unused_table_aliases.iter().map(|span| { - diagnostic( - &UNUSED_TABLE_ALIAS, - Some(*span), - Some(deletion(*span)), - None, - ) - })); - } - if enabled.contains(NULL_REJECTED_LEFT_JOIN.code) { - diagnostics.extend(diagnostics_for_spans( - &NULL_REJECTED_LEFT_JOIN, - &facts.additional.null_rejected_left_joins, - Some("moving the predicate or changing join type requires user intent"), - )); - } - if enabled.contains(IMPLICIT_INNER_JOIN.code) { - diagnostics.extend(facts.additional.implicit_inner_joins.iter().map(|span| { - diagnostic( - &IMPLICIT_INNER_JOIN, - Some(*span), - Some(LintEdit { - start: span.start, - end: span.end, - replacement: "INNER JOIN".to_string(), - }), - None, - ) - })); - } - if enabled.contains(AMBIGUOUS_ORDER_DIRECTIONS.code) { - diagnostics.extend(diagnostics_for_spans( - &AMBIGUOUS_ORDER_DIRECTIONS, - &facts.additional.ambiguous_order_directions, - Some("normalizing every expression requires a multi-range authored rewrite"), - )); - } - if enabled.contains(UNQUALIFIED_MULTI_SOURCE_COLUMN.code) { - diagnostics.extend(diagnostics_for_spans( - &UNQUALIFIED_MULTI_SOURCE_COLUMN, - &facts.additional.unqualified_multi_source_columns, - Some("choosing the correct relation qualifier requires user intent"), - )); - } - if enabled.contains(INCONSISTENT_SINGLE_SOURCE_QUALIFICATION.code) { - diagnostics.extend(diagnostics_for_spans( - &INCONSISTENT_SINGLE_SOURCE_QUALIFICATION, - &facts.additional.inconsistent_single_source_qualification, - Some("qualification style is normalized by the canonical formatter only when safe"), - )); - } - if enabled.contains(UNKNOWN_RELATION_QUALIFIER.code) { - diagnostics.extend(diagnostics_for_spans( - &UNKNOWN_RELATION_QUALIFIER, - &facts.additional.unknown_relation_qualifiers, - Some("the intended relation or alias requires user intent"), - )); - } - if enabled.contains(SIMPLE_BOOLEAN_CASE.code) { - diagnostics.extend(facts.additional.simple_boolean_cases.iter().map(|span| { - diagnostic( - &SIMPLE_BOOLEAN_CASE, - Some(*span), - simple_boolean_case_fix(sql, tokens, *span), - Some("comments or a non-canonical CASE shape prevent a lossless rewrite"), - ) - })); - } - if enabled.contains(NESTED_ELSE_CASE.code) { - diagnostics.extend(diagnostics_for_spans( - &NESTED_ELSE_CASE, - &facts.additional.nested_else_cases, - Some("flattening nested branches requires a multi-range authored rewrite"), - )); - } - if enabled.contains(UNUSED_JOINED_RELATION.code) { - diagnostics.extend(diagnostics_for_spans( - &UNUSED_JOINED_RELATION, - &facts.additional.unused_joined_relations, - Some("the join may intentionally filter or multiply rows"), - )); - } - diagnostics -} - -fn rule_migration_diagnostics(context: &DiagnosticContext<'_>) -> Vec { - let DiagnosticContext { facts, enabled, .. } = context; - let mut diagnostics: Vec = Vec::new(); - if enabled.contains(COMMENT_ATTACHMENT.code) { - diagnostics.extend(diagnostics_for_spans( - &COMMENT_ATTACHMENT, - &facts.invalid_comment_attachments, - Some("comment attachment requires an authored placement decision"), - )); - } - if enabled.contains(CTE_ONLY_BODY.code) { - diagnostics.extend(diagnostics_for_spans( - &CTE_ONLY_BODY, - &facts.cte_only_bodies, - Some("extracting transformation logic into named CTEs requires user intent"), - )); - } - if enabled.contains(TERMINAL_SELECT.code) { - diagnostics.extend(diagnostics_for_spans( - &TERMINAL_SELECT, - &facts.terminal_selects, - Some("choosing the final logical CTE and projection requires user intent"), - )); - } - if enabled.contains(NESTED_CTE.code) { - diagnostics.extend(diagnostics_for_spans( - &NESTED_CTE, - &facts.nested_ctes, - Some("hoisting a nested CTE requires an authored query rewrite"), - )); - } - if enabled.contains(RECURSIVE_CTE.code) { - diagnostics.extend(diagnostics_for_spans( - &RECURSIVE_CTE, - &facts.recursive_ctes, - Some("replacing recursion requires an authored query design"), - )); - } - if enabled.contains(INLINE_QUERY_RELATION.code) { - diagnostics.extend(diagnostics_for_spans( - &INLINE_QUERY_RELATION, - &facts.inline_query_relations, - Some("extracting a derived relation can change correlation and scope"), - )); - } - diagnostics -} - -fn deletion(span: Span) -> LintEdit { - LintEdit { - start: span.start, - end: span.end, - replacement: String::new(), - } -} - -/// `DISTINCT `, only when the parentheses hold the whole select item, so removing them cannot regroup an operator. -fn parenthesized_distinct_fix(sql: &str, tokens: &[Token], span: Span) -> Option { - let source = char_slice(sql, span.start, span.end)?; - let close = tokens.iter().position(|token| token.span.end == span.end)?; - let item_ends = significant_after(tokens, close) - .is_none_or(|next| SELECT_ITEM_END_TOKENS.contains(&tokens[next].token_type)); - if contains_comment(source) || !item_ends { - return None; - } - let open = source.find('(')?; - let close = source.rfind(')')?; - (open < close).then(|| LintEdit { - start: span.start, - end: span.end, - replacement: format!("DISTINCT {}", &source[open + 1..close]), - }) -} - -fn simple_boolean_case_fix(sql: &str, tokens: &[Token], span: Span) -> Option { - let case_index = token_index_for_span(tokens, span).or_else(|| { - tokens - .iter() - .position(|token| token.span.start == span.start && token.token_type == TokenType::Case) - })?; - let end_index = tokens.iter().position(|token| token.span.end == span.end)?; - let source = char_slice(sql, span.start, span.end)?; - if contains_comment(source) { - return None; - } - let case_tokens = &tokens[case_index..=end_index]; - if case_tokens - .iter() - .filter(|token| token.token_type == TokenType::When) - .count() - != 1 - || case_tokens - .iter() - .filter(|token| token.token_type == TokenType::Then) - .count() - != 1 - || case_tokens - .iter() - .filter(|token| token.token_type == TokenType::Else) - .count() - != 1 - { - return None; - } - let when_index = - (case_index + 1..end_index).find(|&index| tokens[index].token_type == TokenType::When)?; - let then_index = - (when_index + 1..end_index).find(|&index| tokens[index].token_type == TokenType::Then)?; - let condition = char_slice( - sql, - tokens[when_index].span.end, - tokens[then_index].span.start, - )? - .trim(); - (!condition.is_empty()).then(|| LintEdit { - start: span.start, - end: span.end, - replacement: format!("COALESCE({condition}, FALSE)"), - }) -} - -fn explicit_union_fix(dialect: DialectType, span: Span) -> Option { - matches!( - dialect, - DialectType::Generic - | DialectType::PostgreSQL - | DialectType::MySQL - | DialectType::BigQuery - | DialectType::Snowflake - | DialectType::DuckDB - | DialectType::Hive - | DialectType::Spark - | DialectType::Trino - | DialectType::Presto - | DialectType::Redshift - | DialectType::ClickHouse - | DialectType::Databricks - | DialectType::Athena - ) - .then(|| LintEdit { - start: span.start, - end: span.end, - replacement: "UNION DISTINCT".to_string(), - }) -} - -fn char_slice(sql: &str, start: usize, end: usize) -> Option<&str> { - if start > end { - return None; - } - let byte_start = char_offset_to_byte_offset(sql, start)?; - let byte_end = char_offset_to_byte_offset(sql, end)?; - sql.get(byte_start..byte_end) -} - -fn char_offset_to_byte_offset(sql: &str, offset: usize) -> Option { - if offset == sql.chars().count() { - return Some(sql.len()); - } - sql.char_indices() - .nth(offset) - .map(|(byte_offset, _)| byte_offset) -} - -fn contains_comment(source: &str) -> bool { - source.contains("--") || source.contains("/*") -} - -fn diagnostics_for_spans( - rule: &LintRuleMetadata, - spans: &[Span], - fix_unavailable_reason: Option<&'static str>, -) -> Vec { - spans - .iter() - .map(|span| diagnostic(rule, Some(*span), None, fix_unavailable_reason)) - .collect() -} - -fn diagnostic( - rule: &LintRuleMetadata, - span: Option, - fix: Option, - fix_unavailable_reason: Option<&'static str>, -) -> LintDiagnostic { - let span = span.unwrap_or_default(); - LintDiagnostic { - code: rule.code, - message: Cow::Borrowed(rule.message), - remediation: Cow::Borrowed(rule.remediation), - start: span.start, - end: span.end, - fix_unavailable_reason: fix.is_none().then_some(fix_unavailable_reason).flatten(), - fix, - } -} - -fn unused_cte_spans(tokens: &[Token], unused_names: &[String]) -> Vec { - let mut remaining: Vec = unused_names - .iter() - .map(|name| name.to_ascii_lowercase()) - .collect(); - let mut spans: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - let name = token.text.to_ascii_lowercase(); - let Some(position) = remaining.iter().position(|candidate| candidate == &name) else { - continue; - }; - if tokens - .get(index + 1) - .is_some_and(|next| next.token_type == TokenType::As) - { - remaining.remove(position); - spans.push(token.span); - } - } - spans -} - -fn null_comparison_spans(tokens: &[Token]) -> Vec { - let comparison_types = [ - TokenType::Eq, - TokenType::Neq, - TokenType::Lt, - TokenType::Lte, - TokenType::Gt, - TokenType::Gte, - ]; - let depths = token_depths(tokens); - let assignments: Vec = assignment_operators(tokens, &depths); - tokens - .iter() - .enumerate() - .filter(|(_, token)| comparison_types.contains(&token.token_type)) - .filter(|(index, _)| !assignments[*index]) - .filter(|(index, _)| { - adjacent_null(tokens, *index, false) || adjacent_null(tokens, *index, true) - }) - .map(|(_, token)| token.span) - .collect() -} - -/// For each token, whether its nearest same-depth clause opener is `SET`, found in one forward pass. -fn assignment_operators(tokens: &[Token], depths: &[usize]) -> Vec { - let mut in_set: Vec = Vec::new(); - let mut assignments: Vec = Vec::with_capacity(tokens.len()); - for (token, &depth) in tokens.iter().zip(depths) { - if in_set.len() <= depth { - in_set.resize(depth + 1, false); - } - assignments.push(in_set[depth]); - match token.token_type { - TokenType::Set => in_set[depth] = true, - TokenType::Where - | TokenType::Having - | TokenType::Qualify - | TokenType::On - | TokenType::Select - | TokenType::Semicolon => in_set[depth] = false, - _ => {} - } - } - assignments -} - -fn adjacent_null(tokens: &[Token], operator_index: usize, forward: bool) -> bool { - let mut index = operator_index as isize + if forward { 1 } else { -1 }; - let mut wrappers = 0_usize; - loop { - if index < 0 { - return false; - } - let Some(token) = tokens.get(index as usize) else { - return false; - }; - if token.token_type == TokenType::Null { - break; - } - let wrapper = if forward { - TokenType::LParen - } else { - TokenType::RParen - }; - if token.token_type != wrapper { - return false; - } - wrappers += 1; - index += if forward { 1 } else { -1 }; - } - index += if forward { 1 } else { -1 }; - let closing_wrapper = if forward { - TokenType::RParen - } else { - TokenType::LParen - }; - for _ in 0..wrappers { - if index < 0 { - return false; - } - let Some(token) = tokens.get(index as usize) else { - return false; - }; - if token.token_type != closing_wrapper { - return false; - } - index += if forward { 1 } else { -1 }; - } - true -} diff --git a/crates/sqlclean/src/sql_lint/_helpers/inline_relations.rs b/crates/sqlclean/src/sql_lint/_helpers/inline_relations.rs deleted file mode 100644 index 7660a13..0000000 --- a/crates/sqlclean/src/sql_lint/_helpers/inline_relations.rs +++ /dev/null @@ -1,223 +0,0 @@ -use polyglot_sql::tokens::{Span, Token, TokenType}; - -use crate::sql_lint::_helpers::engine::is_query_from; - -struct InlineRelationContext<'a> { - tokens: &'a [Token], - depths: &'a [usize], - scopes: Vec>, - significant: &'a [usize], -} - -pub(super) fn collect_inline_query_relation_spans( - tokens: &[Token], - depths: &[usize], - significant: &[usize], -) -> Vec { - let context = InlineRelationContext { - tokens, - depths, - scopes: token_scopes(tokens), - significant, - }; - (0..significant.len()) - .filter_map(|position| inline_query_relation_span(&context, position)) - .collect() -} - -fn inline_query_relation_span( - context: &InlineRelationContext<'_>, - position: usize, -) -> Option { - let &opening = context.significant.get(position)?; - if context.tokens[opening].token_type != TokenType::LParen { - return None; - } - let marker_position = position.checked_sub(1)?; - let previous = context.significant[marker_position]; - if context.tokens[previous].token_type == TokenType::Lateral - || context.tokens[previous].text.eq_ignore_ascii_case("apply") - || !is_query_relation_marker(context, marker_position) - { - return None; - } - - let mut candidate_position = position + 1; - let mut candidate_depth = context.depths[opening] + 1; - loop { - let &candidate = context.significant.get(candidate_position)?; - if context.depths[candidate] != candidate_depth { - return None; - } - if context.tokens[candidate].token_type == TokenType::LParen { - candidate_position += 1; - candidate_depth += 1; - continue; - } - return is_query_start(&context.tokens[candidate]) - .then_some(context.tokens[candidate].span); - } -} - -fn is_query_relation_marker(context: &InlineRelationContext<'_>, marker_position: usize) -> bool { - is_query_relation_marker_base(context, marker_position) - && !query_or_ancestor_is_ignored(context, marker_position) -} - -fn is_query_relation_marker_base( - context: &InlineRelationContext<'_>, - marker_position: usize, -) -> bool { - let marker = context.significant[marker_position]; - if !(is_query_from(context.tokens, marker) - || matches!( - context.tokens[marker].token_type, - TokenType::Join | TokenType::Comma - )) - { - return false; - } - let scope = context.scopes[marker]; - let Some(select_position) = (0..marker_position).rev().find(|&position| { - let index = context.significant[position]; - context.scopes[index] == scope && context.tokens[index].token_type == TokenType::Select - }) else { - return false; - }; - let query_clause = &context.significant[select_position + 1..marker_position]; - if query_clause.iter().any(|&index| { - context.scopes[index] == scope - && matches!( - context.tokens[index].token_type, - TokenType::Semicolon | TokenType::Union | TokenType::Intersect | TokenType::Except - ) - }) { - return false; - } - if is_query_from(context.tokens, marker) { - return true; - } - let Some(from_position) = query_clause - .iter() - .position(|&index| context.scopes[index] == scope && is_query_from(context.tokens, index)) - else { - return false; - }; - !query_clause[from_position + 1..].iter().any(|&index| { - context.scopes[index] == scope && is_post_relation_clause(context.tokens[index].token_type) - }) -} - -fn query_or_ancestor_is_ignored( - context: &InlineRelationContext<'_>, - marker_position: usize, -) -> bool { - let marker = context.significant[marker_position]; - let mut scope = context.scopes[marker]; - while let Some(opening) = scope { - if scope_starts_query(context, opening) && query_scope_is_ignored(context, opening) { - return true; - } - scope = context.scopes[opening]; - } - false -} - -fn scope_starts_query(context: &InlineRelationContext<'_>, opening: usize) -> bool { - let Ok(mut position) = context.significant.binary_search(&opening) else { - return false; - }; - position += 1; - let mut expected_depth = context.depths[opening] + 1; - loop { - let Some(&candidate) = context.significant.get(position) else { - return false; - }; - if context.depths[candidate] != expected_depth { - return false; - } - if context.tokens[candidate].token_type == TokenType::LParen { - position += 1; - expected_depth += 1; - continue; - } - return is_query_start(&context.tokens[candidate]); - } -} - -fn query_scope_is_ignored(context: &InlineRelationContext<'_>, opening: usize) -> bool { - let Ok(mut opening_position) = context.significant.binary_search(&opening) else { - return false; - }; - while opening_position > 0 - && context.tokens[context.significant[opening_position - 1]].token_type == TokenType::LParen - { - opening_position -= 1; - } - let Some(marker_position) = opening_position.checked_sub(1) else { - return false; - }; - let marker = context.significant[marker_position]; - match context.tokens[marker].token_type { - TokenType::Exists | TokenType::In => true, - TokenType::From | TokenType::Join | TokenType::As => false, - TokenType::Comma => !is_query_relation_marker_base(context, marker_position), - TokenType::Union - | TokenType::Intersect - | TokenType::Except - | TokenType::All - | TokenType::Distinct - | TokenType::Semicolon - | TokenType::Lateral => false, - _ if context.tokens[marker].text.eq_ignore_ascii_case("apply") - || context.tokens[marker] - .text - .eq_ignore_ascii_case("materialized") => - { - false - } - _ => true, - } -} - -fn is_query_start(token: &Token) -> bool { - matches!(token.token_type, TokenType::Select | TokenType::Values) - || token.text.eq_ignore_ascii_case("with") -} - -fn is_post_relation_clause(token_type: TokenType) -> bool { - matches!( - token_type, - TokenType::Where - | TokenType::Group - | TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Offset - | TokenType::Fetch - ) -} - -fn token_scopes(tokens: &[Token]) -> Vec> { - let mut openings: Vec = Vec::new(); - tokens - .iter() - .enumerate() - .map(|(index, token)| { - let scope = openings.last().copied(); - if matches!( - token.token_type, - TokenType::LParen | TokenType::LBracket | TokenType::LBrace - ) { - openings.push(index); - } else if matches!( - token.token_type, - TokenType::RParen | TokenType::RBracket | TokenType::RBrace - ) { - let _ = openings.pop(); - } - scope - }) - .collect() -} diff --git a/crates/sqlclean/src/sql_lint/_helpers/join_predicates.rs b/crates/sqlclean/src/sql_lint/_helpers/join_predicates.rs deleted file mode 100644 index a20b443..0000000 --- a/crates/sqlclean/src/sql_lint/_helpers/join_predicates.rs +++ /dev/null @@ -1,734 +0,0 @@ -//! Join predicates, and whether an explicit cross join can multiply rows. - -use std::borrow::Cow; -use std::collections::{HashMap, HashSet}; - -use polyglot_sql::expressions::{Expression, JoinKind, Literal, Select, Subquery, TableRef, With}; -use polyglot_sql::parser::{Parser, ParserConfig}; -use polyglot_sql::tokens::{Span, Token, TokenType}; -use polyglot_sql::{DialectType, ExpressionWalk}; - -use crate::constants::SETTING_SNIPPET_INDENT; -use crate::sql_lint::_helpers::engine::{is_comment, is_layout}; -use crate::sql_lint::constants::{ - CROSS_JOIN_CTE_MARKER, CROSS_JOIN_PATH_PLACEHOLDER, CROSS_JOIN_PREFIX, - CROSS_JOIN_REASON_PLACEHOLDER, CROSS_JOIN_SUFFIX, CROSS_JOIN_UNNAMED_MESSAGE, -}; -use crate::sql_lint::models::{CrossJoinRequest, LintDiagnostic, LintRuleMetadata}; - -const CROSS_JOIN_FIX_REASON: &str = "a keyed relationship requires user intent"; - -/// Tokens in a constant `ON` condition such as `1 = 1`. -const TRIVIAL_EQUALITY_TOKEN_COUNT: usize = 3; -const FIRST_ROW: &str = "1"; -const ROW_GENERATOR_FUNCTIONS: [&str; 16] = [ - "unnest", - "explode", - "explode_outer", - "posexplode", - "posexplode_outer", - "inline", - "inline_outer", - "flatten", - "generate_series", - "generate_subscripts", - "json_each", - "json_tree", - "json_array_elements", - "jsonb_array_elements", - "range", - "stack", -]; - -pub(super) fn join_expression_spans( - tokens: &[Token], - depths: &[usize], - dialect: DialectType, -) -> Vec { - let mut spans: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - if token.token_type != TokenType::On { - continue; - } - let clause = (0..index).rev().find(|&previous| { - depths[previous] == depths[index] - && matches!( - tokens[previous].token_type, - TokenType::Join - | TokenType::Select - | TokenType::Where - | TokenType::From - | TokenType::On - ) - }); - if clause.is_none_or(|previous| tokens[previous].token_type != TokenType::Join) { - continue; - } - let end = (index + 1..tokens.len()) - .find(|&next| { - depths[next] < depths[index] - || (depths[next] == depths[index] - && matches!( - tokens[next].token_type, - TokenType::Join - | TokenType::Where - | TokenType::Group - | TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Union - | TokenType::Intersect - | TokenType::Except - | TokenType::Semicolon - )) - }) - .unwrap_or(tokens.len()); - let mut parser = Parser::with_config( - tokens[index + 1..end].to_vec(), - ParserConfig { - dialect: Some(dialect), - ..Default::default() - }, - ); - if !matches!(parser.parse_expressions(), Ok(Some(ref predicate)) if plain_predicate(predicate)) - { - spans.push(token.span); - } - } - spans -} - -fn plain_column(expression: &Expression) -> bool { - matches!( - expression, - Expression::Column(_) | Expression::Identifier(_) - ) || matches!(expression, Expression::Paren(value) if plain_column(&value.this)) -} - -fn literal(expression: &Expression) -> bool { - matches!( - expression, - Expression::Literal(_) | Expression::Boolean(_) | Expression::Null(_) - ) || matches!(expression, Expression::Neg(value) if matches!(&value.this, Expression::Literal(_))) -} - -fn operand(expression: &Expression) -> bool { - plain_column(expression) || literal(expression) -} - -fn comparison(left: &Expression, right: &Expression) -> bool { - operand(left) && operand(right) -} - -fn plain_predicate(expression: &Expression) -> bool { - match expression { - Expression::Boolean(_) => true, - Expression::Paren(value) => plain_predicate(&value.this), - Expression::And(value) | Expression::Or(value) => { - plain_predicate(&value.left) && plain_predicate(&value.right) - } - Expression::Eq(value) - | Expression::Neq(value) - | Expression::Lt(value) - | Expression::Lte(value) - | Expression::Gt(value) - | Expression::Gte(value) - | Expression::NullSafeEq(value) - | Expression::NullSafeNeq(value) - | Expression::Adjacent(value) - | Expression::ArrayContainsAll(value) - | Expression::ArrayContainedBy(value) - | Expression::ArrayOverlaps(value) - | Expression::ExtendsLeft(value) - | Expression::ExtendsRight(value) - | Expression::Match(value) - | Expression::TsMatch(value) => comparison(&value.left, &value.right), - Expression::Like(value) | Expression::ILike(value) => { - comparison(&value.left, &value.right) - && value.escape.as_ref().is_none_or(literal) - && value.quantifier.is_none() - } - Expression::Between(value) => { - plain_column(&value.this) && operand(&value.low) && operand(&value.high) - } - Expression::In(value) => { - plain_column(&value.this) - && value.query.is_none() - && value.unnest.is_none() - && !value.is_field - && !value.expressions.is_empty() - && value.expressions.iter().all(literal) - } - Expression::IsNull(value) => plain_column(&value.this), - _ => false, - } -} - -/// A `rule` diagnostic for every cross join that can multiply rows, naming its relation and CTE. -pub(super) fn cross_join_diagnostics( - request: &CrossJoinRequest<'_>, - rule: &LintRuleMetadata, -) -> Vec { - let one_row_relations: HashSet = request - .relation_keys - .iter() - .filter(|(_, keys)| keys.iter().any(Vec::is_empty)) - .map(|(name, _)| name.to_ascii_lowercase()) - .collect(); - let relation_names: HashMap = request - .relation_names - .iter() - .map(|(sentinel, name)| (sentinel.to_ascii_lowercase(), name.clone())) - .collect(); - let inputs = OneRowInputs { - one_row_relations: &one_row_relations, - relation_names: &relation_names, - }; - multiplying_cross_joins( - request.statements, - request.tokens, - request.cross_spans, - &inputs, - ) - .into_iter() - .map(|finding| LintDiagnostic { - code: rule.code, - message: finding - .relation - .as_deref() - .map_or(Cow::Borrowed(CROSS_JOIN_UNNAMED_MESSAGE), |relation| { - Cow::Owned(cross_join_message(relation, finding.cte.as_deref())) - }), - remediation: if finding.relation.is_some() { - Cow::Borrowed(rule.remediation) - } else { - Cow::Owned(file_wide_remediation(rule.code)) - }, - start: finding.span.start, - end: finding.span.end, - fix_unavailable_reason: Some(CROSS_JOIN_FIX_REASON), - fix: None, - }) - .collect() -} - -/// Help for a join no relation names: fix it, or record the file-wide exception it allows. -fn file_wide_remediation(code: &str) -> String { - let entry: String = [ - "[[rules.rule_exceptions]]".to_owned(), - format!("rule = \"{code}\""), - format!("path = \"{CROSS_JOIN_PATH_PLACEHOLDER}\""), - format!("reason = \"{CROSS_JOIN_REASON_PLACEHOLDER}\""), - ] - .iter() - .map(|line| format!("\n{SETTING_SNIPPET_INDENT}{line}")) - .collect(); - format!( - "Use a keyed JOIN, or make one side provably at most one row. This join reads no named \ - relation, so a relation-level exception cannot cover it; when the cartesian product is \ - intentional, record a reviewed file-wide exception in the configuration file:{entry}" - ) -} - -/// The finding message naming the joined relation and, inside a CTE, that CTE. -fn cross_join_message(relation: &str, cte: Option<&str>) -> String { - match cte { - Some(cte) => { - format!("{CROSS_JOIN_PREFIX}{relation}{CROSS_JOIN_CTE_MARKER}{cte}{CROSS_JOIN_SUFFIX}") - } - None => format!("{CROSS_JOIN_PREFIX}{relation}{CROSS_JOIN_SUFFIX}"), - } -} - -/// One explicit cross join found in the query tree. -struct CrossJoinTarget { - /// Character offsets where the joined relation may start in the token stream. - anchors: Vec, - /// Whether both joined sides may hold more than one row. - multiplies: bool, - /// The joined relation's name as authored, with dependency sentinels resolved. - relation: Option, - /// The top-level CTE whose query contains the join. - cte: Option, -} - -/// The cross join reported at one `CROSS` keyword. -struct CrossJoinFinding { - span: Span, - relation: Option, - cte: Option, -} - -/// What the analysis knows about relations named in the query. -struct OneRowInputs<'a> { - /// Lowercased dependency sentinels whose relation holds at most one row. - one_row_relations: &'a HashSet, - /// Lowercased dependency sentinels mapped to the relation names they stand for. - relation_names: &'a HashMap, -} - -struct Analysis<'a> { - inputs: &'a OneRowInputs<'a>, - ctes: HashMap, - targets: Vec, -} - -/// Cross joins at `cross_spans` that can multiply rows, each with its relation and CTE. -fn multiplying_cross_joins( - statements: &[Expression], - tokens: &[Token], - cross_spans: &[Span], - inputs: &OneRowInputs<'_>, -) -> Vec { - if cross_spans.is_empty() { - return Vec::new(); - } - let mut analysis = Analysis { - inputs, - ctes: HashMap::new(), - targets: Vec::new(), - }; - for statement in statements { - analysis.record_ctes(statement); - } - for statement in statements { - analysis.visit(statement, None, true); - } - let significant: Vec = (0..tokens.len()) - .filter(|&index| { - !matches!( - tokens[index].token_type, - TokenType::Space - | TokenType::Break - | TokenType::LineComment - | TokenType::BlockComment - ) - }) - .collect(); - let mut findings: Vec = Vec::new(); - for span in cross_spans { - let candidates: Vec = relation_anchor_candidates(tokens, &significant, *span); - let target: Option<&CrossJoinTarget> = analysis - .targets - .iter() - .find(|target| anchored_at(target, &candidates)); - match target { - Some(target) if !target.multiplies => {} - Some(target) => findings.push(CrossJoinFinding { - span: *span, - relation: target.relation.clone(), - cte: target.cte.clone(), - }), - None => findings.push(CrossJoinFinding { - span: *span, - relation: None, - cte: None, - }), - } - } - findings -} - -impl Analysis<'_> { - /// Record whether every CTE holds at most one row, in definition order. - fn record_ctes(&mut self, expression: &Expression) { - for node in expression.dfs() { - let Some(with) = with_clause(node) else { - continue; - }; - for cte in &with.ctes { - let one_row: bool = !with.recursive && self.at_most_one_row(&cte.this); - let name: String = cte.alias.name.to_ascii_lowercase(); - let known: bool = self.ctes.get(&name).copied().unwrap_or(one_row); - self.ctes.insert(name, known && one_row); - } - } - } - - /// Record joins under `expression`, labelled with the top-level CTE containing them. - fn visit(&mut self, expression: &Expression, cte: Option<&str>, top: bool) { - let mut bodies: Vec<&Expression> = Vec::new(); - if let Some(with) = with_clause(expression) { - for definition in &with.ctes { - bodies.push(&definition.this); - let label: Option<&str> = if top { - Some(&definition.alias.name) - } else { - cte - }; - self.visit(&definition.this, label, false); - } - } - if let Expression::Select(select) = expression { - self.record_joins(select, cte); - } - let branches_are_top: bool = top && is_set_operation(expression); - for child in expression.children() { - if bodies.iter().any(|body| std::ptr::eq(*body, child)) { - continue; - } - self.visit(child, cte, branches_are_top); - } - } - - fn record_joins(&mut self, select: &Select, cte: Option<&str>) { - let single_from: Option<&Expression> = select - .from - .as_ref() - .filter(|from| from.expressions.len() == 1) - .and_then(|from| from.expressions.first()); - for (index, join) in select.joins.iter().enumerate() { - if join.kind != JoinKind::Cross { - continue; - } - let left_one_row: bool = - index == 0 && single_from.is_some_and(|from| self.relation_one_row(from)); - let multiplies: bool = !(left_one_row || self.relation_one_row(&join.this)); - self.targets.push(CrossJoinTarget { - anchors: relation_anchors(&join.this), - multiplies, - relation: self.relation_name(&join.this), - cte: cte.map(str::to_owned), - }); - } - } - - fn relation_name(&self, relation: &Expression) -> Option { - match relation { - Expression::Table(table) => Some(self.table_name(table)), - Expression::Subquery(subquery) => { - subquery.alias.as_ref().map(|alias| alias.name.clone()) - } - Expression::Alias(alias) => Some(alias.alias.name.clone()), - _ => None, - } - } - - fn table_name(&self, table: &TableRef) -> String { - if table.schema.is_none() && table.catalog.is_none() { - let lowered: String = table.name.name.to_ascii_lowercase(); - if let Some(name) = self.inputs.relation_names.get(&lowered) { - return name.clone(); - } - } - [&table.catalog, &table.schema] - .into_iter() - .flatten() - .map(|part| part.name.as_str()) - .chain([table.name.name.as_str()]) - .collect::>() - .join(".") - } - - /// Whether a query expression returns at most one row. - fn at_most_one_row(&self, expression: &Expression) -> bool { - match expression { - Expression::Select(select) => self.select_one_row(select), - Expression::Subquery(subquery) => self.subquery_one_row(subquery), - Expression::Paren(paren) => self.at_most_one_row(&paren.this), - Expression::Values(values) => values.expressions.len() <= 1, - _ => false, - } - } - - fn subquery_one_row(&self, subquery: &Subquery) -> bool { - !subquery.lateral - && (subquery - .limit - .as_ref() - .is_some_and(|limit| !limit.percent && at_most_one(&limit.this)) - || self.at_most_one_row(&subquery.this)) - } - - fn select_one_row(&self, select: &Select) -> bool { - if select.limit_by.is_some() || select.connect.is_some() || !select.lateral_views.is_empty() - { - return false; - } - if limited_to_one_row(select) { - return true; - } - if select.expressions.iter().any(generates_rows) { - return false; - } - if select.group_by.is_none() && select.expressions.iter().any(aggregates) { - return true; - } - if select - .qualify - .as_ref() - .is_some_and(|qualify| first_unpartitioned_row(&qualify.this, select)) - { - return true; - } - if select.group_by.is_some() || !select.joins.is_empty() { - return false; - } - match &select.from { - None => true, - Some(from) => match from.expressions.as_slice() { - [relation] => self.relation_one_row(relation), - _ => false, - }, - } - } - - /// Whether a relation in FROM or JOIN holds at most one row. - fn relation_one_row(&self, relation: &Expression) -> bool { - match relation { - Expression::Table(table) => { - if table.schema.is_some() - || table.catalog.is_some() - || table.identifier_func.is_some() - || table.changes.is_some() - { - return false; - } - let name: String = table.name.name.to_ascii_lowercase(); - match self.ctes.get(&name) { - Some(one_row) => *one_row, - None => self.inputs.one_row_relations.contains(&name), - } - } - Expression::Subquery(subquery) => self.subquery_one_row(subquery), - Expression::Paren(paren) => self.relation_one_row(&paren.this), - Expression::Values(values) => values.expressions.len() <= 1, - _ => false, - } - } -} - -/// The WITH clause a query expression owns. -fn with_clause(expression: &Expression) -> Option<&With> { - match expression { - Expression::Select(select) => select.with.as_ref(), - Expression::Union(union) => union.with.as_ref(), - Expression::Intersect(intersect) => intersect.with.as_ref(), - Expression::Except(except) => except.with.as_ref(), - _ => None, - } -} - -fn is_set_operation(expression: &Expression) -> bool { - matches!( - expression, - Expression::Union(_) | Expression::Intersect(_) | Expression::Except(_) - ) -} - -fn limited_to_one_row(select: &Select) -> bool { - let limited: bool = select - .limit - .as_ref() - .is_some_and(|limit| !limit.percent && at_most_one(&limit.this)); - let fetched: bool = select.fetch.as_ref().is_some_and(|fetch| { - !fetch.percent && !fetch.with_ties && fetch.count.as_ref().is_some_and(at_most_one) - }); - let top: bool = select - .top - .as_ref() - .is_some_and(|top| !top.percent && !top.with_ties && at_most_one(&top.this)); - limited || fetched || top -} - -fn at_most_one(expression: &Expression) -> bool { - match expression { - Expression::Literal(literal) => match literal.as_ref() { - Literal::Number(text) => text.parse::().is_ok_and(|count| count <= 1), - _ => false, - }, - Expression::Paren(paren) => at_most_one(&paren.this), - _ => false, - } -} - -/// Whether a projected expression can return several rows per input row. -fn generates_rows(expression: &Expression) -> bool { - expression.dfs().any(|node| match node { - Expression::Unnest(_) - | Expression::Explode(_) - | Expression::ExplodeOuter(_) - | Expression::GenerateSeries(_) => true, - Expression::Function(function) => is_row_generator(&function.name), - _ => false, - }) -} - -fn is_row_generator(name: &str) -> bool { - let name: &str = name.rsplit('.').next().unwrap_or_default(); - ROW_GENERATOR_FUNCTIONS - .iter() - .any(|generator| generator.eq_ignore_ascii_case(name)) -} - -fn anchored_at(target: &CrossJoinTarget, candidates: &[usize]) -> bool { - target - .anchors - .iter() - .any(|anchor| candidates.contains(anchor)) -} - -/// Whether an expression aggregates outside any window function or subquery. -fn aggregates(expression: &Expression) -> bool { - if polyglot_sql::traversal::is_aggregate(expression) { - return true; - } - match expression { - Expression::WindowFunction(_) | Expression::Subquery(_) | Expression::Select(_) => false, - _ => expression.children().into_iter().any(aggregates), - } -} - -/// `QUALIFY ROW_NUMBER() OVER (ORDER BY ...) = 1` without a partition, directly or by alias. -fn first_unpartitioned_row(condition: &Expression, select: &Select) -> bool { - match condition { - Expression::And(operation) => { - first_unpartitioned_row(&operation.left, select) - || first_unpartitioned_row(&operation.right, select) - } - Expression::Paren(paren) => first_unpartitioned_row(&paren.this, select), - Expression::Eq(operation) => { - let compared: &Expression = match (&operation.left, &operation.right) { - (other, Expression::Literal(literal)) | (Expression::Literal(literal), other) - if matches!(literal.as_ref(), Literal::Number(text) if text == FIRST_ROW) => - { - other - } - _ => return false, - }; - match compared { - Expression::WindowFunction(_) => unpartitioned_row_number(compared), - Expression::Column(column) if column.table.is_none() => select - .expressions - .iter() - .any(|projection| match projection { - Expression::Alias(alias) => { - alias.alias.name.eq_ignore_ascii_case(&column.name.name) - && unpartitioned_row_number(&alias.this) - } - _ => false, - }), - _ => false, - } - } - _ => false, - } -} - -fn unpartitioned_row_number(expression: &Expression) -> bool { - matches!( - expression, - Expression::WindowFunction(window) - if matches!(window.this, Expression::RowNumber(_)) - && window.over.window_name.is_none() - && window.over.partition_by.is_empty() - ) -} - -/// Where a joined relation can start: its first name part, or a derived table's alias. -fn relation_anchors(relation: &Expression) -> Vec { - match relation { - Expression::Table(table) => { - let mut anchors: Vec = Vec::new(); - for identifier in [&table.catalog, &table.schema].into_iter().flatten() { - anchors.extend(identifier.span.map(|span| span.start)); - } - anchors.extend(table.name.span.map(|span| span.start)); - anchors.extend(table.span.map(|span| span.start)); - anchors - } - Expression::Subquery(subquery) => subquery - .alias - .as_ref() - .and_then(|alias| alias.span) - .map(|span| span.start) - .into_iter() - .collect(), - Expression::Alias(alias) => alias - .alias - .span - .map(|span| span.start) - .into_iter() - .collect(), - _ => Vec::new(), - } -} - -/// Where the relation after `CROSS JOIN` at `cross` starts, and a derived table's alias. -fn relation_anchor_candidates(tokens: &[Token], significant: &[usize], cross: Span) -> Vec { - let Some(position) = significant - .iter() - .position(|&index| tokens[index].span.start == cross.start) - else { - return Vec::new(); - }; - let Some(&first) = significant.get(position + 2) else { - return Vec::new(); - }; - let mut candidates: Vec = vec![tokens[first].span.start]; - if tokens[first].token_type == TokenType::LParen { - let mut depth: usize = 0; - let mut cursor: usize = position + 2; - while let Some(&index) = significant.get(cursor) { - match tokens[index].token_type { - TokenType::LParen => depth += 1, - TokenType::RParen => { - depth = depth.saturating_sub(1); - if depth == 0 { - break; - } - } - _ => {} - } - cursor += 1; - } - let mut alias: usize = cursor + 1; - if significant - .get(alias) - .is_some_and(|&index| tokens[index].token_type == TokenType::As) - { - alias += 1; - } - if let Some(&index) = significant.get(alias) { - candidates.push(tokens[index].span.start); - } - } - candidates -} - -/// Every token after `ON` before `end`, including those nested in parentheses, stopping at a parenthesis the condition did not open. -pub(super) fn condition_tokens(tokens: &[Token], on: usize, end: usize) -> Vec { - let mut open: usize = 0; - let mut found: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate().take(end).skip(on + 1) { - match token.token_type { - TokenType::LParen => open += 1, - TokenType::RParen if open == 0 => break, - TokenType::RParen => open -= 1, - _ => {} - } - if !is_layout(token) && !is_comment(token) { - found.push(index); - } - } - found -} - -pub(super) fn trivial_on_condition(tokens: &[Token], condition: &[usize]) -> bool { - let significant: Vec<&Token> = condition - .iter() - .map(|&index| &tokens[index]) - .filter(|token| !matches!(token.token_type, TokenType::LParen | TokenType::RParen)) - .collect(); - significant.is_empty() - || significant.len() == 1 && significant[0].token_type == TokenType::True - || significant.len() == TRIVIAL_EQUALITY_TOKEN_COUNT - && significant[1].token_type == TokenType::Eq - && matches!( - significant[0].token_type, - TokenType::Number | TokenType::String | TokenType::True | TokenType::False - ) - && significant[0].token_type == significant[2].token_type - && significant[0].text == significant[2].text -} diff --git a/crates/sqlclean/src/sql_lint/_helpers/mod.rs b/crates/sqlclean/src/sql_lint/_helpers/mod.rs index 4e13c95..64a11d8 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/mod.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/mod.rs @@ -1,10 +1,5 @@ -pub mod additional; pub mod engine; pub mod formatter; pub mod formatter_syntax; -pub mod inline_relations; -pub mod join_predicates; -pub mod parsing; pub mod printed_layout; -pub mod terminal_shape; pub mod token_layout; diff --git a/crates/sqlclean/src/sql_lint/_helpers/parsing.rs b/crates/sqlclean/src/sql_lint/_helpers/parsing.rs deleted file mode 100644 index d9611a1..0000000 --- a/crates/sqlclean/src/sql_lint/_helpers/parsing.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! The one tokenize-and-parse step every lint and fact request shares. - -use std::str::FromStr; - -use polyglot_sql::parser::ParserConfig; - -use crate::sql_lint::models::ParsedSql; -use polyglot_sql::{ - ComplexityGuardOptions, Dialect, DialectType, Expression, ExpressionWalk, Parser, -}; - -const MAX_LINT_FUNCTION_CALL_DEPTH: usize = 128; - -/// Tokenize and parse `sql`, rejecting statements the parser kept as opaque text. -pub(crate) fn parse(sql: &str, dialect_name: &str) -> Result { - let dialect_type = DialectType::from_str(dialect_name).map_err(|error| error.to_string())?; - let dialect = Dialect::get(dialect_type); - let tokens = dialect.tokenize(sql).map_err(|error| error.to_string())?; - let mut parser = Parser::with_config( - tokens.clone(), - ParserConfig { - dialect: Some(dialect_type), - complexity_guard: ComplexityGuardOptions { - max_function_call_depth: Some(MAX_LINT_FUNCTION_CALL_DEPTH), - ..Default::default() - }, - ..Default::default() - }, - ); - let statements = parser.parse().map_err(|error| error.to_string())?; - for statement in &statements { - if statement - .dfs() - .any(|node| matches!(node, Expression::Command(_) | Expression::Raw(_))) - { - return Err("unsupported syntax: native parser retained an opaque SQL node".to_owned()); - } - } - Ok(ParsedSql { - dialect: dialect_type, - tokens, - statements, - }) -} diff --git a/crates/sqlclean/src/sql_lint/_helpers/terminal_shape.rs b/crates/sqlclean/src/sql_lint/_helpers/terminal_shape.rs deleted file mode 100644 index 3c377af..0000000 --- a/crates/sqlclean/src/sql_lint/_helpers/terminal_shape.rs +++ /dev/null @@ -1,373 +0,0 @@ -use polyglot_sql::tokens::{Span, Token, TokenType}; - -use crate::sql_lint::_helpers::engine::{is_query_from, significant_before}; - -const CEREMONIAL_SELECT_LITERAL: &str = "1"; - -pub(super) fn final_cte_name_spans(tokens: &[Token], depths: &[usize]) -> Vec { - let roots = root_with_indices(tokens, depths); - let mut names: Vec<(usize, bool)> = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - if token.token_type != TokenType::With { - continue; - } - let depth = depths[index]; - let first = names.len(); - let mut next = index + 1; - if tokens - .get(next) - .is_some_and(|token| token.text.eq_ignore_ascii_case("recursive")) - { - next += 1; - } - loop { - let name = next; - let Some(open) = cte_body_open(tokens, depths, name) else { - break; - }; - let Some(close) = (open + 1..tokens.len()).find(|&position| { - depths[position] == depth + 1 && tokens[position].token_type == TokenType::RParen - }) else { - break; - }; - names.push((name, false)); - if tokens - .get(close + 1) - .is_none_or(|token| token.token_type != TokenType::Comma) - { - break; - } - next = close + 2; - } - if roots.contains(&index) - && names.len() > first - && let Some((_, terminal)) = names.last_mut() - { - *terminal = true; - } - } - names - .into_iter() - .filter_map(|(index, terminal)| { - let is_final = tokens[index].text.eq_ignore_ascii_case("final"); - (terminal != is_final).then_some(tokens[index].span) - }) - .collect() -} - -pub(super) fn root_with_indices(tokens: &[Token], depths: &[usize]) -> Vec { - let significant: Vec = (0..tokens.len()) - .filter(|&index| { - !matches!( - tokens[index].token_type, - TokenType::Space - | TokenType::Break - | TokenType::LineComment - | TokenType::BlockComment - ) - }) - .collect(); - significant - .split(|&index| tokens[index].token_type == TokenType::Semicolon) - .filter_map(|statement| { - root_query_tokens(tokens, depths, statement) - .first() - .copied() - }) - .filter(|&index| tokens[index].token_type == TokenType::With) - .collect() -} - -fn root_query_tokens<'a>( - tokens: &[Token], - depths: &[usize], - mut significant: &'a [usize], -) -> &'a [usize] { - while significant - .last() - .is_some_and(|&index| tokens[index].token_type == TokenType::Semicolon) - { - significant = &significant[..significant.len() - 1]; - } - while let Some((&first, rest)) = significant.split_first() { - if tokens[first].token_type != TokenType::LParen - || rest - .last() - .is_none_or(|&index| tokens[index].token_type != TokenType::RParen) - || rest.iter().any(|&index| depths[index] <= depths[first]) - { - break; - } - significant = &rest[..rest.len() - 1]; - } - significant -} - -fn cte_body_open(tokens: &[Token], depths: &[usize], name: usize) -> Option { - let mut position = name + 1; - if tokens.get(position)?.token_type == TokenType::LParen { - position = (position + 1..tokens.len()).find(|&index| { - depths[index] == depths[name] + 1 && tokens[index].token_type == TokenType::RParen - })? + 1; - } - if tokens.get(position)?.token_type != TokenType::As { - return None; - } - position += 1; - if tokens.get(position)?.token_type == TokenType::Not { - position += 1; - } - if tokens - .get(position)? - .text - .eq_ignore_ascii_case("materialized") - { - position += 1; - } - (tokens.get(position)?.token_type == TokenType::LParen).then_some(position) -} - -pub(super) fn collect_terminal_shape_facts( - tokens: &[Token], - depths: &[usize], - significant: &[usize], - allows_ceremonial_select: bool, -) -> (Vec, Vec) { - let mut cte_only_bodies: Vec = Vec::new(); - let mut terminal_selects: Vec = Vec::new(); - let significant = root_query_tokens(tokens, depths, significant); - let Some(&first) = significant.first() else { - return (cte_only_bodies, terminal_selects); - }; - let root_depth = depths[first]; - let has_top_level_with = significant.iter().any(|&index| { - depths[index] == root_depth && tokens[index].text.eq_ignore_ascii_case("with") - }); - let terminal_set_operator = significant.iter().find(|&&index| { - depths[index] == root_depth - && matches!( - tokens[index].token_type, - TokenType::Union | TokenType::Intersect | TokenType::Except - ) - }); - if let Some(&operator) = terminal_set_operator { - cte_only_bodies.push(tokens[operator].span); - if has_top_level_with { - terminal_selects.push(tokens[operator].span); - } - return (cte_only_bodies, terminal_selects); - } - let Some(root_select_position) = significant.iter().position(|&index| { - depths[index] == root_depth && tokens[index].token_type == TokenType::Select - }) else { - return (cte_only_bodies, terminal_selects); - }; - let root_select = significant[root_select_position]; - let tail: Vec = significant[root_select_position + 1..] - .iter() - .copied() - .take_while(|&index| tokens[index].token_type != TokenType::Semicolon) - .filter(|&index| depths[index] == root_depth) - .collect(); - let has_terminal_logic = tail.iter().any(|&index| { - matches!( - tokens[index].token_type, - TokenType::Join - | TokenType::Where - | TokenType::Group - | TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Offset - | TokenType::Union - | TokenType::Intersect - | TokenType::Except - ) - }); - if !has_top_level_with || has_terminal_logic { - cte_only_bodies.push(tokens[root_select].span); - } - if !has_top_level_with { - return (cte_only_bodies, terminal_selects); - } - - let final_cte = (0..root_select_position) - .filter(|&position| { - let index = significant[position]; - depths[index] == root_depth - && tokens[index].token_type == TokenType::As - && significant - .get(position + 1) - .is_some_and(|&next| tokens[next].token_type == TokenType::LParen) - }) - .filter_map(|position| cte_name_before_as(tokens, significant, position)) - .next_back(); - let from_position = tail - .iter() - .position(|&index| is_terminal_from(tokens, significant, index)); - let terminal_is_plain = from_position.is_some_and(|position| { - let projection = &tail[..position]; - let relation = tail.get(position + 1); - let remainder = &tail[position.saturating_add(2)..]; - !projection.is_empty() - && plain_terminal_projection(tokens, projection) - && relation.is_some_and(|&index| { - final_cte - .as_ref() - .is_some_and(|name| tokens[index].text.eq_ignore_ascii_case(name)) - }) - && remainder.is_empty() - }); - let terminal_is_ceremonial = allows_ceremonial_select - && tail.len() == 1 - && tokens[tail[0]].text == CEREMONIAL_SELECT_LITERAL; - if !terminal_is_plain && !terminal_is_ceremonial { - terminal_selects.push(tokens[root_select].span); - } - (cte_only_bodies, terminal_selects) -} - -fn cte_name_before_as( - tokens: &[Token], - significant: &[usize], - as_position: usize, -) -> Option { - let previous_position = as_position.checked_sub(1)?; - let previous = significant[previous_position]; - if is_lint_identifier(&tokens[previous]) || tokens[previous].text.eq_ignore_ascii_case("final") - { - return Some(tokens[previous].text.to_ascii_lowercase()); - } - if tokens[previous].token_type != TokenType::RParen { - return None; - } - - let mut balance = 1_usize; - for position in (0..previous_position).rev() { - let index = significant[position]; - match tokens[index].token_type { - TokenType::RParen => balance += 1, - TokenType::LParen => { - balance = balance.saturating_sub(1); - if balance == 0 { - let name = position.checked_sub(1).map(|value| significant[value])?; - return (is_lint_identifier(&tokens[name]) - || tokens[name].text.eq_ignore_ascii_case("final")) - .then(|| tokens[name].text.to_ascii_lowercase()); - } - } - _ => {} - } - } - None -} - -fn plain_terminal_projection(tokens: &[Token], projection: &[usize]) -> bool { - projection.iter().all(|&index| { - is_terminal_projection_identifier(&tokens[index]) - || matches!( - tokens[index].token_type, - TokenType::Star | TokenType::Dot | TokenType::Comma | TokenType::As - ) - }) -} - -fn is_terminal_projection_identifier(token: &Token) -> bool { - is_lint_identifier(token) - || matches!( - token.token_type, - TokenType::Comment - | TokenType::Date - | TokenType::Key - | TokenType::Sequence - | TokenType::Time - ) - || (token.token_type == TokenType::Percent && token.text.eq_ignore_ascii_case("percent")) -} - -fn is_lint_identifier(token: &Token) -> bool { - matches!( - token.token_type, - TokenType::Identifier | TokenType::QuotedIdentifier | TokenType::Var - ) -} - -fn is_terminal_from(tokens: &[Token], significant: &[usize], index: usize) -> bool { - if tokens[index].token_type != TokenType::From { - return false; - } - significant - .iter() - .position(|&candidate| candidate == index) - .and_then(|position| position.checked_sub(1)) - .is_none_or(|position| tokens[significant[position]].token_type != TokenType::Distinct) -} - -pub(super) fn first_row_selection_clause(tokens: &[Token], direct: &[usize]) -> Option { - let from = direct - .iter() - .copied() - .find(|&index| is_query_from(tokens, index)); - direct.iter().copied().find(|&index| { - matches!( - tokens[index].token_type, - TokenType::Limit | TokenType::Offset - ) && from.is_none_or(|from_index| index > from_index) - && significant_before(tokens, index).is_none_or(|previous| { - !matches!( - tokens[previous].token_type, - TokenType::Colon | TokenType::DColon | TokenType::Dot | TokenType::DotColon - ) - }) - }) -} - -/// Whether a row-selection clause is `LIMIT 0`, which selects no rows in any order. -pub(super) fn limits_to_no_rows(tokens: &[Token], index: usize) -> bool { - tokens[index].token_type == TokenType::Limit - && tokens.get(index + 1).is_some_and(|count| { - count.token_type == TokenType::Number && count.text.parse::() == Ok(0) - }) -} - -/// Whether the projection before `limit` holds only constants, so every row it could select is identical. -pub(super) fn projects_only_constants(tokens: &[Token], direct: &[usize], limit: usize) -> bool { - let Some(&first) = direct.first() else { - return false; - }; - let end: usize = direct - .iter() - .copied() - .find(|&index| is_query_from(tokens, index)) - .unwrap_or(limit); - (first..end).all(|index| match tokens[index].token_type { - TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier => { - names_output_or_type(tokens, index) - } - TokenType::Star | TokenType::Select => false, - _ => true, - }) -} - -/// Whether the name at `index` is an alias or a type after `AS`, or a bare alias after a value. -fn names_output_or_type(tokens: &[Token], index: usize) -> bool { - let Some(previous) = index.checked_sub(1).map(|previous| &tokens[previous]) else { - return false; - }; - let followed_by_list_end: bool = tokens - .get(index + 1) - .is_none_or(|next| matches!(next.token_type, TokenType::Comma | TokenType::From)); - previous.token_type == TokenType::As - || (followed_by_list_end - && matches!( - previous.token_type, - TokenType::Number - | TokenType::String - | TokenType::Null - | TokenType::True - | TokenType::False - | TokenType::Parameter - | TokenType::RParen - )) -} diff --git a/crates/sqlclean/src/sql_lint/constants.rs b/crates/sqlclean/src/sql_lint/constants.rs index 908275a..9cd4895 100644 --- a/crates/sqlclean/src/sql_lint/constants.rs +++ b/crates/sqlclean/src/sql_lint/constants.rs @@ -1,5 +1,3 @@ -use polyglot_sql::tokens::TokenType; - pub(crate) const CAST_TYPE_SEPARATOR_KEYWORD: &str = "AS"; pub(crate) const CLOSE_PARENTHESIS: &str = ")"; pub const LINT_API_VERSION: u32 = 1; @@ -7,68 +5,3 @@ pub(crate) const OPEN_PARENTHESIS: &str = "("; pub(crate) const POSITION_ARGUMENT_SEPARATOR: &str = ","; pub(crate) const CTE_MACRO_PLACEHOLDER_LITERAL: &str = "1"; pub(crate) const POSTFIX_CAST_PREFIX_TOKEN_COUNT: usize = 2; -pub(crate) const RELATION_MODIFIER_KEYWORDS: [&str; 43] = [ - "ANTI", - "APPLY", - "ASOF", - "AT", - "BEFORE", - "CHANGES", - "CONNECT", - "CROSS", - "EXCEPT", - "FETCH", - "FINAL", - "FOR", - "FULL", - "GROUP", - "HAVING", - "INNER", - "INTERSECT", - "JOIN", - "LATERAL", - "LEFT", - "LIMIT", - "MATCH_RECOGNIZE", - "MINUS", - "NATURAL", - "OFFSET", - "ON", - "ORDER", - "OUTER", - "PASTE", - "PIVOT", - "POSITIONAL", - "QUALIFY", - "RETURNING", - "RIGHT", - "SAMPLE", - "SEMI", - "START", - "TABLESAMPLE", - "UNION", - "UNPIVOT", - "USING", - "WHERE", - "WINDOW", -]; -/// The parts of a `SQLCLEAN038` message that names its joined relation and CTE. -pub(crate) const CROSS_JOIN_PREFIX: &str = "CROSS JOIN with `"; -pub(crate) const CROSS_JOIN_CTE_MARKER: &str = "` in CTE `"; -pub(crate) const CROSS_JOIN_SUFFIX: &str = "` creates an explicit cartesian product"; -/// The `SQLCLEAN038` message for a join whose relation cannot be named. -pub(crate) const CROSS_JOIN_UNNAMED_MESSAGE: &str = "CROSS JOIN creates an explicit cartesian product with an unnamed relation; only a file-wide exception (rule and path) can cover it"; -/// Stands for the finding's project-relative path until the caller that knows it fills it in. -pub(crate) const CROSS_JOIN_PATH_PLACEHOLDER: &str = ""; -pub(crate) const CROSS_JOIN_REASON_PLACEHOLDER: &str = ""; -/// Tokens that end a select item, so parentheses closed just before one cannot be regrouping an operator. -pub(crate) const SELECT_ITEM_END_TOKENS: [TokenType; 8] = [ - TokenType::Comma, - TokenType::From, - TokenType::As, - TokenType::Semicolon, - TokenType::RParen, - TokenType::Var, - TokenType::Identifier, - TokenType::QuotedIdentifier, -]; diff --git a/crates/sqlclean/src/sql_lint/main/batch_engine.rs b/crates/sqlclean/src/sql_lint/main/batch_engine.rs index 2d7de06..cf5f8c5 100644 --- a/crates/sqlclean/src/sql_lint/main/batch_engine.rs +++ b/crates/sqlclean/src/sql_lint/main/batch_engine.rs @@ -1,19 +1,4 @@ -use crate::sql_lint::_helpers::engine::lint; -use crate::sql_lint::models::{LintRequest, LintResponse}; -use rayon::iter::{IntoParallelIterator, ParallelIterator}; - +/// Lint a JSON array of requests in parallel, with SQLQuack's lint engine. pub fn lint_batch_json(request_json: &str) -> Result { - let requests: Vec = - serde_json::from_str(request_json).map_err(|error| error.to_string())?; - if requests.is_empty() { - return Ok("[]".to_owned()); - } - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(requests.len().min(4)) - .stack_size(16 * 1024 * 1024) - .build() - .map_err(|error| error.to_string())?; - let results: Vec> = - pool.install(|| requests.into_par_iter().map(lint).collect()); - serde_json::to_string(&results).map_err(|error| error.to_string()) + sqlquack_lint::lint_batch_json(request_json) } diff --git a/crates/sqlclean/src/sql_lint/main/catalogue.rs b/crates/sqlclean/src/sql_lint/main/catalogue.rs index 7b534d4..a0bc292 100644 --- a/crates/sqlclean/src/sql_lint/main/catalogue.rs +++ b/crates/sqlclean/src/sql_lint/main/catalogue.rs @@ -1,6 +1,6 @@ -use crate::sql_lint::_helpers::engine::rule_metadata; -use crate::sql_lint::models::LintRuleMetadata; +use crate::sql_lint::types::LintRuleMetadata; +/// Every built-in lint Rule's code, message and remediation. pub fn catalogue() -> Vec { - rule_metadata() + sqlquack_lint::catalogue() } diff --git a/crates/sqlclean/src/sql_lint/main/cross_join_subject.rs b/crates/sqlclean/src/sql_lint/main/cross_join_subject.rs index 7a6d996..01d6a9e 100644 --- a/crates/sqlclean/src/sql_lint/main/cross_join_subject.rs +++ b/crates/sqlclean/src/sql_lint/main/cross_join_subject.rs @@ -1,15 +1,7 @@ //! The relation and CTE a `SQLCLEAN038` finding names, for relation-level Rule exceptions. -use crate::sql_lint::constants::{CROSS_JOIN_CTE_MARKER, CROSS_JOIN_PREFIX, CROSS_JOIN_SUFFIX}; - /// The `(relation, cte)` a cross join finding message names, `None` for the generic message. #[must_use] pub fn cross_join_subject(message: &str) -> Option<(String, Option)> { - let named: &str = message - .strip_prefix(CROSS_JOIN_PREFIX)? - .strip_suffix(CROSS_JOIN_SUFFIX)?; - match named.split_once(CROSS_JOIN_CTE_MARKER) { - Some((relation, cte)) => Some((relation.to_owned(), Some(cte.to_owned()))), - None => Some((named.to_owned(), None)), - } + sqlquack_lint::cross_join_subject(message) } diff --git a/crates/sqlclean/src/sql_lint/main/default_rules.rs b/crates/sqlclean/src/sql_lint/main/default_rules.rs index 33d1cd1..ed6cf43 100644 --- a/crates/sqlclean/src/sql_lint/main/default_rules.rs +++ b/crates/sqlclean/src/sql_lint/main/default_rules.rs @@ -2,5 +2,5 @@ /// The codes enabled when no Rule selection is given. pub fn default_rules() -> Vec<&'static str> { - crate::sql_lint::_helpers::engine::DEFAULT_RULES.to_vec() + sqlquack_lint::default_rules() } diff --git a/crates/sqlclean/src/sql_lint/main/engine.rs b/crates/sqlclean/src/sql_lint/main/engine.rs index 2533f4f..7230267 100644 --- a/crates/sqlclean/src/sql_lint/main/engine.rs +++ b/crates/sqlclean/src/sql_lint/main/engine.rs @@ -1,5 +1,4 @@ -use crate::sql_lint::_helpers::engine::lint_json_impl; - +/// Lint one JSON request with SQLQuack's lint engine and return its JSON response. pub fn lint_json(request_json: &str) -> Result { - lint_json_impl(request_json) + sqlquack_lint::lint_json(request_json) } diff --git a/crates/sqlclean/src/sql_lint/main/fill_finding_path.rs b/crates/sqlclean/src/sql_lint/main/fill_finding_path.rs index d5a6580..ac4a889 100644 --- a/crates/sqlclean/src/sql_lint/main/fill_finding_path.rs +++ b/crates/sqlclean/src/sql_lint/main/fill_finding_path.rs @@ -1,9 +1,7 @@ //! `fill_finding_path`: a SQL finding's help with its file's path written into the entry it prints. -use crate::sql_lint::constants::CROSS_JOIN_PATH_PLACEHOLDER; - /// `remediation` with the path placeholder of a printed exception entry replaced by `path`. #[must_use] pub fn fill_finding_path(remediation: &str, path: &str) -> String { - remediation.replace(CROSS_JOIN_PATH_PLACEHOLDER, path) + sqlquack_lint::fill_finding_path(remediation, path) } diff --git a/crates/sqlclean/src/sql_lint/main/lint.rs b/crates/sqlclean/src/sql_lint/main/lint.rs index ab3be0b..1acb840 100644 --- a/crates/sqlclean/src/sql_lint/main/lint.rs +++ b/crates/sqlclean/src/sql_lint/main/lint.rs @@ -1,8 +1,8 @@ //! Typed, in-process lint of one SQL text. -use crate::sql_lint::models::{LintRequest, LintResponse}; +use crate::sql_lint::types::{LintRequest, LintResponse}; /// Lint `request.sql` with the selected built-in Rules and return findings in source order. pub fn lint(request: LintRequest) -> Result { - crate::sql_lint::_helpers::engine::lint(request) + sqlquack_lint::lint(request) } diff --git a/crates/sqlclean/src/sql_lint/main/parse_sql.rs b/crates/sqlclean/src/sql_lint/main/parse_sql.rs index 14408c0..70bf461 100644 --- a/crates/sqlclean/src/sql_lint/main/parse_sql.rs +++ b/crates/sqlclean/src/sql_lint/main/parse_sql.rs @@ -1,8 +1,8 @@ //! Tokenize and parse SQL the way every lint request does. -use crate::sql_lint::models::ParsedSql; +use crate::sql_lint::types::ParsedSql; -/// Parse `sql` under `dialect`, rejecting statements the parser kept as opaque text. +/// Parse `sql` under `dialect` with SQLQuack, rejecting statements the parser kept as opaque text. pub fn parse_sql(sql: &str, dialect: &str) -> Result { - crate::sql_lint::_helpers::parsing::parse(sql, dialect) + sqlquack_lint::parse_sql(sql, dialect) } diff --git a/crates/sqlclean/src/sql_lint/main/validate_dialect.rs b/crates/sqlclean/src/sql_lint/main/validate_dialect.rs index 6fe0097..8cd151d 100644 --- a/crates/sqlclean/src/sql_lint/main/validate_dialect.rs +++ b/crates/sqlclean/src/sql_lint/main/validate_dialect.rs @@ -1,13 +1,6 @@ -//! Check a dialect name against the dialects the parser supports. +//! Check a dialect name against the dialects lint knows. -use std::str::FromStr; - -use polyglot_sql::DialectType; - -/// `Ok` when `name` names a supported SQL dialect, such as `duckdb` or `snowflake`. +/// `Ok` when `name` names a known SQL dialect, such as `duckdb` or `snowflake`. pub fn validate_dialect(name: &str) -> Result<(), String> { - match DialectType::from_str(name) { - Ok(_) => Ok(()), - Err(error) => Err(format!("unknown SQL dialect '{name}': {error}")), - } + sqlquack_lint::validate_dialect(name) } diff --git a/crates/sqlclean/src/sql_lint/mod.rs b/crates/sqlclean/src/sql_lint/mod.rs index e3460e8..33ed33e 100644 --- a/crates/sqlclean/src/sql_lint/mod.rs +++ b/crates/sqlclean/src/sql_lint/mod.rs @@ -1,9 +1,10 @@ -//! SQL lint Rules and the deterministic formatter over the Polyglot parser. +//! SQL lint Rules and the deterministic formatter; lint runs on SQLQuack, the formatter still on Polyglot. pub mod _helpers; pub mod constants; pub mod main; pub mod models; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlclean/src/sql_lint/models.rs b/crates/sqlclean/src/sql_lint/models.rs index 4d5cdb0..2f27e50 100644 --- a/crates/sqlclean/src/sql_lint/models.rs +++ b/crates/sqlclean/src/sql_lint/models.rs @@ -1,173 +1,7 @@ -use std::borrow::Cow; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; -use polyglot_sql::tokens::{Span, Token}; -use polyglot_sql::{DialectType, Expression}; use serde::{Deserialize, Serialize}; -/// SQL text tokenized and parsed under one dialect. -#[derive(Debug)] -pub struct ParsedSql { - pub dialect: DialectType, - pub tokens: Vec, - pub statements: Vec, -} - -#[derive(Debug, Default)] -pub(crate) struct QueryFacts { - pub null_comparisons: Vec, - pub implicit_cartesian_joins: Vec, - pub joins_without_condition: Vec, - pub unordered_limits: Vec, - pub redundant_distincts: Vec, - pub positional_set_stars: Vec, - pub unused_cte_names: Vec, - pub cross_joins: Vec, - pub cte_only_bodies: Vec, - pub terminal_selects: Vec, - pub nested_ctes: Vec, - pub recursive_ctes: Vec, - pub invalid_comment_attachments: Vec, - pub inline_query_relations: Vec, - pub additional: AdditionalQueryFacts, -} - -#[derive(Debug, Default)] -pub(crate) struct AdditionalQueryFacts { - pub bare_unions: Vec, - pub duplicate_table_aliases: Vec, - pub duplicate_output_aliases: Vec, - pub mixed_group_order_references: Vec, - pub redundant_else_nulls: Vec, - pub parenthesized_distincts: Vec, - pub constant_predicates: Vec, - pub consecutive_semicolons: Vec, - pub redundant_self_aliases: Vec, - pub count_one_literals: Vec, - pub unstable_row_numbers: Vec, - pub null_not_in_predicates: Vec, - pub set_arity_mismatches: Vec, - pub projected_stars: Vec, - pub unaliased_calculations: Vec, - pub unused_table_aliases: Vec, - pub null_rejected_left_joins: Vec, - pub implicit_inner_joins: Vec, - pub ambiguous_order_directions: Vec, - pub unqualified_multi_source_columns: Vec, - pub inconsistent_single_source_qualification: Vec, - pub unknown_relation_qualifiers: Vec, - pub simple_boolean_cases: Vec, - pub nested_else_cases: Vec, - pub unused_joined_relations: Vec, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct LintRequest { - pub version: u32, - pub sql: String, - pub dialect: String, - #[serde(default)] - pub enabled_rules: Option>, - #[serde(default)] - pub ignored_rules: Vec, - #[serde(default)] - pub external_identifiers: Vec, - #[serde(default)] - pub dependency_identifiers: Vec, - #[serde(default)] - pub externally_referenced_ctes: Vec, - /// The harness the host wraps this body in; absent for an ordinary query. - #[serde(default)] - pub fixture_harness: Option, - #[serde(default)] - pub allows_dynamic_output_star: bool, - #[serde(default)] - pub max_literal_length: Option, - #[serde(default)] - pub max_ranking_order_by: Option, - #[serde(default)] - pub relation_keys: HashMap>>, - #[serde(default)] - pub relation_names: HashMap, - /// The SQL each macro call in the query generated, so column qualification is judged as the macro wrote it. - #[serde(default)] - pub sql_macro_outputs: Vec, - /// The configuration file remediation text names; SQLClean's own when absent. - #[serde(default)] - pub config_file: Option, - /// The advice `SQLCLEAN044` gives before its limit setting; SQLClean's own when absent. - #[serde(default)] - pub long_literal_guidance: Option, -} - -/// The generated wrapper around a test or macro body: its terminal SELECT is ceremonial, and these names are generated. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct FixtureHarness { - /// Prefixes of the CTE names the harness generates, which are reachable even when unread. - #[serde(default)] - pub cte_prefixes: Vec, - /// Prefixes of the CTEs standing in for the body's inputs, which may read the empty relation with `SELECT *`. - #[serde(default)] - pub input_cte_prefixes: Vec, - /// The table function an input CTE reads, as `SELECT * FROM ()`, when it has no rows. - #[serde(default)] - pub empty_relation: Option, -} - -/// The character range one macro call's output fills, and the relations its arguments pass, which a plain parameter slot read expands to qualify with. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] -pub struct SqlMacroOutput { - pub start: usize, - pub end: usize, - #[serde(default)] - pub relations: Vec, -} - -#[derive(Debug)] -pub(crate) struct AdditionalFactOptions<'a> { - pub external_identifiers: &'a HashSet, - pub dependency_identifiers: &'a HashSet, - pub sql_macro_outputs: &'a [SqlMacroOutput], - pub allows_ceremonial_select: bool, - pub allows_dynamic_output_star: bool, - pub fixture_harness: Option<&'a FixtureHarness>, -} - -#[derive(Debug, Serialize)] -pub struct LintResponse { - pub version: u32, - pub diagnostics: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct LintDiagnostic { - pub code: &'static str, - pub message: Cow<'static, str>, - pub remediation: Cow<'static, str>, - pub start: usize, - pub end: usize, - #[serde(skip_serializing_if = "Option::is_none")] - pub fix: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub fix_unavailable_reason: Option<&'static str>, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct LintEdit { - pub start: usize, - pub end: usize, - pub replacement: String, -} - -#[derive(Debug, Clone, Copy)] -pub struct LintRuleMetadata { - pub code: &'static str, - pub message: &'static str, - pub remediation: &'static str, -} - #[derive(Debug, Default, Deserialize)] #[serde(deny_unknown_fields)] pub struct FormatRequest { @@ -194,15 +28,3 @@ pub struct FormatResponse { #[serde(skip_serializing_if = "Option::is_none")] pub reason: Option, } - -/// One lint request's explicit cross joins, with what is known about the relations they read. -#[derive(Debug)] -pub(crate) struct CrossJoinRequest<'a> { - pub(crate) statements: &'a [Expression], - pub(crate) tokens: &'a [Token], - pub(crate) cross_spans: &'a [Span], - /// Declared keys by dependency sentinel; the empty key proves at most one row. - pub(crate) relation_keys: &'a HashMap>>, - /// The relation name each dependency sentinel stands for. - pub(crate) relation_names: &'a HashMap, -} diff --git a/crates/sqlclean/src/sql_lint/tests/test_sql_lint.rs b/crates/sqlclean/src/sql_lint/tests/test_sql_lint.rs index 3b72db4..736297b 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_sql_lint.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_sql_lint.rs @@ -1780,6 +1780,7 @@ fn given_ceremonial_select_context_when_linting_terminal_shape_then_rule_035_all } #[test] +#[ignore = "lint is unavailable for tsql until SQLQuack parses it"] fn given_tsql_bare_union_when_linting_then_diagnostic_has_no_invalid_fix() -> Result<(), String> { let test_cases = [test_types::DialectLintRuleTestCase { description: "T-SQL bare UNION is diagnosed without an invalid fix", diff --git a/crates/sqlclean/src/sql_lint/types.rs b/crates/sqlclean/src/sql_lint/types.rs new file mode 100644 index 0000000..1538b43 --- /dev/null +++ b/crates/sqlclean/src/sql_lint/types.rs @@ -0,0 +1,20 @@ +//! The lint request, response and parse types, which SQLQuack's lint engine defines. + +use sqlquack_lint::lint::models as lint; + +/// SQL text tokenized and parsed under one dialect by SQLQuack. +pub type ParsedSql = lint::ParsedSql; +/// One lint request: the SQL, its dialect, the selected Rules and what the host knows. +pub type LintRequest = lint::LintRequest; +/// The harness a host wraps a fixture body in. +pub type FixtureHarness = lint::FixtureHarness; +/// The SQL one macro call generated, located in the query. +pub type SqlMacroOutput = lint::SqlMacroOutput; +/// Findings in source order, or why lint is unavailable for the dialect. +pub type LintResponse = lint::LintResponse; +/// One finding, located by character offsets. +pub type LintDiagnostic = lint::LintDiagnostic; +/// One automatic fix, located by character offsets. +pub type LintEdit = lint::LintEdit; +/// A built-in Rule's code, message and remediation. +pub type LintRuleMetadata = lint::LintRuleMetadata; diff --git a/crates/sqlclean/src/sql_quality/_helpers/cte_columns.rs b/crates/sqlclean/src/sql_quality/_helpers/cte_columns.rs deleted file mode 100644 index 635a244..0000000 --- a/crates/sqlclean/src/sql_quality/_helpers/cte_columns.rs +++ /dev/null @@ -1,457 +0,0 @@ -//! SQLCLEAN042: output columns of non-final CTEs that no later step reads. - -use std::borrow::Cow; - -use polyglot_sql::expressions::{Cte, JoinKind, Select}; -use polyglot_sql::scope::walk_in_scope; -use polyglot_sql::{Expression, ExpressionWalk}; - -use crate::sql_lint::models::{LintDiagnostic, LintEdit}; -use crate::sql_quality::constants::{ - DISTINCT_UNUSED_REMEDIATION, GROUP_BY_ALL_UNUSED_REMEDIATION, GROUP_KEY_UNUSED_REMEDIATION, - UNUSED_CTE_OUTPUT, -}; -use crate::sql_quality::models::QualityRequest; -use crate::sql_quality::removal_safety::{expression_refusal, reads_whole_rows, select_refusal}; -use crate::sql_quality::syntax::{ - Projection, columns, cte_select_tokens, lower, projection_removal, projections, qualifier, - sources, -}; -use crate::sql_tokens::main::projection_spans::projection_spans; -use rustc_hash::{FxHashMap, FxHashSet}; - -const HARNESS_PREFIX: &str = "__"; -const COLUMNS_FUNCTION: &str = "COLUMNS"; -const LAYOUT_FIX_REFUSAL: &str = "The projection list could not be mapped to authored tokens"; - -/// What one later step reads from a CTE. -#[derive(Default)] -struct Usage { - names: FxHashSet, - everything: bool, - passes_to: Vec, - read: bool, -} - -impl Usage { - fn merged(mut self, other: Self) -> Self { - self.names.extend(other.names); - self.everything |= other.everything; - self.passes_to.extend(other.passes_to); - self.read |= other.read; - self - } -} - -/// One CTE whose unused outputs are being reported. -struct CteOutputs<'a> { - cte: &'a Cte, - name: &'a str, - usage: Option<&'a Usage>, -} - -pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { - if request.fixture_body { - return Vec::new(); - } - let mut found: Vec = Vec::new(); - for statement in request.statements { - let Expression::Select(root) = statement else { - continue; - }; - let Some(with) = root.with.as_ref().filter(|with| !with.recursive) else { - continue; - }; - let mut terminal: Select = root.as_ref().clone(); - terminal.with = None; - let terminal: Expression = Expression::Select(Box::new(terminal)); - let names: Vec = with.ctes.iter().map(|cte| lower(&cte.alias.name)).collect(); - let used: FxHashMap = reads(&with.ctes, &names, &terminal); - let select_tokens: FxHashMap = cte_select_tokens(request.tokens); - for (index, cte) in with - .ctes - .iter() - .enumerate() - .take(with.ctes.len().saturating_sub(1)) - { - let name: &str = &names[index]; - if name.starts_with(HARNESS_PREFIX) || request.externally_referenced_ctes.contains(name) - { - continue; - } - found.extend(unused_outputs( - request, - &select_tokens, - CteOutputs { - cte, - name, - usage: used.get(name), - }, - )); - } - } - found -} - -/// Names each CTE's later readers use, following `SELECT *` pass-through to the reader. -fn reads(ctes: &[Cte], names: &[String], terminal: &Expression) -> FxHashMap { - let bodies: Vec<(Option, &Expression)> = ctes - .iter() - .map(|cte| (Some(lower(&cte.alias.name)), &cte.this)) - .chain(std::iter::once((None, terminal))) - .collect(); - let known: FxHashSet<&str> = names.iter().map(String::as_str).collect(); - let outputs: Vec>> = ctes.iter().map(output_names).collect(); - let mut readers: FxHashMap> = FxHashMap::default(); - for (index, (_, body)) in bodies.iter().enumerate() { - for name in read_relations(body) { - if known.contains(name.as_str()) { - readers.entry(name).or_default().push(index); - } - } - } - let mut usage: FxHashMap = FxHashMap::default(); - for (index, name) in names.iter().enumerate() { - let mut current: Usage = Usage::default(); - if index + 1 == names.len() { - current.everything = true; - } - for &reader in readers.get(name.as_str()).into_iter().flatten() { - if reader > index { - let (reader_name, body) = &bodies[reader]; - let target: Target<'_> = Target { - name, - outputs: outputs[index].as_ref(), - }; - current = current.merged(reader_usage(&target, reader_name.clone(), body)); - } - } - current.names.extend(self_reads(&ctes[index].this)); - usage.insert(name.clone(), current); - } - for index in (0..names.len()).rev() { - let passes: Vec = usage[&names[index]].passes_to.clone(); - for reader in passes { - let (everything, inherited): (bool, Vec) = - usage.get(&reader).map_or((true, Vec::new()), |next| { - (next.everything, next.names.iter().cloned().collect()) - }); - if let Some(current) = usage.get_mut(&names[index]) { - current.everything |= everything; - current.names.extend(inherited); - } - } - } - usage -} - -/// The output names of a CTE, or None when a star or unnamed projection hides them. -fn output_names(cte: &Cte) -> Option> { - let Expression::Select(select) = &cte.this else { - return None; - }; - projections(select) - .into_iter() - .map(|item| match item { - Projection::Named { name, .. } => Some(name), - _ => None, - }) - .collect() -} - -fn read_relations(body: &Expression) -> FxHashSet { - body.dfs() - .filter_map(|node| match node { - Expression::Table(table) => Some(lower(&table.name.name)), - _ => None, - }) - .collect() -} - -/// The CTE whose readers are being inspected. -struct Target<'a> { - name: &'a str, - outputs: Option<&'a FxHashSet>, -} - -fn reader_usage(target: &Target<'_>, reader: Option, body: &Expression) -> Usage { - let name: &str = target.name; - let mut usage: Usage = Usage::default(); - let mut aliases: FxHashSet = FxHashSet::default(); - let mut top_level_star: bool = false; - for node in body.dfs() { - let Expression::Select(select) = node else { - continue; - }; - let local: Vec = sources(select) - .into_iter() - .filter(|(_, relation, _)| relation.as_deref() == Some(name)) - .map(|(alias, _, _)| alias) - .collect(); - if local.is_empty() { - continue; - } - if select.joins.iter().any(|join| { - matches!( - join.kind, - JoinKind::Natural - | JoinKind::NaturalLeft - | JoinKind::NaturalRight - | JoinKind::NaturalFull - ) - }) { - usage.everything = true; - } - for join in &select.joins { - usage - .names - .extend(join.using.iter().map(|identifier| lower(&identifier.name))); - } - let top: bool = std::ptr::eq(node, body); - if reads_as_row(select, &local, target.outputs) { - usage.everything = true; - } - for projection in projections(select) { - if let Projection::Star(star) = projection - && star - .table - .as_ref() - .is_none_or(|table| local.contains(&lower(&table.name))) - { - if top - && reader.is_some() - && star.except.is_none() - && star.replace.is_none() - && !reads_whole_rows(select) - { - top_level_star = true; - } else { - usage.everything = true; - } - } - } - aliases.extend(local); - } - if aliases.is_empty() { - return usage; - } - usage.read = true; - usage.everything |= columns(body) - .into_iter() - .any(|column| is_interpolation(&column.name.name)); - if top_level_star && let Some(reader) = reader { - usage.passes_to.push(reader); - } - for column in body.dfs().filter_map(|node| match node { - Expression::Column(column) => Some(column), - _ => None, - }) { - if qualifier(column).is_none_or(|table| aliases.contains(&table)) { - usage.names.insert(lower(&column.name.name)); - } - } - usage -} - -/// Whether a reader consumes whole rows: a nested star, `COLUMNS(...)`, or the alias itself. -fn reads_as_row(select: &Select, local: &[String], outputs: Option<&FxHashSet>) -> bool { - for clause in clause_roots(select) { - for node in walk_in_scope(clause, false) { - if reads_row_at(select, node, local, outputs) { - return true; - } - } - } - false -} - -fn reads_row_at( - select: &Select, - node: &Expression, - local: &[String], - outputs: Option<&FxHashSet>, -) -> bool { - match node { - Expression::Star(star) => { - let projected: bool = select - .expressions - .iter() - .any(|item| std::ptr::eq(item, node)); - let table: Option = star.table.as_ref().map(|table| lower(&table.name)); - !projected && table.is_none_or(|table| local.contains(&table)) - } - Expression::BracedWildcard(_) => true, - Expression::Function(function) => function.name.eq_ignore_ascii_case(COLUMNS_FUNCTION), - Expression::Column(column) => { - let read: String = lower(&column.name.name); - let named_output: bool = outputs.is_some_and(|names| names.contains(&read)); - column.table.is_none() && local.contains(&read) && !named_output - } - _ => false, - } -} - -/// Root expressions of every clause of one select, without its FROM relations. -fn clause_roots(select: &Select) -> Vec<&Expression> { - let mut roots: Vec<&Expression> = select.expressions.iter().collect(); - roots.extend(select.where_clause.iter().map(|clause| &clause.this)); - roots.extend(select.having.iter().map(|clause| &clause.this)); - roots.extend(select.qualify.iter().map(|clause| &clause.this)); - if let Some(group) = &select.group_by { - roots.extend(group.expressions.iter()); - } - if let Some(order) = &select.order_by { - roots.extend(order.expressions.iter().map(|ordered| &ordered.this)); - } - roots.extend(select.joins.iter().filter_map(|join| join.on.as_ref())); - roots -} - -/// Lateral alias reuse: a later projection or a clause of the CTE reading an earlier output. -fn self_reads(body: &Expression) -> FxHashSet { - let mut reads: FxHashSet = FxHashSet::default(); - let Expression::Select(select) = body else { - return reads; - }; - let mut earlier: FxHashSet = FxHashSet::default(); - for item in projections(select) { - if let Projection::Named { name, expression } = item { - for read in unqualified_reads(expression) { - if earlier.contains(&read) && read != name { - reads.insert(read); - } - } - earlier.insert(name); - } - } - let mut clauses: Vec<&Expression> = Vec::new(); - clauses.extend(select.where_clause.iter().map(|clause| &clause.this)); - clauses.extend( - select - .group_by - .iter() - .flat_map(|group| group.expressions.iter()), - ); - clauses.extend(select.having.iter().map(|clause| &clause.this)); - clauses.extend(select.qualify.iter().map(|clause| &clause.this)); - if let Some(order) = &select.order_by { - clauses.extend(order.expressions.iter().map(|ordered| &ordered.this)); - } - for clause in clauses { - reads.extend( - unqualified_reads(clause) - .into_iter() - .filter(|read| earlier.contains(read)), - ); - } - reads -} - -fn unqualified_reads(expression: &Expression) -> Vec { - columns(expression) - .into_iter() - .filter(|column| qualifier(column).is_none()) - .map(|column| lower(&column.name.name)) - .collect() -} - -fn unused_outputs( - request: &QualityRequest<'_>, - select_tokens: &FxHashMap, - outputs: CteOutputs<'_>, -) -> Vec { - let CteOutputs { cte, name, usage } = outputs; - let Expression::Select(select) = &cte.this else { - return Vec::new(); - }; - let Some(usage) = usage.filter(|usage| !usage.everything && usage.read) else { - return Vec::new(); - }; - if !cte.columns.is_empty() { - return Vec::new(); - } - let output: Vec> = projections(select); - if !output - .iter() - .all(|item| matches!(item, Projection::Named { .. })) - { - return Vec::new(); - } - let spans: Vec<(usize, usize)> = select_tokens - .get(name) - .map(|&select_token| projection_spans(request.tokens, select_token)) - .unwrap_or_default(); - let grouped: FxHashSet = select - .group_by - .iter() - .flat_map(|group| group.expressions.iter()) - .flat_map(columns) - .map(|column| lower(&column.name.name)) - .collect(); - let select_refused: Option<&'static str> = select_refusal(select, output.len()); - let mut found: Vec = Vec::new(); - for (index, item) in output.iter().enumerate() { - let Projection::Named { - name: column, - expression, - } = item - else { - continue; - }; - if usage.names.contains(column) || is_interpolation(column) { - continue; - } - let grouping_key: bool = matches!(expression, Expression::Column(source) if grouped.contains(&lower(&source.name.name))); - let (start, end) = spans - .get(index) - .copied() - .filter(|_| spans.len() == output.len()) - .unwrap_or((0, 0)); - let refusal: Option<&'static str> = select_refused - .or_else(|| expression_refusal(expression)) - .or_else(|| (spans.len() != output.len()).then_some(LAYOUT_FIX_REFUSAL)); - let fix: Option = refusal - .is_none() - .then(|| projection_removal(&spans, index)) - .flatten() - .map(|(start, end)| LintEdit { - start, - end, - replacement: String::new(), - }); - found.push(LintDiagnostic { - code: UNUSED_CTE_OUTPUT.code, - message: Cow::Borrowed(UNUSED_CTE_OUTPUT.message), - remediation: Cow::Borrowed(if select.distinct { - DISTINCT_UNUSED_REMEDIATION - } else if select - .group_by - .as_ref() - .is_some_and(|group| group.all == Some(true)) - { - GROUP_BY_ALL_UNUSED_REMEDIATION - } else if grouping_key { - GROUP_KEY_UNUSED_REMEDIATION - } else { - UNUSED_CTE_OUTPUT.remediation - }), - start, - end, - fix_unavailable_reason: if fix.is_none() { - Some(refusal.unwrap_or(LAYOUT_FIX_REFUSAL)) - } else { - None - }, - fix, - }); - } - found -} - -/// Placeholders for template expressions can stand for any columns. -fn is_interpolation(name: &str) -> bool { - let name: String = lower(name); - crate::constants::GENERATED_PLACEHOLDER_PREFIXES - .iter() - .any(|prefix| name.starts_with(prefix)) -} diff --git a/crates/sqlclean/src/sql_quality/_helpers/json_parses.rs b/crates/sqlclean/src/sql_quality/_helpers/json_parses.rs deleted file mode 100644 index 1caaf5d..0000000 --- a/crates/sqlclean/src/sql_quality/_helpers/json_parses.rs +++ /dev/null @@ -1,201 +0,0 @@ -//! SQLCLEAN045: the same JSON payload column parsed more than once in one SELECT scope. - -use std::borrow::Cow; -use std::collections::HashMap; - -use polyglot_sql::expressions::{Column, Identifier}; -use polyglot_sql::scope::walk_in_scope; -use polyglot_sql::tokens::Token; -use polyglot_sql::{DialectType, Expression, ExpressionWalk}; - -use crate::sql_lint::models::LintDiagnostic; -use crate::sql_quality::constants::REPEATED_JSON_PARSE; -use crate::sql_quality::models::QualityRequest; -use crate::sql_quality::syntax::{is_trivia, lower}; - -/// Significant tokens searched backwards from the column for the function name. -const MAX_FUNCTION_NAME_DISTANCE: usize = 32; -const PARSE_ONCE_FIX_REFUSAL: &str = - "Parsing the payload once requires naming a column in the CTE that reads it"; - -/// An identifier normalised the way the dialect resolves it. -type IdentifierKey = String; - -/// One JSON parse call whose argument resolves to a plain column. -struct ParseCall<'a> { - function: &'static str, - column: &'a Column, - column_start: usize, -} - -pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { - let functions: &[&'static str] = parse_functions(request.dialect); - if functions.is_empty() { - return Vec::new(); - } - let mut found: Vec = Vec::new(); - for statement in request.statements { - for node in statement.dfs() { - if matches!(node, Expression::Select(_)) { - found.extend(scope_diagnostics( - request.tokens, - node, - functions, - request.dialect, - )); - } - } - } - found -} - -/// JSON parse functions that take one string argument, by dialect, in upper case. -fn parse_functions(dialect: DialectType) -> &'static [&'static str] { - match dialect { - DialectType::Snowflake | DialectType::Databricks => &["PARSE_JSON", "TRY_PARSE_JSON"], - DialectType::BigQuery => &["PARSE_JSON", "SAFE.PARSE_JSON"], - DialectType::DuckDB => &["JSON"], - _ => &[], - } -} - -fn scope_diagnostics( - tokens: &[Token], - select: &Expression, - functions: &[&'static str], - dialect: DialectType, -) -> Vec { - let quote: char = identifier_quote(dialect); - let mut calls: Vec> = walk_in_scope(select, false) - .filter_map(|node| parse_call(node, functions)) - .collect(); - calls.sort_by_key(|call| call.column_start); - let mut groups: HashMap< - (&'static str, Option, IdentifierKey), - Vec<&ParseCall<'_>>, - > = HashMap::new(); - for call in &calls { - let qualifier: Option = call - .column - .table - .as_ref() - .map(|table| identifier_key(table, dialect)); - groups - .entry(( - call.function, - qualifier, - identifier_key(&call.column.name, dialect), - )) - .or_default() - .push(call); - } - let mut found: Vec = Vec::new(); - for group in groups.values().filter(|group| group.len() > 1) { - let second: &ParseCall<'_> = group[1]; - let (start, end) = function_span(tokens, second); - found.push(LintDiagnostic { - code: REPEATED_JSON_PARSE.code, - message: Cow::Borrowed(REPEATED_JSON_PARSE.message), - remediation: Cow::Owned(remediation(second, group.len(), quote)), - start, - end, - fix: None, - fix_unavailable_reason: Some(PARSE_ONCE_FIX_REFUSAL), - }); - } - found -} - -/// Normalises a name the way the dialect resolves it (Snowflake folds unquoted names upward). -fn identifier_key(identifier: &Identifier, dialect: DialectType) -> IdentifierKey { - match dialect { - DialectType::Snowflake if identifier.quoted => identifier.name.clone(), - DialectType::Snowflake => identifier.name.to_uppercase(), - _ => lower(&identifier.name), - } -} - -fn parse_call<'a>(node: &'a Expression, functions: &[&'static str]) -> Option> { - let (name, argument): (&str, &Expression) = match node { - Expression::ParseJson(function) => ("PARSE_JSON", &function.this), - Expression::Function(function) if function.args.len() == 1 => { - (function.name.as_str(), &function.args[0]) - } - _ => return None, - }; - let function: &'static str = functions - .iter() - .copied() - .find(|candidate| candidate.eq_ignore_ascii_case(name))?; - let column: &Column = unwrap_column(argument)?; - let column_start: usize = column.span.or(column.name.span)?.start; - Some(ParseCall { - function, - column, - column_start, - }) -} - -/// The column under wrappers that keep the parsed value tied to that column. -fn unwrap_column(expression: &Expression) -> Option<&Column> { - let mut current: &Expression = expression; - loop { - current = match current { - Expression::Column(column) => return Some(column), - Expression::Paren(paren) => &paren.this, - Expression::Lower(function) | Expression::Upper(function) => &function.this, - Expression::Trim(trim) if trim.characters.is_none() => &trim.this, - Expression::Cast(cast) | Expression::TryCast(cast) | Expression::SafeCast(cast) => { - &cast.this - } - Expression::Coalesce(coalesce) if coalesce.expressions.len() == 1 => { - &coalesce.expressions[0] - } - _ => return None, - }; - } -} - -/// Span of the function-name token that opens `call`, or of its column when not found. -fn function_span(tokens: &[Token], call: &ParseCall<'_>) -> (usize, usize) { - let bare: &str = call.function.rsplit('.').next().unwrap_or(call.function); - let before: usize = tokens.partition_point(|token| token.span.start < call.column_start); - tokens[..before] - .iter() - .rev() - .filter(|token| !is_trivia(token)) - .take(MAX_FUNCTION_NAME_DISTANCE) - .find(|token| token.text.eq_ignore_ascii_case(bare)) - .map_or((call.column_start, call.column_start), |token| { - (token.span.start, token.span.end) - }) -} - -fn remediation(call: &ParseCall<'_>, count: usize, quote: char) -> String { - let name: String = written(&call.column.name, quote); - let column: String = call.column.table.as_ref().map_or_else( - || name.clone(), - |table| format!("{}.{name}", written(table, quote)), - ); - let function: &str = call.function; - format!( - "{function}({column}) is parsed {count} times in one SELECT. Parse it once in the CTE that reads it, for example `{function}({name}) AS payload`, and read every field from that column in a later step." - ) -} - -/// The identifier as authored, re-quoted with the dialect's identifier quote when it was quoted. -fn written(identifier: &Identifier, quote: char) -> String { - if identifier.quoted { - let escaped: String = identifier.name.replace(quote, &format!("{quote}{quote}")); - format!("{quote}{escaped}{quote}") - } else { - identifier.name.clone() - } -} - -fn identifier_quote(dialect: DialectType) -> char { - match dialect { - DialectType::BigQuery | DialectType::Databricks => '`', - _ => '"', - } -} diff --git a/crates/sqlclean/src/sql_quality/_helpers/keys.rs b/crates/sqlclean/src/sql_quality/_helpers/keys.rs deleted file mode 100644 index 2e006e5..0000000 --- a/crates/sqlclean/src/sql_quality/_helpers/keys.rs +++ /dev/null @@ -1,437 +0,0 @@ -//! Unique-key propagation through one model's relations, for ranking determinism proofs. - -use std::collections::HashMap; - -use polyglot_sql::Expression; -use polyglot_sql::expressions::{JoinKind, Literal, Select}; - -use crate::sql_quality::constants::FIRST_ROW; -use crate::sql_quality::models::QualityRequest; -use crate::sql_quality::syntax::{Projection, columns, lower, projections, qualifier, sources}; -use crate::sql_quality::types::{InputKey, Keys}; - -/// Keys known for relations by name: declared upstream keys and earlier CTE outputs. -pub(crate) struct KeyEnvironment { - pub(crate) declared: HashMap, - pub(crate) ctes: HashMap, - /// Per CTE, `(row number output, partition output columns)` for deduplication filters. - pub(crate) row_numbers: HashMap)>>, -} - -impl KeyEnvironment { - /// Output keys of a query expression, by output column name. - pub(crate) fn query_keys(&self, expression: &Expression) -> Keys { - match expression { - Expression::Select(select) => self.select_keys(select), - Expression::Subquery(subquery) => self.query_keys(&subquery.this), - Expression::Paren(paren) => self.query_keys(&paren.this), - Expression::Union(union) if !union.all => all_columns_key(&union.left), - Expression::Intersect(intersect) => self.query_keys(&intersect.left), - Expression::Except(except) => self.query_keys(&except.left), - _ => Vec::new(), - } - } - - fn relation_keys(&self, relation: &Expression) -> Keys { - match relation { - Expression::Table(table) => { - let name: String = lower(&table.name.name); - self.ctes - .get(&name) - .cloned() - .or_else(|| self.declared.get(&name).cloned()) - .unwrap_or_default() - } - Expression::Subquery(subquery) => self.query_keys(&subquery.this), - _ => Vec::new(), - } - } - - /// Keys of a select's joined and filtered input rows, before projection. - pub(crate) fn input_keys(&self, select: &Select) -> Vec { - if select - .from - .as_ref() - .is_none_or(|from| from.expressions.len() != 1) - { - return Vec::new(); - } - let found: Vec<(String, Option, &Expression)> = sources(select); - let Some((first_alias, _, first)) = found.first() else { - return Vec::new(); - }; - let mut keys: Vec = qualify(first_alias, self.relation_keys(first)); - keys.extend(qualify(first_alias, self.deduplicated_keys(select, first))); - for (join, (alias, _, relation)) in select.joins.iter().zip(found.iter().skip(1)) { - let right: Vec = qualify(alias, self.relation_keys(relation)); - let equated: Vec<(String, String)> = join_equalities(join, alias); - let left_columns: Vec<&str> = equated.iter().map(|(_, left)| left.as_str()).collect(); - let right_columns: Vec<&str> = - equated.iter().map(|(right, _)| right.as_str()).collect(); - let left_unique: bool = keys.iter().any(|key| key_within(key, &left_columns)); - let right_unique: bool = right.iter().any(|key| key_within(key, &right_columns)); - let mut joined: Vec = Vec::new(); - if right_unique - && !matches!( - join.kind, - JoinKind::Right | JoinKind::Full | JoinKind::Cross - ) - { - joined.extend(keys.iter().cloned()); - } - if left_unique && matches!(join.kind, JoinKind::Inner | JoinKind::Right) { - joined.extend(right.iter().cloned()); - } - keys = joined; - } - keys - } - - /// Input keys plus a `QUALIFY ROW_NUMBER() ... = n` partition, which holds only for output rows. - fn filtered_input_keys(&self, select: &Select) -> Vec { - let mut keys: Vec = self.input_keys(select); - if let Some(qualify_clause) = &select.qualify { - keys.extend(qualified_rank_keys(&qualify_clause.this, &sources(select))); - } - keys - } - - /// Partition keys of a `ROW_NUMBER()` output that this select filters to `= 1`. - fn deduplicated_keys(&self, select: &Select, relation: &Expression) -> Keys { - let Expression::Table(table) = relation else { - return Vec::new(); - }; - let Some(ranks) = self.row_numbers.get(&lower(&table.name.name)) else { - return Vec::new(); - }; - let filtered: Vec = select - .where_clause - .as_ref() - .map(|clause| first_row_filters(&clause.this)) - .unwrap_or_default(); - ranks - .iter() - .filter(|(rank, _)| filtered.contains(rank)) - .map(|(_, key)| key.clone()) - .collect() - } - - fn select_keys(&self, select: &Select) -> Keys { - let output: Vec> = projections(select); - let mut keys: Keys = Vec::new(); - if select.distinct - && output - .iter() - .all(|item| matches!(item, Projection::Named { .. })) - { - keys.push(output_names(&output)); - } - if let Some(group_by) = &select.group_by - && group_by.all.is_none() - && let Some(grain) = output_key_of(&group_by.expressions, &output) - { - keys.push(grain); - return keys; - } - let star: bool = output - .iter() - .any(|item| matches!(item, Projection::Star(_))); - for input in self.filtered_input_keys(select) { - if let Some(key) = project_key(&input, &output, star) { - keys.push(key); - } - } - keys - } -} - -fn qualify(alias: &str, keys: Keys) -> Vec { - keys.into_iter() - .map(|key| qualified_key(alias, key)) - .collect() -} - -fn qualified_key(alias: &str, key: Vec) -> InputKey { - key.into_iter() - .map(|column| (alias.to_owned(), column)) - .collect() -} - -fn key_within(key: &InputKey, columns: &[&str]) -> bool { - key.iter() - .all(|(_, column)| columns.contains(&column.as_str())) -} - -/// `(right column, left column)` pairs equated by a join's ON or USING condition. -fn join_equalities( - join: &polyglot_sql::expressions::Join, - right_alias: &str, -) -> Vec<(String, String)> { - let mut pairs: Vec<(String, String)> = join - .using - .iter() - .map(|identifier| (lower(&identifier.name), lower(&identifier.name))) - .collect(); - let mut pending: Vec<&Expression> = join.on.iter().collect(); - while let Some(expression) = pending.pop() { - match expression { - Expression::And(operation) => { - pending.push(&operation.left); - pending.push(&operation.right); - } - Expression::Paren(paren) => pending.push(&paren.this), - Expression::Eq(operation) => { - if let (Expression::Column(left), Expression::Column(right)) = - (&operation.left, &operation.right) - { - let (inner, outer) = if qualifier(left).as_deref() == Some(right_alias) { - (left, right) - } else { - (right, left) - }; - pairs.push((lower(&inner.name.name), lower(&outer.name.name))); - } - } - _ => {} - } - } - pairs -} - -fn qualified_rank_keys( - condition: &Expression, - found: &[(String, Option, &Expression)], -) -> Vec { - let Expression::Eq(operation) = condition else { - return Vec::new(); - }; - let window = match (&operation.left, &operation.right) { - (Expression::WindowFunction(window), Expression::Literal(_)) - | (Expression::Literal(_), Expression::WindowFunction(window)) => window, - _ => return Vec::new(), - }; - if !matches!(window.this, Expression::RowNumber(_)) || window.over.partition_by.is_empty() { - return Vec::new(); - } - let mut key: InputKey = InputKey::new(); - for expression in &window.over.partition_by { - let Expression::Column(column) = expression else { - return Vec::new(); - }; - let alias: String = qualifier(column) - .or_else(|| (found.len() == 1).then(|| found[0].0.clone())) - .unwrap_or_default(); - key.insert((alias, lower(&column.name.name))); - } - vec![key] -} - -fn output_names(output: &[Projection<'_>]) -> Vec { - output - .iter() - .filter_map(|item| match item { - Projection::Named { name, .. } => Some(name.clone()), - _ => None, - }) - .collect() -} - -fn output_key_of(expressions: &[Expression], output: &[Projection<'_>]) -> Option> { - let mut key: Vec = Vec::new(); - for expression in expressions { - if let Expression::Literal(literal) = expression - && let Literal::Number(text) = literal.as_ref() - { - let Ok(ordinal) = text.parse::() else { - return None; - }; - let Some(Projection::Named { name, .. }) = ordinal - .checked_sub(1) - .and_then(|position| output.get(position)) - else { - return None; - }; - key.push(name.clone()); - continue; - } - let Expression::Column(column) = expression else { - return None; - }; - let name: String = lower(&column.name.name); - let projected: Option = output.iter().find_map(|item| match item { - Projection::Named { - name: output_name, - expression: Expression::Column(source), - } if lower(&source.name.name) == name => Some(output_name.clone()), - Projection::Named { - name: output_name, .. - } if *output_name == name => Some(output_name.clone()), - _ => None, - }); - key.push(projected?); - } - Some(key) -} - -fn project_key(input: &InputKey, output: &[Projection<'_>], star: bool) -> Option> { - let mut key: Vec = Vec::new(); - for (alias, column) in input { - let projected: Option = output.iter().find_map(|item| match item { - Projection::Named { - name, - expression: Expression::Column(source), - } if lower(&source.name.name) == *column - && qualifier(source).is_none_or(|table| table == *alias) => - { - Some(name.clone()) - } - Projection::Star(star) - if star - .table - .as_ref() - .is_none_or(|table| lower(&table.name) == *alias) => - { - Some(column.clone()) - } - _ => None, - }); - match projected { - Some(name) => key.push(name), - None if star => key.push(column.clone()), - None => return None, - } - } - Some(key) -} - -fn all_columns_key(left: &Expression) -> Keys { - let Expression::Select(select) = left else { - return Vec::new(); - }; - let output: Vec> = projections(select); - if output - .iter() - .all(|item| matches!(item, Projection::Named { .. })) - { - vec![output_names(&output)] - } else { - Vec::new() - } -} - -/// Columns a ranking window partitions and orders by, generously resolved to input aliases. -pub(crate) fn window_columns( - partition_by: &[Expression], - order_by: &[&Expression], -) -> Vec<(Option, String)> { - partition_by - .iter() - .chain(order_by.iter().copied()) - .flat_map(columns) - .map(|column| (qualifier(column), lower(&column.name.name))) - .collect() -} - -/// Whether some input key is covered by the window's partition and order columns. -pub(crate) fn covers(keys: &[InputKey], window: &[(Option, String)]) -> bool { - keys.iter().any(|key| key_covered(key, window)) -} - -fn key_covered(key: &InputKey, window: &[(Option, String)]) -> bool { - key.iter() - .all(|(alias, column)| window_has(window, alias, column)) -} - -fn window_has(window: &[(Option, String)], alias: &str, column: &str) -> bool { - window.iter().any(|(qualifier, name)| { - name == column && qualifier.as_deref().is_none_or(|table| table == alias) - }) -} - -/// Keys of declared relations and of each top-level CTE, in definition order. -pub(crate) fn key_environment(request: &QualityRequest<'_>) -> KeyEnvironment { - let mut environment: KeyEnvironment = KeyEnvironment { - declared: lowered_keys(request.relation_keys), - ctes: HashMap::new(), - row_numbers: HashMap::new(), - }; - for statement in request.statements { - let Expression::Select(select) = statement else { - continue; - }; - for cte in select.with.iter().flat_map(|with| with.ctes.iter()) { - let keys = environment.query_keys(&cte.this); - environment.ctes.insert(lower(&cte.alias.name), keys); - if let Expression::Select(body) = &cte.this { - environment - .row_numbers - .insert(lower(&cte.alias.name), row_number_outputs(body)); - } - } - } - environment -} - -fn lowered_keys(keys: &HashMap) -> HashMap { - let mut lowered: HashMap = HashMap::new(); - for (name, relation_keys) in keys { - let mut converted: Keys = Vec::new(); - for key in relation_keys { - let mut columns: Vec = Vec::new(); - for column in key { - columns.push(lower(column)); - } - converted.push(columns); - } - lowered.insert(lower(name), converted); - } - lowered -} - -/// `(output name, partition output columns)` for each `ROW_NUMBER() OVER (PARTITION BY ...)` output. -fn row_number_outputs(select: &Select) -> Vec<(String, Vec)> { - let output: Vec> = projections(select); - let mut found: Vec<(String, Vec)> = Vec::new(); - for item in &output { - let Projection::Named { - name, - expression: Expression::WindowFunction(window), - } = item - else { - continue; - }; - if !matches!(window.this, Expression::RowNumber(_)) || window.over.partition_by.is_empty() { - continue; - } - if let Some(key) = output_key_of(&window.over.partition_by, &output) { - found.push((name.clone(), key)); - } - } - found -} - -/// Columns a filter pins to the first row, as in `WHERE rn = 1 AND ...`. -fn first_row_filters(condition: &Expression) -> Vec { - let mut found: Vec = Vec::new(); - let mut pending: Vec<&Expression> = vec![condition]; - while let Some(expression) = pending.pop() { - match expression { - Expression::And(operation) => { - pending.push(&operation.left); - pending.push(&operation.right); - } - Expression::Paren(paren) => pending.push(&paren.this), - Expression::Eq(operation) => { - if let (Expression::Column(column), Expression::Literal(literal)) - | (Expression::Literal(literal), Expression::Column(column)) = - (&operation.left, &operation.right) - && matches!(literal.as_ref(), Literal::Number(text) if text == FIRST_ROW) - { - found.push(lower(&column.name.name)); - } - } - _ => {} - } - } - found -} diff --git a/crates/sqlclean/src/sql_quality/_helpers/literals.rs b/crates/sqlclean/src/sql_quality/_helpers/literals.rs deleted file mode 100644 index 90a7999..0000000 --- a/crates/sqlclean/src/sql_quality/_helpers/literals.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! SQLCLEAN044: string literals longer than the project limit. - -use std::borrow::Cow; - -use polyglot_sql::tokens::TokenType; - -use crate::sql_lint::models::LintDiagnostic; -use crate::sql_quality::constants::{ - LONG_LITERAL, MAX_LITERAL_LENGTH_KEY, RULES_THRESHOLDS_SECTION, -}; -use crate::sql_quality::models::QualityRequest; -use crate::sql_quality::setting_help::{Setting, setting_help}; - -const CONSTANT_FIX_REFUSAL: &str = - "Moving the value to a constant requires choosing its name and placement"; - -pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { - request - .tokens - .iter() - .filter(|token| is_string(token.token_type)) - .filter_map(|token| { - let length: usize = token.text.chars().count(); - (length > request.max_literal_length).then(|| LintDiagnostic { - code: LONG_LITERAL.code, - message: Cow::Borrowed(LONG_LITERAL.message), - remediation: Cow::Owned(remediation(length, request)), - start: token.span.start, - end: token.span.end, - fix: None, - fix_unavailable_reason: Some(CONSTANT_FIX_REFUSAL), - }) - }) - .collect() -} - -fn remediation(length: usize, request: &QualityRequest<'_>) -> String { - format!( - "{} {}", - request.long_literal_guidance, - setting_help(&Setting { - purpose: &format!( - "To allow literals of {length} characters (the current value is {})", - request.max_literal_length - ), - config_file: request.config_file, - section: RULES_THRESHOLDS_SECTION, - key: MAX_LITERAL_LENGTH_KEY, - value: &length.to_string(), - }) - ) -} - -fn is_string(kind: TokenType) -> bool { - matches!( - kind, - TokenType::String - | TokenType::DollarString - | TokenType::EscapeString - | TokenType::RawString - | TokenType::NationalString - | TokenType::UnicodeString - | TokenType::ByteString - | TokenType::TripleSingleQuotedString - | TokenType::TripleDoubleQuotedString - | TokenType::HeredocString - | TokenType::HeredocStringAlternative - ) -} diff --git a/crates/sqlclean/src/sql_quality/_helpers/ranking.rs b/crates/sqlclean/src/sql_quality/_helpers/ranking.rs deleted file mode 100644 index 23dcb2e..0000000 --- a/crates/sqlclean/src/sql_quality/_helpers/ranking.rs +++ /dev/null @@ -1,218 +0,0 @@ -//! SQLCLEAN043 ranking sort caps and SQLCLEAN018 ranking determinism proofs. - -use std::borrow::Cow; -use std::collections::{HashMap, HashSet}; - -use polyglot_sql::expressions::{Over, Select, WindowFunction}; -use polyglot_sql::scope::walk_in_scope; -use polyglot_sql::tokens::Token; -use polyglot_sql::{Expression, ExpressionWalk}; - -use crate::sql_lint::models::LintDiagnostic; -use crate::sql_quality::constants::{ - MAX_RANKING_ORDER_BY_KEY, RANKING_SORT_CAP, RANKING_SORT_GUIDANCE, RULES_THRESHOLDS_SECTION, - TIE_SENSITIVE_FUNCTIONS, UNPROVEN_RANKING_MESSAGE, UNPROVEN_RANKING_REMEDIATION, - UNSTABLE_ROW_NUMBER_CODE, -}; -use crate::sql_quality::keys::{KeyEnvironment, covers, window_columns}; -use crate::sql_quality::models::QualityRequest; -use crate::sql_quality::setting_help::{Setting, setting_help}; -use crate::sql_quality::syntax::{columns, lower}; -use crate::sql_quality::types::InputKey; - -const PRIORITY_FIX_REFUSAL: &str = - "Choosing a priority column and tie-breaker requires author intent"; -const KEY_FIX_REFUSAL: &str = "Declaring a key or adding a tie-breaker requires author intent"; - -struct RankingWindow<'a> { - function: &'static str, - window: &'a WindowFunction, - select: &'a Select, -} - -pub(crate) fn diagnostics( - request: &QualityRequest<'_>, - environment: &KeyEnvironment, -) -> Vec { - let cap: bool = request.enabled.contains(RANKING_SORT_CAP.code); - let proof: bool = request.enabled.contains(UNSTABLE_ROW_NUMBER_CODE); - let mut used: HashSet = HashSet::new(); - let mut function_tokens: HashMap> = HashMap::new(); - for (index, token) in request.tokens.iter().enumerate() { - if TIE_SENSITIVE_FUNCTIONS - .iter() - .chain(["RANK", "DENSE_RANK"].iter()) - .any(|name| token.text.eq_ignore_ascii_case(name)) - { - function_tokens - .entry(token.text.to_ascii_uppercase()) - .or_default() - .push(index); - } - } - let mut found: Vec = Vec::new(); - for ranking in ranking_windows(request.statements) { - let over: Option<&Over> = effective_over(&ranking.window.over, ranking.select); - let order: Vec<&Expression> = over.map_or_else(Vec::new, |over| { - over.order_by.iter().map(|ordered| &ordered.this).collect() - }); - let partition: &[Expression] = over.map_or(&[], |over| over.partition_by.as_slice()); - let exceeds: bool = cap && order.len() > request.max_ranking_order_by; - let unproven: bool = proof - && !order.is_empty() - && TIE_SENSITIVE_FUNCTIONS.contains(&ranking.function) - && !proven(environment, ranking.select, partition, &order); - if !exceeds && !unproven { - continue; - } - let candidates: &[usize] = function_tokens - .get(ranking.function) - .map_or(&[], Vec::as_slice); - let token: Option = function_token( - request.tokens, - candidates, - window_anchor(partition, &order), - &used, - ); - let (start, end) = token.map_or((0, 0), |index| { - ( - request.tokens[index].span.start, - request.tokens[index].span.end, - ) - }); - used.extend(token); - if exceeds { - found.push(LintDiagnostic { - code: RANKING_SORT_CAP.code, - message: Cow::Borrowed(RANKING_SORT_CAP.message), - remediation: Cow::Owned(sort_cap_remediation( - order.len(), - request.max_ranking_order_by, - request.config_file, - )), - start, - end, - fix: None, - fix_unavailable_reason: Some(PRIORITY_FIX_REFUSAL), - }); - } - if unproven { - found.push(LintDiagnostic { - code: UNSTABLE_ROW_NUMBER_CODE, - message: Cow::Borrowed(UNPROVEN_RANKING_MESSAGE), - remediation: Cow::Borrowed(UNPROVEN_RANKING_REMEDIATION), - start, - end, - fix: None, - fix_unavailable_reason: Some(KEY_FIX_REFUSAL), - }); - } - } - found -} - -fn sort_cap_remediation(sort_keys: usize, limit: usize, config_file: &str) -> String { - format!( - "{RANKING_SORT_GUIDANCE} {}", - setting_help(&Setting { - purpose: &format!("To allow {sort_keys} sort keys (the current value is {limit})"), - config_file, - section: RULES_THRESHOLDS_SECTION, - key: MAX_RANKING_ORDER_BY_KEY, - value: &sort_keys.to_string(), - }) - ) -} - -fn proven( - environment: &KeyEnvironment, - select: &Select, - partition: &[Expression], - order: &[&Expression], -) -> bool { - let keys: Vec = environment.input_keys(select); - covers(&keys, &window_columns(partition, order)) -} - -fn ranking_windows(statements: &[Expression]) -> Vec> { - let mut found: Vec> = Vec::new(); - for statement in statements { - for node in statement.dfs() { - let Expression::Select(select) = node else { - continue; - }; - for local in walk_in_scope(node, false) { - let Expression::WindowFunction(window) = local else { - continue; - }; - let function: &'static str = match &window.this { - Expression::RowNumber(_) => "ROW_NUMBER", - Expression::Rank(_) => "RANK", - Expression::DenseRank(_) => "DENSE_RANK", - Expression::FirstValue(_) => "FIRST_VALUE", - Expression::LastValue(_) => "LAST_VALUE", - _ => continue, - }; - found.push(RankingWindow { - function, - window, - select, - }); - } - } - } - found -} - -fn effective_over<'a>(over: &'a Over, select: &'a Select) -> Option<&'a Over> { - let mut current: &Over = over; - let mut seen: HashSet = HashSet::new(); - loop { - if !current.order_by.is_empty() || current.window_name.is_none() { - return Some(current); - } - let name: String = lower(¤t.window_name.as_ref()?.name); - if !seen.insert(name.clone()) { - return None; - } - current = &select - .windows - .iter() - .flatten() - .find(|window| lower(&window.name.name) == name)? - .spec; - } -} - -/// Start offset of the first located column a window partitions or orders by. -fn window_anchor(partition: &[Expression], order: &[&Expression]) -> Option { - let mut expressions: Vec<&Expression> = partition.iter().collect(); - expressions.extend(order.iter().copied()); - for expression in expressions { - for column in columns(expression) { - if let Some(span) = column.span { - return Some(span.start); - } - } - } - None -} - -/// The function-name token before the anchor, or the first token not reported yet. -fn function_token( - tokens: &[Token], - candidates: &[usize], - anchor: Option, - used: &HashSet, -) -> Option { - if let Some(anchor) = anchor { - let before: usize = candidates.partition_point(|&index| tokens[index].span.start < anchor); - if let Some(position) = before.checked_sub(1) { - return Some(candidates[position]); - } - } - candidates - .iter() - .copied() - .find(|index| !used.contains(index)) -} diff --git a/crates/sqlclean/src/sql_quality/_helpers/removal_safety.rs b/crates/sqlclean/src/sql_quality/_helpers/removal_safety.rs deleted file mode 100644 index 24b7641..0000000 --- a/crates/sqlclean/src/sql_quality/_helpers/removal_safety.rs +++ /dev/null @@ -1,102 +0,0 @@ -//! SQLCLEAN042 allow-list: when removing an unused CTE output provably keeps every row. - -use polyglot_sql::expressions::Select; -use polyglot_sql::traversal::contains_aggregate; -use polyglot_sql::{Expression, ExpressionWalk}; - -use crate::sql_quality::constants::{ - DETERMINISTIC_SCALAR_FUNCTIONS, MIN_REMOVABLE_OUTPUTS, ROW_NEUTRAL_EXPRESSIONS, -}; - -const ORDER_BY_ALL: &str = "all"; -const DISTINCT_REFUSAL: &str = "SELECT DISTINCT depends on every output column"; -const DISTINCT_ON_REFUSAL: &str = "SELECT DISTINCT ON keeps one row per key chosen by the select"; -const GROUP_BY_ALL_REFUSAL: &str = "GROUP BY ALL groups by every non-aggregate output column"; -const ORDER_BY_ALL_REFUSAL: &str = "ORDER BY ALL sorts by every output column"; -const UNGROUPED_AGGREGATE_REFUSAL: &str = - "An aggregate select without GROUP BY depends on its whole projection"; -const POSITIONAL_REFUSAL: &str = "Positional GROUP BY or ORDER BY would shift"; -const ONLY_OUTPUT_REFUSAL: &str = "Removing the only output column leaves an empty select"; -const EXPRESSION_REFUSAL: &str = "Only column references and deterministic scalar expressions are removed automatically; this expression may change the rows the CTE returns"; - -/// Why removing any output of `select` could change its rows, if it could. -pub(super) fn select_refusal(select: &Select, outputs: usize) -> Option<&'static str> { - if select.distinct { - Some(DISTINCT_REFUSAL) - } else if select.distinct_on.is_some() { - Some(DISTINCT_ON_REFUSAL) - } else if select - .group_by - .as_ref() - .is_some_and(|group| group.all == Some(true)) - { - Some(GROUP_BY_ALL_REFUSAL) - } else if orders_by_all(select) { - Some(ORDER_BY_ALL_REFUSAL) - } else if select.group_by.is_none() - && (select.having.is_some() || select.expressions.iter().any(contains_aggregate)) - { - Some(UNGROUPED_AGGREGATE_REFUSAL) - } else if positional_clauses(select) { - Some(POSITIONAL_REFUSAL) - } else if outputs < MIN_REMOVABLE_OUTPUTS { - Some(ONLY_OUTPUT_REFUSAL) - } else { - None - } -} - -/// Why removing this one projection could change the rows, if it could. -pub(super) fn expression_refusal(expression: &Expression) -> Option<&'static str> { - (!row_neutral(expression)).then_some(EXPRESSION_REFUSAL) -} - -/// Whether a star select depends on every input column or on their positions. -pub(super) fn reads_whole_rows(select: &Select) -> bool { - select.distinct - || select.distinct_on.is_some() - || select.group_by.is_some() - || orders_by_all(select) - || positional_clauses(select) -} - -/// A column reference or deterministic scalar expression (no aggregate, window, SRF or subquery). -fn row_neutral(expression: &Expression) -> bool { - expression.dfs().all(|node| match node { - Expression::Function(function) => { - let name: String = function.name.to_ascii_uppercase(); - !function.distinct && DETERMINISTIC_SCALAR_FUNCTIONS.contains(&name.as_str()) - } - other => ROW_NEUTRAL_EXPRESSIONS.contains(&other.variant_name()), - }) -} - -fn orders_by_all(select: &Select) -> bool { - select - .order_by - .iter() - .flat_map(|order| order.expressions.iter()) - .any(|ordered| match &ordered.this { - Expression::Column(column) => { - column.table.is_none() - && !column.name.quoted - && column.name.name.eq_ignore_ascii_case(ORDER_BY_ALL) - } - Expression::Var(var) => var.this.eq_ignore_ascii_case(ORDER_BY_ALL), - _ => false, - }) -} - -fn positional_clauses(select: &Select) -> bool { - let numeric = |expression: &Expression| matches!(expression, Expression::Literal(_)); - select - .group_by - .iter() - .flat_map(|group| group.expressions.iter()) - .any(numeric) - || select - .order_by - .iter() - .flat_map(|order| order.expressions.iter()) - .any(|ordered| numeric(&ordered.this)) -} diff --git a/crates/sqlclean/src/sql_quality/_helpers/safety.rs b/crates/sqlclean/src/sql_quality/_helpers/safety.rs deleted file mode 100644 index 49d638c..0000000 --- a/crates/sqlclean/src/sql_quality/_helpers/safety.rs +++ /dev/null @@ -1,709 +0,0 @@ -//! SQLCLEAN046 to SQLCLEAN053: report-only safety Rules for queries that cannot do what they appear to. - -use std::borrow::Cow; -use std::collections::HashSet; - -use polyglot_sql::expressions::{Delete, Identifier, Insert, Join, Update, With}; -use polyglot_sql::tokens::{Token, TokenType}; -use polyglot_sql::traversal::is_aggregate; -use polyglot_sql::{DialectType, Expression, ExpressionWalk}; - -use crate::sql_lint::models::{LintDiagnostic, LintRuleMetadata}; -use crate::sql_quality::constants::{ - DUPLICATE_CTE_NAME, INSERT_WITHOUT_COLUMNS, LITERAL_ZERO_DIVISION, MISPLACED_AGGREGATE, - NATURAL_JOIN, NULL_MATCH, REVERSED_BETWEEN, UNFILTERED_WRITE, -}; -use crate::sql_quality::models::QualityRequest; -use crate::sql_quality::syntax::is_trivia; - -const RENAME_REFUSAL: &str = "Renaming a CTE requires choosing which definition its readers mean"; -const NULL_TEST_REFUSAL: &str = "Rewriting to IS NULL changes what the query returns"; -const DIVISOR_REFUSAL: &str = "Choosing a divisor or a NULL guard requires the intended result"; -const WRITE_SCOPE_REFUSAL: &str = "Only the author knows which rows the statement should change"; -const COLUMN_LIST_REFUSAL: &str = "Listing the target columns requires the table's schema"; -const JOIN_CONDITION_REFUSAL: &str = "Writing the join condition requires both relations' schemas"; -const BOUNDS_REFUSAL: &str = "Swapping the bounds changes what the query returns"; -const AGGREGATE_REFUSAL: &str = - "Moving the expression to HAVING, QUALIFY or an outer query requires restructuring the query"; -const ASYMMETRIC: &str = "ASYMMETRIC"; -const MODULO_FUNCTION: &str = "MOD"; -const SINGLE_ARGUMENT_AGGREGATES: [&str; 2] = ["MIN", "MAX"]; -const ISO_DATE_LENGTH: usize = 10; - -/// A finding's start and end character offsets. -type CharSpan = (usize, usize); - -/// Significant tokens with their parenthesis depth. -struct Significant<'a> { - tokens: Vec<&'a Token>, - depths: Vec, -} - -impl<'a> Significant<'a> { - fn new(tokens: &'a [Token]) -> Self { - let tokens: Vec<&Token> = tokens.iter().filter(|token| !is_trivia(token)).collect(); - let mut depths: Vec = Vec::with_capacity(tokens.len()); - let mut depth: usize = 0; - for token in &tokens { - if token.token_type == TokenType::RParen { - depth = depth.saturating_sub(1); - } - depths.push(depth); - if token.token_type == TokenType::LParen { - depth += 1; - } - } - Self { tokens, depths } - } - - fn kind(&self, index: usize) -> Option { - self.tokens.get(index).map(|token| token.token_type) - } - - fn is(&self, index: usize, kind: TokenType) -> bool { - self.kind(index) == Some(kind) - } - - fn text_is(&self, index: usize, text: &str) -> bool { - self.tokens - .get(index) - .is_some_and(|token| token.text.eq_ignore_ascii_case(text)) - } - - fn close(&self, open: usize) -> Option { - (open + 1..self.tokens.len()).find(|&index| { - self.is(index, TokenType::RParen) && self.depths[index] == self.depths[open] - }) - } - - fn span(&self, first: usize, last: usize) -> CharSpan { - (self.tokens[first].span.start, self.tokens[last].span.end) - } - - /// Top-level comma positions inside the parentheses that open at `open`. - fn arguments_commas(&self, open: usize, close: usize) -> Vec { - (open + 1..close) - .filter(|&index| { - self.is(index, TokenType::Comma) && self.depths[index] == self.depths[open] + 1 - }) - .collect() - } - - /// Non-empty statements split at top-level semicolons, as index ranges. - fn statements(&self) -> Vec<(usize, usize)> { - let mut ranges: Vec<(usize, usize)> = Vec::new(); - let mut start: usize = 0; - for index in 0..=self.tokens.len() { - let ends = index == self.tokens.len() - || (self.is(index, TokenType::Semicolon) && self.depths[index] == 0); - if ends { - if index > start { - ranges.push((start, index)); - } - start = index + 1; - } - } - ranges - } -} - -/// A literal bound whose order is exact: a number, or an ISO `DATE` literal. -#[derive(PartialEq, PartialOrd)] -enum Bound { - Number(f64), - Date(String), -} - -pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { - let significant = Significant::new(request.tokens); - let enabled = |rule: &LintRuleMetadata| request.enabled.contains(rule.code); - let mut found: Vec = Vec::new(); - let mut report = - |rule: &'static LintRuleMetadata, reason: &'static str, spans: Vec| { - found.extend(spans.into_iter().map(|(start, end)| LintDiagnostic { - code: rule.code, - message: Cow::Borrowed(rule.message), - remediation: Cow::Borrowed(rule.remediation), - start, - end, - fix: None, - fix_unavailable_reason: Some(reason), - })); - }; - if enabled(&DUPLICATE_CTE_NAME) { - report( - &DUPLICATE_CTE_NAME, - RENAME_REFUSAL, - duplicate_cte_spans(request.statements, request.dialect), - ); - } - if enabled(&NULL_MATCH) { - report( - &NULL_MATCH, - NULL_TEST_REFUSAL, - null_match_spans(&significant), - ); - } - if enabled(&LITERAL_ZERO_DIVISION) { - report( - &LITERAL_ZERO_DIVISION, - DIVISOR_REFUSAL, - zero_division_spans(&significant), - ); - } - if enabled(&UNFILTERED_WRITE) || enabled(&INSERT_WITHOUT_COLUMNS) { - let (writes, inserts) = statement_spans(&significant, request.statements); - if enabled(&UNFILTERED_WRITE) { - report(&UNFILTERED_WRITE, WRITE_SCOPE_REFUSAL, writes); - } - if enabled(&INSERT_WITHOUT_COLUMNS) { - report(&INSERT_WITHOUT_COLUMNS, COLUMN_LIST_REFUSAL, inserts); - } - } - if enabled(&NATURAL_JOIN) { - report( - &NATURAL_JOIN, - JOIN_CONDITION_REFUSAL, - natural_join_spans(&significant), - ); - } - if enabled(&REVERSED_BETWEEN) { - report( - &REVERSED_BETWEEN, - BOUNDS_REFUSAL, - reversed_between_spans(&significant), - ); - } - if enabled(&MISPLACED_AGGREGATE) { - report( - &MISPLACED_AGGREGATE, - AGGREGATE_REFUSAL, - misplaced_aggregate_spans(&significant, &aggregate_names(request.statements)), - ); - } - found -} - -fn duplicate_cte_spans(statements: &[Expression], dialect: DialectType) -> Vec { - let mut spans: Vec = Vec::new(); - for statement in statements { - for with in statement.dfs().filter_map(with_clause) { - let mut seen: HashSet = HashSet::new(); - for cte in &with.ctes { - if !seen.insert(cte_key(&cte.alias, dialect)) - && let Some(span) = cte.alias.span - { - spans.push((span.start, span.end)); - } - } - } - } - spans -} - -fn with_clause(node: &Expression) -> Option<&With> { - match node { - Expression::Select(select) => select.with.as_ref(), - Expression::Union(operation) => operation.with.as_ref(), - Expression::Intersect(operation) => operation.with.as_ref(), - Expression::Except(operation) => operation.with.as_ref(), - Expression::Insert(insert) => insert.with.as_ref(), - Expression::Update(update) => update.with.as_ref(), - Expression::Delete(delete) => delete.with.as_ref(), - _ => None, - } -} - -/// A CTE name as the dialect resolves it, so only names the engine treats as equal collide. -fn cte_key(identifier: &Identifier, dialect: DialectType) -> String { - let quoted_names_ignore_case = matches!( - dialect, - DialectType::DuckDB - | DialectType::BigQuery - | DialectType::Databricks - | DialectType::Spark - | DialectType::Hive - | DialectType::SQLite - | DialectType::TSQL - | DialectType::Trino - | DialectType::Presto - | DialectType::Athena - ); - match dialect { - DialectType::ClickHouse => identifier.name.clone(), - _ if identifier.quoted && !quoted_names_ignore_case => identifier.name.clone(), - DialectType::Snowflake | DialectType::Oracle => identifier.name.to_uppercase(), - _ => identifier.name.to_lowercase(), - } -} - -/// `CASE x WHEN NULL` and a singleton positive `IN (NULL)`; SQLCLEAN019 owns `NOT IN`. -fn null_match_spans(significant: &Significant<'_>) -> Vec { - let mut spans: Vec = Vec::new(); - for index in 0..significant.tokens.len() { - match significant.kind(index) { - Some(TokenType::In) - if !(index > 0 && significant.is(index - 1, TokenType::Not)) - && significant.is(index + 1, TokenType::LParen) - && significant.is(index + 2, TokenType::Null) - && significant.is(index + 3, TokenType::RParen) => - { - spans.push(significant.span(index, index + 3)); - } - Some(TokenType::Case) if !significant.is(index + 1, TokenType::When) => { - spans.extend(simple_case_null_spans(significant, index)); - } - _ => {} - } - } - spans -} - -fn simple_case_null_spans(significant: &Significant<'_>, case: usize) -> Vec { - let depth: usize = significant.depths[case]; - let mut nested: usize = 0; - let mut spans: Vec = Vec::new(); - for index in case + 1..significant.tokens.len() { - if significant.depths[index] < depth { - break; - } - if significant.depths[index] > depth { - continue; - } - match significant.kind(index) { - Some(TokenType::Case) => nested += 1, - Some(TokenType::End) if nested == 0 => break, - Some(TokenType::End) => nested -= 1, - Some(TokenType::When) - if nested == 0 - && significant.is(index + 1, TokenType::Null) - && significant.is(index + 2, TokenType::Then) => - { - spans.push(significant.span(index, index + 1)); - } - _ => {} - } - } - spans -} - -fn zero_division_spans(significant: &Significant<'_>) -> Vec { - let mut spans: Vec = Vec::new(); - for index in 0..significant.tokens.len() { - let operator = matches!( - significant.kind(index), - Some(TokenType::Slash | TokenType::Percent | TokenType::Div) - ); - if operator - && let Some((number, end)) = numeric_operand(significant, index + 1) - && is_zero(&significant.tokens[number].text) - && !binds_tighter(significant.kind(end + 1)) - { - spans.push(significant.span(index, end)); - } - if significant.text_is(index, MODULO_FUNCTION) - && significant.is(index + 1, TokenType::LParen) - && let Some(close) = significant.close(index + 1) - && let [comma] = significant.arguments_commas(index + 1, close)[..] - && let Some((number, end)) = numeric_operand(significant, comma + 1) - && is_zero(&significant.tokens[number].text) - && end + 1 == close - { - spans.push(significant.span(index, close)); - } - } - spans -} - -/// Operators that bind tighter than division, so a following one changes the divisor. -fn binds_tighter(kind: Option) -> bool { - matches!( - kind, - Some( - TokenType::DColon - | TokenType::DStar - | TokenType::Caret - | TokenType::LBracket - | TokenType::Dot - ) - ) -} - -/// Whether a number literal's digits are all zero, whatever its exponent. -fn is_zero(text: &str) -> bool { - let mantissa: &str = text.split(['e', 'E']).next().unwrap_or(text); - mantissa.chars().any(|character| character.is_ascii_digit()) - && mantissa - .chars() - .all(|character| matches!(character, '0' | '.' | '_')) -} - -/// An optionally signed and parenthesised number literal at `start`: its number token and last index. -fn numeric_operand(significant: &Significant<'_>, start: usize) -> Option<(usize, usize)> { - let mut index: usize = start; - let mut opens: usize = 0; - while significant.is(index, TokenType::LParen) { - opens += 1; - index += 1; - } - if matches!( - significant.kind(index), - Some(TokenType::Dash | TokenType::Plus) - ) { - index += 1; - } - if !significant.is(index, TokenType::Number) { - return None; - } - let mut end: usize = index; - for _ in 0..opens { - if !significant.is(end + 1, TokenType::RParen) { - return None; - } - end += 1; - } - Some((index, end)) -} - -/// A number or an ISO `DATE 'YYYY-MM-DD'` literal starting at `start`, and its last index. -fn bound(significant: &Significant<'_>, start: usize) -> Option<(Bound, usize)> { - if significant.is(start, TokenType::Date) && significant.is(start + 1, TokenType::String) { - let text: &str = &significant.tokens[start + 1].text; - let iso: bool = text.len() == ISO_DATE_LENGTH - && text - .char_indices() - .all(|(position, character)| match position { - 4 | 7 => character == '-', - _ => character.is_ascii_digit(), - }); - return iso.then(|| (Bound::Date(text.to_owned()), start + 1)); - } - let (number, end) = numeric_operand(significant, start)?; - let Ok(magnitude) = significant.tokens[number] - .text - .replace('_', "") - .parse::() - else { - return None; - }; - let negative: bool = significant.is(number - 1, TokenType::Dash); - Some(( - Bound::Number(if negative { -magnitude } else { magnitude }), - end, - )) -} - -fn reversed_between_spans(significant: &Significant<'_>) -> Vec { - let mut spans: Vec = Vec::new(); - for index in 0..significant.tokens.len() { - if !significant.is(index, TokenType::Between) { - continue; - } - let low_start: usize = index + 1 + usize::from(significant.text_is(index + 1, ASYMMETRIC)); - if let Some((low, low_end)) = bound(significant, low_start) - && significant.is(low_end + 1, TokenType::And) - && let Some((high, high_end)) = bound(significant, low_end + 2) - && ends_bound(significant.kind(high_end + 1)) - && matches!( - (&low, &high), - (Bound::Number(_), Bound::Number(_)) | (Bound::Date(_), Bound::Date(_)) - ) - && low > high - { - spans.push(significant.span(index, high_end)); - } - } - spans -} - -/// Tokens that bind looser than BETWEEN, so the upper bound ends before them. -fn ends_bound(kind: Option) -> bool { - match kind { - None => true, - Some(kind) => matches!( - kind, - TokenType::RParen - | TokenType::Comma - | TokenType::Semicolon - | TokenType::And - | TokenType::Or - | TokenType::Then - | TokenType::When - | TokenType::Else - | TokenType::End - | TokenType::As - | TokenType::From - | TokenType::Where - | TokenType::Group - | TokenType::Having - | TokenType::Qualify - | TokenType::Window - | TokenType::Order - | TokenType::Limit - | TokenType::Union - | TokenType::Except - | TokenType::Intersect - ), - } -} - -/// UPDATE or DELETE without a limiting predicate, and INSERT without a column list. -fn statement_spans( - significant: &Significant<'_>, - statements: &[Expression], -) -> (Vec, Vec) { - let mut writes: Vec = Vec::new(); - let mut inserts: Vec = Vec::new(); - let ranges: Vec<(usize, usize)> = significant.statements(); - if ranges.len() != statements.len() { - return (writes, inserts); - } - for (statement, (start, end)) in statements.iter().zip(ranges) { - let keyword: usize = (start..end) - .find(|&index| { - significant.depths[index] == 0 - && matches!( - significant.kind(index), - Some(TokenType::Update | TokenType::Delete | TokenType::Insert) - ) - }) - .unwrap_or(start); - let span = significant.span(keyword, keyword); - match statement { - Expression::Update(update) if !update_is_limited(update) => writes.push(span), - Expression::Delete(delete) if !delete_is_limited(delete) => writes.push(span), - Expression::Insert(insert) if lacks_column_list(insert) => inserts.push(span), - _ => {} - } - } - (writes, inserts) -} - -fn update_is_limited(update: &Update) -> bool { - update - .where_clause - .as_ref() - .is_some_and(|clause| reads_rows(&clause.this)) - || update.limit.is_some() - || update - .table_joins - .iter() - .chain(&update.from_joins) - .any(join_is_limiting) -} - -fn delete_is_limited(delete: &Delete) -> bool { - delete - .where_clause - .as_ref() - .is_some_and(|clause| reads_rows(&clause.this)) - || delete.limit.is_some() - || delete.joins.iter().any(join_is_limiting) -} - -/// Whether a predicate depends on the rows: it reads a column or a subquery, unlike `WHERE TRUE`. -fn reads_rows(predicate: &Expression) -> bool { - predicate.dfs().any(|node| { - matches!( - node, - Expression::Column(_) | Expression::Select(_) | Expression::Subquery(_) - ) - }) -} - -fn join_is_limiting(join: &Join) -> bool { - join.on.as_ref().is_some_and(reads_rows) || !join.using.is_empty() -} - -fn lacks_column_list(insert: &Insert) -> bool { - insert.columns.is_empty() - && !insert.default_values - && !insert.by_name - && insert.directory.is_none() - && (insert.query.is_some() || !insert.values.is_empty()) -} - -fn natural_join_spans(significant: &Significant<'_>) -> Vec { - let mut spans: Vec = Vec::new(); - for index in 0..significant.tokens.len() { - if !significant.is(index, TokenType::Natural) { - continue; - } - let mut join: usize = index + 1; - while matches!( - significant.kind(join), - Some( - TokenType::Left - | TokenType::Right - | TokenType::Full - | TokenType::Inner - | TokenType::Outer - ) - ) { - join += 1; - } - if significant.is(join, TokenType::Join) { - spans.push(significant.span(index, join)); - } - } - spans -} - -/// Function names the parser classified as aggregates, upper-cased without underscores. -fn aggregate_names(statements: &[Expression]) -> HashSet { - statements - .iter() - .flat_map(|statement| statement.dfs()) - .filter(|node| is_aggregate(node)) - .map(|node| match node { - Expression::AggregateFunction(function) => normalized_name(&function.name), - other => normalized_name(other.variant_name()), - }) - .collect() -} - -fn normalized_name(name: &str) -> String { - name.chars() - .filter(|character| *character != '_') - .flat_map(char::to_uppercase) - .collect() -} - -/// Aggregate and window calls directly in WHERE, ON or GROUP BY, outside nested queries. -fn misplaced_aggregate_spans( - significant: &Significant<'_>, - aggregates: &HashSet, -) -> Vec { - let mut spans: Vec = Vec::new(); - for index in 0..significant.tokens.len() { - let first: usize = match significant.kind(index) { - Some(TokenType::Where | TokenType::On) => index + 1, - Some(TokenType::Group) if significant.is(index + 1, TokenType::By) => index + 2, - _ => continue, - }; - spans.extend(clause_call_spans( - significant, - (first, significant.depths[index]), - significant.is(index, TokenType::On), - aggregates, - )); - } - spans -} - -fn clause_call_spans( - significant: &Significant<'_>, - (first, depth): (usize, usize), - comma_ends: bool, - aggregates: &HashSet, -) -> Vec { - let mut spans: Vec = Vec::new(); - let mut index: usize = first; - while index < significant.tokens.len() { - if significant.depths[index] < depth - || (significant.depths[index] == depth && ends_clause(significant, index, comma_ends)) - { - break; - } - if significant.is(index, TokenType::LParen) - && matches!( - significant.kind(index + 1), - Some(TokenType::Select | TokenType::With | TokenType::Values) - ) - { - index = significant - .close(index) - .map_or(significant.tokens.len(), |close| close + 1); - continue; - } - if let Some((close, end)) = call(significant, index) { - if significant.is(close + 1, TokenType::Over) { - spans.push(significant.span(index, end)); - index = end + 1; - continue; - } - if is_aggregate_call(significant, index, close, aggregates) { - spans.push(significant.span(index, close)); - index = close + 1; - continue; - } - } - index += 1; - } - spans -} - -/// A function call at `name`: its closing parenthesis and the end of any OVER specification. -fn call(significant: &Significant<'_>, name: usize) -> Option<(usize, usize)> { - let text: &str = &significant.tokens[name].text; - let is_name: bool = text - .chars() - .next() - .is_some_and(|character| character.is_ascii_alphabetic() || character == '_') - && text - .chars() - .all(|character| character.is_ascii_alphanumeric() || character == '_'); - if !is_name - || !significant.is(name + 1, TokenType::LParen) - || (name > 0 && significant.is(name - 1, TokenType::Dot)) - { - return None; - } - let close: usize = significant.close(name + 1)?; - let end: usize = if significant.is(close + 1, TokenType::Over) { - if significant.is(close + 2, TokenType::LParen) { - significant.close(close + 2)? - } else { - close + 2 - } - } else { - close - }; - (end < significant.tokens.len()).then_some((close, end)) -} - -fn is_aggregate_call( - significant: &Significant<'_>, - name: usize, - close: usize, - aggregates: &HashSet, -) -> bool { - let normalized: String = normalized_name(&significant.tokens[name].text); - aggregates.contains(&normalized) - && (!SINGLE_ARGUMENT_AGGREGATES.contains(&normalized.as_str()) - || significant.arguments_commas(name + 1, close).is_empty()) -} - -/// Whether `index` ends the clause; only an ON condition ends at a comma, before the next relation. -fn ends_clause(significant: &Significant<'_>, index: usize, comma_ends: bool) -> bool { - match significant.kind(index) { - Some(TokenType::Group) => significant.is(index + 1, TokenType::By), - Some(TokenType::Comma) => comma_ends, - Some(kind) => matches!( - kind, - TokenType::Where - | TokenType::Having - | TokenType::Qualify - | TokenType::Window - | TokenType::Order - | TokenType::Limit - | TokenType::Union - | TokenType::Except - | TokenType::Intersect - | TokenType::Returning - | TokenType::Fetch - | TokenType::Offset - | TokenType::Semicolon - | TokenType::Join - | TokenType::Inner - | TokenType::Left - | TokenType::Right - | TokenType::Full - | TokenType::Cross - | TokenType::Natural - | TokenType::Set - | TokenType::Using - ), - None => true, - } -} diff --git a/crates/sqlclean/src/sql_quality/_helpers/setting_help.rs b/crates/sqlclean/src/sql_quality/_helpers/setting_help.rs deleted file mode 100644 index a456ab7..0000000 --- a/crates/sqlclean/src/sql_quality/_helpers/setting_help.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Remediation text that shows the exact configuration setting to write. - -use crate::constants::SETTING_SNIPPET_INDENT; - -/// One setting a remediation tells the user to write. -pub(crate) struct Setting<'a> { - pub(crate) purpose: &'a str, - pub(crate) config_file: &'a str, - pub(crate) section: &'a str, - pub(crate) key: &'a str, - pub(crate) value: &'a str, -} - -/// `, set this in :` followed by the section and `key = value`. -pub(crate) fn setting_help(setting: &Setting<'_>) -> String { - let Setting { - purpose, - config_file, - section, - key, - value, - } = setting; - format!( - "{purpose}, set this in {config_file}:\n{SETTING_SNIPPET_INDENT}[{section}]\n{SETTING_SNIPPET_INDENT}{key} = {value}" - ) -} diff --git a/crates/sqlclean/src/sql_quality/_helpers/syntax.rs b/crates/sqlclean/src/sql_quality/_helpers/syntax.rs deleted file mode 100644 index 7f79990..0000000 --- a/crates/sqlclean/src/sql_quality/_helpers/syntax.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! Shared syntax helpers for SQL quality Rules. - -use polyglot_sql::expressions::{Column, Select, Star}; -use polyglot_sql::tokens::{Token, TokenType}; -use polyglot_sql::{Expression, ExpressionWalk}; - -use crate::sql_quality::constants::MIN_REMOVABLE_OUTPUTS; -use rustc_hash::FxHashMap; - -/// One projection of a select list. -pub(super) enum Projection<'a> { - Named { - name: String, - expression: &'a Expression, - }, - Star(&'a Star), - Unnamed, -} - -pub(super) fn lower(name: &str) -> String { - name.to_ascii_lowercase() -} - -pub(super) fn projections(select: &Select) -> Vec> { - select - .expressions - .iter() - .map(|expression| match expression { - Expression::Alias(alias) => Projection::Named { - name: lower(&alias.alias.name), - expression: &alias.this, - }, - Expression::Column(column) => Projection::Named { - name: lower(&column.name.name), - expression, - }, - Expression::Star(star) => Projection::Star(star), - _ => Projection::Unnamed, - }) - .collect() -} - -pub(super) fn columns(expression: &Expression) -> Vec<&Column> { - expression - .dfs() - .filter_map(|node| match node { - Expression::Column(column) => Some(column.as_ref()), - _ => None, - }) - .collect() -} - -pub(super) fn qualifier(column: &Column) -> Option { - column.table.as_ref().map(|table| lower(&table.name)) -} - -/// Relations read directly by one select: `(alias, relation name, expression)`. -pub(super) fn sources(select: &Select) -> Vec<(String, Option, &Expression)> { - let mut found: Vec<(String, Option, &Expression)> = Vec::new(); - let from = select.from.iter().flat_map(|from| from.expressions.iter()); - for expression in from.chain(select.joins.iter().map(|join| &join.this)) { - match expression { - Expression::Table(table) => { - let name: String = lower(&table.name.name); - let alias: String = table - .alias - .as_ref() - .map_or_else(|| name.clone(), |alias| lower(&alias.name)); - found.push((alias, Some(name), expression)); - } - Expression::Subquery(subquery) => { - let alias: String = subquery - .alias - .as_ref() - .map_or_else(String::new, |alias| lower(&alias.name)); - found.push((alias, None, expression)); - } - _ => found.push((String::new(), None, expression)), - } - } - found -} - -/// Token index of the SELECT opening each `name AS (SELECT` CTE body, by lower-cased name. -pub(super) fn cte_select_tokens(tokens: &[Token]) -> FxHashMap { - let significant: Vec = (0..tokens.len()) - .filter(|&index| !is_trivia(&tokens[index])) - .collect(); - let mut found: FxHashMap = FxHashMap::default(); - for window in significant.windows(4) { - let [name_index, as_index, paren_index, select_index] = window else { - continue; - }; - if tokens[*as_index].token_type == TokenType::As - && tokens[*paren_index].token_type == TokenType::LParen - && tokens[*select_index].token_type == TokenType::Select - { - found - .entry(lower(&tokens[*name_index].text)) - .or_insert(*select_index); - } - } - found -} - -/// The comma-inclusive deletion span that removes projection `index` from `spans`. -pub(super) fn projection_removal(spans: &[(usize, usize)], index: usize) -> Option<(usize, usize)> { - if spans.len() < MIN_REMOVABLE_OUTPUTS { - return None; - } - let (start, end) = spans[index]; - if index + 1 < spans.len() { - Some((start, spans[index + 1].0)) - } else { - Some((spans[index - 1].1, end)) - } -} - -pub(super) fn is_trivia(token: &Token) -> bool { - matches!( - token.token_type, - TokenType::Space | TokenType::Break | TokenType::LineComment | TokenType::BlockComment - ) -} diff --git a/crates/sqlclean/src/sql_quality/constants.rs b/crates/sqlclean/src/sql_quality/constants.rs deleted file mode 100644 index fa1cbd2..0000000 --- a/crates/sqlclean/src/sql_quality/constants.rs +++ /dev/null @@ -1,228 +0,0 @@ -use crate::sql_lint::models::LintRuleMetadata; - -pub(crate) const UNUSED_CTE_OUTPUT: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN042", - message: "CTE output column is never read later in the query", - remediation: "Remove the column from the CTE output, or read it in a later CTE or the final query.", -}; -pub(crate) const RANKING_SORT_CAP: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN043", - message: "Ranking window sorts by more expressions than max_ranking_order_by", - remediation: "Order by the one or two columns that decide which row wins, then a unique row key as the final tie-breaker.", -}; -pub(crate) const LONG_LITERAL: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN044", - message: "String literal is longer than max_literal_length", - remediation: "Shorten the literal, or move the value into a reference table or a query parameter.", -}; -pub(crate) const REPEATED_JSON_PARSE: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN045", - message: "JSON payload is parsed more than once in one SELECT", - remediation: "Parse the payload once in the CTE that reads it, for example `TRY_PARSE_JSON(value) AS payload`, and read every field from that column in a later step.", -}; -pub(crate) const DUPLICATE_CTE_NAME: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN046", - message: "WITH defines the same CTE name more than once", - remediation: "Give each CTE in one WITH clause its own name; engines reject the duplicate or silently read only one definition.", -}; -pub(crate) const NULL_MATCH: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN047", - message: "CASE WHEN NULL or IN (NULL) never matches", - remediation: "A simple CASE compares with =, and x IN (NULL) means x = NULL, so neither is ever true. Test the value with IS NULL, for example CASE WHEN x IS NULL THEN ... END.", -}; -pub(crate) const LITERAL_ZERO_DIVISION: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN048", - message: "Expression divides by a literal zero", - remediation: "Depending on the dialect this raises an error or returns NULL. Remove the division, or divide by NULLIF(divisor, 0) when the divisor can legitimately be zero.", -}; -pub(crate) const UNFILTERED_WRITE: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN049", - message: "UPDATE or DELETE has no limiting predicate and changes every row", - remediation: "Add a WHERE condition that reads the target rows; a constant such as WHERE TRUE does not limit them. Use TRUNCATE, or an exception for this file, when changing every row is intended.", -}; -pub(crate) const INSERT_WITHOUT_COLUMNS: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN050", - message: "INSERT does not list its target columns", - remediation: "List the target columns, for example INSERT INTO orders (order_id, status) SELECT ..., so values cannot shift into the wrong column when the table changes.", -}; -pub(crate) const NATURAL_JOIN: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN051", - message: "NATURAL JOIN matches on every column the relations share", - remediation: "Join with an explicit ON or USING condition; adding a column with a shared name to either relation silently changes a NATURAL JOIN.", -}; -pub(crate) const REVERSED_BETWEEN: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN052", - message: "BETWEEN lower bound is greater than its upper bound", - remediation: "BETWEEN low AND high is empty when low > high, so it never matches, and NOT BETWEEN matches every non-NULL value. Swap the bounds, or write BETWEEN SYMMETRIC where the dialect supports it.", -}; -pub(crate) const MISPLACED_AGGREGATE: LintRuleMetadata = LintRuleMetadata { - code: "SQLCLEAN053", - message: "Aggregate or window function is used in WHERE, ON or GROUP BY", - remediation: "Rows are filtered and grouped before aggregates and windows are computed. Filter an aggregate in HAVING, filter a window result in QUALIFY or an outer query, and group by the underlying columns.", -}; -pub(crate) const UNSTABLE_ROW_NUMBER_CODE: &str = "SQLCLEAN018"; -pub(crate) const UNPROVEN_RANKING_MESSAGE: &str = - "Ranking window order is not proven unique within its partition"; -pub(crate) const UNPROVEN_RANKING_REMEDIATION: &str = "PARTITION BY and ORDER BY must together cover a unique key of the ranked rows. End the ORDER BY with a unique, non-null tie-breaker column. If the rows have no unique column, build one upstream from the columns that identify a row (for example `HASH(source_partition, source_offset, line_index) AS row_key`) and order by it last."; -/// SQLCLEAN043 guidance before the `max_ranking_order_by` setting snippet. -pub(crate) const RANKING_SORT_GUIDANCE: &str = "Order by the one or two columns that decide which row wins, then a unique row key as the final tie-breaker. If the rows have no unique column, build one upstream from the columns that identify a row (for example `HASH(source_partition, source_offset, line_index) AS row_key`), so SQLCLEAN018 can prove the order is deterministic. If the order genuinely needs more keys, raise `max_ranking_order_by`."; -/// SQLCLEAN044 guidance before the `max_literal_length` setting snippet. -pub(crate) const LONG_LITERAL_GUIDANCE: &str = "Shorten the literal, or move the value into a reference table or a query parameter so it is defined once."; -pub(crate) const RULES_THRESHOLDS_SECTION: &str = "rules.thresholds"; -pub(crate) const MAX_RANKING_ORDER_BY_KEY: &str = "max_ranking_order_by"; -pub(crate) const MAX_LITERAL_LENGTH_KEY: &str = "max_literal_length"; -pub(crate) const DISTINCT_UNUSED_REMEDIATION: &str = "This CTE uses SELECT DISTINCT, so the column still decides which rows are kept. Make the deduplication independent of the column, or restructure the CTE so it outputs only what later steps read."; -pub(crate) const GROUP_KEY_UNUSED_REMEDIATION: &str = "Drop the column from the CTE output but keep it in GROUP BY; the grouping grain stays the same."; -pub(crate) const GROUP_BY_ALL_UNUSED_REMEDIATION: &str = "This CTE uses GROUP BY ALL, so the column is part of the grouping grain. List the grouping columns explicitly and keep this one in GROUP BY, then drop it from the output."; -/// Expression kinds that compute one value per input row without changing the row set. -pub(crate) const ROW_NEUTRAL_EXPRESSIONS: &[&str] = &[ - "literal", - "boolean", - "null", - "identifier", - "column", - "paren", - "dot", - "subscript", - "data_type", - "interval", - "cast", - "try_cast", - "safe_cast", - "collation", - "case", - "and", - "or", - "not", - "neg", - "add", - "sub", - "mul", - "div", - "mod", - "eq", - "neq", - "lt", - "lte", - "gt", - "gte", - "like", - "i_like", - "is", - "is_null", - "is_true", - "is_false", - "in", - "between", - "null_safe_eq", - "bitwise_and", - "bitwise_or", - "bitwise_xor", - "concat", - "concat_ws", - "substring", - "upper", - "lower", - "length", - "trim", - "replace", - "left", - "right", - "lpad", - "rpad", - "split", - "split_part", - "regexp_replace", - "initcap", - "position", - "contains", - "starts_with", - "ends_with", - "abs", - "round", - "floor", - "ceil", - "greatest", - "least", - "coalesce", - "null_if", - "if_func", - "if_null", - "nvl", - "nvl2", - "decode", - "date_add", - "date_diff", - "date_trunc", - "timestamp_trunc", - "extract", - "to_date", - "to_timestamp", - "to_char", - "to_number", - "date", - "year", - "month", - "last_day", - "struct_extract", -]; -/// Generic function names known to be deterministic scalar functions in supported dialects. -pub(crate) const DETERMINISTIC_SCALAR_FUNCTIONS: &[&str] = &[ - "ABS", - "ARRAY_SIZE", - "CEIL", - "CHARINDEX", - "COALESCE", - "CONCAT", - "CONCAT_WS", - "DATEADD", - "DATEDIFF", - "DATE_ADD", - "DATE_DIFF", - "DATE_PART", - "DATE_TRUNC", - "DAY", - "DIV0", - "FLOOR", - "GREATEST", - "HOUR", - "IFF", - "IFNULL", - "LEAST", - "LEFT", - "LENGTH", - "LOWER", - "LPAD", - "LTRIM", - "MD5", - "MOD", - "MONTH", - "NULLIF", - "NVL", - "NVL2", - "POWER", - "REGEXP_REPLACE", - "REPLACE", - "RIGHT", - "ROUND", - "RPAD", - "RTRIM", - "SPLIT_PART", - "SQRT", - "STRPOS", - "SUBSTR", - "SUBSTRING", - "TO_CHAR", - "TO_DATE", - "TO_VARCHAR", - "TRIM", - "TRUNC", - "UPPER", - "YEAR", - "ZEROIFNULL", -]; -pub(crate) const DEFAULT_MAX_LITERAL_LENGTH: usize = 100; -pub(crate) const DEFAULT_MAX_RANKING_ORDER_BY: usize = 5; -pub(crate) const TIE_SENSITIVE_FUNCTIONS: [&str; 3] = ["ROW_NUMBER", "FIRST_VALUE", "LAST_VALUE"]; -pub(crate) const MIN_REMOVABLE_OUTPUTS: usize = 2; -pub(crate) const FIRST_ROW: &str = "1"; diff --git a/crates/sqlclean/src/sql_quality/main/diagnostics.rs b/crates/sqlclean/src/sql_quality/main/diagnostics.rs deleted file mode 100644 index 0dc8266..0000000 --- a/crates/sqlclean/src/sql_quality/main/diagnostics.rs +++ /dev/null @@ -1,33 +0,0 @@ -use crate::sql_lint::models::LintDiagnostic; -use crate::sql_quality::constants::{ - LONG_LITERAL, RANKING_SORT_CAP, REPEATED_JSON_PARSE, UNSTABLE_ROW_NUMBER_CODE, - UNUSED_CTE_OUTPUT, -}; -use crate::sql_quality::keys::{KeyEnvironment, key_environment}; -use crate::sql_quality::models::QualityRequest; -use crate::sql_quality::{cte_columns, json_parses, literals, ranking, safety}; - -/// Run every enabled SQL quality Rule over one parsed body. -pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Result, String> { - if request.max_literal_length == 0 || request.max_ranking_order_by == 0 { - return Err("max_literal_length and max_ranking_order_by must be positive".to_owned()); - } - let mut found: Vec = Vec::new(); - if request.enabled.contains(LONG_LITERAL.code) { - found.extend(literals::diagnostics(request)); - } - if request.enabled.contains(REPEATED_JSON_PARSE.code) { - found.extend(json_parses::diagnostics(request)); - } - found.extend(safety::diagnostics(request)); - if request.enabled.contains(UNUSED_CTE_OUTPUT.code) { - found.extend(cte_columns::diagnostics(request)); - } - if request.enabled.contains(RANKING_SORT_CAP.code) - || request.enabled.contains(UNSTABLE_ROW_NUMBER_CODE) - { - let environment: KeyEnvironment = key_environment(request); - found.extend(ranking::diagnostics(request, &environment)); - } - Ok(found) -} diff --git a/crates/sqlclean/src/sql_quality/main/mod.rs b/crates/sqlclean/src/sql_quality/main/mod.rs deleted file mode 100644 index 4c63048..0000000 --- a/crates/sqlclean/src/sql_quality/main/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod diagnostics; diff --git a/crates/sqlclean/src/sql_quality/mod.rs b/crates/sqlclean/src/sql_quality/mod.rs deleted file mode 100644 index 1881d4e..0000000 --- a/crates/sqlclean/src/sql_quality/mod.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Enforceable SQL quality Rules with project-wide limits and ranking determinism proofs. - -pub mod constants; -#[path = "_helpers/cte_columns.rs"] -mod cte_columns; -#[path = "_helpers/json_parses.rs"] -mod json_parses; -#[path = "_helpers/keys.rs"] -mod keys; -#[path = "_helpers/literals.rs"] -mod literals; -pub mod main; -pub mod models; -#[path = "_helpers/ranking.rs"] -mod ranking; -#[path = "_helpers/removal_safety.rs"] -mod removal_safety; -#[path = "_helpers/safety.rs"] -mod safety; -#[path = "_helpers/setting_help.rs"] -mod setting_help; -#[path = "_helpers/syntax.rs"] -mod syntax; -pub mod types; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlclean/src/sql_quality/models.rs b/crates/sqlclean/src/sql_quality/models.rs deleted file mode 100644 index 0e49301..0000000 --- a/crates/sqlclean/src/sql_quality/models.rs +++ /dev/null @@ -1,22 +0,0 @@ -use std::collections::{HashMap, HashSet}; - -use polyglot_sql::tokens::Token; -use polyglot_sql::{DialectType, Expression}; - -/// Inputs for one body's quality Rules, borrowed from the native lint request. -#[derive(Debug)] -pub(crate) struct QualityRequest<'a> { - pub(crate) statements: &'a [Expression], - pub(crate) tokens: &'a [Token], - pub(crate) dialect: DialectType, - pub(crate) enabled: &'a HashSet, - pub(crate) max_literal_length: usize, - pub(crate) max_ranking_order_by: usize, - pub(crate) relation_keys: &'a HashMap>>, - pub(crate) externally_referenced_ctes: &'a HashSet, - pub(crate) fixture_body: bool, - /// The configuration file remediation text tells the user to edit. - pub(crate) config_file: &'a str, - /// The advice a long literal finding gives before its limit setting. - pub(crate) long_literal_guidance: &'a str, -} diff --git a/crates/sqlclean/src/sql_quality/tests.rs b/crates/sqlclean/src/sql_quality/tests.rs deleted file mode 100644 index 8b02834..0000000 --- a/crates/sqlclean/src/sql_quality/tests.rs +++ /dev/null @@ -1,12 +0,0 @@ -#[path = "tests/helpers.rs"] -mod helpers; -#[path = "tests/test_json_parses.rs"] -mod json_parses; -#[path = "tests/test_quality_rules.rs"] -mod quality_rules; -#[path = "tests/test_quality_scale.rs"] -mod quality_scale; -#[path = "tests/test_safety_rules.rs"] -mod safety_rules; -#[path = "tests/test_types.rs"] -mod test_types; diff --git a/crates/sqlclean/src/sql_quality/tests/helpers.rs b/crates/sqlclean/src/sql_quality/tests/helpers.rs deleted file mode 100644 index 5b62492..0000000 --- a/crates/sqlclean/src/sql_quality/tests/helpers.rs +++ /dev/null @@ -1,135 +0,0 @@ -use serde_json::{Value, json}; - -use crate::sql_lint::main::engine::lint_json; -use crate::sql_quality::tests::test_types::QualityRuleTestCase; - -pub(crate) fn lint(test_case: &QualityRuleTestCase) -> Result, String> { - lint_with_host_fields(test_case, "{}") -} - -/// Lint `test_case` with `host_fields`, a JSON object, merged into the request. -pub(crate) fn lint_with_host_fields( - test_case: &QualityRuleTestCase, - host_fields: &str, -) -> Result, String> { - let relation_keys: Value = - serde_json::from_str(test_case.relation_keys).map_err(|error| error.to_string())?; - let host_fields: Value = - serde_json::from_str(host_fields).map_err(|error| error.to_string())?; - let mut request: Value = json!({ - "version": 1, - "sql": test_case.sql, - "dialect": "duckdb", - "enabled_rules": [test_case.rule], - "max_literal_length": 24, - "max_ranking_order_by": 2, - "relation_keys": relation_keys, - }); - request - .as_object_mut() - .ok_or("the request is not an object")? - .extend( - host_fields - .as_object() - .cloned() - .ok_or("host fields are not an object")?, - ); - let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) - .map_err(|error| error.to_string())?; - Ok(response["diagnostics"] - .as_array() - .cloned() - .unwrap_or_default()) -} - -/// Lint `sql` in `dialect` with only `rule` enabled. -pub(crate) fn lint_dialect(sql: &str, dialect: &str, rule: &str) -> Result, String> { - let request: Value = json!({ - "version": 1, - "sql": sql, - "dialect": dialect, - "enabled_rules": [rule], - }); - let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) - .map_err(|error| error.to_string())?; - Ok(response["diagnostics"] - .as_array() - .cloned() - .unwrap_or_default()) -} - -/// One DuckDB SELECT that parses each of `column_count` payload columns twice. -pub(crate) fn wide_json_select(column_count: usize) -> String { - let projections: Vec = (0..column_count * 2) - .map(|index| format!("json(e.c{})->>'f{index}' AS f{index}", index % column_count)) - .collect(); - format!("SELECT {} FROM events AS e", projections.join(", ")) -} - -pub(crate) fn anchors(sql: &str, diagnostics: &[Value]) -> Vec { - diagnostics - .iter() - .map(|diagnostic| { - let start: usize = diagnostic["start"].as_u64().unwrap_or(0) as usize; - let end: usize = diagnostic["end"].as_u64().unwrap_or(0) as usize; - sql.chars() - .skip(start) - .take(end.saturating_sub(start)) - .collect() - }) - .collect() -} - -pub(crate) fn fixed(sql: &str, diagnostics: &[Value]) -> Option { - let mut edits: Vec<(usize, usize, String)> = diagnostics - .iter() - .filter_map(|diagnostic| { - let fix: &Value = diagnostic.get("fix")?; - Some(( - fix["start"].as_u64()? as usize, - fix["end"].as_u64()? as usize, - fix["replacement"].as_str()?.to_owned(), - )) - }) - .collect(); - edits.sort_by_key(|edit| std::cmp::Reverse(edit.0)); - let applied: bool = !edits.is_empty(); - let mut characters: Vec = sql.chars().collect(); - for (start, end, replacement) in edits { - characters.splice(start..end, replacement.chars()); - } - applied.then(|| characters.into_iter().collect()) -} - -/// A chain of CTEs that each drop their last column and rank without a declared key. -pub(crate) fn wide_cte_chain(cte_count: usize, column_count: usize) -> String { - let columns: Vec = (0..column_count).map(|index| format!("c{index}")).collect(); - let mut ctes: Vec = vec![format!( - "step_0 AS (SELECT {} FROM source_rows AS s)", - columns - .iter() - .map(|column| format!("s.{column}")) - .collect::>() - .join(", ") - )]; - for step in 1..cte_count { - let mut outputs: Vec = columns[..column_count - 1] - .iter() - .map(|column| format!("p.{column}")) - .collect(); - outputs.push(format!( - "ROW_NUMBER() OVER (PARTITION BY p.c0 ORDER BY p.c1) AS c{}", - column_count - 1 - )); - ctes.push(format!( - "step_{step} AS (SELECT {} FROM step_{} AS p)", - outputs.join(", "), - step - 1 - )); - } - format!( - "WITH {} SELECT * FROM step_{}", - ctes.join(",\n"), - cte_count - 1 - ) -} diff --git a/crates/sqlclean/src/sql_quality/tests/test_json_parses.rs b/crates/sqlclean/src/sql_quality/tests/test_json_parses.rs deleted file mode 100644 index 3586cc6..0000000 --- a/crates/sqlclean/src/sql_quality/tests/test_json_parses.rs +++ /dev/null @@ -1,282 +0,0 @@ -use std::time::Instant; - -use serde_json::Value; - -use crate::sql_quality::tests::helpers::{anchors, lint_dialect, wide_json_select}; -use crate::sql_quality::tests::test_types::{JsonParseScaleTestCase, JsonParseTestCase}; - -const RULE: &str = "SQLCLEAN045"; - -#[test] -fn given_json_parse_calls_when_linting_then_repeats_in_one_select_are_reported() --> Result<(), String> { - let test_cases = [ - JsonParseTestCase { - description: "same column parsed twice reports the second call", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(e.value):order_id AS order_id, TRY_PARSE_JSON(e.value):status AS status FROM events AS e", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(e.value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "the finding is located at the second call", - dialect: "snowflake", - sql: "SELECT try_parse_json(value):a AS a, TRY_PARSE_JSON(value):b AS b FROM events", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "three parses give one finding with the count", - dialect: "snowflake", - sql: "SELECT PARSE_JSON(value):a AS a, PARSE_JSON(value):b AS b FROM events WHERE PARSE_JSON(value):c IS NOT NULL", - expected_anchors: &["PARSE_JSON"], - expected_details: &["PARSE_JSON(value) is parsed 3 times in one SELECT."], - }, - JsonParseTestCase { - description: "column casing and wrappers do not hide a repeat", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(LOWER(E.Value)):a AS a, TRY_PARSE_JSON((CAST(TRIM(e.value) AS VARCHAR))):b AS b, TRY_PARSE_JSON(COALESCE(UPPER(e.VALUE))):c AS c FROM events AS e", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(e.value) is parsed 3 times in one SELECT."], - }, - JsonParseTestCase { - description: "multi-argument COALESCE is not a plain column", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(COALESCE(e.value, e.backup)):a AS a, TRY_PARSE_JSON(COALESCE(e.value, e.fallback)):b AS b, TRY_PARSE_JSON(e.value):c AS c FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "TRIM with characters is not a plain column", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(TRIM(e.value, ' ')):a AS a, TRY_PARSE_JSON(e.value):b AS b FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "quoted names differing in case are different columns", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(e.\"Value\"):a AS a, TRY_PARSE_JSON(e.\"value\"):b AS b FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "quoted and unquoted names are not merged", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(e.\"value\"):a AS a, TRY_PARSE_JSON(e.value):b AS b FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "quoted qualifiers differing in case are different relations", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(\"E\".value):a AS a, TRY_PARSE_JSON(\"e\".value):b AS b FROM events AS \"E\" INNER JOIN events AS \"e\" ON \"E\".id = \"e\".id", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "identical quoted names are reported as written", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(\"E\".\"Value\"):a AS a, TRY_PARSE_JSON(\"E\".\"Value\"):b AS b FROM events AS \"E\"", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(\"E\".\"Value\") is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "bigquery quoted names keep backticks", - dialect: "bigquery", - sql: "SELECT JSON_VALUE(PARSE_JSON(e.`Value`), '$.a') AS a, JSON_VALUE(PARSE_JSON(e.`Value`), '$.b') AS b FROM events AS e", - expected_anchors: &["PARSE_JSON"], - expected_details: &["PARSE_JSON(e.`Value`) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "snowflake quoted upper case matches the folded unquoted name", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(e.\"VALUE\"):a AS a, TRY_PARSE_JSON(e.value):b AS b FROM events AS e", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(e.value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "duckdb quoted names are case-insensitive", - dialect: "duckdb", - sql: "SELECT json(e.\"Value\") -> 'a' AS a, json(e.value) -> 'b' AS b FROM events AS e", - expected_anchors: &["json"], - expected_details: &["JSON(e.value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "bigquery backticked names are case-insensitive", - dialect: "bigquery", - sql: "SELECT JSON_VALUE(PARSE_JSON(e.`Value`), '$.a') AS a, JSON_VALUE(PARSE_JSON(e.value), '$.b') AS b FROM events AS e", - expected_anchors: &["PARSE_JSON"], - expected_details: &["PARSE_JSON(e.value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "databricks backticked names are case-insensitive", - dialect: "databricks", - sql: "SELECT try_parse_json(e.`v`) AS a, try_parse_json(e.V) AS b FROM events AS e", - expected_anchors: &["try_parse_json"], - expected_details: &["TRY_PARSE_JSON(e.V) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "references are grouped as written", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(value):a AS a, TRY_PARSE_JSON(e.value):b AS b, TRY_PARSE_JSON(events.value):c AS c FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "different columns pass", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(e.payload):a AS a, TRY_PARSE_JSON(e.metadata):b AS b FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "same column name on different tables passes", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(o.payload):a AS a, TRY_PARSE_JSON(c.payload):b AS b FROM orders AS o INNER JOIN customers AS c ON o.customer_id = c.customer_id", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "different parse functions are not merged", - dialect: "snowflake", - sql: "SELECT PARSE_JSON(value):a AS a, TRY_PARSE_JSON(value):b AS b FROM events", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "one parse per CTE scope passes", - dialect: "snowflake", - sql: "WITH parsed AS (SELECT TRY_PARSE_JSON(value) AS payload FROM events), final AS (SELECT TRY_PARSE_JSON(value) AS payload FROM events) SELECT payload FROM final", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "nested SELECT is its own scope", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(e.value):a AS a, (SELECT MAX(TRY_PARSE_JSON(x.value):b) FROM events AS x) AS b FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "repeat inside a subquery is reported in that scope", - dialect: "snowflake", - sql: "SELECT e.order_id FROM events AS e WHERE e.order_id IN (SELECT TRY_PARSE_JSON(value):id FROM events WHERE TRY_PARSE_JSON(value):ok)", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "union branches are separate scopes", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(value) AS payload FROM events UNION ALL SELECT TRY_PARSE_JSON(value) AS payload FROM events", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "a repeat inside one CTE body is reported", - dialect: "snowflake", - sql: "WITH parsed AS (SELECT TRY_PARSE_JSON(value):a AS a, TRY_PARSE_JSON(value):b AS b FROM events) SELECT a, b FROM parsed", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "non-column arguments pass", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(value || '') AS a, TRY_PARSE_JSON(value || '') AS b FROM events", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "bigquery safe parse is matched", - dialect: "bigquery", - sql: "SELECT JSON_VALUE(SAFE.PARSE_JSON(e.value), '$.a') AS a, JSON_VALUE(SAFE.PARSE_JSON(e.value), '$.b') AS b FROM events AS e", - expected_anchors: &["PARSE_JSON"], - expected_details: &["SAFE.PARSE_JSON(e.value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "databricks try parse is matched", - dialect: "databricks", - sql: "SELECT try_parse_json(value) AS a, try_parse_json(value) AS b FROM events", - expected_anchors: &["try_parse_json"], - expected_details: &["TRY_PARSE_JSON(value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "duckdb json is matched", - dialect: "duckdb", - sql: "SELECT json(e.value)->>'a' AS a, json(e.value)->>'b' AS b FROM events AS e", - expected_anchors: &["json"], - expected_details: &["JSON(e.value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "dialects without an equivalent are not checked", - dialect: "postgres", - sql: "SELECT CAST(value AS JSONB) AS a, CAST(value AS JSONB) AS b FROM events", - expected_anchors: &[], - expected_details: &[], - }, - ]; - for test_case in &test_cases { - let diagnostics: Vec = lint_dialect(test_case.sql, test_case.dialect, RULE)?; - assert_eq!( - anchors(test_case.sql, &diagnostics), - test_case.expected_anchors, - "{}", - test_case.description - ); - let details: Vec = diagnostics - .iter() - .map(|diagnostic| { - diagnostic["remediation"] - .as_str() - .unwrap_or_default() - .split_inclusive(". ") - .next() - .unwrap_or_default() - .trim_end() - .to_owned() - }) - .collect(); - assert_eq!( - details, test_case.expected_details, - "{}", - test_case.description - ); - assert!( - diagnostics - .iter() - .all(|diagnostic| diagnostic["fix"].is_null()), - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_wide_select_with_many_repeated_parses_when_linting_then_it_stays_linear() --> Result<(), String> { - let test_cases = [JsonParseScaleTestCase { - description: "3,000 projections parsing 1,500 payload columns twice each", - column_count: 1_500, - expected_findings: 1_500, - expected_max_seconds: 10.0, - }]; - for test_case in &test_cases { - let started: Instant = Instant::now(); - let diagnostics: Vec = - lint_dialect(&wide_json_select(test_case.column_count), "duckdb", RULE)?; - let elapsed: f64 = started.elapsed().as_secs_f64(); - assert_eq!( - diagnostics.len(), - test_case.expected_findings, - "{}", - test_case.description - ); - assert!( - elapsed <= test_case.expected_max_seconds, - "{}: {elapsed:.2}s", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlclean/src/sql_quality/tests/test_quality_rules.rs b/crates/sqlclean/src/sql_quality/tests/test_quality_rules.rs deleted file mode 100644 index 966f575..0000000 --- a/crates/sqlclean/src/sql_quality/tests/test_quality_rules.rs +++ /dev/null @@ -1,435 +0,0 @@ -use serde_json::Value; - -use crate::sql_quality::tests::helpers::{anchors, fixed, lint, lint_with_host_fields}; -use crate::sql_quality::tests::test_types::{QualityRemediationTestCase, QualityRuleTestCase}; - -#[test] -fn given_quality_rule_inputs_when_linting_then_findings_and_fixes_match() -> Result<(), String> { - let test_cases = [ - QualityRuleTestCase { - description: "long literal is reported without a split fix", - sql: "SELECT 'pending orders awaiting shipment' AS label", - rule: "SQLCLEAN044", - relation_keys: "{}", - expected_anchors: &["'pending orders awaiting shipment'"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "short literal passes", - sql: "SELECT 'pending' AS label", - rule: "SQLCLEAN044", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "typed literal is reported without a fix", - sql: "SELECT TIMESTAMP '2026-04-01 10:00:00.000000000' AS ordered_at", - rule: "SQLCLEAN044", - relation_keys: "{}", - expected_anchors: &["'2026-04-01 10:00:00.000000000'"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "ranking sort above the cap", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.a, o.b, o.order_id) AS rn FROM orders AS o", - rule: "SQLCLEAN043", - relation_keys: "{}", - expected_anchors: &["ROW_NUMBER"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "ranking sort within the cap", - sql: "SELECT RANK() OVER (ORDER BY o.a, o.order_id) AS rn FROM orders AS o", - rule: "SQLCLEAN043", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "ordering covers a declared key", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.ordered_at, o.order_id) AS rn FROM orders AS o", - rule: "SQLCLEAN018", - relation_keys: r#"{"orders": [["order_id"]]}"#, - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "ordering without a key is unproven", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.ordered_at) AS rn FROM orders AS o", - rule: "SQLCLEAN018", - relation_keys: r#"{"orders": [["order_id"]]}"#, - expected_anchors: &["ROW_NUMBER"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "key proven through a CTE and a one-to-one join", - sql: "WITH lines AS (SELECT l.line_id, l.order_id FROM order_lines AS l), final AS (SELECT l.line_id, o.customer_id, ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY l.line_id) AS rn FROM lines AS l INNER JOIN orders AS o ON l.order_id = o.order_id) SELECT f.line_id, f.customer_id, f.rn FROM final AS f", - rule: "SQLCLEAN018", - relation_keys: r#"{"orders": [["order_id"]], "order_lines": [["line_id"]]}"#, - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "key lost through a fan-out join", - sql: "WITH final AS (SELECT o.order_id, ROW_NUMBER() OVER (ORDER BY o.order_id) AS rn FROM orders AS o INNER JOIN order_lines AS l ON l.order_id = o.order_id) SELECT f.order_id, f.rn FROM final AS f", - rule: "SQLCLEAN018", - relation_keys: r#"{"orders": [["order_id"]], "order_lines": [["line_id"]]}"#, - expected_anchors: &["ROW_NUMBER"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "GROUP BY grain is a key", - sql: "WITH totals AS (SELECT o.customer_id, SUM(o.amount) AS amount FROM orders AS o GROUP BY o.customer_id), final AS (SELECT t.customer_id, ROW_NUMBER() OVER (ORDER BY t.amount, t.customer_id) AS rn FROM totals AS t) SELECT f.customer_id, f.rn FROM final AS f", - rule: "SQLCLEAN018", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "positional GROUP BY grain is a key", - sql: "WITH totals AS (SELECT o.customer_id, SUM(o.amount) AS amount FROM orders AS o GROUP BY 1), final AS (SELECT t.customer_id, ROW_NUMBER() OVER (ORDER BY t.amount, t.customer_id) AS rn FROM totals AS t) SELECT f.customer_id, f.rn FROM final AS f", - rule: "SQLCLEAN018", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a later rn = 1 filter makes the partition a key", - sql: "WITH ranked AS (SELECT o.order_id, o.customer_id, ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.order_id) AS rn FROM orders AS o), latest AS (SELECT r.customer_id, r.order_id FROM ranked AS r WHERE r.rn = 1), final AS (SELECT l.customer_id, ROW_NUMBER() OVER (ORDER BY l.customer_id) AS position FROM latest AS l) SELECT f.customer_id, f.position FROM final AS f", - rule: "SQLCLEAN018", - relation_keys: r#"{"orders": [["order_id"]]}"#, - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a QUALIFY window cannot prove itself", - sql: "SELECT o.customer_id, o.order_id FROM orders AS o QUALIFY ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.amount DESC) = 1", - rule: "SQLCLEAN018", - relation_keys: r#"{"orders": [["order_id"]]}"#, - expected_anchors: &["ROW_NUMBER"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a QUALIFY partition is a key for later steps", - sql: "WITH latest AS (SELECT o.customer_id, o.order_id FROM orders AS o QUALIFY ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.order_id) = 1), final AS (SELECT l.customer_id, ROW_NUMBER() OVER (ORDER BY l.customer_id) AS position FROM latest AS l) SELECT f.customer_id, f.position FROM final AS f", - rule: "SQLCLEAN018", - relation_keys: r#"{"orders": [["order_id"]]}"#, - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "RANK ties are deterministic", - sql: "SELECT RANK() OVER (ORDER BY o.ordered_at) AS rn FROM orders AS o", - rule: "SQLCLEAN018", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "unused CTE output is removed", - sql: "WITH orders AS (SELECT o.order_id, o.status, o.note FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &["o.note"], - expected_fixed_sql: Some( - "WITH orders AS (SELECT o.order_id, o.status FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f", - ), - }, - QualityRuleTestCase { - description: "star pass-through into the final CTE keeps every column", - sql: "WITH orders AS (SELECT o.order_id, o.note FROM raw_orders AS o), final AS (SELECT * FROM orders) SELECT * FROM final", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "star pass-through follows the reader's own reads", - sql: "WITH orders AS (SELECT o.order_id, o.note FROM raw_orders AS o), staged AS (SELECT * FROM orders), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &["o.note"], - expected_fixed_sql: Some( - "WITH orders AS (SELECT o.order_id FROM raw_orders AS o), staged AS (SELECT * FROM orders), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", - ), - }, - QualityRuleTestCase { - description: "a same-named cast of a source column is not a lateral read", - sql: "WITH orders AS (SELECT o.order_id, CAST(note AS VARCHAR) AS note, o.note || 'x' AS tagged FROM raw_orders AS o), final AS (SELECT r.order_id, r.tagged FROM orders AS r) SELECT f.order_id, f.tagged FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &["CAST(note AS VARCHAR) AS note"], - expected_fixed_sql: Some( - "WITH orders AS (SELECT o.order_id, o.note || 'x' AS tagged FROM raw_orders AS o), final AS (SELECT r.order_id, r.tagged FROM orders AS r) SELECT f.order_id, f.tagged FROM final AS f", - ), - }, - QualityRuleTestCase { - description: "an interpolated expression in a reader may read any column", - sql: "WITH orders AS (SELECT o.order_id, o.status FROM raw_orders AS o), final AS (SELECT r.order_id FROM orders AS r WHERE __sqlclean_relation_parameter_0__) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "an interpolated output is never reported", - sql: "WITH orders AS (SELECT o.order_id, __SQLCLEAN_INTERPOLATION_0002 FROM raw_orders AS o), final AS (SELECT r.order_id FROM orders AS r) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "DISTINCT CTE reports without a fix", - sql: "WITH orders AS (SELECT DISTINCT o.order_id, o.note FROM raw_orders AS o), final AS (SELECT r.order_id FROM orders AS r) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &["o.note"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "columns read in later join and filter clauses are used", - sql: "WITH orders AS (SELECT o.order_id, o.customer_id, o.status FROM raw_orders AS o), final AS (SELECT c.name FROM orders AS r INNER JOIN customers AS c ON r.customer_id = c.customer_id WHERE r.status = 'open' QUALIFY ROW_NUMBER() OVER (PARTITION BY r.order_id ORDER BY c.name) = 1) SELECT f.name FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a column read only through a nested star is used", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT g.order_id, HASH(g.*) AS row_hash FROM g) SELECT f.order_id, f.row_hash FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a whole-row reference reads every column", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT g.order_id, HASH(g) AS row_hash FROM g) SELECT f.order_id, f.row_hash FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "COUNT(DISTINCT row) reads every column", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT COUNT(DISTINCT g) AS n FROM g) SELECT f.n FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "COLUMNS(*) reads every column", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT g.order_id FROM g WHERE COLUMNS(*) IS NOT NULL) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "COLUMNS with a pattern reads every column", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT MAX(COLUMNS('order.*')) FROM g) SELECT * FROM final", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "DISTINCT star pass-through reads every column", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), staged AS (SELECT DISTINCT * FROM g), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a positional ORDER BY over a star pass-through reads every column", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), staged AS (SELECT * FROM g ORDER BY 2 LIMIT 1), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "an output alias read in WHERE or GROUP BY is used", - sql: "WITH g AS (SELECT r.order_id, r.amount * 2 AS doubled, r.customer_id % 2 AS bucket FROM raw_orders AS r WHERE doubled > 1 GROUP BY r.order_id, doubled, bucket), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "GROUP BY ALL grouping column is reported without a fix", - sql: "WITH g AS (SELECT r.customer_id, r.status, SUM(r.amount) AS total FROM raw_orders AS r GROUP BY ALL), final AS (SELECT g.customer_id, g.total FROM g) SELECT f.customer_id, f.total FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &["r.status"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "an aggregate in an ungrouped select is reported without a fix", - sql: "WITH g AS (SELECT 'all' AS label, COUNT(*) AS n FROM raw_orders AS r), final AS (SELECT g.label FROM g) SELECT f.label FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &["COUNT(*) AS n"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a set-returning projection is reported without a fix", - sql: "WITH g AS (SELECT r.order_id, UNNEST(r.tags) AS tag FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &["UNNEST(r.tags) AS tag"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "ORDER BY ALL with LIMIT is reported without a fix", - sql: "WITH g AS (SELECT r.order_id, r.customer_id FROM raw_orders AS r ORDER BY ALL LIMIT 1), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &["r.customer_id"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "DISTINCT ON is reported without a fix", - sql: "WITH g AS (SELECT DISTINCT ON (r.customer_id) r.order_id, r.amount FROM raw_orders AS r ORDER BY r.customer_id, r.order_id), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &["r.amount"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a volatile projection is reported without a fix", - sql: "WITH g AS (SELECT r.order_id, RANDOM() AS draw FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &["RANDOM() AS draw"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a deterministic scalar projection is removed", - sql: "WITH g AS (SELECT r.order_id, COALESCE(UPPER(r.status), 'none') AS label FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - rule: "SQLCLEAN042", - relation_keys: "{}", - expected_anchors: &["COALESCE(UPPER(r.status), 'none') AS label"], - expected_fixed_sql: Some( - "WITH g AS (SELECT r.order_id FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - ), - }, - ]; - for test_case in &test_cases { - let diagnostics: Vec = lint(test_case)?; - assert_eq!( - anchors(test_case.sql, &diagnostics), - test_case.expected_anchors, - "{}", - test_case.description - ); - assert_eq!( - fixed(test_case.sql, &diagnostics).as_deref(), - test_case.expected_fixed_sql, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_threshold_findings_when_linting_then_remediation_shows_the_exact_setting() --> Result<(), String> { - let test_cases = [ - QualityRemediationTestCase { - description: "long literal points at shortening it and the length setting", - sql: "SELECT 'pending orders awaiting shipment' AS label", - rule: "SQLCLEAN044", - host_fields: "{}", - expected_fragments: &[ - "Shorten the literal", - "set this in sqlclean.toml:", - "the current value is 24", - "[rules.thresholds]\n max_literal_length = 32", - ], - absent_fragments: &[], - }, - QualityRemediationTestCase { - description: "a host's long literal guidance replaces SQLClean's and names its file", - sql: "SELECT 'pending orders awaiting shipment' AS label", - rule: "SQLCLEAN044", - host_fields: r#"{"long_literal_guidance": "Declare it once as a constant.", "config_file": "host.toml"}"#, - expected_fragments: &[ - "Declare it once as a constant. To allow literals of 32 characters", - "set this in host.toml:", - ], - absent_fragments: &["Shorten the literal", "sqlclean.toml"], - }, - QualityRemediationTestCase { - description: "long ranking sort points at a row key and the sort setting", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.a, o.b, o.order_id) AS rn FROM orders AS o", - rule: "SQLCLEAN043", - host_fields: "{}", - expected_fragments: &[ - "unique row key as the final tie-breaker", - "HASH(source_partition, source_offset, line_index) AS row_key", - "the current value is 2", - "[rules.thresholds]\n max_ranking_order_by = 3", - ], - absent_fragments: &[], - }, - QualityRemediationTestCase { - description: "unproven ranking points at a unique tie-breaker and a built row key", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.a) AS rn FROM orders AS o", - rule: "SQLCLEAN018", - host_fields: "{}", - expected_fragments: &[ - "cover a unique key of the ranked rows", - "a unique, non-null tie-breaker column", - "HASH(source_partition, source_offset, line_index) AS row_key", - ], - absent_fragments: &[], - }, - ]; - for test_case in &test_cases { - let diagnostics: Vec = lint_with_host_fields( - &QualityRuleTestCase { - description: test_case.description, - sql: test_case.sql, - rule: test_case.rule, - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - test_case.host_fields, - )?; - let remediation: &str = diagnostics - .first() - .and_then(|diagnostic| diagnostic["remediation"].as_str()) - .ok_or_else(|| format!("{}: no finding", test_case.description))?; - let present: Vec<&str> = test_case - .expected_fragments - .iter() - .copied() - .filter(|fragment| remediation.contains(fragment)) - .collect(); - assert_eq!( - present, test_case.expected_fragments, - "{}: {remediation}", - test_case.description - ); - let leaked: Vec<&str> = test_case - .absent_fragments - .iter() - .copied() - .filter(|fragment| remediation.contains(fragment)) - .collect(); - assert_eq!( - leaked, - Vec::<&str>::new(), - "{}: {remediation}", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlclean/src/sql_quality/tests/test_quality_scale.rs b/crates/sqlclean/src/sql_quality/tests/test_quality_scale.rs deleted file mode 100644 index 512e814..0000000 --- a/crates/sqlclean/src/sql_quality/tests/test_quality_scale.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::time::Instant; - -use serde_json::{Value, json}; - -use crate::sql_lint::main::engine::lint_json; -use crate::sql_quality::tests::helpers::wide_cte_chain; -use crate::sql_quality::tests::test_types::QualityScaleTestCase; - -#[test] -fn given_many_wide_ctes_with_findings_when_linting_then_quality_rules_stay_linear() --> Result<(), String> { - let test_cases = [QualityScaleTestCase { - description: "1,500 wide CTEs, each dropping a column and ranking without a key", - cte_count: 1_500, - column_count: 24, - expected_findings: 2_998, - expected_max_seconds: 20.0, - }]; - for test_case in &test_cases { - let request: Value = json!({ - "version": 1, - "sql": wide_cte_chain(test_case.cte_count, test_case.column_count), - "dialect": "duckdb", - "enabled_rules": ["SQLCLEAN018", "SQLCLEAN042", "SQLCLEAN043", "SQLCLEAN044"], - }); - let started: Instant = Instant::now(); - let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) - .map_err(|error| error.to_string())?; - let elapsed: f64 = started.elapsed().as_secs_f64(); - assert_eq!( - response["diagnostics"].as_array().map_or(0, Vec::len), - test_case.expected_findings, - "{}", - test_case.description - ); - assert!( - elapsed <= test_case.expected_max_seconds, - "{}: {elapsed:.2}s", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlclean/src/sql_quality/tests/test_safety_rules.rs b/crates/sqlclean/src/sql_quality/tests/test_safety_rules.rs deleted file mode 100644 index b04263d..0000000 --- a/crates/sqlclean/src/sql_quality/tests/test_safety_rules.rs +++ /dev/null @@ -1,207 +0,0 @@ -use serde_json::Value; - -use crate::sql_quality::tests::helpers::{anchors, lint_dialect}; -use crate::sql_quality::tests::test_types::SafetyRuleTestCase; - -#[test] -fn given_safety_rule_inputs_when_linting_then_exactly_the_mistakes_are_reported() --> Result<(), String> { - let test_cases = [ - SafetyRuleTestCase { - description: "unquoted CTE names collide regardless of case", - dialect: "duckdb", - sql: "WITH paid AS (SELECT 1 AS x), PAID AS (SELECT 2 AS x) SELECT x FROM paid", - rule: "SQLCLEAN046", - expected_anchors: &["PAID"], - }, - SafetyRuleTestCase { - description: "quoted names that differ in case are distinct in PostgreSQL", - dialect: "postgres", - sql: r#"WITH "paid" AS (SELECT 1 AS x), "PAID" AS (SELECT 2 AS x) SELECT x FROM "paid""#, - rule: "SQLCLEAN046", - expected_anchors: &[], - }, - SafetyRuleTestCase { - description: "a quoted name equals the folded unquoted name in Snowflake", - dialect: "snowflake", - sql: r#"WITH "PAID" AS (SELECT 1 AS x), paid AS (SELECT 2 AS x) SELECT x FROM paid"#, - rule: "SQLCLEAN046", - expected_anchors: &["paid"], - }, - SafetyRuleTestCase { - description: "a nested WITH may reuse an outer CTE name", - dialect: "duckdb", - sql: "WITH paid AS (WITH paid AS (SELECT 1 AS x) SELECT x FROM paid) SELECT x FROM paid", - rule: "SQLCLEAN046", - expected_anchors: &[], - }, - SafetyRuleTestCase { - description: "simple CASE compares with NULL, and a singleton IN list holds only NULL", - dialect: "duckdb", - sql: "SELECT CASE status WHEN 'open' THEN 1 WHEN NULL THEN 2 END AS s FROM orders WHERE region IN (NULL)", - rule: "SQLCLEAN047", - expected_anchors: &["WHEN NULL", "IN (NULL)"], - }, - SafetyRuleTestCase { - description: "searched CASE, IS NULL, NOT IN and a longer IN list pass", - dialect: "duckdb", - sql: "SELECT CASE WHEN status IS NULL THEN 1 END AS s FROM orders WHERE region NOT IN (NULL) AND kind IN (1, NULL)", - rule: "SQLCLEAN047", - expected_anchors: &[], - }, - SafetyRuleTestCase { - description: "only the CASE whose operand compares with NULL is reported", - dialect: "duckdb", - sql: "SELECT CASE status WHEN 'open' THEN CASE WHEN NULL THEN 1 END END AS a, CASE WHEN amount > 0 THEN CASE region WHEN NULL THEN 2 END END AS b FROM orders", - rule: "SQLCLEAN047", - expected_anchors: &["WHEN NULL"], - }, - SafetyRuleTestCase { - description: "division and modulo by literal zero in operator and function form", - dialect: "duckdb", - sql: "SELECT amount / 0 AS a, amount % 0.0 AS b, MOD(amount, 0) AS c, amount / (-0) AS d FROM orders", - rule: "SQLCLEAN048", - expected_anchors: &["/ 0", "% 0.0", "MOD(amount, 0)", "/ (-0)"], - }, - SafetyRuleTestCase { - description: "a non-zero or computed divisor passes", - dialect: "duckdb", - sql: "SELECT amount / 0.5 AS a, amount / 1e-400 AS e, amount / 0 ** 2 AS b, MOD(amount, 0 + 1) AS c, amount / NULLIF(quantity, 0) AS d FROM orders", - rule: "SQLCLEAN048", - expected_anchors: &[], - }, - SafetyRuleTestCase { - description: "a missing or constant WHERE does not limit an UPDATE or DELETE", - dialect: "duckdb", - sql: "UPDATE orders SET status = 'closed'; DELETE FROM orders WHERE TRUE; DELETE FROM orders WHERE 1 = 1", - rule: "SQLCLEAN049", - expected_anchors: &["UPDATE", "DELETE", "DELETE"], - }, - SafetyRuleTestCase { - description: "a row predicate or a subquery limits the write", - dialect: "duckdb", - sql: "UPDATE orders SET status = 'closed' WHERE closed_at IS NOT NULL; DELETE FROM orders WHERE EXISTS (SELECT 1 FROM refunds)", - rule: "SQLCLEAN049", - expected_anchors: &[], - }, - SafetyRuleTestCase { - description: "a joined DELETE and a LIMIT bound the rows in MySQL", - dialect: "mysql", - sql: "DELETE o FROM orders o JOIN customers c ON c.id = o.customer_id; DELETE FROM orders LIMIT 10", - rule: "SQLCLEAN049", - expected_anchors: &[], - }, - SafetyRuleTestCase { - description: "UPDATE FROM without WHERE, and DELETE after a WITH clause", - dialect: "postgres", - sql: "UPDATE orders SET status = 'closed' FROM customers; WITH stale AS (SELECT 1 AS id) DELETE FROM orders", - rule: "SQLCLEAN049", - expected_anchors: &["UPDATE", "DELETE"], - }, - SafetyRuleTestCase { - description: "an INSERT without a column list in query and VALUES form", - dialect: "duckdb", - sql: "INSERT INTO orders SELECT * FROM staged_orders; INSERT INTO orders VALUES (1, 'open')", - rule: "SQLCLEAN050", - expected_anchors: &["INSERT", "INSERT"], - }, - SafetyRuleTestCase { - description: "listed columns and BY NAME pass", - dialect: "duckdb", - sql: "INSERT INTO orders (order_id) SELECT order_id FROM staged_orders; INSERT INTO orders BY NAME SELECT 1 AS order_id", - rule: "SQLCLEAN050", - expected_anchors: &[], - }, - SafetyRuleTestCase { - description: "DEFAULT VALUES needs no column list", - dialect: "postgres", - sql: "INSERT INTO orders DEFAULT VALUES", - rule: "SQLCLEAN050", - expected_anchors: &[], - }, - SafetyRuleTestCase { - description: "NATURAL joins are reported with their join keywords", - dialect: "duckdb", - sql: "SELECT * FROM orders NATURAL JOIN customers NATURAL LEFT OUTER JOIN regions JOIN items USING (order_id)", - rule: "SQLCLEAN051", - expected_anchors: &["NATURAL JOIN", "NATURAL LEFT OUTER JOIN"], - }, - SafetyRuleTestCase { - description: "reversed number, negative, decimal and date bounds, with or without NOT", - dialect: "duckdb", - sql: "SELECT * FROM orders WHERE a BETWEEN 10 AND 1 OR b NOT BETWEEN -1 AND -5 OR c BETWEEN 1.5 AND 1.25 OR d BETWEEN DATE '2026-02-01' AND DATE '2026-01-31'", - rule: "SQLCLEAN052", - expected_anchors: &[ - "BETWEEN 10 AND 1", - "BETWEEN -1 AND -5", - "BETWEEN 1.5 AND 1.25", - "BETWEEN DATE '2026-02-01' AND DATE '2026-01-31'", - ], - }, - SafetyRuleTestCase { - description: "ASYMMETRIC keeps the ordered meaning in PostgreSQL", - dialect: "postgres", - sql: "SELECT * FROM orders WHERE a BETWEEN ASYMMETRIC 10 AND 1", - rule: "SQLCLEAN052", - expected_anchors: &["BETWEEN ASYMMETRIC 10 AND 1"], - }, - SafetyRuleTestCase { - description: "ordered, equal, SYMMETRIC, computed, text and frame bounds pass", - dialect: "duckdb", - sql: "SELECT SUM(a) OVER (ORDER BY b ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS s FROM orders WHERE a BETWEEN 9 AND 10 OR b BETWEEN -5 AND -1 OR c BETWEEN 5 AND 5 OR d BETWEEN SYMMETRIC 10 AND 1 OR e BETWEEN 10 AND 1 + 20 OR f BETWEEN 'b' AND 'a'", - rule: "SQLCLEAN052", - expected_anchors: &[], - }, - SafetyRuleTestCase { - description: "aggregates and windows in WHERE, ON and after a GROUP BY comma", - dialect: "duckdb", - sql: "SELECT o.customer_id FROM orders AS o JOIN customers AS c ON c.id = o.customer_id AND COUNT(*) > 1 WHERE SUM(o.amount) > 100 AND ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.ordered_at) = 1 GROUP BY o.customer_id, MAX(o.amount)", - rule: "SQLCLEAN053", - expected_anchors: &[ - "COUNT(*)", - "SUM(o.amount)", - "ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.ordered_at)", - "MAX(o.amount)", - ], - }, - SafetyRuleTestCase { - description: "HAVING, QUALIFY, FILTER, subqueries and unknown functions pass", - dialect: "duckdb", - sql: "SELECT customer_id, COUNT(*) FILTER (WHERE amount > 0) AS paid FROM orders WHERE amount > (SELECT AVG(amount) FROM orders) AND order_total(amount) > 0 GROUP BY customer_id HAVING COUNT(*) > 1 QUALIFY ROW_NUMBER() OVER (ORDER BY customer_id) = 1", - rule: "SQLCLEAN053", - expected_anchors: &[], - }, - SafetyRuleTestCase { - description: "DuckDB PIVOT USING aggregates end the pivot ON clause", - dialect: "duckdb", - sql: "PIVOT orders ON status IN ('placed') USING MAX(amount), SUM(amount) GROUP BY customer_id", - rule: "SQLCLEAN053", - expected_anchors: &[], - }, - SafetyRuleTestCase { - description: "a join ON aggregate is still reported beside a pivot", - dialect: "duckdb", - sql: "SELECT o.id FROM orders AS o JOIN (PIVOT items ON kind USING MAX(qty)) AS p ON p.id = o.id AND MIN(o.amount) > 0", - rule: "SQLCLEAN053", - expected_anchors: &["MIN(o.amount)"], - }, - SafetyRuleTestCase { - description: "multi-argument MAX is the scalar function in SQLite", - dialect: "sqlite", - sql: "SELECT order_id FROM orders WHERE max(amount, refunded) > 100", - rule: "SQLCLEAN053", - expected_anchors: &[], - }, - ]; - for test_case in &test_cases { - let diagnostics: Vec = - lint_dialect(test_case.sql, test_case.dialect, test_case.rule)?; - assert_eq!( - anchors(test_case.sql, &diagnostics), - test_case.expected_anchors, - "{}", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlclean/src/sql_quality/tests/test_types.rs b/crates/sqlclean/src/sql_quality/tests/test_types.rs deleted file mode 100644 index 6c02eab..0000000 --- a/crates/sqlclean/src/sql_quality/tests/test_types.rs +++ /dev/null @@ -1,49 +0,0 @@ -pub(crate) struct QualityRuleTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) rule: &'static str, - pub(crate) relation_keys: &'static str, - pub(crate) expected_anchors: &'static [&'static str], - pub(crate) expected_fixed_sql: Option<&'static str>, -} - -pub(crate) struct QualityScaleTestCase { - pub(crate) description: &'static str, - pub(crate) cte_count: usize, - pub(crate) column_count: usize, - pub(crate) expected_findings: usize, - pub(crate) expected_max_seconds: f64, -} - -pub(crate) struct QualityRemediationTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) rule: &'static str, - /// Extra request fields as a JSON object, as a host sends them. - pub(crate) host_fields: &'static str, - pub(crate) expected_fragments: &'static [&'static str], - pub(crate) absent_fragments: &'static [&'static str], -} - -pub(crate) struct JsonParseTestCase { - pub(crate) description: &'static str, - pub(crate) dialect: &'static str, - pub(crate) sql: &'static str, - pub(crate) expected_anchors: &'static [&'static str], - pub(crate) expected_details: &'static [&'static str], -} - -pub(crate) struct JsonParseScaleTestCase { - pub(crate) description: &'static str, - pub(crate) column_count: usize, - pub(crate) expected_findings: usize, - pub(crate) expected_max_seconds: f64, -} - -pub(crate) struct SafetyRuleTestCase { - pub(crate) description: &'static str, - pub(crate) dialect: &'static str, - pub(crate) sql: &'static str, - pub(crate) rule: &'static str, - pub(crate) expected_anchors: &'static [&'static str], -} diff --git a/crates/sqlclean/src/sql_quality/types.rs b/crates/sqlclean/src/sql_quality/types.rs deleted file mode 100644 index 717e03a..0000000 --- a/crates/sqlclean/src/sql_quality/types.rs +++ /dev/null @@ -1,6 +0,0 @@ -use std::collections::BTreeSet; - -/// Column sets that are unique for one relation, by lower-cased output column name. -pub(crate) type Keys = Vec>; -/// A unique column set over a select's joined input, as `(source alias, column)` pairs. -pub(crate) type InputKey = BTreeSet<(String, String)>; diff --git a/fensu.toml b/fensu.toml index 7f53731..72bf6fb 100644 --- a/fensu.toml +++ b/fensu.toml @@ -43,6 +43,11 @@ rule = "FPRSL105" path = "crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs" reason = "This test-support entry checks that fixes preserve meaning in the sql_lint tests, which the analyzer does not count as importers." +[[targets.rust.rule_exceptions]] +rule = "FPRSL105" +path = "crates/sqlclean/src/sql_tokens/main/projection_spans.rs" +reason = "Integration surface: SQLBuild reads projection spans through this entry; SQLQuack's lint engine has its own copy." + [targets.rust.rule_options.FPRSR311] # `skill/` is the agent skill the `sqlclean skills` command embeds; scripts.generate_docs_skill writes its references. allowed_entries = ["Cargo.toml", "build.rs", "src/", "tests/", "benches/", "examples/", "README*", "LICENSE*", "skill/"] From 6c7791430e6ef0c31a57dee838815ce459779241 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:48:17 +0000 Subject: [PATCH 04/21] wip(format): lay out around CTE macro placeholders, and fingerprint with SQLQuack's tokens --- Cargo.toml | 1 + crates/sqlclean/Cargo.toml | 1 + .../src/sql_lint/_helpers/cte_macros.rs | 271 ++++++++++++++++++ .../src/sql_lint/_helpers/formatter.rs | 30 +- crates/sqlclean/src/sql_lint/_helpers/mod.rs | 1 + .../sqlclean/src/sql_tokens/_helpers/mod.rs | 1 + .../sql_tokens/_helpers/reserved_keywords.rs | 13 + .../_helpers/sqlquack_canonical_tokens.rs | 99 +++++++ crates/sqlclean/src/sql_tokens/constants.rs | 45 +++ .../src/sql_tokens/main/query_fingerprint.rs | 31 +- crates/sqlclean/src/sql_tokens/models.rs | 26 ++ crates/sqlclean/src/sql_tokens/tests.rs | 2 + .../tests/test_query_fingerprint.rs | 66 +++++ 13 files changed, 576 insertions(+), 11 deletions(-) create mode 100644 crates/sqlclean/src/sql_lint/_helpers/cte_macros.rs create mode 100644 crates/sqlclean/src/sql_tokens/_helpers/sqlquack_canonical_tokens.rs create mode 100644 crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs diff --git a/Cargo.toml b/Cargo.toml index 62634da..37649af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ sqlclean = { path = "crates/sqlclean" } # SQLQuack's formatter, which lays out the dialects it parses natively without Polyglot. sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "f2985b374694f3da56c4f1e5b36fbc5d81e101ef" } sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "f2985b374694f3da56c4f1e5b36fbc5d81e101ef" } +sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "f2985b374694f3da56c4f1e5b36fbc5d81e101ef" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sqlclean/Cargo.toml b/crates/sqlclean/Cargo.toml index 9b69421..53d64af 100644 --- a/crates/sqlclean/Cargo.toml +++ b/crates/sqlclean/Cargo.toml @@ -18,6 +18,7 @@ polyglot-sql.workspace = true polyglot-sql-function-catalogs.workspace = true sqlquack-format.workspace = true sqlquack-syntax.workspace = true +sqlquack-syntax-postgres.workspace = true rayon.workspace = true rustc-hash.workspace = true serde.workspace = true diff --git a/crates/sqlclean/src/sql_lint/_helpers/cte_macros.rs b/crates/sqlclean/src/sql_lint/_helpers/cte_macros.rs new file mode 100644 index 0000000..68021f4 --- /dev/null +++ b/crates/sqlclean/src/sql_lint/_helpers/cte_macros.rs @@ -0,0 +1,271 @@ +//! CTE macro placeholders, which SQLQuack does not parse, taken out before layout and put back after. + +use std::collections::BTreeSet; + +use sqlquack_syntax::tokens::models::TokenType; +use sqlquack_syntax::{Dialect, LosslessToken, TokenKind}; + +use crate::constants::INTERPOLATION_PLACEHOLDER_PREFIX; + +const RESTORE_FAILURE: &str = "native formatter could not restore CTE macro placeholders"; + +/// SQL with its CTE macro placeholders taken out, and the groups to put back. +pub(super) struct WithoutCteMacros { + pub sql: String, + groups: Vec, + /// The significant and comment texts of the original SQL, in order. + original: Vec<(bool, String)>, +} + +/// Placeholders, with the commas taken out with them, that stood before one significant token. +struct MacroGroup { + /// The index, among the significant tokens of the stripped SQL, of the token they preceded. + anchor: usize, + /// How many comments stood between the previous significant token and the group. + comments_before: usize, + texts: Vec, +} + +/// Take the CTE macro placeholders out of `sql`; `None` when it has none. +pub(super) fn take_out_cte_macros( + sql: &str, + dialect: Dialect, +) -> Result, String> { + if !sql.contains(INTERPOLATION_PLACEHOLDER_PREFIX) { + return Ok(None); + } + let Ok(tokens) = sqlquack_syntax::tokenize(sql, dialect) else { + return Ok(None); + }; + let significant: Vec = (0..tokens.len()) + .filter(|&index| matches!(tokens[index].kind, TokenKind::Significant(_))) + .collect(); + let macros = macro_positions(sql, &tokens, &significant); + if macros.is_empty() { + return Ok(None); + } + let removed = removed_positions(&tokens, &significant, ¯os); + let mut groups: Vec = Vec::new(); + let mut stripped = String::with_capacity(sql.len()); + let mut kept = 0_usize; + let mut cursor = 0_usize; + let mut position = 0_usize; + while position < significant.len() { + if !removed.contains(&position) { + kept += 1; + position += 1; + continue; + } + let previous_end = position + .checked_sub(1) + .map_or(0, |previous| tokens[significant[previous]].end); + let comments_before = tokens[..significant[position]] + .iter() + .filter(|token| token.start >= previous_end && is_comment(token)) + .count(); + let mut texts: Vec = Vec::new(); + while removed.contains(&position) { + let index = significant[position]; + stripped.push_str(&sql[cursor..tokens[index].start]); + texts.push(tokens[index].text(sql).to_string()); + cursor = tokens + .get(index + 1) + .filter(|next| next.kind == TokenKind::Whitespace) + .map_or(tokens[index].end, |next| next.end); + position += 1; + } + groups.push(MacroGroup { + anchor: kept, + comments_before, + texts, + }); + } + stripped.push_str(&sql[cursor..]); + Ok(Some(WithoutCteMacros { + sql: stripped, + groups, + original: token_texts(sql, &tokens), + })) +} + +impl WithoutCteMacros { + /// Put the placeholders back into `formatted`, the laid-out stripped SQL, each on its own line. + pub(super) fn put_back(&self, formatted: &str, dialect: Dialect) -> Result { + let tokens = sqlquack_syntax::tokenize(formatted, dialect) + .map_err(|_| RESTORE_FAILURE.to_string())?; + let significant: Vec = (0..tokens.len()) + .filter(|&index| matches!(tokens[index].kind, TokenKind::Significant(_))) + .collect(); + let mut restored = String::with_capacity(formatted.len() + 64); + let mut cursor = 0_usize; + for group in &self.groups { + let gap_start = group + .anchor + .checked_sub(1) + .map_or(0, |previous| tokens[significant[previous]].end); + let gap_end = significant + .get(group.anchor) + .map_or(formatted.len(), |&anchor| tokens[anchor].start); + // After the comments that stood before the group; the token check below confirms it. + let insert_at = match group.comments_before.checked_sub(1) { + None => gap_start, + Some(last) => tokens + .iter() + .filter(|token| { + token.start >= gap_start && token.end <= gap_end && is_comment(token) + }) + .nth(last) + .map_or(gap_start, |comment| comment.end), + }; + let rest = &formatted[insert_at..gap_end]; + let lead_end = insert_at + (rest.len() - rest.trim_start().len()); + let indent = " ".repeat(line_indentation(formatted, gap_end)); + let lead = &formatted[insert_at..lead_end]; + let mut separator = if lead.contains('\n') { + lead.to_string() + } else { + format!("\n{indent}") + }; + restored.push_str(&formatted[cursor..insert_at]); + for text in &group.texts { + if text != "," { + restored.push_str(&separator); + separator = format!("\n\n{indent}"); + } + restored.push_str(text); + } + if lead_end < formatted.len() { + restored.push_str(&separator); + } + cursor = lead_end; + } + restored.push_str(&formatted[cursor..]); + let restored_tokens = sqlquack_syntax::tokenize(&restored, dialect) + .map_err(|_| RESTORE_FAILURE.to_string())?; + // SQLQuack checked the recasing of the stripped SQL; here only the order and text matter. + let restored_texts = token_texts(&restored, &restored_tokens); + let same_tokens = restored_texts.len() == self.original.len() + && restored_texts.iter().zip(&self.original).all( + |((comment, text), (original_comment, original))| { + comment == original_comment + && if *comment { + text == original + } else { + text.eq_ignore_ascii_case(original) + } + }, + ); + if !same_tokens { + return Err(RESTORE_FAILURE.to_string()); + } + Ok(restored) + } +} + +/// Positions, among significant tokens, of placeholders that stand where a CTE would. +fn macro_positions(sql: &str, tokens: &[LosslessToken], significant: &[usize]) -> Vec { + let kinds: Vec = significant + .iter() + .map(|&index| match tokens[index].kind { + TokenKind::Significant(kind) => kind, + _ => TokenType::Var, + }) + .collect(); + let text = |position: usize| tokens[significant[position]].text(sql); + let is_macro = |position: usize| text(position).starts_with(INTERPOLATION_PLACEHOLDER_PREFIX); + let starts_cte = |position: usize| { + position < kinds.len() + && (is_macro(position) + || (text(position) + .chars() + .all(|character| character.is_alphanumeric() || character == '_') + && matches!(kinds.get(position + 1), Some(TokenType::As | TokenType::LParen)))) + }; + let mut depth = 0_usize; + let mut scopes: Vec = Vec::new(); + let mut positions: Vec = Vec::new(); + for (position, kind) in kinds.iter().enumerate() { + match kind { + TokenType::LParen | TokenType::LBracket | TokenType::LBrace => depth += 1, + TokenType::RParen | TokenType::RBracket | TokenType::RBrace => { + depth = depth.saturating_sub(1); + } + TokenType::With => scopes.push(depth), + TokenType::Select => scopes.retain(|scope| *scope != depth), + _ if is_macro(position) + && scopes.contains(&depth) + && position > 0 + && (matches!( + kinds[position - 1], + TokenType::With | TokenType::Comma | TokenType::Recursive + ) || positions.last() == Some(&(position - 1))) + && (matches!( + kinds.get(position + 1), + Some(TokenType::Comma | TokenType::Select) + ) || starts_cte(position + 1)) => + { + positions.push(position); + } + _ => {} + } + } + positions +} + +/// The placeholders and the commas that go with them: each one's following comma, or, for a +/// group that ends the CTE list, the comma before the group. +fn removed_positions( + tokens: &[LosslessToken], + significant: &[usize], + macros: &[usize], +) -> BTreeSet { + let is_comma = |position: usize| { + significant + .get(position) + .is_some_and(|&index| tokens[index].kind == TokenKind::Significant(TokenType::Comma)) + }; + let mut removed: BTreeSet = macros.iter().copied().collect(); + for &position in macros { + if is_comma(position + 1) { + removed.insert(position + 1); + } + } + for &position in macros { + let group_end = !removed.contains(&(position + 1)); + if group_end && !is_comma(position + 1) { + let mut first = position; + while first > 0 && removed.contains(&(first - 1)) { + first -= 1; + } + let ends_cte_list = significant.get(position + 1).is_none_or(|&index| { + tokens[index].kind == TokenKind::Significant(TokenType::Select) + }); + if first > 0 && is_comma(first - 1) && ends_cte_list { + removed.insert(first - 1); + } + } + } + removed +} + +fn is_comment(token: &LosslessToken) -> bool { + matches!(token.kind, TokenKind::LineComment | TokenKind::BlockComment) +} + +/// Each significant or comment token's text, with whether it is a comment. +fn token_texts(sql: &str, tokens: &[LosslessToken]) -> Vec<(bool, String)> { + tokens + .iter() + .filter(|token| token.kind != TokenKind::Whitespace) + .map(|token| (is_comment(token), token.text(sql).to_string())) + .collect() +} + +/// The spaces and tabs that start the line holding byte `offset`. +fn line_indentation(sql: &str, offset: usize) -> usize { + let line_start = sql[..offset].rfind('\n').map_or(0, |position| position + 1); + sql[line_start..offset] + .chars() + .take_while(|character| *character == ' ' || *character == '\t') + .count() +} diff --git a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs index 37c55ab..55ee103 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs @@ -8,6 +8,7 @@ use sqlquack_format::FormatOptions; use sqlquack_syntax::Dialect as SqlQuackDialect; use crate::constants::INTERPOLATION_PLACEHOLDER_PREFIX; +use crate::sql_lint::_helpers::cte_macros::take_out_cte_macros; use crate::sql_lint::_helpers::formatter_syntax::{cte_macro_indices, protect_syntax}; use crate::sql_lint::_helpers::printed_layout::{PrintedSql, WrapOptions, wrap_lines}; use crate::sql_lint::_helpers::token_layout::{ @@ -140,17 +141,38 @@ fn format_with_sqlquack( }, ..FormatOptions::new(dialect) }; - let formatted = - sqlquack_format::format_sql(&request.sql, &options).map_err(|refusal| refusal.reason)?; + let sql = format_around_cte_macros(&request.sql, &options)?; + // SQLQuack checks its own layout; putting placeholders back is checked here. + if request.sql.contains(INTERPOLATION_PLACEHOLDER_PREFIX) + && sql != request.sql + && format_around_cte_macros(&sql, &options)? != sql + { + return Err(NOT_IDEMPOTENT_FAILURE.to_string()); + } Ok(FormatResponse { version: LINT_API_VERSION, - changed: formatted.changed, - sql: formatted.sql, + changed: sql != request.sql, + sql, formatted: true, reason: None, }) } +/// SQLQuack's layout of `sql`, which it checks for token identity and idempotency. +fn sqlquack_layout(sql: &str, options: &FormatOptions) -> Result { + sqlquack_format::format_sql(sql, options) + .map(|formatted| formatted.sql) + .map_err(|refusal| refusal.reason) +} + +/// Lay out `sql` with SQLQuack, taking CTE macro placeholders out first and putting them back. +fn format_around_cte_macros(sql: &str, options: &FormatOptions) -> Result { + match take_out_cte_macros(sql, options.dialect)? { + None => sqlquack_layout(sql, options), + Some(without) => without.put_back(&sqlquack_layout(&without.sql, options)?, options.dialect), + } +} + /// The printed output of [`format_layout`], its tokens, and the oracle it was printed against. type FormattedLayout = (String, Vec, Option); diff --git a/crates/sqlclean/src/sql_lint/_helpers/mod.rs b/crates/sqlclean/src/sql_lint/_helpers/mod.rs index 4e13c95..9cdd3b7 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/mod.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/mod.rs @@ -1,4 +1,5 @@ pub mod additional; +pub mod cte_macros; pub mod engine; pub mod formatter; pub mod formatter_syntax; diff --git a/crates/sqlclean/src/sql_tokens/_helpers/mod.rs b/crates/sqlclean/src/sql_tokens/_helpers/mod.rs index a2df8fb..d75d204 100644 --- a/crates/sqlclean/src/sql_tokens/_helpers/mod.rs +++ b/crates/sqlclean/src/sql_tokens/_helpers/mod.rs @@ -1,3 +1,4 @@ pub mod builtin_functions; pub mod reserved_keywords; +pub mod sqlquack_canonical_tokens; pub mod token_context; diff --git a/crates/sqlclean/src/sql_tokens/_helpers/reserved_keywords.rs b/crates/sqlclean/src/sql_tokens/_helpers/reserved_keywords.rs index ccf9004..a15eb1d 100644 --- a/crates/sqlclean/src/sql_tokens/_helpers/reserved_keywords.rs +++ b/crates/sqlclean/src/sql_tokens/_helpers/reserved_keywords.rs @@ -5,6 +5,8 @@ use std::sync::LazyLock; use polyglot_sql::DialectType; +use crate::sql_tokens::models::KeywordList; + static DUCKDB: LazyLock> = LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/duckdb.txt"))); static POSTGRES: LazyLock> = @@ -34,6 +36,17 @@ pub(crate) fn is_reserved_keyword(dialect: DialectType, upper_word: &str) -> boo keywords.contains(upper_word) } +/// Whether the upper-cased word is reserved in `list`, a SQLQuack fingerprint dialect's list. +pub(crate) fn is_listed_keyword(list: KeywordList, upper_word: &str) -> bool { + let keywords: &HashSet = match list { + KeywordList::DuckDB => &DUCKDB, + KeywordList::PostgreSQL => &POSTGRES, + KeywordList::Snowflake => &SNOWFLAKE, + KeywordList::Generic => &GENERIC, + }; + keywords.contains(upper_word) +} + fn keyword_set(data: &str) -> HashSet { data.lines() .map(str::trim) diff --git a/crates/sqlclean/src/sql_tokens/_helpers/sqlquack_canonical_tokens.rs b/crates/sqlclean/src/sql_tokens/_helpers/sqlquack_canonical_tokens.rs new file mode 100644 index 0000000..77a1551 --- /dev/null +++ b/crates/sqlclean/src/sql_tokens/_helpers/sqlquack_canonical_tokens.rs @@ -0,0 +1,99 @@ +//! The canonical token stream of SQL under SQLQuack's tokenizers. + +use sqlquack_syntax::TokenKind; +use sqlquack_syntax_postgres::scan::main::scan::scan; +use sqlquack_syntax_postgres::scan::models::TokenKind as PostgresTokenKind; + +use crate::sql_tokens::_helpers::reserved_keywords::is_listed_keyword; +use crate::sql_tokens::constants::{ + CALL_CLOSER, CALL_OPENER, COLUMN_LIST_RELATION_KEYWORDS, CTE_BODY_OPENERS, CTE_BODY_PREFIX, + PLACEHOLDER_PREFIX, +}; +use crate::sql_tokens::main::is_unquoted_word::is_unquoted_word; +use crate::sql_tokens::main::name_positions::name_positions; +use crate::sql_tokens::models::{FingerprintTokenizer, SqlquackFingerprintDialect}; + +/// Significant token texts with reserved keywords and called names, except `__` placeholders, upper-cased. +pub(crate) fn sqlquack_canonical_tokens( + sql: &str, + dialect: SqlquackFingerprintDialect, +) -> Result, String> { + let raws: Vec<&str> = significant_texts(sql, dialect.tokenizer)?; + let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); + let names: Vec = name_positions(&uppers); + let closers: Vec> = matching_closers(&uppers); + let mut texts: Vec = Vec::with_capacity(raws.len()); + for (index, raw) in raws.iter().enumerate() { + let folds = !names[index] + && is_unquoted_word(raw) + && (is_listed_keyword(dialect.keywords, &uppers[index]) + || calls(raw, &uppers, &closers, index)); + texts.push(if folds { + uppers[index].clone() + } else { + (*raw).to_owned() + }); + } + Ok(texts) +} + +/// The text of each significant token of `sql`, in order. +fn significant_texts(sql: &str, tokenizer: FingerprintTokenizer) -> Result, String> { + match tokenizer { + FingerprintTokenizer::Lossless(dialect) => Ok(sqlquack_syntax::tokenize(sql, dialect) + .map_err(|error| format!("SQL tokenization failed: {error}"))? + .iter() + .filter(|token| matches!(token.kind, TokenKind::Significant(_))) + .map(|token| token.text(sql)) + .collect()), + FingerprintTokenizer::PostgresScanner => scan(sql) + .into_iter() + .map(|token| match token.kind { + PostgresTokenKind::Fault(_) => { + Err(format!("SQL tokenization failed: {}", token.value)) + } + _ => sql + .get(token.start..token.end) + .ok_or_else(|| format!("SQL tokenization failed at byte {}", token.start)), + }) + .collect(), + } +} + +/// The index of the `)` closing each `(`, by the index of the `(`. +fn matching_closers(uppers: &[String]) -> Vec> { + let mut closers: Vec> = vec![None; uppers.len()]; + let mut open: Vec = Vec::new(); + for (index, token) in uppers.iter().enumerate() { + if token == CALL_OPENER { + open.push(index); + } else if token == CALL_CLOSER + && let Some(opener) = open.pop() + { + closers[opener] = Some(index); + } + } + closers +} + +/// Whether the word at `index` is called: followed by `(` and neither a placeholder nor a relation. +fn calls(raw: &str, uppers: &[String], closers: &[Option], index: usize) -> bool { + if uppers.get(index + 1).is_none_or(|next| next != CALL_OPENER) + || raw.starts_with(PLACEHOLDER_PREFIX) + { + return false; + } + let previous: Option<&str> = index.checked_sub(1).map(|before| uppers[before].as_str()); + if previous.is_some_and(|word| COLUMN_LIST_RELATION_KEYWORDS.contains(&word)) { + return false; + } + let Some(closer) = closers[index + 1] else { + return true; + }; + !(uppers + .get(closer + 1) + .is_some_and(|next| next == CTE_BODY_PREFIX) + && uppers + .get(closer + 2) + .is_some_and(|next| CTE_BODY_OPENERS.contains(&next.as_str()))) +} diff --git a/crates/sqlclean/src/sql_tokens/constants.rs b/crates/sqlclean/src/sql_tokens/constants.rs index 6036eb9..2ca7bdb 100644 --- a/crates/sqlclean/src/sql_tokens/constants.rs +++ b/crates/sqlclean/src/sql_tokens/constants.rs @@ -2,6 +2,8 @@ use polyglot_sql::tokens::TokenType; +use crate::sql_tokens::models::{FingerprintTokenizer, KeywordList, SqlquackFingerprintDialect}; + /// Column aliases after `AS` may be any keyword in DuckDB and PostgreSQL. pub(crate) const ALIAS_KEYWORD: &str = "AS"; /// The token that turns a preceding name into a function call. @@ -33,3 +35,46 @@ pub(crate) const PROJECTION_ENDS: [TokenType; 13] = [ TokenType::Except, TokenType::Intersect, ]; +/// Fingerprint dialects SQLQuack tokenizes; generic uses the DuckDB tokenizer, PostgreSQL its native scanner. +pub(crate) const SQLQUACK_FINGERPRINT_DIALECTS: [SqlquackFingerprintDialect; 6] = [ + SqlquackFingerprintDialect { + name: "generic", + tokenizer: FingerprintTokenizer::Lossless(sqlquack_syntax::Dialect::DuckDB), + keywords: KeywordList::Generic, + }, + SqlquackFingerprintDialect { + name: "duckdb", + tokenizer: FingerprintTokenizer::Lossless(sqlquack_syntax::Dialect::DuckDB), + keywords: KeywordList::DuckDB, + }, + SqlquackFingerprintDialect { + name: "postgres", + tokenizer: FingerprintTokenizer::PostgresScanner, + keywords: KeywordList::PostgreSQL, + }, + SqlquackFingerprintDialect { + name: "postgresql", + tokenizer: FingerprintTokenizer::PostgresScanner, + keywords: KeywordList::PostgreSQL, + }, + SqlquackFingerprintDialect { + name: "snowflake", + tokenizer: FingerprintTokenizer::Lossless(sqlquack_syntax::Dialect::Snowflake), + keywords: KeywordList::Snowflake, + }, + SqlquackFingerprintDialect { + name: "sqlite", + tokenizer: FingerprintTokenizer::Lossless(sqlquack_syntax::Dialect::SQLite), + keywords: KeywordList::Generic, + }, +]; +/// The token that closes a call's arguments or a column list. +pub(crate) const CALL_CLOSER: &str = ")"; +/// Keywords after which a word followed by `(` names a relation and its column list. +pub(crate) const COLUMN_LIST_RELATION_KEYWORDS: [&str; 2] = ["INTO", "TABLE"]; +/// The tokens that can follow `AS` after a CTE's column list. +pub(crate) const CTE_BODY_OPENERS: [&str; 3] = ["(", "MATERIALIZED", "NOT"]; +/// The prefix of SQLBuild placeholder calls such as `__ref` and `__source`. +pub(crate) const PLACEHOLDER_PREFIX: &str = "__"; +/// The keyword between a CTE's column list and its body. +pub(crate) const CTE_BODY_PREFIX: &str = "AS"; diff --git a/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs b/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs index 4cf7751..bbc4b03 100644 --- a/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs +++ b/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs @@ -5,21 +5,38 @@ use std::str::FromStr; use polyglot_sql::{Dialect, DialectType}; use sha2::{Digest, Sha256}; +use crate::sql_tokens::_helpers::sqlquack_canonical_tokens::sqlquack_canonical_tokens; +use crate::sql_tokens::constants::SQLQUACK_FINGERPRINT_DIALECTS; use crate::sql_tokens::main::canonical_tokens::canonical_tokens; /// Return the SHA-256 hex digest of the canonical token stream of `sql` under `dialect_name`. pub fn query_fingerprint(sql: &str, dialect_name: &str) -> Result { + let texts: Vec = match SQLQUACK_FINGERPRINT_DIALECTS + .iter() + .find(|dialect| dialect.name == dialect_name) + { + Some(dialect) => sqlquack_canonical_tokens(sql, *dialect)?, + None => polyglot_canonical_tokens(sql, dialect_name)?, + }; + let mut digest = Sha256::new(); + for text in texts { + digest.update(text.len().to_string().as_bytes()); + digest.update(b":"); + digest.update(text.as_bytes()); + } + Ok(format!("{:x}", digest.finalize())) +} + +/// Dialects SQLQuack cannot tokenize yet keep the Polyglot token stream. +fn polyglot_canonical_tokens(sql: &str, dialect_name: &str) -> Result, String> { let dialect_type = DialectType::from_str(dialect_name) .map_err(|_| format!("unknown SQL fingerprint dialect '{dialect_name}'"))?; let dialect = Dialect::get(dialect_type); - let mut digest = Sha256::new(); let tokens = dialect .tokenize(sql) .map_err(|error| format!("SQL tokenization failed: {error}"))?; - for token in canonical_tokens(sql, &tokens, &dialect)? { - digest.update(token.text.len().to_string().as_bytes()); - digest.update(b":"); - digest.update(token.text.as_bytes()); - } - Ok(format!("{:x}", digest.finalize())) + Ok(canonical_tokens(sql, &tokens, &dialect)? + .into_iter() + .map(|token| token.text) + .collect()) } diff --git a/crates/sqlclean/src/sql_tokens/models.rs b/crates/sqlclean/src/sql_tokens/models.rs index b18ebb1..1f9088f 100644 --- a/crates/sqlclean/src/sql_tokens/models.rs +++ b/crates/sqlclean/src/sql_tokens/models.rs @@ -5,3 +5,29 @@ pub struct CanonicalToken { pub text: String, } + +/// The reserved-keyword list a SQLQuack fingerprint dialect folds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum KeywordList { + DuckDB, + PostgreSQL, + Snowflake, + Generic, +} + +/// The SQLQuack tokenizer a fingerprint dialect reads its tokens with. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum FingerprintTokenizer { + /// `sqlquack_syntax`'s lossless tokenizer for one dialect. + Lossless(sqlquack_syntax::Dialect), + /// The native PostgreSQL scanner of `sqlquack_syntax_postgres`. + PostgresScanner, +} + +/// A fingerprint dialect SQLQuack tokenizes: its tokenizer and its reserved keywords. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SqlquackFingerprintDialect { + pub(crate) name: &'static str, + pub(crate) tokenizer: FingerprintTokenizer, + pub(crate) keywords: KeywordList, +} diff --git a/crates/sqlclean/src/sql_tokens/tests.rs b/crates/sqlclean/src/sql_tokens/tests.rs index 5468fc5..b7029e6 100644 --- a/crates/sqlclean/src/sql_tokens/tests.rs +++ b/crates/sqlclean/src/sql_tokens/tests.rs @@ -2,3 +2,5 @@ mod canonical_tokens; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_query_fingerprint.rs"] +mod query_fingerprint; diff --git a/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs b/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs new file mode 100644 index 0000000..23514b3 --- /dev/null +++ b/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs @@ -0,0 +1,66 @@ +use crate::sql_tokens::main::query_fingerprint::query_fingerprint; + +/// SQLBuild's golden fingerprints for the dialects SQLQuack tokenizes; SQLBuild stores these +/// hashes, so SQLClean must compute the same ones. +#[test] +fn given_sqlbuild_golden_queries_when_fingerprinting_then_hashes_match_sqlbuild() -> Result<(), String> +{ + let test_cases: [(&str, &str, &str); 5] = [ + ( + "generic", + "SELECT id, name FROM orders", + "264bcb1f81ce5ba25394a052016edc265aefba85c0698f616d3037f9d872d2bd", + ), + ( + "duckdb", + "SELECT o.id, $$Customer's note$$ AS note, amount::DECIMAL(10, 2) AS amount FROM orders AS o", + "07786643d9593498ac2b2394e6801cb578b23ba0f9a0e70dbcdaf0d3a3cbf9db", + ), + ( + "postgresql", + "SELECT o.id, $tag$it's$tag$ AS note, amount::NUMERIC AS amount FROM orders AS o", + "8db865c6f239b990c917754df176bd3c9b2c0c322b9af1437298ec98b67241f9", + ), + ( + "snowflake", + "SELECT o.id, payload:customer.name::STRING AS customer FROM orders AS o QUALIFY ROW_NUMBER() OVER (PARTITION BY o.id ORDER BY o.id) = 1", + "c75f7b5931a18e1a074ffa7ab113c256fc838285af75a3f9feb62d61cc76d94d", + ), + ( + "duckdb", + "select o.id from __ref(\"orders\") as o where o.id > 0", + "fda88b9b0f26486bdd25b8e3bab5a982fbb387901ab229091b17772d044aebd0", + ), + ]; + for (dialect, sql, expected) in test_cases { + assert_eq!(query_fingerprint(sql, dialect)?, expected, "{dialect}: {sql}"); + } + Ok(()) +} + +/// Called names fold to upper case, except placeholders, quoted names and relation or CTE column lists. +#[test] +fn given_called_names_when_fingerprinting_then_only_unquoted_calls_ignore_case() -> Result<(), String> +{ + let test_cases: [(&str, &str, &str, bool); 6] = [ + ("duckdb", "SELECT MyFunc(id) FROM orders", "SELECT myfunc(id) FROM orders", true), + ("snowflake", "SELECT \"MyFunc\"(id) FROM orders", "SELECT \"myfunc\"(id) FROM orders", false), + ("duckdb", "SELECT id FROM __ref('orders')", "SELECT id FROM __REF('orders')", false), + ( + "duckdb", + "WITH Tree(id) AS (SELECT 1) SELECT id FROM orders", + "WITH tree(id) AS (SELECT 1) SELECT id FROM orders", + false, + ), + ("postgres", "INSERT INTO Paid(id) SELECT 1", "INSERT INTO paid(id) SELECT 1", false), + ("sqlite", "SELECT Upper(name) FROM orders", "SELECT UPPER(name) FROM orders", true), + ]; + for (dialect, left, right, expected_equal) in test_cases { + assert_eq!( + query_fingerprint(left, dialect)? == query_fingerprint(right, dialect)?, + expected_equal, + "{dialect}: {left} | {right}" + ); + } + Ok(()) +} From 9d10096bea22e5044b7bc7485a835ce307e40bb9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:57:45 +0000 Subject: [PATCH 05/21] wip(facts): keep Snowflake date parts out of the column facts SQLQuack's Snowflake parse keeps DATEPART, LAST_DAY and DATE_DIFF as the calls written, so their bare date part, such as day in DATEPART(day, ts), parsed as a column, where Polyglot lowered the call and read it as a unit. Statement facts now leave that argument out of the columns: any bare name for DATEPART and LAST_DAY, and only a known unit name for DATE_DIFF, as Polyglot did. Against SQLClean on Polyglot, the Snowflake facts of 12,313 inputs both parse now differ on 6, all JSON_OBJECTAGG(t.x), whose argument SQLQuack reports as a column and Polyglot dropped; DuckDB's 5,352 are identical. --- .../src/sql_facts/_helpers/collection.rs | 50 ++++++++++- crates/sqlclean/src/sql_facts/constants.rs | 89 +++++++++++++++++++ crates/sqlclean/src/sql_facts/mod.rs | 1 + 3 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 crates/sqlclean/src/sql_facts/constants.rs diff --git a/crates/sqlclean/src/sql_facts/_helpers/collection.rs b/crates/sqlclean/src/sql_facts/_helpers/collection.rs index e38aa97..2d25783 100644 --- a/crates/sqlclean/src/sql_facts/_helpers/collection.rs +++ b/crates/sqlclean/src/sql_facts/_helpers/collection.rs @@ -9,6 +9,7 @@ use crate::sql_facts::_helpers::statements::{StatementTokens, statement_token_ra use crate::sql_facts::_helpers::tokens::{ function_nodes, join_clause_span, literal_nodes, star_nodes, }; +use crate::sql_facts::constants::{DATE_PART_CALLS, DATE_PART_NAMES}; use crate::sql_facts::models::{ ColumnNode, FileFacts, JoinNode, NamedNode, StatementFacts, TableNode, }; @@ -23,6 +24,7 @@ pub(crate) fn file_facts(sql: &str, dialect: &str) -> Result ranges, ) = parsed_statements(sql, dialect)?; let characters: Vec = sql.chars().collect(); + let snowflake: bool = dialect.eq_ignore_ascii_case("snowflake"); let facts: Vec = statements .iter() .zip(ranges) @@ -34,6 +36,7 @@ pub(crate) fn file_facts(sql: &str, dialect: &str) -> Result statement, tokens: &tokens[range.first..=range.last], terminated_end: range.terminated_end, + snowflake, }, &characters, ) @@ -70,6 +73,8 @@ struct StatementSource<'a> { statement: &'a Expression, tokens: &'a [Token], terminated_end: usize, + /// Whether the dialect is Snowflake, whose date functions take a bare date part. + snowflake: bool, } fn statement_facts(source: &StatementSource<'_>, characters: &[char]) -> StatementFacts { @@ -78,8 +83,16 @@ fn statement_facts(source: &StatementSource<'_>, characters: &[char]) -> Stateme statement, tokens, terminated_end, + snowflake, } = *source; let nodes: Vec<&Expression> = statement_nodes(statement); + let date_parts: Vec<&Expression> = match snowflake { + true => nodes + .iter() + .filter_map(|node| date_part_argument(node)) + .collect(), + false => Vec::new(), + }; StatementFacts { index, kind: statement.variant_name().to_owned(), @@ -103,7 +116,13 @@ fn statement_facts(source: &StatementSource<'_>, characters: &[char]) -> Stateme ), stars: star_nodes(tokens), functions: function_nodes(tokens), - columns: sorted(nodes.iter().filter_map(|node| column_node(node)).collect()), + columns: sorted( + nodes + .iter() + .filter(|node| !is_date_part(node, &date_parts)) + .filter_map(|node| column_node(node)) + .collect(), + ), literals: literal_nodes(tokens, characters), } } @@ -210,6 +229,35 @@ fn join_node(join: &Join, tokens: &[Token]) -> Option { }) } +/// A Snowflake date call's bare date part, such as `day` in `DATEPART(day, ts)`: a unit, as Polyglot read it. +fn date_part_argument(node: &Expression) -> Option<&Expression> { + let Expression::Function(function) = node else { + return None; + }; + let (_, position, any_name) = DATE_PART_CALLS + .iter() + .find(|(name, _, _)| name.eq_ignore_ascii_case(&function.name))?; + function + .args + .get(*position) + .filter(|argument| reads_as_unit(argument, *any_name)) +} + +/// Whether `argument` is a bare name that reads as a date part: any name, or only a known unit. +fn reads_as_unit(argument: &Expression, any_name: bool) -> bool { + match argument { + Expression::Column(column) if column.table.is_none() && !column.name.quoted => { + any_name || DATE_PART_NAMES.contains(&column.name.name.to_ascii_uppercase().as_str()) + } + _ => false, + } +} + +/// Whether `node` is one of the date parts found in its statement. +fn is_date_part(node: &Expression, date_parts: &[&Expression]) -> bool { + date_parts.iter().any(|part| std::ptr::eq(*part, node)) +} + fn column_node(node: &Expression) -> Option { let Expression::Column(column) = node else { return None; diff --git a/crates/sqlclean/src/sql_facts/constants.rs b/crates/sqlclean/src/sql_facts/constants.rs new file mode 100644 index 0000000..44cc5c6 --- /dev/null +++ b/crates/sqlclean/src/sql_facts/constants.rs @@ -0,0 +1,89 @@ +//! Constants for statement facts. + +/// Calls whose bare date part Polyglot read as a unit: name, argument position, any name or only a known unit. +pub(crate) const DATE_PART_CALLS: [(&str, usize, bool); 3] = [ + ("DATEPART", 0, true), + ("LAST_DAY", 1, true), + ("DATE_DIFF", 0, false), +]; +/// The date part names, abbreviations included, that Polyglot read as a DATEDIFF unit. +pub(crate) const DATE_PART_NAMES: [&str; 78] = [ + "D", + "DAY", + "DAYOFMONTH", + "DAYS", + "DD", + "H", + "HH", + "HOUR", + "HOURS", + "HR", + "HRS", + "M", + "MCS", + "MI", + "MICROSEC", + "MICROSECOND", + "MICROSECONDS", + "MICROSECS", + "MILLISEC", + "MILLISECON", + "MILLISECOND", + "MILLISECONDS", + "MILLISECS", + "MIN", + "MINS", + "MINUTE", + "MINUTES", + "MM", + "MON", + "MONS", + "MONTH", + "MONTHS", + "MS", + "MSEC", + "MSECOND", + "MSECONDS", + "MSECS", + "N", + "NANOSEC", + "NANOSECOND", + "NANOSECS", + "NS", + "NSEC", + "NSECOND", + "NSECONDS", + "Q", + "QQ", + "QTR", + "QTRS", + "QUARTER", + "QUARTERS", + "S", + "SEC", + "SECOND", + "SECONDS", + "SECS", + "SS", + "US", + "USEC", + "USECOND", + "USECONDS", + "USECS", + "W", + "WEEK", + "WEEKOFYEAR", + "WEEKS", + "WK", + "WOY", + "WW", + "WY", + "Y", + "YEAR", + "YEARS", + "YR", + "YRS", + "YY", + "YYY", + "YYYY", +]; diff --git a/crates/sqlclean/src/sql_facts/mod.rs b/crates/sqlclean/src/sql_facts/mod.rs index 1d1f678..0e911a1 100644 --- a/crates/sqlclean/src/sql_facts/mod.rs +++ b/crates/sqlclean/src/sql_facts/mod.rs @@ -1,6 +1,7 @@ //! Structural facts about each SQL statement, which custom Python Rules read. pub mod _helpers; +pub mod constants; pub mod main; pub mod models; From 66af5d58d1d26f0156a790e4cb27b3cced9b4abf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:02:27 +0000 Subject: [PATCH 06/21] wip(facts): report facts as unavailable for a dialect SQLQuack does not parse file_facts now answers a dialect SQLQuack does not parse yet, such as MySQL, with no statements and an unavailable reason, the way lint's response does, instead of an error; an unknown dialect is still an error. statement_ends answers None for such a dialect. Both ask lint whether the dialect is unavailable, so facts and lint always agree. The CLI keeps failing those files for now: the custom Rule host turns an unavailable reason into its error, and format_source turns None into one, until the per-file unavailable report replaces them. --- .../src/checking/_helpers/custom.rs | 6 ++- .../src/formatting/main/format_source.rs | 4 +- .../src/sql_facts/_helpers/collection.rs | 39 ++++++++++++++-- .../sqlclean/src/sql_facts/main/file_facts.rs | 2 +- .../src/sql_facts/main/statement_ends.rs | 4 +- crates/sqlclean/src/sql_facts/models.rs | 3 ++ .../src/sql_facts/tests/test_file_facts.rs | 46 +++++++++++++++++-- .../src/sql_facts/tests/test_types.rs | 6 +++ 8 files changed, 98 insertions(+), 12 deletions(-) diff --git a/crates/sqlclean-cli/src/checking/_helpers/custom.rs b/crates/sqlclean-cli/src/checking/_helpers/custom.rs index e3b881e..693a66f 100644 --- a/crates/sqlclean-cli/src/checking/_helpers/custom.rs +++ b/crates/sqlclean-cli/src/checking/_helpers/custom.rs @@ -189,11 +189,15 @@ fn file_key( } fn host_file(file: &SourceFile, rules: Vec) -> Result { + let facts = file_facts(&file.text, &file.dialect)?; + if let Some(reason) = facts.unavailable { + return Err(reason); + } Ok(HostFile { path: file.path.clone(), source: file.text.clone(), dialect: file.dialect.clone(), - facts: file_facts(&file.text, &file.dialect)?, + facts, rules, }) } diff --git a/crates/sqlclean-cli/src/formatting/main/format_source.rs b/crates/sqlclean-cli/src/formatting/main/format_source.rs index 117a2fd..c235b0f 100644 --- a/crates/sqlclean-cli/src/formatting/main/format_source.rs +++ b/crates/sqlclean-cli/src/formatting/main/format_source.rs @@ -15,7 +15,9 @@ pub fn format_source( indent: usize, ) -> Result { let characters: Vec = text.chars().collect(); - let mut boundaries: Vec = statement_ends(text, dialect)?; + let mut boundaries: Vec = statement_ends(text, dialect)?.ok_or_else(|| { + format!("SQL formatting is unavailable for the {dialect} dialect: SQLQuack does not parse it yet") + })?; if let Some(last) = boundaries.last_mut() { *last = characters.len(); } diff --git a/crates/sqlclean/src/sql_facts/_helpers/collection.rs b/crates/sqlclean/src/sql_facts/_helpers/collection.rs index 2d25783..cecb255 100644 --- a/crates/sqlclean/src/sql_facts/_helpers/collection.rs +++ b/crates/sqlclean/src/sql_facts/_helpers/collection.rs @@ -13,10 +13,18 @@ use crate::sql_facts::constants::{DATE_PART_CALLS, DATE_PART_NAMES}; use crate::sql_facts::models::{ ColumnNode, FileFacts, JoinNode, NamedNode, StatementFacts, TableNode, }; +use crate::sql_lint::constants::LINT_API_VERSION; +use crate::sql_lint::main::lint::lint; use crate::sql_lint::main::parse_sql::parse_sql; -use crate::sql_lint::types::ParsedSql; +use crate::sql_lint::types::{LintRequest, LintResponse, ParsedSql}; pub(crate) fn file_facts(sql: &str, dialect: &str) -> Result { + if let Some(reason) = unavailable(dialect)? { + return Ok(FileFacts { + statements: Vec::new(), + unavailable: Some(reason), + }); + } let ( ParsedSql { tokens, statements, .. @@ -42,12 +50,35 @@ pub(crate) fn file_facts(sql: &str, dialect: &str) -> Result ) }) .collect(); - Ok(FileFacts { statements: facts }) + Ok(FileFacts { + statements: facts, + unavailable: None, + }) } -pub(crate) fn statement_ends(sql: &str, dialect: &str) -> Result, String> { +pub(crate) fn statement_ends(sql: &str, dialect: &str) -> Result>, String> { + if unavailable(dialect)?.is_some() { + return Ok(None); + } let (_, ranges) = parsed_statements(sql, dialect)?; - Ok(ranges.iter().map(|range| range.terminated_end).collect()) + Ok(Some( + ranges.iter().map(|range| range.terminated_end).collect(), + )) +} + +/// Why facts are unavailable for `dialect`: lint's own answer for a dialect SQLQuack does not parse yet. +fn unavailable(dialect: &str) -> Result, String> { + let probe: LintResponse = lint(LintRequest { + version: LINT_API_VERSION, + dialect: dialect.to_owned(), + enabled_rules: Some(Vec::new()), + ..LintRequest::default() + })?; + Ok(probe.unavailable.map(|_| { + format!( + "SQL facts are unavailable for the {dialect} dialect: SQLQuack does not parse it yet" + ) + })) } /// Parse `sql` and split its tokens into one range per parsed statement. diff --git a/crates/sqlclean/src/sql_facts/main/file_facts.rs b/crates/sqlclean/src/sql_facts/main/file_facts.rs index 31bae43..0b3824b 100644 --- a/crates/sqlclean/src/sql_facts/main/file_facts.rs +++ b/crates/sqlclean/src/sql_facts/main/file_facts.rs @@ -2,7 +2,7 @@ use crate::sql_facts::models::FileFacts; -/// Parse `sql` under `dialect` and return each statement's structural facts. +/// Each statement's structural facts, or why they are unavailable for a dialect SQLQuack does not parse yet. pub fn file_facts(sql: &str, dialect: &str) -> Result { crate::sql_facts::_helpers::collection::file_facts(sql, dialect) } diff --git a/crates/sqlclean/src/sql_facts/main/statement_ends.rs b/crates/sqlclean/src/sql_facts/main/statement_ends.rs index e8bcbb3..adeddc4 100644 --- a/crates/sqlclean/src/sql_facts/main/statement_ends.rs +++ b/crates/sqlclean/src/sql_facts/main/statement_ends.rs @@ -1,6 +1,6 @@ //! Where each statement of one SQL file ends, without collecting its facts. -/// Parse `sql` under `dialect` and return the end of each statement, after its terminator. -pub fn statement_ends(sql: &str, dialect: &str) -> Result, String> { +/// The end of each statement of `sql`, after its terminator; `None` when SQLQuack does not parse `dialect` yet. +pub fn statement_ends(sql: &str, dialect: &str) -> Result>, String> { crate::sql_facts::_helpers::collection::statement_ends(sql, dialect) } diff --git a/crates/sqlclean/src/sql_facts/models.rs b/crates/sqlclean/src/sql_facts/models.rs index bd8e28e..de21cb1 100644 --- a/crates/sqlclean/src/sql_facts/models.rs +++ b/crates/sqlclean/src/sql_facts/models.rs @@ -6,6 +6,9 @@ use serde::Serialize; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)] pub struct FileFacts { pub statements: Vec, + /// Why there are no facts: SQLQuack does not parse the dialect yet, as lint reports; absent otherwise. + #[serde(skip_serializing_if = "Option::is_none")] + pub unavailable: Option, } /// One statement's source range, kind and the common structures it contains. diff --git a/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs b/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs index 59e7194..5334436 100644 --- a/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs +++ b/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs @@ -3,7 +3,7 @@ use crate::sql_facts::main::statement_ends::statement_ends; use crate::sql_facts::models::{FileFacts, StatementFacts}; use crate::sql_facts::tests::helpers::text; use crate::sql_facts::tests::test_types::{ - ParseOutcomeTestCase, StatementEndsTestCase, StatementFactsTestCase, + ParseOutcomeTestCase, StatementEndsTestCase, StatementFactsTestCase, UnavailableFactsTestCase, }; const TWO_STATEMENTS: &str = "WITH recent AS (SELECT o.order_id, o.* FROM shop.orders AS o WHERE o.total > 10)\n\ @@ -260,14 +260,15 @@ fn given_sql_text_when_finding_statement_ends_then_each_ends_after_its_terminato for test_case in &test_cases { let characters: Vec = test_case.sql.chars().collect(); let statements: Option> = statement_ends(test_case.sql, "duckdb") + .ok() + .flatten() .map(|ends| { let starts = std::iter::once(0).chain(ends.iter().copied()); starts .zip(ends.iter().copied()) .map(|(start, end)| characters[start..end].iter().collect()) .collect() - }) - .ok(); + }); let texts: Option> = statements .as_ref() .map(|statements| statements.iter().map(String::as_str).collect()); @@ -279,3 +280,42 @@ fn given_sql_text_when_finding_statement_ends_then_each_ends_after_its_terminato ); } } + +#[test] +fn given_a_dialect_sqlquack_does_not_parse_when_collecting_facts_then_they_are_unavailable_not_empty() +-> Result<(), String> { + let test_cases = [ + UnavailableFactsTestCase { + description: "MySQL has no SQLQuack parser, so its facts are unavailable", + dialect: "mysql", + expected_unavailable: Some( + "SQL facts are unavailable for the mysql dialect: SQLQuack does not parse it yet", + ), + }, + UnavailableFactsTestCase { + description: "DuckDB parses, so its facts are available", + dialect: "duckdb", + expected_unavailable: None, + }, + ]; + for test_case in &test_cases { + let facts: FileFacts = file_facts("SELECT 1; SELECT 2", test_case.dialect)?; + let ends: Option> = statement_ends("SELECT 1; SELECT 2", test_case.dialect)?; + assert_eq!( + facts.unavailable.as_deref(), + test_case.expected_unavailable, + "{}", + test_case.description + ); + assert_eq!( + (facts.statements.len(), ends.map(|ends| ends.len())), + match test_case.expected_unavailable { + Some(_) => (0, None), + None => (2, Some(2)), + }, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlclean/src/sql_facts/tests/test_types.rs b/crates/sqlclean/src/sql_facts/tests/test_types.rs index f6d6024..f06b5fd 100644 --- a/crates/sqlclean/src/sql_facts/tests/test_types.rs +++ b/crates/sqlclean/src/sql_facts/tests/test_types.rs @@ -20,6 +20,12 @@ pub(crate) struct ParseOutcomeTestCase { pub(crate) expected_statements: Option, } +pub(crate) struct UnavailableFactsTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) expected_unavailable: Option<&'static str>, +} + pub(crate) struct StatementEndsTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, From 3800b96e472f2aa142fcd188a911c772009d4df2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:44:14 +0000 Subject: [PATCH 07/21] wip(facts): read Postgres facts as written, and pin SQLQuack with the Postgres lint slice SQLQuack is now pinned at the lint slice that parses PostgreSQL and Generic natively (wip/lint-postgres bd308904a), so their lint and facts are available. The native PostgreSQL parser folds unquoted names to lower case, so the facts read table, column, CTE and alias names from the source as written. A column's span now starts at its qualifier when the parser located the name alone, as for a subscripted column. The CREATE and ALTER statements the native tree keeps as a generic object take the kind Polyglot's tree gave them, such as create_index or alter_table. The Rule test harness fails a case whose dialect has no facts, rather than evaluating the Rule over an empty file. --- Cargo.lock | 15 +++- Cargo.toml | 4 +- .../src/sql_facts/_helpers/collection.rs | 86 ++++++++++++++----- crates/sqlclean/src/sql_facts/constants.rs | 16 ++++ .../rule_testing/_helpers/native_facts.py | 5 +- src/sqlclean/rule_testing/constants.py | 1 + 6 files changed, 100 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e2f644a..0bb9c40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -653,19 +653,20 @@ dependencies = [ [[package]] name = "sqlquack-lint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db#4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=bd308904a1b88f97922b09a02a96369e632b1881#bd308904a1b88f97922b09a02a96369e632b1881" dependencies = [ "rayon", "rustc-hash", "serde", "serde_json", "sqlquack-syntax", + "sqlquack-syntax-postgres", ] [[package]] name = "sqlquack-syntax" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db#4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=bd308904a1b88f97922b09a02a96369e632b1881#bd308904a1b88f97922b09a02a96369e632b1881" dependencies = [ "serde", "serde_json", @@ -678,13 +679,21 @@ dependencies = [ [[package]] name = "sqlquack-syntax-derive" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db#4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=bd308904a1b88f97922b09a02a96369e632b1881#bd308904a1b88f97922b09a02a96369e632b1881" dependencies = [ "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "sqlquack-syntax-postgres" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=bd308904a1b88f97922b09a02a96369e632b1881#bd308904a1b88f97922b09a02a96369e632b1881" +dependencies = [ + "sqlquack-syntax", +] + [[package]] name = "stacker" version = "0.1.25" diff --git a/Cargo.toml b/Cargo.toml index ca72390..dd4c05c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,8 +41,8 @@ sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } # SQLQuack's lint engine and the tokens and trees it reads, which replace Polyglot for lint and facts. -sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db" } -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4f9ac36d8382e8a1e15cb24fcceb6f5e5a4677db" } +sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "bd308904a1b88f97922b09a02a96369e632b1881" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "bd308904a1b88f97922b09a02a96369e632b1881" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sqlclean/src/sql_facts/_helpers/collection.rs b/crates/sqlclean/src/sql_facts/_helpers/collection.rs index cecb255..0434197 100644 --- a/crates/sqlclean/src/sql_facts/_helpers/collection.rs +++ b/crates/sqlclean/src/sql_facts/_helpers/collection.rs @@ -1,6 +1,6 @@ //! Walk each parsed statement once and collect its structural facts. -use sqlquack_syntax::expressions::names::models::{Column, TableRef}; +use sqlquack_syntax::expressions::names::models::{Column, Identifier, TableRef}; use sqlquack_syntax::expressions::queries::models::{Cte, Join, With}; use sqlquack_syntax::tokens::models::{Span, Token, TokenType}; use sqlquack_syntax::{Expression, ExpressionWalk}; @@ -9,7 +9,9 @@ use crate::sql_facts::_helpers::statements::{StatementTokens, statement_token_ra use crate::sql_facts::_helpers::tokens::{ function_nodes, join_clause_span, literal_nodes, star_nodes, }; -use crate::sql_facts::constants::{DATE_PART_CALLS, DATE_PART_NAMES}; +use crate::sql_facts::constants::{ + ALTER_OBJECT_KINDS, CREATE_OBJECT_KINDS, DATE_PART_CALLS, DATE_PART_NAMES, +}; use crate::sql_facts::models::{ ColumnNode, FileFacts, JoinNode, NamedNode, StatementFacts, TableNode, }; @@ -126,7 +128,7 @@ fn statement_facts(source: &StatementSource<'_>, characters: &[char]) -> Stateme }; StatementFacts { index, - kind: statement.variant_name().to_owned(), + kind: statement_kind(statement), start: tokens.first().map_or(0, |token| token.span.start), end: tokens.last().map_or(0, |token| token.span.end), terminated_end, @@ -134,15 +136,20 @@ fn statement_facts(source: &StatementSource<'_>, characters: &[char]) -> Stateme nodes .iter() .flat_map(|node| with_ctes(node)) - .filter_map(|cte| cte_node(cte, tokens)) + .filter_map(|cte| cte_node(cte, tokens, characters)) + .collect(), + ), + tables: sorted( + nodes + .iter() + .filter_map(|node| table_node(node, characters)) .collect(), ), - tables: sorted(nodes.iter().filter_map(|node| table_node(node)).collect()), joins: sorted( nodes .iter() .flat_map(|node| select_joins(node)) - .filter_map(|join| join_node(join, tokens)) + .filter_map(|join| join_node(join, tokens, characters)) .collect(), ), stars: star_nodes(tokens), @@ -151,7 +158,7 @@ fn statement_facts(source: &StatementSource<'_>, characters: &[char]) -> Stateme nodes .iter() .filter(|node| !is_date_part(node, &date_parts)) - .filter_map(|node| column_node(node)) + .filter_map(|node| column_node(node, characters)) .collect(), ), literals: literal_nodes(tokens, characters), @@ -198,7 +205,7 @@ fn select_joins(node: &Expression) -> &[Join] { } /// The CTE located at its name; the parser leaves CTE names unlocated, so read `name AS (`. -fn cte_node(cte: &Cte, tokens: &[Token]) -> Option { +fn cte_node(cte: &Cte, tokens: &[Token], characters: &[char]) -> Option { let span: Span = cte.alias.span.or_else(|| { tokens .windows(2) @@ -212,13 +219,13 @@ fn cte_node(cte: &Cte, tokens: &[Token]) -> Option { .map(|pair| pair[0].span) })?; Some(NamedNode { - name: cte.alias.name.clone(), + name: authored(&cte.alias, characters), start: span.start, end: span.end, }) } -fn table_node(node: &Expression) -> Option { +fn table_node(node: &Expression, characters: &[char]) -> Option { let Expression::Table(table) = node else { return None; }; @@ -232,20 +239,23 @@ fn table_node(node: &Expression) -> Option { let qualified_name: String = [&table.catalog, &table.schema] .into_iter() .flatten() - .map(|part| part.name.as_str()) - .chain([table.name.name.as_str()]) - .collect::>() + .chain([&table.name]) + .map(|part| authored(part, characters)) + .collect::>() .join("."); Some(TableNode { start, end: span.end, - name: table.name.name.clone(), + name: authored(&table.name, characters), qualified_name, - alias: table.alias.as_ref().map(|alias| alias.name.clone()), + alias: table + .alias + .as_ref() + .map(|alias| authored(alias, characters)), }) } -fn join_node(join: &Join, tokens: &[Token]) -> Option { +fn join_node(join: &Join, tokens: &[Token], characters: &[char]) -> Option { let relation: Option<&TableRef> = match &join.this { Expression::Table(table) => Some(table), _ => None, @@ -255,7 +265,7 @@ fn join_node(join: &Join, tokens: &[Token]) -> Option { start, end, kind: format!("{:?}", join.kind).to_ascii_uppercase(), - relation: relation.map(|table| table.name.name.clone()), + relation: relation.map(|table| authored(&table.name, characters)), has_condition: join.on.is_some() || !join.using.is_empty(), }) } @@ -289,20 +299,54 @@ fn is_date_part(node: &Expression, date_parts: &[&Expression]) -> bool { date_parts.iter().any(|part| std::ptr::eq(*part, node)) } -fn column_node(node: &Expression) -> Option { +fn column_node(node: &Expression, characters: &[char]) -> Option { let Expression::Column(column) = node else { return None; }; let column: &Column = column; let span: Span = column.span.or(column.name.span)?; + let start: usize = column + .table + .as_ref() + .and_then(|table| table.span) + .map_or(span.start, |table| table.start.min(span.start)); Some(ColumnNode { - start: span.start, + start, end: span.end, - name: column.name.name.clone(), - qualifier: column.table.as_ref().map(|table| table.name.clone()), + name: authored(&column.name, characters), + qualifier: column + .table + .as_ref() + .map(|table| authored(table, characters)), }) } +/// A name as written: a parser that folds unquoted names keeps the folded name, so read the source. +fn authored(identifier: &Identifier, characters: &[char]) -> String { + let written: Option = identifier + .span + .filter(|_| !identifier.quoted) + .and_then(|span| characters.get(span.start..span.end)) + .map(|text| text.iter().collect()); + written + .filter(|text| text.eq_ignore_ascii_case(&identifier.name)) + .unwrap_or_else(|| identifier.name.clone()) +} + +/// The statement's kind, naming a generic CREATE or ALTER by its object as Polyglot's tree did. +fn statement_kind(statement: &Expression) -> String { + let (kind, known): (&str, &[(&str, &str)]) = match statement { + Expression::CreateObject(node) => (&node.kind, &CREATE_OBJECT_KINDS), + Expression::AlterObject(node) => (&node.kind, &ALTER_OBJECT_KINDS), + _ => ("", &[]), + }; + known + .iter() + .find(|(object, _)| object.eq_ignore_ascii_case(kind)) + .map_or(statement.variant_name(), |(_, name)| name) + .to_owned() +} + /// The first located table or column inside `expression`, in source order. fn first_span(expression: &Expression) -> Option { expression diff --git a/crates/sqlclean/src/sql_facts/constants.rs b/crates/sqlclean/src/sql_facts/constants.rs index 44cc5c6..aae0fa6 100644 --- a/crates/sqlclean/src/sql_facts/constants.rs +++ b/crates/sqlclean/src/sql_facts/constants.rs @@ -87,3 +87,19 @@ pub(crate) const DATE_PART_NAMES: [&str; 78] = [ "YYY", "YYYY", ]; +/// CREATE statements the native PostgreSQL tree keeps as a generic object, by object kind, and their kind. +pub(crate) const CREATE_OBJECT_KINDS: [(&str, &str); 6] = [ + ("INDEX", "create_index"), + ("TRIGGER", "create_trigger"), + ("TYPE", "create_type"), + ("DOMAIN", "create_type"), + ("SCHEMA", "create_schema"), + ("SEQUENCE", "create_sequence"), +]; +/// ALTER statements the native PostgreSQL tree keeps as a generic object, by object kind, and their kind. +pub(crate) const ALTER_OBJECT_KINDS: [(&str, &str); 4] = [ + ("TABLE", "alter_table"), + ("INDEX", "alter_index"), + ("SEQUENCE", "alter_sequence"), + ("VIEW", "alter_view"), +]; diff --git a/src/sqlclean/rule_testing/_helpers/native_facts.py b/src/sqlclean/rule_testing/_helpers/native_facts.py index fad7500..c45ab16 100644 --- a/src/sqlclean/rule_testing/_helpers/native_facts.py +++ b/src/sqlclean/rule_testing/_helpers/native_facts.py @@ -6,7 +6,7 @@ import sysconfig from pathlib import Path -from sqlclean.rule_testing.constants import EXECUTABLE_NAME, FACTS_COMMAND +from sqlclean.rule_testing.constants import EXECUTABLE_NAME, FACTS_COMMAND, UNAVAILABLE_KEY from sqlclean.rule_testing.exceptions import RuleCaseError @@ -23,6 +23,9 @@ def native_facts(*, source: str, dialect: str) -> dict[str, object]: stderr: str = completed.stderr.decode("utf-8", errors="replace").strip() raise RuleCaseError(f"sqlclean could not parse the case SQL: {stderr}") facts: dict[str, object] = json.loads(completed.stdout.decode("utf-8")) + unavailable: object = facts.get(UNAVAILABLE_KEY) + if unavailable is not None: + raise RuleCaseError(f"sqlclean could not parse the case SQL: {unavailable}") return facts diff --git a/src/sqlclean/rule_testing/constants.py b/src/sqlclean/rule_testing/constants.py index f700b4c..db1febd 100644 --- a/src/sqlclean/rule_testing/constants.py +++ b/src/sqlclean/rule_testing/constants.py @@ -4,3 +4,4 @@ DEFAULT_CASE_DIALECT: str = "generic" EXECUTABLE_NAME: str = "sqlclean" FACTS_COMMAND: str = "facts" +UNAVAILABLE_KEY: str = "unavailable" From 65b28ba708d54b9e85736ef98558a3ab73efbb68 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:01:01 +0000 Subject: [PATCH 08/21] wip(tokens): fingerprint with sqlquack-fingerprint, as SQLBuild does Dialects SQLQuack fingerprints use its shared function; the others keep the Polyglot token stream SQLBuild also uses for them. A comma-less run of CTE macro placeholders is now separated by blank lines like any other CTE. --- Cargo.lock | 22 +++-- Cargo.toml | 8 +- crates/sqlclean/Cargo.toml | 2 +- .../src/sql_lint/_helpers/cte_macros.rs | 5 +- .../src/sql_lint/_helpers/formatter.rs | 4 +- .../src/sql_lint/tests/test_layout_rules.rs | 2 +- .../sqlclean/src/sql_tokens/_helpers/mod.rs | 1 - .../sql_tokens/_helpers/reserved_keywords.rs | 13 --- .../_helpers/sqlquack_canonical_tokens.rs | 99 ------------------- crates/sqlclean/src/sql_tokens/constants.rs | 48 +-------- .../src/sql_tokens/main/query_fingerprint.rs | 42 ++++---- crates/sqlclean/src/sql_tokens/models.rs | 26 ----- crates/sqlclean/src/sql_tokens/tests.rs | 4 +- .../tests/test_query_fingerprint.rs | 81 ++++++++++++--- 14 files changed, 122 insertions(+), 235 deletions(-) delete mode 100644 crates/sqlclean/src/sql_tokens/_helpers/sqlquack_canonical_tokens.rs diff --git a/Cargo.lock b/Cargo.lock index a60bc1a..b91df81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -628,10 +628,10 @@ dependencies = [ "serde", "serde_json", "sha2", + "sqlquack-fingerprint", "sqlquack-format", "sqlquack-lint", "sqlquack-syntax", - "sqlquack-syntax-postgres", ] [[package]] @@ -652,10 +652,20 @@ dependencies = [ "toml", ] +[[package]] +name = "sqlquack-fingerprint" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=2736a815f821ca6c152bb76160cdf5972c73b9a3#2736a815f821ca6c152bb76160cdf5972c73b9a3" +dependencies = [ + "sha2", + "sqlquack-syntax", + "sqlquack-syntax-postgres", +] + [[package]] name = "sqlquack-format" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=c2f8b2b1b218bc82c2935629110ff4bac6a5620b#c2f8b2b1b218bc82c2935629110ff4bac6a5620b" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=2736a815f821ca6c152bb76160cdf5972c73b9a3#2736a815f821ca6c152bb76160cdf5972c73b9a3" dependencies = [ "serde_json", "sqlquack-syntax", @@ -664,7 +674,7 @@ dependencies = [ [[package]] name = "sqlquack-lint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=c2f8b2b1b218bc82c2935629110ff4bac6a5620b#c2f8b2b1b218bc82c2935629110ff4bac6a5620b" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=2736a815f821ca6c152bb76160cdf5972c73b9a3#2736a815f821ca6c152bb76160cdf5972c73b9a3" dependencies = [ "rayon", "rustc-hash", @@ -676,7 +686,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=c2f8b2b1b218bc82c2935629110ff4bac6a5620b#c2f8b2b1b218bc82c2935629110ff4bac6a5620b" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=2736a815f821ca6c152bb76160cdf5972c73b9a3#2736a815f821ca6c152bb76160cdf5972c73b9a3" dependencies = [ "serde", "serde_json", @@ -689,7 +699,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-derive" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=c2f8b2b1b218bc82c2935629110ff4bac6a5620b#c2f8b2b1b218bc82c2935629110ff4bac6a5620b" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=2736a815f821ca6c152bb76160cdf5972c73b9a3#2736a815f821ca6c152bb76160cdf5972c73b9a3" dependencies = [ "proc-macro2", "quote", @@ -699,7 +709,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-postgres" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=c2f8b2b1b218bc82c2935629110ff4bac6a5620b#c2f8b2b1b218bc82c2935629110ff4bac6a5620b" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=2736a815f821ca6c152bb76160cdf5972c73b9a3#2736a815f821ca6c152bb76160cdf5972c73b9a3" dependencies = [ "sqlquack-syntax", ] diff --git a/Cargo.toml b/Cargo.toml index a1a75c2..5fd3b0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,10 +41,10 @@ sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } # SQLQuack's formatter and lint engine, and the tokens and trees they read, which replace Polyglot. -sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "c2f8b2b1b218bc82c2935629110ff4bac6a5620b" } -sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "c2f8b2b1b218bc82c2935629110ff4bac6a5620b" } -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "c2f8b2b1b218bc82c2935629110ff4bac6a5620b" } -sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "c2f8b2b1b218bc82c2935629110ff4bac6a5620b" } +sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "2736a815f821ca6c152bb76160cdf5972c73b9a3" } +sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "2736a815f821ca6c152bb76160cdf5972c73b9a3" } +sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "2736a815f821ca6c152bb76160cdf5972c73b9a3" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "2736a815f821ca6c152bb76160cdf5972c73b9a3" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sqlclean/Cargo.toml b/crates/sqlclean/Cargo.toml index 1f098d2..bd79287 100644 --- a/crates/sqlclean/Cargo.toml +++ b/crates/sqlclean/Cargo.toml @@ -16,10 +16,10 @@ test-support = [] [dependencies] polyglot-sql.workspace = true polyglot-sql-function-catalogs.workspace = true +sqlquack-fingerprint.workspace = true sqlquack-format.workspace = true sqlquack-lint.workspace = true sqlquack-syntax.workspace = true -sqlquack-syntax-postgres.workspace = true rayon.workspace = true rustc-hash.workspace = true serde.workspace = true diff --git a/crates/sqlclean/src/sql_lint/_helpers/cte_macros.rs b/crates/sqlclean/src/sql_lint/_helpers/cte_macros.rs index 68021f4..4a84384 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/cte_macros.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/cte_macros.rs @@ -179,7 +179,10 @@ fn macro_positions(sql: &str, tokens: &[LosslessToken], significant: &[usize]) - || (text(position) .chars() .all(|character| character.is_alphanumeric() || character == '_') - && matches!(kinds.get(position + 1), Some(TokenType::As | TokenType::LParen)))) + && matches!( + kinds.get(position + 1), + Some(TokenType::As | TokenType::LParen) + ))) }; let mut depth = 0_usize; let mut scopes: Vec = Vec::new(); diff --git a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs index 55ee103..6e46a5c 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs @@ -169,7 +169,9 @@ fn sqlquack_layout(sql: &str, options: &FormatOptions) -> Result fn format_around_cte_macros(sql: &str, options: &FormatOptions) -> Result { match take_out_cte_macros(sql, options.dialect)? { None => sqlquack_layout(sql, options), - Some(without) => without.put_back(&sqlquack_layout(&without.sql, options)?, options.dialect), + Some(without) => { + without.put_back(&sqlquack_layout(&without.sql, options)?, options.dialect) + } } } diff --git a/crates/sqlclean/src/sql_lint/tests/test_layout_rules.rs b/crates/sqlclean/src/sql_lint/tests/test_layout_rules.rs index d8597cd..b746db4 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_layout_rules.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_layout_rules.rs @@ -181,7 +181,7 @@ fn given_clause_layout_rules_when_formatting_then_each_rule_applies_idempotently dialect: "duckdb", sql: "WITH a AS (SELECT 1 AS x), __sqlclean_interpolation_0__ __sqlclean_interpolation_1__ paid AS (SELECT * FROM a) SELECT * FROM paid", line_width: 100, - expected_sql: "WITH a AS (\n SELECT 1 AS x\n),\n\n__sqlclean_interpolation_0__\n\n__sqlclean_interpolation_1__\npaid AS (\n SELECT *\n FROM a\n)\nSELECT *\nFROM paid", + expected_sql: "WITH a AS (\n SELECT 1 AS x\n),\n\n__sqlclean_interpolation_0__\n\n__sqlclean_interpolation_1__\n\npaid AS (\n SELECT *\n FROM a\n)\n\nSELECT *\nFROM paid", }, test_types::LayoutRuleTestCase { description: "an alias and comma that push a call over the width break the call", diff --git a/crates/sqlclean/src/sql_tokens/_helpers/mod.rs b/crates/sqlclean/src/sql_tokens/_helpers/mod.rs index d75d204..a2df8fb 100644 --- a/crates/sqlclean/src/sql_tokens/_helpers/mod.rs +++ b/crates/sqlclean/src/sql_tokens/_helpers/mod.rs @@ -1,4 +1,3 @@ pub mod builtin_functions; pub mod reserved_keywords; -pub mod sqlquack_canonical_tokens; pub mod token_context; diff --git a/crates/sqlclean/src/sql_tokens/_helpers/reserved_keywords.rs b/crates/sqlclean/src/sql_tokens/_helpers/reserved_keywords.rs index a15eb1d..ccf9004 100644 --- a/crates/sqlclean/src/sql_tokens/_helpers/reserved_keywords.rs +++ b/crates/sqlclean/src/sql_tokens/_helpers/reserved_keywords.rs @@ -5,8 +5,6 @@ use std::sync::LazyLock; use polyglot_sql::DialectType; -use crate::sql_tokens::models::KeywordList; - static DUCKDB: LazyLock> = LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/duckdb.txt"))); static POSTGRES: LazyLock> = @@ -36,17 +34,6 @@ pub(crate) fn is_reserved_keyword(dialect: DialectType, upper_word: &str) -> boo keywords.contains(upper_word) } -/// Whether the upper-cased word is reserved in `list`, a SQLQuack fingerprint dialect's list. -pub(crate) fn is_listed_keyword(list: KeywordList, upper_word: &str) -> bool { - let keywords: &HashSet = match list { - KeywordList::DuckDB => &DUCKDB, - KeywordList::PostgreSQL => &POSTGRES, - KeywordList::Snowflake => &SNOWFLAKE, - KeywordList::Generic => &GENERIC, - }; - keywords.contains(upper_word) -} - fn keyword_set(data: &str) -> HashSet { data.lines() .map(str::trim) diff --git a/crates/sqlclean/src/sql_tokens/_helpers/sqlquack_canonical_tokens.rs b/crates/sqlclean/src/sql_tokens/_helpers/sqlquack_canonical_tokens.rs deleted file mode 100644 index 77a1551..0000000 --- a/crates/sqlclean/src/sql_tokens/_helpers/sqlquack_canonical_tokens.rs +++ /dev/null @@ -1,99 +0,0 @@ -//! The canonical token stream of SQL under SQLQuack's tokenizers. - -use sqlquack_syntax::TokenKind; -use sqlquack_syntax_postgres::scan::main::scan::scan; -use sqlquack_syntax_postgres::scan::models::TokenKind as PostgresTokenKind; - -use crate::sql_tokens::_helpers::reserved_keywords::is_listed_keyword; -use crate::sql_tokens::constants::{ - CALL_CLOSER, CALL_OPENER, COLUMN_LIST_RELATION_KEYWORDS, CTE_BODY_OPENERS, CTE_BODY_PREFIX, - PLACEHOLDER_PREFIX, -}; -use crate::sql_tokens::main::is_unquoted_word::is_unquoted_word; -use crate::sql_tokens::main::name_positions::name_positions; -use crate::sql_tokens::models::{FingerprintTokenizer, SqlquackFingerprintDialect}; - -/// Significant token texts with reserved keywords and called names, except `__` placeholders, upper-cased. -pub(crate) fn sqlquack_canonical_tokens( - sql: &str, - dialect: SqlquackFingerprintDialect, -) -> Result, String> { - let raws: Vec<&str> = significant_texts(sql, dialect.tokenizer)?; - let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); - let names: Vec = name_positions(&uppers); - let closers: Vec> = matching_closers(&uppers); - let mut texts: Vec = Vec::with_capacity(raws.len()); - for (index, raw) in raws.iter().enumerate() { - let folds = !names[index] - && is_unquoted_word(raw) - && (is_listed_keyword(dialect.keywords, &uppers[index]) - || calls(raw, &uppers, &closers, index)); - texts.push(if folds { - uppers[index].clone() - } else { - (*raw).to_owned() - }); - } - Ok(texts) -} - -/// The text of each significant token of `sql`, in order. -fn significant_texts(sql: &str, tokenizer: FingerprintTokenizer) -> Result, String> { - match tokenizer { - FingerprintTokenizer::Lossless(dialect) => Ok(sqlquack_syntax::tokenize(sql, dialect) - .map_err(|error| format!("SQL tokenization failed: {error}"))? - .iter() - .filter(|token| matches!(token.kind, TokenKind::Significant(_))) - .map(|token| token.text(sql)) - .collect()), - FingerprintTokenizer::PostgresScanner => scan(sql) - .into_iter() - .map(|token| match token.kind { - PostgresTokenKind::Fault(_) => { - Err(format!("SQL tokenization failed: {}", token.value)) - } - _ => sql - .get(token.start..token.end) - .ok_or_else(|| format!("SQL tokenization failed at byte {}", token.start)), - }) - .collect(), - } -} - -/// The index of the `)` closing each `(`, by the index of the `(`. -fn matching_closers(uppers: &[String]) -> Vec> { - let mut closers: Vec> = vec![None; uppers.len()]; - let mut open: Vec = Vec::new(); - for (index, token) in uppers.iter().enumerate() { - if token == CALL_OPENER { - open.push(index); - } else if token == CALL_CLOSER - && let Some(opener) = open.pop() - { - closers[opener] = Some(index); - } - } - closers -} - -/// Whether the word at `index` is called: followed by `(` and neither a placeholder nor a relation. -fn calls(raw: &str, uppers: &[String], closers: &[Option], index: usize) -> bool { - if uppers.get(index + 1).is_none_or(|next| next != CALL_OPENER) - || raw.starts_with(PLACEHOLDER_PREFIX) - { - return false; - } - let previous: Option<&str> = index.checked_sub(1).map(|before| uppers[before].as_str()); - if previous.is_some_and(|word| COLUMN_LIST_RELATION_KEYWORDS.contains(&word)) { - return false; - } - let Some(closer) = closers[index + 1] else { - return true; - }; - !(uppers - .get(closer + 1) - .is_some_and(|next| next == CTE_BODY_PREFIX) - && uppers - .get(closer + 2) - .is_some_and(|next| CTE_BODY_OPENERS.contains(&next.as_str()))) -} diff --git a/crates/sqlclean/src/sql_tokens/constants.rs b/crates/sqlclean/src/sql_tokens/constants.rs index 2ca7bdb..beee14b 100644 --- a/crates/sqlclean/src/sql_tokens/constants.rs +++ b/crates/sqlclean/src/sql_tokens/constants.rs @@ -2,8 +2,6 @@ use polyglot_sql::tokens::TokenType; -use crate::sql_tokens::models::{FingerprintTokenizer, KeywordList, SqlquackFingerprintDialect}; - /// Column aliases after `AS` may be any keyword in DuckDB and PostgreSQL. pub(crate) const ALIAS_KEYWORD: &str = "AS"; /// The token that turns a preceding name into a function call. @@ -35,46 +33,6 @@ pub(crate) const PROJECTION_ENDS: [TokenType; 13] = [ TokenType::Except, TokenType::Intersect, ]; -/// Fingerprint dialects SQLQuack tokenizes; generic uses the DuckDB tokenizer, PostgreSQL its native scanner. -pub(crate) const SQLQUACK_FINGERPRINT_DIALECTS: [SqlquackFingerprintDialect; 6] = [ - SqlquackFingerprintDialect { - name: "generic", - tokenizer: FingerprintTokenizer::Lossless(sqlquack_syntax::Dialect::DuckDB), - keywords: KeywordList::Generic, - }, - SqlquackFingerprintDialect { - name: "duckdb", - tokenizer: FingerprintTokenizer::Lossless(sqlquack_syntax::Dialect::DuckDB), - keywords: KeywordList::DuckDB, - }, - SqlquackFingerprintDialect { - name: "postgres", - tokenizer: FingerprintTokenizer::PostgresScanner, - keywords: KeywordList::PostgreSQL, - }, - SqlquackFingerprintDialect { - name: "postgresql", - tokenizer: FingerprintTokenizer::PostgresScanner, - keywords: KeywordList::PostgreSQL, - }, - SqlquackFingerprintDialect { - name: "snowflake", - tokenizer: FingerprintTokenizer::Lossless(sqlquack_syntax::Dialect::Snowflake), - keywords: KeywordList::Snowflake, - }, - SqlquackFingerprintDialect { - name: "sqlite", - tokenizer: FingerprintTokenizer::Lossless(sqlquack_syntax::Dialect::SQLite), - keywords: KeywordList::Generic, - }, -]; -/// The token that closes a call's arguments or a column list. -pub(crate) const CALL_CLOSER: &str = ")"; -/// Keywords after which a word followed by `(` names a relation and its column list. -pub(crate) const COLUMN_LIST_RELATION_KEYWORDS: [&str; 2] = ["INTO", "TABLE"]; -/// The tokens that can follow `AS` after a CTE's column list. -pub(crate) const CTE_BODY_OPENERS: [&str; 3] = ["(", "MATERIALIZED", "NOT"]; -/// The prefix of SQLBuild placeholder calls such as `__ref` and `__source`. -pub(crate) const PLACEHOLDER_PREFIX: &str = "__"; -/// The keyword between a CTE's column list and its body. -pub(crate) const CTE_BODY_PREFIX: &str = "AS"; +/// Dialect names fingerprinted under another name, as SQLBuild resolves them. +pub(crate) const FINGERPRINT_DIALECT_ALIASES: [(&str, &str); 2] = + [("motherduck", "duckdb"), ("sqlserver", "tsql")]; diff --git a/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs b/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs index bbc4b03..b1452b4 100644 --- a/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs +++ b/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs @@ -4,39 +4,33 @@ use std::str::FromStr; use polyglot_sql::{Dialect, DialectType}; use sha2::{Digest, Sha256}; +use sqlquack_fingerprint::FingerprintDialect; -use crate::sql_tokens::_helpers::sqlquack_canonical_tokens::sqlquack_canonical_tokens; -use crate::sql_tokens::constants::SQLQUACK_FINGERPRINT_DIALECTS; +use crate::sql_tokens::constants::FINGERPRINT_DIALECT_ALIASES; use crate::sql_tokens::main::canonical_tokens::canonical_tokens; -/// Return the SHA-256 hex digest of the canonical token stream of `sql` under `dialect_name`. +/// Return the fingerprint of `sql` under `dialect_name` that SQLBuild stores: SQLQuack's for the +/// dialects it fingerprints, and the Polyglot token stream's for the others, as SQLBuild does. pub fn query_fingerprint(sql: &str, dialect_name: &str) -> Result { - let texts: Vec = match SQLQUACK_FINGERPRINT_DIALECTS + let name: &str = FINGERPRINT_DIALECT_ALIASES .iter() - .find(|dialect| dialect.name == dialect_name) - { - Some(dialect) => sqlquack_canonical_tokens(sql, *dialect)?, - None => polyglot_canonical_tokens(sql, dialect_name)?, - }; - let mut digest = Sha256::new(); - for text in texts { - digest.update(text.len().to_string().as_bytes()); - digest.update(b":"); - digest.update(text.as_bytes()); + .find(|(alias, _)| *alias == dialect_name) + .map_or(dialect_name, |(_, name)| name); + if let Ok(dialect) = FingerprintDialect::from_name(name) { + return sqlquack_fingerprint::query_fingerprint(sql, dialect) + .map_err(|error| error.to_string()); } - Ok(format!("{:x}", digest.finalize())) -} - -/// Dialects SQLQuack cannot tokenize yet keep the Polyglot token stream. -fn polyglot_canonical_tokens(sql: &str, dialect_name: &str) -> Result, String> { - let dialect_type = DialectType::from_str(dialect_name) + let dialect_type = DialectType::from_str(name) .map_err(|_| format!("unknown SQL fingerprint dialect '{dialect_name}'"))?; let dialect = Dialect::get(dialect_type); let tokens = dialect .tokenize(sql) .map_err(|error| format!("SQL tokenization failed: {error}"))?; - Ok(canonical_tokens(sql, &tokens, &dialect)? - .into_iter() - .map(|token| token.text) - .collect()) + let mut digest = Sha256::new(); + for token in canonical_tokens(sql, &tokens, &dialect)? { + digest.update(token.text.len().to_string().as_bytes()); + digest.update(b":"); + digest.update(token.text.as_bytes()); + } + Ok(format!("{:x}", digest.finalize())) } diff --git a/crates/sqlclean/src/sql_tokens/models.rs b/crates/sqlclean/src/sql_tokens/models.rs index 1f9088f..b18ebb1 100644 --- a/crates/sqlclean/src/sql_tokens/models.rs +++ b/crates/sqlclean/src/sql_tokens/models.rs @@ -5,29 +5,3 @@ pub struct CanonicalToken { pub text: String, } - -/// The reserved-keyword list a SQLQuack fingerprint dialect folds. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum KeywordList { - DuckDB, - PostgreSQL, - Snowflake, - Generic, -} - -/// The SQLQuack tokenizer a fingerprint dialect reads its tokens with. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum FingerprintTokenizer { - /// `sqlquack_syntax`'s lossless tokenizer for one dialect. - Lossless(sqlquack_syntax::Dialect), - /// The native PostgreSQL scanner of `sqlquack_syntax_postgres`. - PostgresScanner, -} - -/// A fingerprint dialect SQLQuack tokenizes: its tokenizer and its reserved keywords. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct SqlquackFingerprintDialect { - pub(crate) name: &'static str, - pub(crate) tokenizer: FingerprintTokenizer, - pub(crate) keywords: KeywordList, -} diff --git a/crates/sqlclean/src/sql_tokens/tests.rs b/crates/sqlclean/src/sql_tokens/tests.rs index b7029e6..b4e9827 100644 --- a/crates/sqlclean/src/sql_tokens/tests.rs +++ b/crates/sqlclean/src/sql_tokens/tests.rs @@ -1,6 +1,6 @@ #[path = "tests/test_canonical_tokens.rs"] mod canonical_tokens; -#[path = "tests/test_types.rs"] -mod test_types; #[path = "tests/test_query_fingerprint.rs"] mod query_fingerprint; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs b/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs index 23514b3..72dc61f 100644 --- a/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs +++ b/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs @@ -3,9 +3,9 @@ use crate::sql_tokens::main::query_fingerprint::query_fingerprint; /// SQLBuild's golden fingerprints for the dialects SQLQuack tokenizes; SQLBuild stores these /// hashes, so SQLClean must compute the same ones. #[test] -fn given_sqlbuild_golden_queries_when_fingerprinting_then_hashes_match_sqlbuild() -> Result<(), String> -{ - let test_cases: [(&str, &str, &str); 5] = [ +fn given_sqlbuild_golden_queries_when_fingerprinting_then_hashes_match_sqlbuild() +-> Result<(), String> { + let test_cases: [(&str, &str, &str); 8] = [ ( "generic", "SELECT id, name FROM orders", @@ -31,29 +31,88 @@ fn given_sqlbuild_golden_queries_when_fingerprinting_then_hashes_match_sqlbuild( "select o.id from __ref(\"orders\") as o where o.id > 0", "fda88b9b0f26486bdd25b8e3bab5a982fbb387901ab229091b17772d044aebd0", ), + ( + "bigquery", + "SELECT o.id, r'raw\\d' AS pattern FROM `project.dataset.orders` AS o", + "ce893f2fd9ab11ba453fc3c02e77b6690df6c3d1d8bc141fc19cf3891d9a61b4", + ), + ( + "databricks", + "SELECT /*+ BROADCAST(o) */ o.id FROM `catalog`.`orders` AS o", + "7cd96f42bcb764136a7f423181302ba13cead02bb6cfb7382502a3a2aa4107c3", + ), + ( + "sqlserver", + "SELECT TOP 10 [o].[id], N'order' AS kind FROM [dbo].[orders] AS o", + "f4fac3edb7682bbeddf4f73cc199493d45f1419bc8e2cd1d972fead2d67e6b71", + ), ]; for (dialect, sql, expected) in test_cases { - assert_eq!(query_fingerprint(sql, dialect)?, expected, "{dialect}: {sql}"); + assert_eq!( + query_fingerprint(sql, dialect)?, + expected, + "{dialect}: {sql}" + ); } Ok(()) } +/// MotherDuck fingerprints as DuckDB; `qualify` is reserved in DuckDB but not generic SQL. +#[test] +fn given_the_motherduck_alias_when_fingerprinting_then_it_hashes_as_duckdb() -> Result<(), String> { + let sql = "select a from t qualify b"; + assert_eq!( + query_fingerprint(sql, "motherduck")?, + query_fingerprint(sql, "duckdb")? + ); + assert_ne!( + query_fingerprint(sql, "duckdb")?, + query_fingerprint(sql, "generic")? + ); + Ok(()) +} + /// Called names fold to upper case, except placeholders, quoted names and relation or CTE column lists. #[test] -fn given_called_names_when_fingerprinting_then_only_unquoted_calls_ignore_case() -> Result<(), String> -{ +fn given_called_names_when_fingerprinting_then_only_unquoted_calls_ignore_case() +-> Result<(), String> { let test_cases: [(&str, &str, &str, bool); 6] = [ - ("duckdb", "SELECT MyFunc(id) FROM orders", "SELECT myfunc(id) FROM orders", true), - ("snowflake", "SELECT \"MyFunc\"(id) FROM orders", "SELECT \"myfunc\"(id) FROM orders", false), - ("duckdb", "SELECT id FROM __ref('orders')", "SELECT id FROM __REF('orders')", false), + ( + "duckdb", + "SELECT MyFunc(id) FROM orders", + "SELECT myfunc(id) FROM orders", + true, + ), + ( + "snowflake", + "SELECT \"MyFunc\"(id) FROM orders", + "SELECT \"myfunc\"(id) FROM orders", + false, + ), + ( + "duckdb", + "SELECT id FROM __ref('orders')", + "SELECT id FROM __REF('orders')", + false, + ), ( "duckdb", "WITH Tree(id) AS (SELECT 1) SELECT id FROM orders", "WITH tree(id) AS (SELECT 1) SELECT id FROM orders", false, ), - ("postgres", "INSERT INTO Paid(id) SELECT 1", "INSERT INTO paid(id) SELECT 1", false), - ("sqlite", "SELECT Upper(name) FROM orders", "SELECT UPPER(name) FROM orders", true), + ( + "postgres", + "INSERT INTO Paid(id) SELECT 1", + "INSERT INTO paid(id) SELECT 1", + false, + ), + ( + "sqlite", + "SELECT Upper(name) FROM orders", + "SELECT UPPER(name) FROM orders", + true, + ), ]; for (dialect, left, right, expected_equal) in test_cases { assert_eq!( From 4c54330167fa6873b242db99ee760e7edd88d29c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:28:03 +0000 Subject: [PATCH 09/21] wip(format): report formatting unavailable where SQLQuack has no formatter, and drop the Polyglot oracle Dialects without a native SQLQuack formatter now return their SQL unchanged with an unavailable reason instead of a Polyglot layout, so the oracle formatter and its token layout, printed layout and syntax protection are removed. Fingerprints use SQLQuack's generic tokens for dialects without their own tokenizer, as SQLBuild does; the SQL Server golden changes to the generic digest. The recording demo's custom Rule reads statement facts instead of the removed AST. --- .../sqlclean/src/sql_lint/_helpers/engine.rs | 26 - .../src/sql_lint/_helpers/formatter.rs | 354 +----- .../src/sql_lint/_helpers/formatter_syntax.rs | 459 ------- crates/sqlclean/src/sql_lint/_helpers/mod.rs | 4 - .../src/sql_lint/_helpers/printed_layout.rs | 1080 ----------------- .../src/sql_lint/_helpers/token_layout.rs | 815 ------------- crates/sqlclean/src/sql_lint/constants.rs | 6 - crates/sqlclean/src/sql_lint/models.rs | 3 + crates/sqlclean/src/sql_lint/tests.rs | 4 - crates/sqlclean/src/sql_lint/tests/helpers.rs | 78 +- .../sql_lint/tests/test_authored_tokens.rs | 416 ------- .../src/sql_lint/tests/test_format_corpus.rs | 48 +- .../tests/test_function_call_spacing.rs | 70 -- .../tests/test_idempotency_reprint.rs | 48 - .../src/sql_lint/tests/test_layout_rules.rs | 84 -- .../tests/test_set_operation_formatter.rs | 9 +- .../sqlclean/src/sql_lint/tests/test_types.rs | 22 - crates/sqlclean/src/sql_tokens/constants.rs | 4 +- .../src/sql_tokens/main/query_fingerprint.rs | 28 +- .../tests/test_query_fingerprint.rs | 4 +- website/recordings/README.md | 2 +- website/recordings/prepare.sh | 39 +- 22 files changed, 124 insertions(+), 3479 deletions(-) delete mode 100644 crates/sqlclean/src/sql_lint/_helpers/engine.rs delete mode 100644 crates/sqlclean/src/sql_lint/_helpers/formatter_syntax.rs delete mode 100644 crates/sqlclean/src/sql_lint/_helpers/printed_layout.rs delete mode 100644 crates/sqlclean/src/sql_lint/_helpers/token_layout.rs delete mode 100644 crates/sqlclean/src/sql_lint/tests/test_authored_tokens.rs delete mode 100644 crates/sqlclean/src/sql_lint/tests/test_idempotency_reprint.rs diff --git a/crates/sqlclean/src/sql_lint/_helpers/engine.rs b/crates/sqlclean/src/sql_lint/_helpers/engine.rs deleted file mode 100644 index caed3ec..0000000 --- a/crates/sqlclean/src/sql_lint/_helpers/engine.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Token nesting depths, the one lint helper the formatter still reads. - -use polyglot_sql::tokens::{Token, TokenType}; - -/// Each token's bracket nesting depth, counted before the token itself. -pub(crate) fn token_depths(tokens: &[Token]) -> Vec { - let mut depth = 0_usize; - tokens - .iter() - .map(|token| { - let current = depth; - if matches!( - token.token_type, - TokenType::LParen | TokenType::LBracket | TokenType::LBrace - ) { - depth += 1; - } else if matches!( - token.token_type, - TokenType::RParen | TokenType::RBracket | TokenType::RBrace - ) { - depth = depth.saturating_sub(1); - } - current - }) - .collect() -} diff --git a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs index 6e46a5c..edf8e7c 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs @@ -1,121 +1,61 @@ -use std::collections::{BTreeMap, HashMap}; -use std::str::FromStr; - -use polyglot_sql::tokens::{Token, TokenType}; -use polyglot_sql::{ComplexityGuardOptions, Dialect, DialectType, ParseOptions}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use sqlquack_format::FormatOptions; use sqlquack_syntax::Dialect as SqlQuackDialect; use crate::constants::INTERPOLATION_PLACEHOLDER_PREFIX; use crate::sql_lint::_helpers::cte_macros::take_out_cte_macros; -use crate::sql_lint::_helpers::formatter_syntax::{cte_macro_indices, protect_syntax}; -use crate::sql_lint::_helpers::printed_layout::{PrintedSql, WrapOptions, wrap_lines}; -use crate::sql_lint::_helpers::token_layout::{ - AuthoredSql, KeywordRecase, OracleLayout, TokenTexts, comments_in, neutralize_comments, - oracle_layout, print_authored_tokens, tokenize, verify_printed_tokens, -}; -use crate::sql_lint::constants::{ - CAST_TYPE_SEPARATOR_KEYWORD, CLOSE_PARENTHESIS, DUCKDB_DIALECT, LINT_API_VERSION, - OPEN_PARENTHESIS, SNOWFLAKE_DIALECT, -}; +use crate::sql_lint::constants::{DUCKDB_DIALECT, LINT_API_VERSION, SNOWFLAKE_DIALECT}; use crate::sql_lint::models::{FormatRequest, FormatResponse}; -#[cfg(test)] -thread_local! { - /// Idempotency re-prints on this thread, for tests that assert which bodies are re-printed. - pub(crate) static RELAYOUTS: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - /// Spaces per indent step when a request does not choose. const DEFAULT_INDENT: usize = 2; const NOT_IDEMPOTENT_FAILURE: &str = "native formatter output is not idempotent"; -const UNSUPPORTED_SQL_FAILURE: &str = - "native formatter could not safely restore literal or cast spelling"; pub(crate) fn format_json_impl(request_json: &str) -> Result { let request: FormatRequest = serde_json::from_str(request_json).map_err(|error| error.to_string())?; - let response = format_sql(request)?; + let response = format_sql(request); serde_json::to_string(&response).map_err(|error| error.to_string()) } /// Format requests in parallel, in order; a request that fails is declined with its reason. pub(crate) fn format_requests(requests: Vec) -> Vec { - requests - .into_par_iter() - .map(|request| { - let original: String = request.sql.clone(); - format_sql(request).unwrap_or_else(|reason| FormatResponse { - version: LINT_API_VERSION, - sql: original, - changed: false, - formatted: false, - reason: Some(reason), - }) - }) - .collect() + requests.into_par_iter().map(format_sql).collect() } -fn format_sql(request: FormatRequest) -> Result { - let original = request.sql.clone(); - match try_format_sql(request) { - Ok(value) => Ok(value), - Err(reason) => Ok(FormatResponse { +/// Lay out `request`, or return its SQL unchanged with why it was declined. +fn format_sql(request: FormatRequest) -> FormatResponse { + let declined = + |sql: String, reason: Option, unavailable: Option| FormatResponse { version: LINT_API_VERSION, - sql: original, + sql, changed: false, formatted: false, - reason: Some(reason), - }), - } -} - -fn try_format_sql(request: FormatRequest) -> Result { + reason, + unavailable, + }; if request.version != LINT_API_VERSION { - return Err(format!( + let reason = format!( "unsupported native format request version {}; expected {LINT_API_VERSION}", request.version - )); - } - if let Some(dialect) = sqlquack_dialect(&request.dialect) { - return format_with_sqlquack(request, dialect); + ); + return declined(request.sql, Some(reason), None); } - let original = request.sql; - let dialect_type = - DialectType::from_str(&request.dialect).map_err(|error| error.to_string())?; - let dialect = Dialect::get(dialect_type); - let parse_options = ParseOptions { - complexity_guard: Some(ComplexityGuardOptions { - max_function_call_depth: request.max_function_call_depth, - ..Default::default() - }), - }; - let wrap = request.line_width.map(|line_width| WrapOptions { - line_width, - token_widths: &request.token_widths, - }); - let layout = Layout { - dialect: &dialect, - wrap: wrap.as_ref(), - indent: request.indent.unwrap_or(DEFAULT_INDENT), + let Some(dialect) = sqlquack_dialect(&request.dialect) else { + let unavailable = format!( + "SQL formatting is unavailable for the {} dialect: SQLQuack does not format it yet", + request.dialect + ); + return declined(request.sql, None, Some(unavailable)); }; - let (formatted, formatted_tokens, oracle) = format_layout(&original, &layout, &parse_options)?; - let changed = formatted != original; - if changed && relayout((&formatted, &formatted_tokens), oracle.as_ref(), &layout)? != formatted - { - return Err(NOT_IDEMPOTENT_FAILURE.to_string()); + let original = request.sql.clone(); + match format_with_sqlquack(request, dialect) { + Ok(response) => response, + Err(reason) => declined(original, Some(reason), None), } - Ok(FormatResponse { - version: LINT_API_VERSION, - changed, - sql: formatted, - formatted: true, - reason: None, - }) } -/// The SQLQuack dialect that lays out `dialect`, for the dialects SQLQuack parses natively. +/// The SQLQuack dialect that lays out `dialect`, for the dialects SQLQuack formats natively. fn sqlquack_dialect(dialect: &str) -> Option { match dialect.to_ascii_lowercase().as_str() { SNOWFLAKE_DIALECT => Some(SqlQuackDialect::Snowflake), @@ -155,6 +95,7 @@ fn format_with_sqlquack( sql, formatted: true, reason: None, + unavailable: None, }) } @@ -174,246 +115,3 @@ fn format_around_cte_macros(sql: &str, options: &FormatOptions) -> Result, Option); - -/// Print the tokens of `sql` with parse-tree whitespace; return the output and its oracle. -fn format_layout( - sql: &str, - layout: &Layout<'_>, - parse_options: &ParseOptions, -) -> Result { - let dialect = layout.dialect; - let tokens = dialect - .tokenize(sql) - .map_err(|error| format!("native formatter tokenization failed: {error}"))?; - if tokens.is_empty() { - return Ok((sql.to_string(), tokens, None)); - } - let comments = comments_in(sql, &tokens); - let neutral = neutralize_comments(sql, &comments); - let validation_sql = protect_syntax(&neutral, &tokens, &[], false)?; - dialect - .parse_with_options(&validation_sql.sql, parse_options) - .map_err(|error| format!("native formatter could not parse SQL: {error}"))?; - let semantic_tokens = without_statement_terminators(&tokens); - let oracle = oracle_layout( - &layout_oracle(&neutral, &semantic_tokens, dialect, parse_options)?, - dialect, - )?; - let authored = AuthoredSql { - sql, - tokens: &tokens, - comments: &comments, - }; - let (printed, recases, authored_texts) = layout.print(&authored, &oracle)?; - if printed == sql { - return Ok((printed, tokens, Some(oracle))); - } - let printed_tokens = tokenize(&printed, dialect)?; - verify_printed_tokens( - (sql, &tokens, &authored_texts), - (&printed, &printed_tokens), - dialect, - &recases, - )?; - Ok((printed, printed_tokens, Some(oracle))) -} - -/// Re-print formatted SQL against its oracle; unchanged tokens imply an unchanged layout. -fn relayout( - (formatted, tokens): (&str, &[Token]), - oracle: Option<&OracleLayout>, - layout: &Layout<'_>, -) -> Result { - #[cfg(test)] - RELAYOUTS.with(|relayouts| relayouts.set(relayouts.get() + 1)); - let Some(oracle) = oracle.filter(|_| !tokens.is_empty()) else { - return Ok(formatted.to_string()); - }; - let comments = comments_in(formatted, tokens); - let authored = AuthoredSql { - sql: formatted, - tokens, - comments: &comments, - }; - Ok(layout.print(&authored, oracle)?.0) -} - -/// The dialect, indent and optional line wrapping every printed layout uses. -struct Layout<'a> { - dialect: &'a Dialect, - wrap: Option<&'a WrapOptions<'a>>, - indent: usize, -} - -impl Layout<'_> { - fn print( - &self, - authored: &AuthoredSql<'_>, - oracle: &OracleLayout, - ) -> Result<(String, BTreeMap, TokenTexts), String> { - let print = print_authored_tokens(authored, oracle, self.dialect)?; - let no_widths: HashMap = HashMap::new(); - let token_widths = self.wrap.map_or(&no_widths, |wrap| wrap.token_widths); - let Some(mut sql) = PrintedSql::new(&print.sql, &print.tokens, token_widths) else { - return Ok((print.sql, print.recases, print.texts)); - }; - sql.set_indent(self.indent); - sql.attach_commas_before_comments(); - if let Some(wrap) = self.wrap { - sql.join_fitting_groups(wrap.line_width); - } - sql.arrange_clauses(self.wrap.map_or(usize::MAX, |wrap| wrap.line_width)); - if let Some(wrap) = self.wrap { - wrap_lines(&mut sql, wrap.line_width); - } - sql.lead_operators_of_multiline_operands(); - Ok((sql.render(), print.recases, print.texts)) - } -} - -/// Return the parse-tree layout of `neutral_sql`; only its whitespace is ever printed. -fn layout_oracle( - neutral_sql: &str, - semantic_tokens: &[Token], - dialect: &Dialect, - parse_options: &ParseOptions, -) -> Result { - let protected = protect_syntax( - neutral_sql, - semantic_tokens, - &cast_type_token_ranges(neutral_sql, semantic_tokens)?, - true, - )?; - let expressions = dialect - .parse_with_options(&protected.sql, parse_options) - .map_err(|error| error.to_string())?; - let statements: Result, String> = expressions - .iter() - .map(|expression| { - dialect - .generate_pretty(expression) - .map_err(|error| error.to_string()) - }) - .collect(); - let restored = protected.restore(statements?.join(";\n"), dialect)?; - layout_cte_macros(restored, dialect) -} - -fn layout_cte_macros(mut formatted: String, dialect: &Dialect) -> Result { - if !formatted.contains(INTERPOLATION_PLACEHOLDER_PREFIX) { - return Ok(formatted); - } - let tokens = dialect - .tokenize(&formatted) - .map_err(|error| error.to_string())?; - let macros = cte_macro_indices(&tokens); - if macros.is_empty() { - return Ok(formatted); - } - let depths = crate::sql_lint::_helpers::engine::token_depths(&tokens); - let chars: Vec = formatted.chars().collect(); - let mut gaps: BTreeMap<(usize, usize), String> = BTreeMap::new(); - for index in macros { - let with = (0..index) - .rev() - .find(|&previous| { - tokens[previous].token_type == TokenType::With && depths[previous] == depths[index] - }) - .ok_or_else(|| "native formatter lost a CTE macro scope".to_string())?; - let indentation = line_indentation(&chars, tokens[with].span.start); - let newline = format!("\n{}", " ".repeat(indentation)); - gaps.insert( - (tokens[index - 1].span.end, tokens[index].span.start), - newline.clone(), - ); - if tokens - .get(index + 1) - .is_some_and(|token| token.token_type == TokenType::Comma) - && let Some(next) = tokens.get(index + 2) - { - gaps.insert((tokens[index + 1].span.end, next.span.start), newline); - } - } - for ((start, end), replacement) in gaps.into_iter().rev() { - let start = char_to_byte(&formatted, start)?; - let end = char_to_byte(&formatted, end)?; - formatted.replace_range(start..end, &replacement); - } - Ok(formatted) -} - -fn line_indentation(characters: &[char], offset: usize) -> usize { - let line_start = characters[..offset] - .iter() - .rposition(|character| *character == '\n') - .map_or(0, |position| position + 1); - characters[line_start..offset] - .iter() - .take_while(|character| **character == ' ' || **character == '\t') - .count() -} - -fn cast_type_token_ranges(sql: &str, tokens: &[Token]) -> Result, String> { - let characters: Vec = sql.chars().collect(); - let mut ranges: Vec<(usize, usize)> = Vec::new(); - let mut cast_type_starts: Vec> = Vec::new(); - let mut previous: Option = None; - for (index, token) in tokens.iter().enumerate() { - let text: String = characters - .get(token.span.start..token.span.end) - .ok_or_else(|| UNSUPPORTED_SQL_FAILURE.to_string())? - .iter() - .collect(); - let normalized = text.to_ascii_uppercase(); - if normalized == OPEN_PARENTHESIS { - let opens_cast = previous - .as_deref() - .is_some_and(|value| matches!(value, "CAST" | "TRY_CAST")); - cast_type_starts.push(opens_cast.then_some(usize::MAX)); - } else if normalized == CLOSE_PARENTHESIS { - if let Some(Some(type_start)) = cast_type_starts.pop() - && type_start != usize::MAX - && type_start < index - { - ranges.push((type_start, index - 1)); - } - } else if normalized == CAST_TYPE_SEPARATOR_KEYWORD - && let Some(Some(type_start)) = cast_type_starts.last_mut() - && *type_start == usize::MAX - { - *type_start = index + 1; - } - previous = Some(normalized); - } - ranges.sort_unstable(); - Ok(ranges) -} - -fn without_statement_terminators(tokens: &[Token]) -> Vec { - tokens - .iter() - .enumerate() - .filter(|(index, token)| { - token.token_type != TokenType::Semicolon - && !(token.token_type == TokenType::Comma - && tokens - .get(index + 1) - .is_some_and(|next| next.token_type == TokenType::From)) - }) - .map(|(_, token)| token.clone()) - .collect() -} - -fn char_to_byte(value: &str, char_offset: usize) -> Result { - if char_offset == value.chars().count() { - return Ok(value.len()); - } - value - .char_indices() - .nth(char_offset) - .map(|(offset, _)| offset) - .ok_or_else(|| "native formatter produced an invalid character offset".to_string()) -} diff --git a/crates/sqlclean/src/sql_lint/_helpers/formatter_syntax.rs b/crates/sqlclean/src/sql_lint/_helpers/formatter_syntax.rs deleted file mode 100644 index 86071f4..0000000 --- a/crates/sqlclean/src/sql_lint/_helpers/formatter_syntax.rs +++ /dev/null @@ -1,459 +0,0 @@ -use crate::constants::INTERPOLATION_PLACEHOLDER_PREFIX; -use crate::sql_lint::_helpers::engine::token_depths; -use crate::sql_lint::constants::{ - CTE_MACRO_PLACEHOLDER_LITERAL, POSITION_ARGUMENT_SEPARATOR, POSTFIX_CAST_PREFIX_TOKEN_COUNT, -}; -use polyglot_sql::Dialect; -use polyglot_sql::tokens::{Token, TokenType}; -use std::collections::{HashMap, HashSet}; - -pub(super) struct ProtectedSql { - pub sql: String, - /// The lower-case prefix every sentinel name starts with. - prefix: String, - fragments: Vec<(String, String)>, - in_functions: HashSet, -} - -impl ProtectedSql { - pub(super) fn restore(&self, formatted: String, dialect: &Dialect) -> Result { - let tokens = dialect - .tokenize(&formatted) - .map_err(|error| error.to_string())?; - let mut positions: HashMap> = HashMap::new(); - for (index, token) in tokens.iter().enumerate() { - let prefixed = token - .text - .as_bytes() - .get(..self.prefix.len()) - .is_some_and(|start| start.eq_ignore_ascii_case(self.prefix.as_bytes())); - if !prefixed { - continue; - } - positions - .entry(token.text.to_ascii_lowercase()) - .or_default() - .push(index); - } - let mut edits: Vec<(usize, usize, String)> = Vec::new(); - for (placeholder, original) in &self.fragments { - let key = cte_placeholder_sentinel(placeholder) - .unwrap_or(placeholder) - .trim_start_matches(':') - .trim_start_matches('.') - .trim_matches('\'') - .to_ascii_lowercase(); - let found = positions - .get(&key) - .filter(|found| found.len() == 1) - .ok_or_else(|| { - "native formatter could not restore a protected SQL fragment".to_string() - })?; - let index = found[0]; - if self.in_functions.contains(&key) { - let mut depth = 0; - let separator = (index + 1..tokens.len()) - .find(|&position| { - match tokens[position].token_type { - TokenType::LParen => depth += 1, - TokenType::RParen => depth -= 1, - _ => {} - } - depth == 1 && tokens[position].token_type == TokenType::Comma - }) - .ok_or_else(|| "native formatter lost a POSITION separator".to_string())?; - edits.push(( - tokens[separator].span.start, - tokens[separator].span.end, - " IN".to_string(), - )); - } - if let Some(sentinel) = cte_placeholder_sentinel(placeholder) { - let _ = sentinel; - edits.push(restore_cte_macro(&tokens, index, original)?); - continue; - } - if placeholder.starts_with("::") { - edits.extend(restore_postfix_cast(&tokens, index, original)?); - continue; - } - let start = if placeholder.starts_with('.') { - if index == 0 || tokens[index - 1].token_type != TokenType::Dot { - return Err("native formatter changed a protected path accessor".to_string()); - } - tokens[index - 1].span.start - } else { - tokens[index].span.start - }; - edits.push((start, tokens[index].span.end, original.clone())); - } - edits.sort_by_key(|edit| edit.0); - let chars: Vec = formatted.chars().collect(); - let mut result = String::new(); - let mut cursor = 0; - for (start, end, replacement) in edits { - if start < cursor { - return Err("native formatter produced overlapping syntax restorations".to_string()); - } - result.extend(&chars[cursor..start]); - result.push_str(&replacement); - cursor = end; - } - result.extend(&chars[cursor..]); - Ok(result) - } -} - -/// The sentinel of a CTE macro placeholder, with or without the comma added after it. -fn cte_placeholder_sentinel(placeholder: &str) -> Option<&str> { - let placeholder: &str = placeholder.strip_suffix(',').unwrap_or(placeholder); - placeholder.strip_suffix(" AS (SELECT 1)") -} - -pub(super) fn protect_syntax( - sql: &str, - tokens: &[Token], - cast_types: &[(usize, usize)], - expressions: bool, -) -> Result { - let mut prefix = "__sqlclean_format_".to_string(); - let lowercase: String = sql.to_ascii_lowercase(); - while lowercase.contains(&prefix) { - prefix.push('_'); - } - let mut cast_type_ends: HashMap = HashMap::new(); - for (start, end) in cast_types { - let _ = cast_type_ends.entry(*start).or_insert(*end); - } - let mut ranges: Vec<(usize, usize, String)> = Vec::new(); - let cte_macros = cte_macro_indices(tokens); - let depths = token_depths(tokens); - let mut separators: HashSet = HashSet::new(); - let mut in_functions: HashSet = HashSet::new(); - let mut index = 0; - while index < tokens.len() { - let token = &tokens[index]; - let sentinel = || format!("{prefix}{}__", ranges.len()); - if separators.contains(&index) { - ranges.push((token.span.start, token.span.end, ",".to_string())); - index += 1; - continue; - } - let (end, replacement) = if cte_macros.contains(&index) { - let separator: &str = if cte_macro_needs_comma(tokens, index) { - "," - } else { - "" - }; - (index, format!("{} AS (SELECT 1){separator}", sentinel())) - } else if !expressions { - index += 1; - continue; - } else if let Some(end) = cast_type_ends.get(&index) { - (*end, sentinel()) - } else if token.token_type == TokenType::DColon { - ( - postfix_type_end(tokens, index + 1)?, - format!("::{}", sentinel()), - ) - } else if token.token_type == TokenType::Colon - && index > 0 - && tokens[index - 1].token_type != TokenType::String - && tokens.get(index + 1).is_some_and(|next| { - next.text - .chars() - .next() - .is_some_and(|character| character.is_alphanumeric() || character == '_') - || matches!( - next.token_type, - TokenType::QuotedIdentifier | TokenType::String - ) - }) - { - (index + 1, format!(".{}", sentinel())) - } else if is_typed_literal(tokens, index) - || (matches!(token.token_type, TokenType::Var | TokenType::Identifier) - && tokens.get(index + 1).is_some_and(|next| { - matches!(next.token_type, TokenType::Var | TokenType::Identifier) - }) - && tokens - .get(index + 2) - .is_some_and(|next| next.token_type == TokenType::Arrow)) - { - (index + 1, sentinel()) - } else if (token.text.eq_ignore_ascii_case("MOD") - || token.text.eq_ignore_ascii_case("POSITION")) - && tokens - .get(index + 1) - .is_some_and(|next| next.token_type == TokenType::LParen) - { - let sentinel: String = sentinel(); - if token.text.eq_ignore_ascii_case("POSITION") { - let separator = (index + 2..tokens.len()) - .take_while(|&position| { - !(tokens[position].token_type == TokenType::RParen - && depths[position] == depths[index] + 1) - }) - .find(|&position| { - tokens[position].token_type == TokenType::In - && depths[position] == depths[index] + 1 - }); - if let Some(separator) = separator { - separators.insert(separator); - in_functions.insert(sentinel.clone()); - } - } - (index, sentinel) - } else if matches!( - token.token_type, - TokenType::String | TokenType::DollarString - ) { - (index, format!("'{}'", sentinel())) - } else { - index += 1; - continue; - }; - ranges.push((token.span.start, tokens[end].span.end, replacement)); - index = end + 1; - } - let chars: Vec = sql.chars().collect(); - let token_indices: HashMap = tokens - .iter() - .enumerate() - .map(|(index, token)| (token.span.start, index)) - .collect(); - let mut prepared = String::new(); - let mut fragments: Vec<(String, String)> = Vec::new(); - let mut cursor = 0; - for (start, end, replacement) in ranges { - prepared.extend(&chars[cursor..start]); - prepared.push_str(&replacement); - if replacement != POSITION_ARGUMENT_SEPARATOR { - let first = token_indices[&start]; - let last = (first..tokens.len()) - .find(|&index| tokens[index].span.end == end) - .ok_or_else(|| "native formatter lost a source fragment boundary".to_string())?; - fragments.push(( - replacement, - canonical_fragment(&chars, &tokens[first..=last]), - )); - } - cursor = end; - } - prepared.extend(&chars[cursor..]); - Ok(ProtectedSql { - sql: prepared, - prefix, - fragments, - in_functions, - }) -} - -fn canonical_fragment(chars: &[char], tokens: &[Token]) -> String { - let mut result = String::new(); - for (index, token) in tokens.iter().enumerate() { - if index > 0 - && !matches!( - token.token_type, - TokenType::LParen - | TokenType::RParen - | TokenType::Comma - | TokenType::Dot - | TokenType::Colon - | TokenType::DColon - | TokenType::LBracket - | TokenType::RBracket - | TokenType::Lt - | TokenType::Gt - ) - && !matches!( - tokens[index - 1].token_type, - TokenType::LParen - | TokenType::Dot - | TokenType::Colon - | TokenType::DColon - | TokenType::LBracket - | TokenType::Lt - ) - { - result.push(' '); - } - result.extend(&chars[token.span.start..token.span.end]); - } - result -} - -pub(super) fn cte_macro_indices(tokens: &[Token]) -> HashSet { - let depths = token_depths(tokens); - let mut scopes: HashSet = HashSet::new(); - let mut indices: HashSet = HashSet::new(); - for (index, token) in tokens.iter().enumerate() { - if token.token_type == TokenType::With { - scopes.insert(depths[index]); - } else if token.token_type == TokenType::Select { - scopes.remove(&depths[index]); - } else if token.text.starts_with(INTERPOLATION_PLACEHOLDER_PREFIX) - && scopes.contains(&depths[index]) - && index > 0 - && (matches!( - tokens[index - 1].token_type, - TokenType::With | TokenType::Comma - ) || indices.contains(&(index - 1))) - && tokens.get(index + 1).is_some_and(|next| { - matches!(next.token_type, TokenType::Comma | TokenType::Select) - || starts_cte(tokens, index + 1) - }) - { - indices.insert(index); - } - } - indices -} - -/// Whether a CTE definition, `name AS (` or another macro sentinel, starts at token `index`. -fn starts_cte(tokens: &[Token], index: usize) -> bool { - let Some(token) = tokens.get(index) else { - return false; - }; - if token.text.starts_with(INTERPOLATION_PLACEHOLDER_PREFIX) { - return true; - } - let named: bool = matches!( - token.token_type, - TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier - ) || token - .text - .chars() - .all(|character| character.is_alphanumeric() || character == '_'); - named - && tokens - .get(index + 1) - .is_some_and(|next| next.token_type == TokenType::As) - && tokens - .get(index + 2) - .is_some_and(|open| open.token_type == TokenType::LParen) -} - -/// Whether the CTE macro sentinel at token `index` needs a comma to separate it from the next CTE. -pub(super) fn cte_macro_needs_comma(tokens: &[Token], index: usize) -> bool { - starts_cte(tokens, index + 1) -} - -fn restore_cte_macro( - tokens: &[Token], - start: usize, - original: &str, -) -> Result<(usize, usize, String), String> { - let end = start + 5; - if tokens - .get(end) - .is_none_or(|token| token.token_type != TokenType::RParen) - || tokens[start + 1].token_type != TokenType::As - || tokens[start + 2].token_type != TokenType::LParen - || tokens[start + 3].token_type != TokenType::Select - || tokens[start + 4].text != CTE_MACRO_PLACEHOLDER_LITERAL - { - return Err("native formatter changed a CTE macro placeholder".to_string()); - } - Ok(( - tokens[start].span.start, - tokens[end].span.end, - original.to_string(), - )) -} - -fn restore_postfix_cast( - tokens: &[Token], - index: usize, - original: &str, -) -> Result, String> { - if index < POSTFIX_CAST_PREFIX_TOKEN_COUNT - || tokens[index - 1].token_type != TokenType::As - || tokens - .get(index + 1) - .is_none_or(|token| token.token_type != TokenType::RParen) - { - return Err("native formatter changed a postfix cast wrapper".to_string()); - } - let mut depth = 0; - let mut open = None; - for position in (0..index - 1).rev() { - match tokens[position].token_type { - TokenType::RParen => depth += 1, - TokenType::LParen if depth == 0 => { - open = Some(position); - break; - } - TokenType::LParen => depth -= 1, - _ => {} - } - } - let open = open - .filter(|&position| position > 0 && tokens[position - 1].token_type == TokenType::Cast) - .ok_or_else(|| "native formatter could not identify a postfix cast wrapper".to_string())?; - Ok(vec![ - ( - tokens[open - 1].span.start, - tokens[open + 1].span.start, - String::new(), - ), - ( - tokens[index - 2].span.end, - tokens[index + 1].span.end, - original.to_string(), - ), - ]) -} - -fn is_typed_literal(tokens: &[Token], index: usize) -> bool { - matches!( - tokens[index].text.to_ascii_uppercase().as_str(), - "DATE" - | "TIME" - | "TIMESTAMP" - | "TIMESTAMPTZ" - | "TIMESTAMP_NTZ" - | "TIMESTAMP_LTZ" - | "TIMESTAMP_TZ" - ) && tokens.get(index + 1).is_some_and(|token| { - matches!( - token.token_type, - TokenType::String | TokenType::DollarString - ) - }) -} - -fn postfix_type_end(tokens: &[Token], start: usize) -> Result { - let Some(_) = tokens.get(start) else { - return Err("native formatter could not locate a postfix cast type".to_string()); - }; - let mut end = start; - while tokens.get(end + 1).is_some_and(|token| { - matches!( - token.text.to_ascii_uppercase().as_str(), - "PRECISION" | "VARYING" | "WITH" | "WITHOUT" | "TIME" | "ZONE" - ) - }) { - end += 1; - } - if tokens - .get(end + 1) - .is_some_and(|token| token.token_type == TokenType::LParen) - { - let mut depth = 0; - for (index, token) in tokens.iter().enumerate().skip(end + 1) { - match token.token_type { - TokenType::LParen => depth += 1, - TokenType::RParen => { - depth -= 1; - if depth == 0 { - return Ok(index); - } - } - _ => {} - } - } - return Err("native formatter could not locate postfix cast parameters".to_string()); - } - Ok(end) -} diff --git a/crates/sqlclean/src/sql_lint/_helpers/mod.rs b/crates/sqlclean/src/sql_lint/_helpers/mod.rs index 30f7eb7..9ea5498 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/mod.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/mod.rs @@ -1,6 +1,2 @@ pub mod cte_macros; -pub mod engine; pub mod formatter; -pub mod formatter_syntax; -pub mod printed_layout; -pub mod token_layout; diff --git a/crates/sqlclean/src/sql_lint/_helpers/printed_layout.rs b/crates/sqlclean/src/sql_lint/_helpers/printed_layout.rs deleted file mode 100644 index a26645f..0000000 --- a/crates/sqlclean/src/sql_lint/_helpers/printed_layout.rs +++ /dev/null @@ -1,1080 +0,0 @@ -//! Printed SQL as tokens and gaps, with clause layout rules and line-width wrapping. - -use std::collections::HashMap; -use std::ops::Range; - -use polyglot_sql::tokens::TokenType; - -use crate::sql_lint::constants::OPEN_PARENTHESIS; - -/// The indent step of the parse-tree layout every printed statement starts from. -const PARSED_INDENT: usize = 2; -const COMMA: &str = ","; -const SINGLE_SPACE: &str = " "; -const BETWEEN_KEYWORD: &str = "BETWEEN"; -const BETWEEN_SEPARATOR: &str = "AND"; -const VALUES_KEYWORD: &str = "VALUES"; -const DECODE_FUNCTION: &str = "DECODE"; -const CASE_KEYWORD: &str = "CASE"; -const SINGLE_ITEM_CLAUSES: [&str; 4] = ["SELECT", "WHERE", "QUALIFY", "HAVING"]; -const SELECT_MODIFIERS: [&str; 10] = [ - "DISTINCT", "ALL", "TOP", "PERCENT", "WITH", "TIES", "ON", "AS", "STRUCT", "VALUE", -]; -const LEADING_OPERATORS: [&str; 8] = ["+", "-", "*", "/", "%", "||", "AND", "OR"]; -const CHAIN_KEYWORDS: [&str; 2] = ["AND", "OR"]; -const BETWEEN_BOUNDARIES: [&str; 12] = [ - "AND", "OR", "WHEN", "THEN", "ELSE", "WHERE", "ON", "HAVING", "QUALIFY", "SELECT", ",", "NOT", -]; -const ARITHMETIC_OPERATORS: [&str; 5] = ["+", "-", "||", "/", "%"]; -const WINDOW_CLAUSE_KEYWORDS: [&str; 4] = ["ORDER", "ROWS", "RANGE", "GROUPS"]; -const CASE_BRANCH_KEYWORDS: [&str; 3] = ["WHEN", "THEN", "ELSE"]; -const TYPE_PARAMETER_WORDS: [&str; 1] = ["MAX"]; -const TYPE_NAMES: [&str; 30] = [ - "BIGNUMERIC", - "BINARY", - "BIT", - "CHAR", - "CHARACTER", - "DATETIME", - "DATETIME2", - "DATETIMEOFFSET", - "DEC", - "DECIMAL", - "DOUBLE", - "FLOAT", - "INTERVAL", - "NCHAR", - "NUMBER", - "NUMERIC", - "NVARCHAR", - "STRING", - "TEXT", - "TIME", - "TIMESTAMP", - "TIMESTAMPTZ", - "TIMESTAMP_LTZ", - "TIMESTAMP_NTZ", - "TIMESTAMP_TZ", - "TIMETZ", - "VARBINARY", - "VARCHAR", - "BYTES", - "BPCHAR", -]; - -/// Printed SQL whose layout passes only ever rewrite the gaps between tokens. -pub(crate) struct PrintedSql<'a> { - tokens: Vec, - kinds: Vec, - /// The text before each token, plus the text after the last token. - gaps: Vec, - depths: Vec, - partners: Vec>, - /// Gaps that line wrapping must keep: inside VALUES rows and type parameter lists. - locked: Vec, - token_widths: &'a HashMap, - /// Spaces per indent step. - indent: usize, -} - -/// One printed line; end columns are absent when comments or multi-line tokens pin it. -struct Line { - first: usize, - last: usize, - indentation: usize, - end_columns: Option>, -} - -impl<'a> PrintedSql<'a> { - /// Split printed `sql` at its token byte ranges; `None` when it has no tokens. - pub(crate) fn new( - sql: &str, - printed_tokens: &[(Range, TokenType)], - token_widths: &'a HashMap, - ) -> Option { - if printed_tokens.is_empty() { - return None; - } - let mut texts: Vec = Vec::with_capacity(printed_tokens.len()); - let mut gaps: Vec = Vec::with_capacity(printed_tokens.len() + 1); - let mut previous_end = 0_usize; - for (range, _) in printed_tokens { - gaps.push(sql[previous_end..range.start].to_owned()); - texts.push(sql[range.clone()].to_owned()); - previous_end = range.end; - } - gaps.push(sql[previous_end..].to_owned()); - let (depths, partners) = bracket_structure(&texts); - let locked = vec![false; texts.len() + 1]; - Some(Self { - kinds: printed_tokens.iter().map(|(_, kind)| *kind).collect(), - tokens: texts, - gaps, - depths, - partners, - locked, - token_widths, - indent: PARSED_INDENT, - }) - } - - fn len(&self) -> usize { - self.tokens.len() - } - - fn upper(&self, index: usize) -> String { - self.tokens[index].to_ascii_uppercase() - } - - fn is(&self, index: usize, text: &str) -> bool { - self.tokens - .get(index) - .is_some_and(|token| token.eq_ignore_ascii_case(text)) - } - - fn line_ranges(&self, first: usize, last: usize) -> Vec<(usize, usize)> { - let mut ranges: Vec<(usize, usize)> = Vec::new(); - let mut start = first; - for index in first + 1..=last { - if self.gaps[index].contains('\n') { - ranges.push((start, index - 1)); - start = index; - } - } - ranges.push((start, last)); - ranges - } - - /// The first and last token of the printed line holding `index`. - fn line_of(&self, index: usize) -> (usize, usize) { - let first = (1..=index) - .rev() - .find(|&position| self.gaps[position].contains('\n')) - .unwrap_or(0); - let last = (index + 1..self.len()) - .find(|&position| self.gaps[position].contains('\n')) - .map_or(self.len() - 1, |position| position - 1); - (first, last) - } - - fn line(&self, first: usize, last: usize) -> Line { - let indentation = self.indentation(first); - let lead = self.gaps[first].rsplit('\n').next().unwrap_or_default(); - let trailing = self.gaps[last + 1].split('\n').next().unwrap_or_default(); - let commented = !lead.trim().is_empty() - || !trailing.trim().is_empty() - || (first + 1..=last).any(|index| !self.gaps[index].trim().is_empty()); - let multiline = (first..=last).any(|index| self.tokens[index].contains('\n')); - let end_columns = (!commented && !multiline).then(|| { - let mut column = indentation; - let mut ends: Vec = Vec::with_capacity(last - first + 1); - for index in first..=last { - if index > first { - column += self.gaps[index].chars().count(); - } - column += self.width(index); - ends.push(column); - } - ends - }); - Line { - first, - last, - indentation, - end_columns, - } - } - - /// Indentation of the line that starts at token `first`. - fn indentation(&self, first: usize) -> usize { - self.gaps[first] - .rsplit('\n') - .next() - .unwrap_or_default() - .chars() - .count() - } - - fn width(&self, index: usize) -> usize { - let token = &self.tokens[index]; - self.token_widths - .get(&token.to_ascii_uppercase()) - .copied() - .unwrap_or_else(|| token.chars().count()) - } - - /// Whether any gap of the tokens after `first` up to `last` holds a line break. - fn spans_lines(&self, first: usize, last: usize) -> bool { - (first + 1..=last).any(|index| self.gaps[index].contains('\n')) - } - - pub(crate) fn render(&self) -> String { - let mut output = String::new(); - for (gap, token) in self.gaps.iter().zip(&self.tokens) { - output.push_str(gap); - output.push_str(token); - } - output.push_str(&self.gaps[self.tokens.len()]); - output - } -} - -fn bracket_structure(tokens: &[String]) -> (Vec, Vec>) { - let mut depths: Vec = Vec::with_capacity(tokens.len()); - let mut partners: Vec> = vec![None; tokens.len()]; - let mut open: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - match token.as_str() { - "(" | "[" | "{" => { - depths.push(open.len()); - open.push(index); - } - ")" | "]" | "}" => { - let opener = open.pop(); - depths.push(open.len()); - if let Some(opener) = opener { - partners[opener] = Some(index); - partners[index] = Some(opener); - } - } - _ => depths.push(open.len()), - } - } - (depths, partners) -} - -fn newline(indentation: usize) -> String { - format!("\n{}", " ".repeat(indentation)) -} - -impl PrintedSql<'_> { - /// Re-indent every line from the parse-tree indent step to `indent` spaces per step. - pub(crate) fn set_indent(&mut self, indent: usize) { - if indent == self.indent { - return; - } - let step = self.indent; - let scale = |spaces: usize| spaces / step * indent + spaces % step; - for gap in &mut self.gaps { - if !gap.contains('\n') { - continue; - } - let comment_start = if gap.contains("/*") { - gap.rfind('\n').unwrap_or(0) - } else { - 0 - }; - let (kept, scaled) = gap.split_at(comment_start); - let mut rebuilt = kept.to_string(); - for (position, segment) in scaled.split('\n').enumerate() { - if position > 0 { - rebuilt.push('\n'); - let spaces = segment.len() - segment.trim_start_matches(' ').len(); - rebuilt.push_str(&" ".repeat(scale(spaces))); - rebuilt.push_str(&segment[spaces..]); - } else { - rebuilt.push_str(segment); - } - } - *gap = rebuilt; - } - self.indent = indent; - } - - /// Rejoin each multi-line expression group, innermost first, whose joined line fits. - pub(crate) fn join_fitting_groups(&mut self, line_width: usize) { - let mut openers: Vec = (0..self.len()) - .filter(|&index| self.tokens[index] == OPEN_PARENTHESIS) - .collect(); - openers.sort_by_key(|&index| std::cmp::Reverse(self.depths[index])); - for opener in openers { - let Some(closer) = self.partners[opener] else { - continue; - }; - if !self.spans_lines(opener, closer) || !self.joinable_group(opener, closer) { - continue; - } - let (first, _) = self.line_of(opener); - let (_, last) = self.line_of(closer); - let indentation = self.indentation(first); - let fits = |start: usize| { - self.joined_width(start, last, indentation) - .is_some_and(|width| width <= line_width) - }; - let start = if fits(first) { - first - } else if let Some(operator) = self.leading_operator_before(first, opener) - && fits(operator) - { - operator - } else { - continue; - }; - for index in opener + 1..=closer { - if self.gaps[index].contains('\n') { - self.gaps[index] = self.joined_gap(index).to_string(); - } - } - if start != first { - self.gaps[start] = newline(indentation); - } - } - } - - /// The last binary operator on the line from `first` that applies to the group at `opener`. - fn leading_operator_before(&self, first: usize, opener: usize) -> Option { - (first + 1..opener).rev().find(|&index| { - self.depths[index] == self.depths[opener] - && LEADING_OPERATORS.contains(&self.upper(index).as_str()) - && self.gaps[index] == SINGLE_SPACE - && self.ends_operand(index - 1) - }) - } - - /// Move a comma that follows a line comment to the end of the commented line. - pub(crate) fn attach_commas_before_comments(&mut self) { - for index in 1..self.len().saturating_sub(1) { - if self.tokens[index] != COMMA { - continue; - } - let Some((comment, rest)) = self.gaps[index].split_once('\n') else { - continue; - }; - if !comment.trim_start().starts_with("--") || !rest.trim().is_empty() { - continue; - } - let (comment, rest) = (comment.to_string(), rest.to_string()); - let next = self.gaps[index + 1].trim_start_matches(' ').to_string(); - self.gaps[index] = String::new(); - self.gaps[index + 1] = if next.starts_with('\n') { - comment + &next - } else { - format!("{comment}\n{rest}{next}") - }; - } - } - - fn joinable_group(&self, opener: usize, closer: usize) -> bool { - (opener + 1..closer).all(|index| { - !(self.kinds[index] == TokenType::Select - || self.is(index, VALUES_KEYWORD) - || (self.is(index, CASE_KEYWORD) - && self - .case_end(index) - .is_none_or(|end| self.spans_lines(index, end)))) - }) - } - - /// The width of tokens `first..=last` on one line at `indentation`; `None` when comments pin. - fn joined_width(&self, first: usize, last: usize, indentation: usize) -> Option { - let trailing = self.gaps[last + 1].split('\n').next().unwrap_or_default(); - let pinned = !trailing.trim().is_empty() - || (first + 1..=last).any(|index| !self.gaps[index].trim().is_empty()) - || (first..=last).any(|index| self.tokens[index].contains('\n')); - if pinned { - return None; - } - let mut width = indentation; - for index in first..=last { - if index > first { - width += if self.gaps[index].contains('\n') { - self.joined_gap(index).len() - } else { - self.gaps[index].chars().count() - }; - } - width += self.width(index); - } - Some(width) - } - - /// The gap before `index` once its line break is removed. - fn joined_gap(&self, index: usize) -> &'static str { - if self.tokens[index - 1] == OPEN_PARENTHESIS - || matches!(self.tokens[index].as_str(), ")" | ",") - { - "" - } else { - SINGLE_SPACE - } - } - - /// Apply the layout rules that only move existing line breaks before wrapping. - pub(crate) fn arrange_clauses(&mut self, line_width: usize) { - for index in 0..self.len() { - if self.is(index, DECODE_FUNCTION) { - self.pair_decode_arguments(index); - } - } - for index in 1..self.len() { - if CHAIN_KEYWORDS.contains(&self.upper(index).as_str()) { - self.join_chain_break_in_group(index); - } - } - for index in 0..self.len() { - if self.is(index, VALUES_KEYWORD) { - self.break_values_rows_unless_fitting(index, line_width); - } - } - for index in 0..self.len() { - if self.kinds[index] == TokenType::With { - self.separate_common_table_expressions(index); - } - } - for index in 0..self.len() { - if SINGLE_ITEM_CLAUSES.contains(&self.upper(index).as_str()) { - self.join_single_line_body(index, line_width); - } - } - } - - /// Start the next line with any binary operator whose operand spans several lines. - pub(crate) fn lead_operators_of_multiline_operands(&mut self) { - for index in 1..self.len().saturating_sub(1) { - if !LEADING_OPERATORS.contains(&self.upper(index).as_str()) - || self.gaps[index] != SINGLE_SPACE - || self.gaps[index + 1].contains('\n') - || !self.ends_operand(index - 1) - || (self.is(index, "AND") && self.follows_between(index)) - { - continue; - } - let multiline = self - .left_operand_start(index - 1) - .is_some_and(|start| self.spans_lines(start, index - 1)) - || self - .right_operand_end(index + 1) - .is_some_and(|end| self.spans_lines(index + 1, end)); - if multiline { - let (first, _) = self.line_of(index); - self.gaps[index] = newline(self.indentation(first)); - } - } - } - - /// Put the first DECODE argument on its own line and each search/result pair on one line. - fn pair_decode_arguments(&mut self, name: usize) { - let opener = name + 1; - if !self.is(opener, OPEN_PARENTHESIS) || (name > 0 && self.is(name - 1, ".")) { - return; - } - let Some(closer) = self.partners[opener] else { - return; - }; - if !self.gaps[opener + 1].contains('\n') { - return; - } - let mut argument = 0_usize; - for index in opener + 2..closer { - if self.depths[index] != self.depths[opener] + 1 || self.tokens[index - 1] != COMMA { - continue; - } - argument += 1; - if argument.is_multiple_of(2) && self.gaps[index].trim().is_empty() { - self.gaps[index] = SINGLE_SPACE.to_string(); - } - } - } - - /// Join an AND or OR line break inside a bracket group whose arguments share its line. - fn join_chain_break_in_group(&mut self, index: usize) { - if !self.gaps[index].contains('\n') || !self.gaps[index].trim().is_empty() { - return; - } - let depth = self.depths[index]; - let Some(opener) = (0..index) - .rev() - .find(|&previous| self.depths[previous] < depth) - else { - return; - }; - if self.tokens[opener] == OPEN_PARENTHESIS && !self.spans_lines(opener, index - 1) { - self.gaps[index] = SINGLE_SPACE.to_string(); - } - } - - /// Start every row of a VALUES list on its own line unless the whole list fits on its line. - fn break_values_rows_unless_fitting(&mut self, values: usize, line_width: usize) { - let rows = self.values_rows(values); - let Some(&(_, last_closer)) = rows.last() else { - return; - }; - let (first, last) = self.line_of(values); - let fits = !self.spans_lines(values, last_closer) - && self - .line(first, last) - .end_columns - .is_some_and(|ends| ends.last().is_some_and(|end| *end <= line_width)); - if fits { - return; - } - let mut indentation = self.indentation(first); - if self.gaps[values].contains('\n') { - indentation = self.indentation(values); - } else if values > 0 - && self.tokens[values - 1] == OPEN_PARENTHESIS - && let Some(closer) = self.partners[values - 1] - { - self.set_break(values, &newline(indentation + self.indent)); - self.set_break(closer, &newline(indentation)); - indentation += self.indent; - } - let row_line = newline(indentation + self.indent); - for (opener, _) in rows { - self.set_break(opener, &row_line); - } - } - - /// Separate the CTEs of the WITH at `with`, and the statement after them, by one blank line. - fn separate_common_table_expressions(&mut self, with: usize) { - let (first, _) = self.line_of(with); - let indentation = self.indentation(first); - let mut item = with + 1; - if self.is(item, "RECURSIVE") { - item += 1; - } - while let Some(end) = self.common_table_expression_end(item) { - let next = end + 1; - if self.is(next, ",") { - self.blank_line_before(next + 1, indentation); - item = next + 1; - continue; - } - if next < self.len() && !self.is(next, ";") && !self.is(next, ")") { - self.blank_line_before(next, indentation); - } - break; - } - } - - /// The last token of the CTE, or CTE macro placeholder, that starts at `item`. - fn common_table_expression_end(&self, item: usize) -> Option { - let name = self.tokens.get(item)?; - if name - .to_ascii_lowercase() - .starts_with(crate::constants::INTERPOLATION_PLACEHOLDER_PREFIX) - && !self.is(item + 1, "AS") - && !self.is(item + 1, OPEN_PARENTHESIS) - { - return Some(item); - } - if !matches!( - self.kinds[item], - TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier - ) && !name.chars().all(|c| c.is_alphanumeric() || c == '_') - { - return None; - } - let mut index = item + 1; - if self.is(index, OPEN_PARENTHESIS) { - index = self.partners[index]? + 1; - } - if !self.is(index, "AS") { - return None; - } - index += 1; - if self.is(index, "NOT") { - index += 1; - } - if self.is(index, "MATERIALIZED") { - index += 1; - } - if !self.is(index, OPEN_PARENTHESIS) { - return None; - } - self.partners[index] - } - - /// Replace the line break before `index` with exactly one blank line, keeping comments. - fn blank_line_before(&mut self, index: usize, indentation: usize) { - let gap = &self.gaps[index]; - let (trailing, rest) = gap.split_once('\n').unwrap_or((gap.as_str(), "")); - let rest = rest.trim_start(); - let margin = " ".repeat(indentation); - self.gaps[index] = if rest.is_empty() { - format!("{}\n\n{margin}", trailing.trim_end()) - } else { - format!("{}\n\n{margin}{rest}", trailing.trim_end()) - }; - } - - /// Move a clause body that is one line under its clause keyword onto the keyword line. - fn join_single_line_body(&mut self, clause: usize, line_width: usize) { - let (first, last) = self.line_of(clause); - let body = last + 1; - let is_select = self.is(clause, "SELECT"); - let modifiers_only = (clause + 1..=last).all(|index| { - SELECT_MODIFIERS.contains(&self.upper(index).as_str()) - || self.kinds[index] == TokenType::Number - || matches!(self.tokens[index].as_str(), "(" | ")" | ",") - || self.depths[index] > self.depths[clause] - }); - if body >= self.len() - || clause != first - || (!is_select && clause != last) - || (is_select && !modifiers_only) - || !self.gaps[body].contains('\n') - || !self.gaps[body].trim().is_empty() - { - return; - } - let clause_indentation = self.indentation(first); - if self.indentation(body) <= clause_indentation { - return; - } - let (_, body_last) = self.line_of(body); - if body_last + 1 < self.len() && self.indentation(body_last + 1) > clause_indentation { - return; - } - let pinned = (body + 1..=body_last).any(|index| !self.gaps[index].trim().is_empty()) - || (first..=body_last).any(|index| self.tokens[index].contains('\n')); - if pinned || (!is_select && self.has_top_level_chain(body, body_last)) { - return; - } - let mut width = clause_indentation; - for index in first..=body_last { - if index > first { - width += if index == body { - 1 - } else { - self.gaps[index].chars().count() - }; - } - width += self.width(index); - } - width += self.gaps[body_last + 1] - .split('\n') - .next() - .unwrap_or_default() - .chars() - .count(); - if width <= line_width { - self.gaps[body] = SINGLE_SPACE.to_string(); - } - } - - fn has_top_level_chain(&self, first: usize, last: usize) -> bool { - let depth = (first..=last).map(|index| self.depths[index]).min(); - (first..=last).any(|index| { - Some(self.depths[index]) == depth - && CHAIN_KEYWORDS.contains(&self.upper(index).as_str()) - && !(self.is(index, "AND") && self.follows_between(index)) - }) - } - - /// Whether the AND at `index` separates the bounds of a BETWEEN. - fn follows_between(&self, index: usize) -> bool { - let depth = self.depths[index]; - for previous in (0..index).rev() { - if self.depths[previous] < depth { - return false; - } - if self.depths[previous] > depth { - continue; - } - let upper = self.upper(previous); - if upper == BETWEEN_KEYWORD { - return true; - } - if BETWEEN_BOUNDARIES.contains(&upper.as_str()) { - return false; - } - } - false - } - - fn ends_operand(&self, index: usize) -> bool { - matches!(self.tokens[index].as_str(), ")" | "]") - || self.is(index, "END") - || matches!( - self.kinds[index], - TokenType::Var - | TokenType::Identifier - | TokenType::QuotedIdentifier - | TokenType::Number - | TokenType::String - ) - } - - /// The first token of a bracket group or CASE expression that ends at `last`. - fn left_operand_start(&self, last: usize) -> Option { - if matches!(self.tokens[last].as_str(), ")" | "]") { - return self.partners[last]; - } - if !self.is(last, "END") { - return None; - } - let depth = self.depths[last]; - let mut nested = 0_usize; - for previous in (0..last).rev() { - if self.depths[previous] != depth { - continue; - } - if self.is(previous, "END") { - nested += 1; - } else if self.is(previous, "CASE") { - if nested == 0 { - return Some(previous); - } - nested -= 1; - } - } - None - } - - /// The last token of a bracket group, call, or CASE expression that starts at `first`. - fn right_operand_end(&self, first: usize) -> Option { - if self.tokens[first] == OPEN_PARENTHESIS { - return self.partners[first]; - } - let word = self.tokens[first] - .chars() - .all(|character| character.is_alphanumeric() || character == '_'); - if word && self.is(first + 1, OPEN_PARENTHESIS) { - return self.partners[first + 1]; - } - if !self.is(first, CASE_KEYWORD) { - return None; - } - self.case_end(first) - } -} - -/// Line width and the printed widths of tokens that stand in for longer authored text. -pub(crate) struct WrapOptions<'a> { - pub(crate) line_width: usize, - pub(crate) token_widths: &'a HashMap, -} - -/// Re-break every line of `printed` that is longer than the line width and can be shortened. -pub(crate) fn wrap_lines(printed: &mut PrintedSql<'_>, line_width: usize) { - printed.lock_unbreakable_groups(); - let mut pending: Vec<(usize, usize)> = printed.line_ranges(0, printed.len() - 1); - pending.reverse(); - while let Some((first, last)) = pending.pop() { - let line = printed.line(first, last); - if !overflows(&line, line_width) || !printed.break_line(&line) { - continue; - } - let mut lines = printed.line_ranges(first, last); - lines.reverse(); - pending.extend(lines); - } -} - -fn overflows(line: &Line, line_width: usize) -> bool { - line.end_columns - .as_ref() - .is_some_and(|ends| ends.last().is_some_and(|end| *end > line_width)) -} - -impl PrintedSql<'_> { - /// Lock the inside of VALUES rows and of type parameter lists such as `NUMBER(38, 2)`. - fn lock_unbreakable_groups(&mut self) { - for index in 0..self.len() { - if self.is(index, VALUES_KEYWORD) { - for (opener, closer) in self.values_rows(index) { - self.lock(opener, closer); - } - } else if let Some(closer) = self.type_parameters_closer(index) { - self.lock(index - 1, closer); - } - } - } - - fn lock(&mut self, first: usize, last: usize) { - for gap in first + 1..=last { - self.locked[gap] = true; - } - } - - /// The bracketed rows that directly follow the VALUES keyword at `values`. - fn values_rows(&self, values: usize) -> Vec<(usize, usize)> { - let mut rows: Vec<(usize, usize)> = Vec::new(); - let mut opener = values + 1; - while opener < self.len() && self.tokens[opener] == OPEN_PARENTHESIS { - let Some(closer) = self.partners[opener] else { - break; - }; - rows.push((opener, closer)); - if !(self.is(closer + 1, COMMA) && self.is(closer + 2, OPEN_PARENTHESIS)) { - break; - } - opener = closer + 2; - } - rows - } - - /// The closer of a type parameter list opened at `opener`, such as `(38, 2)` of `NUMBER`. - fn type_parameters_closer(&self, opener: usize) -> Option { - if opener < 1 || self.tokens[opener] != OPEN_PARENTHESIS { - return None; - } - let closer = self.partners[opener]?; - let name = &self.tokens[opener - 1]; - if closer == opener + 1 || !name.chars().all(|c| c.is_alphanumeric() || c == '_') { - return None; - } - let type_position = opener > 1 && (self.is(opener - 2, "::") || self.is(opener - 2, "AS")); - if !type_position && !TYPE_NAMES.contains(&name.to_ascii_uppercase().as_str()) { - return None; - } - let parameters = (opener + 1..closer).all(|index| { - let token = &self.tokens[index]; - token == COMMA - || is_digits(token) - || TYPE_PARAMETER_WORDS.contains(&token.to_ascii_uppercase().as_str()) - }); - parameters.then_some(closer) - } - - /// Apply the outermost applicable break: chains, VALUES rows, bracket groups, then commas. - fn break_line(&mut self, line: &Line) -> bool { - let values_row = self.tokens[line.first] == OPEN_PARENTHESIS && self.locked[line.first + 1]; - !values_row - && (self.break_before(line, &CHAIN_KEYWORDS, 0) - || self.break_before(line, &ARITHMETIC_OPERATORS, self.indent) - || self.break_values_rows(line) - || self.explode_group(line) - || self.break_after_commas(line)) - } - - /// Put each row of a VALUES list that starts this line on its own line. - fn break_values_rows(&mut self, line: &Line) -> bool { - let depth = self.line_depth(line); - let Some(values) = (line.first..=line.last).find(|&index| { - self.is(index, VALUES_KEYWORD) - && (self.depths[index] == depth || index == line.first) - && !self.locked[index + 1] - }) else { - return false; - }; - let rows = self.values_rows(values); - if rows.is_empty() - || rows - .iter() - .all(|(opener, _)| self.gaps[*opener].contains('\n')) - { - return false; - } - let row_line = newline(line.indentation + self.indent); - for (opener, _) in rows { - self.gaps[opener] = row_line.clone(); - } - true - } - - /// Put the arguments or CASE branches of the outermost, widest group on their own lines. - fn explode_group(&mut self, line: &Line) -> bool { - let Some(ends) = line.end_columns.as_ref() else { - return false; - }; - let start_column = |index: usize| { - if index == line.first { - line.indentation - } else { - ends[index - 1 - line.first] - } - }; - let line_end = ends[ends.len() - 1]; - let mut chosen: Option<(usize, usize, usize)> = None; - for opener in line.first..=line.last { - let Some(closer) = self.group_closer(opener, line) else { - continue; - }; - let end_column = if closer > line.last { - line_end - } else { - ends[closer - line.first] - }; - let width = end_column - start_column(opener); - let better = chosen.is_none_or(|(best, _, best_width)| { - self.depths[opener] < self.depths[best] - || (self.depths[opener] == self.depths[best] && width >= best_width) - }); - if better { - chosen = Some((opener, closer, width)); - } - } - let Some((opener, closer, _)) = chosen else { - return false; - }; - if closer > line.last { - self.indent_lines(opener + 1, closer - 1, self.indent); - } - let inner = newline(line.indentation + self.indent); - if self.is(opener, CASE_KEYWORD) { - self.explode_case(opener, closer, line.indentation); - return true; - } - self.set_break(opener + 1, &inner); - self.set_break(closer, &newline(line.indentation)); - if self.is(opener + 1, VALUES_KEYWORD) { - return true; - } - let window = opener > 0 && self.is(opener - 1, "OVER"); - let decode = opener > 0 && self.is(opener - 1, DECODE_FUNCTION); - let mut argument = 0_usize; - for index in opener + 2..closer { - if self.depths[index] != self.depths[opener] + 1 { - continue; - } - if window { - if WINDOW_CLAUSE_KEYWORDS.contains(&self.upper(index).as_str()) { - self.set_break(index, &inner); - } - } else if self.tokens[index - 1] == COMMA { - argument += 1; - if !decode || !argument.is_multiple_of(2) { - self.set_break(index, &inner); - } - } - } - true - } - - /// The closer of an explodable group opened at `opener` on `line`. - fn group_closer(&self, opener: usize, line: &Line) -> Option { - if self.is(opener, CASE_KEYWORD) { - let end = self.case_end(opener)?; - return (end <= line.last && end > opener + 2).then_some(end); - } - let closer = self.partners[opener]?; - let explodable = closer > opener + 2 - && matches!(self.tokens[opener].as_str(), "(" | "[" | "{") - && !self.locked[opener + 1] - && (closer <= line.last || opener < line.last); - explodable.then_some(closer) - } - - /// The END of the CASE expression at `case`. - fn case_end(&self, case: usize) -> Option { - let depth = self.depths[case]; - let mut nested = 0_usize; - for index in case + 1..self.len() { - if self.depths[index] < depth { - return None; - } - if self.depths[index] != depth { - continue; - } - if self.is(index, CASE_KEYWORD) { - nested += 1; - } else if self.is(index, "END") { - if nested == 0 { - return Some(index); - } - nested -= 1; - } - } - None - } - - /// Put each WHEN, THEN and ELSE of a one-line CASE on its own line and END under CASE. - fn explode_case(&mut self, case: usize, end: usize, indentation: usize) { - let inner = newline(indentation + self.indent); - let depth = self.depths[case]; - let mut nested = 0_usize; - for index in case + 1..end { - if self.depths[index] != depth { - continue; - } - if self.is(index, CASE_KEYWORD) { - nested += 1; - } else if self.is(index, "END") { - nested -= 1; - } else if nested == 0 && CASE_BRANCH_KEYWORDS.contains(&self.upper(index).as_str()) { - self.set_break(index, &inner); - } - } - self.set_break(end, &newline(indentation)); - } - - /// Replace a whitespace-only gap; comments keep their gaps. - fn set_break(&mut self, index: usize, gap: &str) { - if self.gaps[index].trim().is_empty() { - self.gaps[index] = gap.to_string(); - } - } - - /// Indent lines starting within `first..=last` by `extra`, except gaps with block comments. - fn indent_lines(&mut self, first: usize, last: usize, extra: usize) { - let margin = format!("\n{}", " ".repeat(extra)); - for index in first..=last { - if self.gaps[index].contains('\n') && !self.gaps[index].contains("/*") { - self.gaps[index] = self.gaps[index].replace('\n', &margin); - } - } - } - - /// Break before every operator of `operators` at the line's outermost depth. - fn break_before(&mut self, line: &Line, operators: &[&str], extra_indent: usize) -> bool { - let depth = self.line_depth(line); - let mut between = false; - let mut breaks: Vec = Vec::new(); - for index in line.first..=line.last { - if self.depths[index] != depth || self.locked[index] { - continue; - } - let upper = self.upper(index); - if upper == BETWEEN_KEYWORD { - between = true; - continue; - } - if upper == BETWEEN_SEPARATOR && between { - between = false; - continue; - } - if index > line.first - && operators.contains(&upper.as_str()) - && self.gaps[index] == SINGLE_SPACE - && ends_operand(&self.tokens[index - 1]) - { - breaks.push(index); - } - } - let indentation = newline(line.indentation + extra_indent); - for index in &breaks { - self.gaps[*index] = indentation.clone(); - } - !breaks.is_empty() - } - - /// Break after every comma at the outermost comma depth of the line. - fn break_after_commas(&mut self, line: &Line) -> bool { - let Some(depth) = (line.first..line.last) - .filter(|&index| self.tokens[index] == COMMA && !self.locked[index + 1]) - .map(|index| self.depths[index]) - .min() - else { - return false; - }; - let indentation = newline(line.indentation + self.indent); - let mut changed = false; - for index in line.first..line.last { - if self.depths[index] == depth && self.tokens[index] == COMMA && !self.locked[index + 1] - { - self.gaps[index + 1] = indentation.clone(); - changed = true; - } - } - changed - } - - fn line_depth(&self, line: &Line) -> usize { - (line.first..=line.last) - .map(|index| self.depths[index]) - .min() - .unwrap_or_default() - } -} - -fn is_digits(token: &str) -> bool { - token.chars().all(|character| character.is_ascii_digit()) -} - -fn ends_operand(token: &str) -> bool { - token.chars().last().is_some_and(|character| { - character.is_alphanumeric() || matches!(character, '_' | '\'' | '"' | '`' | ')' | ']') - }) -} diff --git a/crates/sqlclean/src/sql_lint/_helpers/token_layout.rs b/crates/sqlclean/src/sql_lint/_helpers/token_layout.rs deleted file mode 100644 index 9c3ca01..0000000 --- a/crates/sqlclean/src/sql_lint/_helpers/token_layout.rs +++ /dev/null @@ -1,815 +0,0 @@ -//! Print authored SQL tokens with whitespace taken from a parse-tree layout oracle. - -use std::collections::{BTreeMap, BTreeSet}; -use std::ops::Range; - -use polyglot_sql::Dialect; -use polyglot_sql::tokens::{Token, TokenType}; - -use crate::sql_lint::constants::{CLOSE_PARENTHESIS, OPEN_PARENTHESIS}; -use crate::sql_tokens::main::canonical_tokens::canonical_tokens; -use crate::sql_tokens::main::case_folding::foldable_tokens; -use crate::sql_tokens::main::is_unquoted_word::is_unquoted_word; -use crate::sql_tokens::main::name_positions::name_positions; -use crate::sql_tokens::main::token_texts::token_texts; - -const TOKEN_PRESERVATION_FAILURE: &str = - "native formatter would change authored SQL tokens, not only layout"; -const COMMENT_ATTACHMENT_FAILURE: &str = - "native formatter could not preserve comment token attachments"; -const STRING_BREAK_FAILURE: &str = - "native formatter would join adjacent string literals across a line break"; -const ALIGNMENT_FAILURE: &str = "native formatter could not align its layout with authored tokens"; -const RESYNC_WINDOW: usize = 48; -const RESYNC_CONFIRMATION: usize = 1; -const NO_SPACE_BEFORE: [&str; 7] = [",", ")", "]", ".", ";", "::", ":"]; -const NO_SPACE_AFTER: [&str; 5] = ["(", "[", ".", "::", ":"]; -const STRUCTURAL_PUNCTUATION: [&str; 6] = ["(", ")", "[", "]", ",", ";"]; - -#[derive(Debug, Clone)] -pub(crate) struct Comment { - pub(crate) start: usize, - pub(crate) end: usize, - pub(crate) text: String, - pub(crate) line: bool, - pub(crate) leading: bool, -} - -/// One authored or oracle token with its raw source text and alignment key. -struct Lexeme { - start: usize, - end: usize, - raw: String, - key: LexemeKey, - /// In the reserved-keyword and built-in fold set shared with the change fingerprint. - foldable: bool, - /// An unquoted authored word outside the name positions that keep reserved words exact. - recasable: bool, - /// An unquoted word the dialect tokenizer types as a keyword. - keyword: bool, -} - -/// How lexemes compare: words ignoring ASCII case, punctuation by type, the rest by raw text. -#[derive(Clone, Copy, PartialEq, Eq)] -enum LexemeKey { - Word, - Punctuation(TokenType), - Raw, -} - -impl Lexeme { - fn same_key(&self, other: &Lexeme) -> bool { - self.key == other.key - && match self.key { - LexemeKey::Word => self.raw.eq_ignore_ascii_case(&other.raw), - LexemeKey::Punctuation(_) => true, - LexemeKey::Raw => self.raw == other.raw, - } - } -} - -/// Authored tokens printed against an oracle, with what the token check needs about them. -pub(crate) struct AuthoredPrint { - pub(crate) sql: String, - /// The byte range and type of each printed token; comments and whitespace lie between them. - pub(crate) tokens: Vec<(Range, TokenType)>, - pub(crate) recases: BTreeMap, - pub(crate) texts: TokenTexts, -} - -/// The source text and case-fold flag of every token, read once for the token check. -pub(crate) struct TokenTexts { - texts: Vec, - foldable: Vec, -} - -impl TokenTexts { - #[cfg(test)] - pub(crate) fn read(sql: &str, tokens: &[Token], dialect: &Dialect) -> Result { - let texts = token_texts(sql, tokens)?; - let foldable = foldable_tokens(&texts, tokens, dialect); - Ok(Self { texts, foldable }) - } - - /// Whether token `index` here, upper-cased when foldable, spells `canonical`. - fn same_canonical(&self, index: usize, canonical: &str) -> bool { - let (text, fold) = (&self.texts[index], self.foldable[index]); - text.len() == canonical.len() - && text.bytes().zip(canonical.bytes()).all(|(byte, expected)| { - expected - == if fold { - byte.to_ascii_uppercase() - } else { - byte - } - }) - } -} - -/// An authored token the printer recased to the layout guide's keyword spelling. -pub(crate) struct KeywordRecase { - pub(crate) generated: String, - pub(crate) keyword: bool, -} - -/// Comments attached to the authored token they precede or follow. -#[derive(Default)] -struct Attached<'a> { - leading: Vec<&'a Comment>, - trailing: Vec<&'a Comment>, -} - -pub(crate) fn comments_in(sql: &str, tokens: &[Token]) -> Vec { - let characters: Vec = sql.chars().collect(); - let mut comments: Vec = Vec::new(); - let mut gap_start = 0_usize; - for token in tokens { - comments.extend(comments_in_gap(&characters, gap_start, token.span.start)); - gap_start = token.span.end; - } - comments.extend(comments_in_gap(&characters, gap_start, characters.len())); - comments -} - -/// Split the non-whitespace text the tokenizer skipped between two tokens into comments. -fn comments_in_gap(characters: &[char], start: usize, end: usize) -> Vec { - let mut comments: Vec = Vec::new(); - let mut index = start; - while index < end { - if characters[index].is_whitespace() { - index += 1; - continue; - } - let start = index; - let line_start = (0..start) - .rev() - .find(|&position| characters[position] == '\n') - .map_or(0, |position| position + 1); - let leading = characters[line_start..start] - .iter() - .all(|character| character.is_whitespace()); - let block = characters[index] == '/' && characters.get(index + 1) == Some(&'*'); - index = if block { - block_comment_end(characters, start, end) - } else { - (start..end) - .find(|&position| characters[position] == '\n') - .unwrap_or(end) - }; - comments.push(Comment { - start, - end: index, - text: characters[start..index].iter().collect(), - line: !block, - leading, - }); - } - comments -} - -/// End of a block comment: nested comments when they close inside the gap, else the first `*/`. -fn block_comment_end(characters: &[char], start: usize, end: usize) -> usize { - let closes_at = |position: usize| { - position + 1 < end && characters[position] == '*' && characters[position + 1] == '/' - }; - let opens_at = |position: usize| { - position + 1 < end && characters[position] == '/' && characters[position + 1] == '*' - }; - let mut depth = 0_usize; - let mut position = start; - while position < end { - if opens_at(position) { - depth += 1; - position += 2; - } else if closes_at(position) { - depth -= 1; - position += 2; - if depth == 0 { - return position; - } - } else { - position += 1; - } - } - (start + 2..end) - .find(|&position| closes_at(position)) - .map_or(end, |position| position + 2) -} - -pub(crate) fn neutralize_comments(sql: &str, comments: &[Comment]) -> String { - let mut characters: Vec = sql.chars().collect(); - for comment in comments { - for character in &mut characters[comment.start..comment.end] { - if *character != '\n' && *character != '\r' { - *character = ' '; - } - } - } - characters.into_iter().collect() -} - -/// Authored SQL text with its dialect tokens and the comments between them. -pub(crate) struct AuthoredSql<'a> { - pub(crate) sql: &'a str, - pub(crate) tokens: &'a [Token], - pub(crate) comments: &'a [Comment], -} - -/// The lexemes and characters of a layout oracle, read once for every print against it. -pub(crate) struct OracleLayout { - generated: Vec, - characters: Vec, -} - -/// Tokenize `oracle` for printing against it. -pub(crate) fn oracle_layout(oracle: &str, dialect: &Dialect) -> Result { - let oracle_tokens = dialect - .tokenize(oracle) - .map_err(|error| error.to_string())?; - Ok(OracleLayout { - generated: lexemes(oracle, &oracle_tokens, None)?, - characters: oracle.chars().collect(), - }) -} - -/// Print every authored token in order, taking only whitespace and keyword case from `oracle`. -pub(crate) fn print_authored_tokens( - source: &AuthoredSql<'_>, - oracle: &OracleLayout, - dialect: &Dialect, -) -> Result { - let AuthoredSql { - sql, - tokens, - comments, - } = *source; - let authored = lexemes(sql, tokens, Some(dialect))?; - let generated = &oracle.generated; - let alignment = align(&authored, generated)?; - let separators = separators(&authored, generated, &alignment, &oracle.characters)?; - let attached = attach_comments(tokens, comments); - let mut output = String::with_capacity(sql.len() + sql.len() / 4); - let string_breaks = string_breaks(sql, tokens); - let mut recases: BTreeMap = BTreeMap::new(); - let mut printed_tokens: Vec<(Range, TokenType)> = Vec::with_capacity(tokens.len()); - for (index, lexeme) in authored.iter().enumerate() { - let forced_break = string_breaks.contains(&index) && !separators[index].contains('\n'); - let forced: String; - let separator: &str = if forced_break { - forced = format!("\n{}", current_indentation(&output)); - &forced - } else { - &separators[index] - }; - let leading = &attached[index].leading; - if leading.is_empty() { - output.push_str(separator); - } else if index == 0 || separator.contains('\n') { - output.push_str(separator); - let indentation = current_indentation(&output); - for comment in leading { - output.push_str(&comment.text); - output.push('\n'); - output.push_str(&indentation); - } - } else { - let indentation = current_indentation(&output); - for comment in leading { - output.push('\n'); - output.push_str(&indentation); - output.push_str(&comment.text); - } - output.push('\n'); - output.push_str(&indentation); - } - let guide = alignment[index].map(|at| &generated[at]); - if let Some(guide) = guide.filter(|guide| confirms_keyword(lexeme, guide)) { - recases.insert( - index, - KeywordRecase { - generated: guide.raw.clone(), - keyword: guide.keyword, - }, - ); - } - let start = output.len(); - output.push_str(&printed_text(lexeme, guide)); - printed_tokens.push((start..output.len(), tokens[index].token_type)); - let trailing = &attached[index].trailing; - for (position, comment) in trailing.iter().enumerate() { - output.push(' '); - output.push_str(&comment.text); - if !comment.line { - continue; - } - let next_starts_line = if position + 1 < trailing.len() { - false - } else { - separators - .get(index + 1) - .is_none_or(|next| next.starts_with('\n')) - }; - if !next_starts_line { - output.push('\n'); - if position + 1 < trailing.len() { - output.push_str(¤t_indentation(&output)); - } - } - } - } - let mut texts = TokenTexts { - texts: Vec::with_capacity(authored.len()), - foldable: Vec::with_capacity(authored.len()), - }; - for lexeme in authored { - texts.texts.push(lexeme.raw); - texts.foldable.push(lexeme.foldable); - } - Ok(AuthoredPrint { - sql: output, - tokens: printed_tokens, - recases, - texts, - }) -} - -/// Refuse output that changes anything but layout, fold-set case, and recorded keyword recases. -pub(crate) fn verify_printed_tokens( - (before, before_tokens, before_texts): (&str, &[Token], &TokenTexts), - (after, after_tokens): (&str, &[Token]), - dialect: &Dialect, - recases: &BTreeMap, -) -> Result<(), String> { - let after_canonical = canonical_tokens(after, after_tokens, dialect)?; - if before_texts.texts.len() != after_canonical.len() { - return Err(TOKEN_PRESERVATION_FAILURE.to_string()); - } - let mut after_texts: Option> = None; - for (index, canonical) in after_canonical.iter().enumerate() { - if before_texts.same_canonical(index, &canonical.text) { - continue; - } - let Some(recase) = recases.get(&index) else { - return Err(TOKEN_PRESERVATION_FAILURE.to_string()); - }; - let texts: &Vec = match after_texts { - Some(ref texts) => texts, - None => after_texts.insert(token_texts(after, after_tokens)?), - }; - let printed: &str = &texts[index]; - if !recase.allows(&before_texts.texts[index], printed) { - return Err(TOKEN_PRESERVATION_FAILURE.to_string()); - } - } - let attached = comments_stay_attached( - &comment_positions(before, before_tokens), - &comment_positions(after, after_tokens), - after_tokens, - ); - if !attached { - return Err(COMMENT_ATTACHMENT_FAILURE.to_string()); - } - if !string_breaks(before, before_tokens).is_subset(&string_breaks(after, after_tokens)) { - return Err(STRING_BREAK_FAILURE.to_string()); - } - Ok(()) -} - -impl KeywordRecase { - /// Whether `authored` became `printed` only by taking the guide's upper-case keyword spelling. - fn allows(&self, authored: &str, printed: &str) -> bool { - self.keyword - && printed == self.generated - && printed == printed.to_ascii_uppercase() - && authored.eq_ignore_ascii_case(printed) - } -} - -pub(crate) fn tokenize(sql: &str, dialect: &Dialect) -> Result, String> { - dialect.tokenize(sql).map_err(|error| error.to_string()) -} - -/// Whether every comment keeps its place, allowing a line comment to move just past a comma. -fn comments_stay_attached( - before: &[(String, usize)], - after: &[(String, usize)], - after_tokens: &[Token], -) -> bool { - before.len() == after.len() - && before.iter().zip(after).all( - |((before_text, before_position), (after_text, after_position))| { - before_text == after_text - && (before_position == after_position - || (*after_position == before_position + 1 - && before_text.starts_with("--") - && after_tokens - .get(*before_position) - .is_some_and(|token| token.token_type == TokenType::Comma))) - }, - ) -} - -fn comment_positions(sql: &str, tokens: &[Token]) -> Vec<(String, usize)> { - let mut positions: Vec<(String, usize)> = Vec::new(); - for comment in comments_in(sql, tokens) { - let preceding = tokens.partition_point(|token| token.span.end <= comment.start); - positions.push((comment.text, preceding)); - } - positions -} - -/// Lexemes of `sql`; only authored lexemes carry the recase classification. -fn lexemes(sql: &str, tokens: &[Token], dialect: Option<&Dialect>) -> Result, String> { - let raws = token_texts(sql, tokens)?; - let (foldable, recasable) = match dialect { - Some(dialect) => { - let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); - let names = name_positions(&uppers); - let foldable = foldable_tokens(&raws, tokens, dialect); - let recasable: Vec = raws - .iter() - .zip(names) - .map(|(raw, name)| !name && is_unquoted_word(raw)) - .collect(); - (foldable, recasable) - } - None => (vec![false; tokens.len()], vec![false; tokens.len()]), - }; - let mut lexemes = Vec::with_capacity(tokens.len()); - for (index, (raw, token)) in raws.into_iter().zip(tokens).enumerate() { - lexemes.push(lexeme(raw, token, foldable[index], recasable[index])); - } - Ok(lexemes) -} - -fn lexeme(raw: String, token: &Token, foldable: bool, recasable: bool) -> Lexeme { - let word = is_unquoted_word(&raw); - let key = if word { - LexemeKey::Word - } else if raw - .chars() - .all(|character| character.is_ascii_punctuation()) - { - LexemeKey::Punctuation(token.token_type) - } else { - LexemeKey::Raw - }; - let keyword = word - && !matches!( - token.token_type, - TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier - ); - Lexeme { - start: token.span.start, - end: token.span.end, - raw, - key, - foldable, - recasable, - keyword, - } -} - -fn is_string_literal(token: &Token) -> bool { - matches!( - token.token_type, - TokenType::String - | TokenType::NationalString - | TokenType::EscapeString - | TokenType::RawString - | TokenType::ByteString - | TokenType::UnicodeString - | TokenType::HexString - | TokenType::BitString - | TokenType::DollarString - | TokenType::TripleSingleQuotedString - | TokenType::TripleDoubleQuotedString - ) -} - -/// Indices of string literals that follow another string literal across a line break. -fn string_breaks(sql: &str, tokens: &[Token]) -> BTreeSet { - let characters: Vec = sql.chars().collect(); - (1..tokens.len()) - .filter(|&index| { - is_string_literal(&tokens[index - 1]) - && is_string_literal(&tokens[index]) - && characters[tokens[index - 1].span.end..tokens[index].span.start].contains(&'\n') - }) - .collect() -} - -/// Map authored tokens to matching oracle tokens; a closer only matches its opener's partner. -fn align(authored: &[Lexeme], generated: &[Lexeme]) -> Result>, String> { - let mut state = Alignment { - authored, - generated, - authored_partners: bracket_partners(authored), - generated_partners: bracket_partners(generated), - forward: vec![None; authored.len()], - backward: vec![None; generated.len()], - }; - let (mut left, mut right) = (0_usize, 0_usize); - while left < authored.len() && right < generated.len() { - if state.matches(left, right) { - state.forward[left] = Some(right); - state.backward[right] = Some(left); - left += 1; - right += 1; - continue; - } - let (skip_left, skip_right) = state - .resync(left, right) - .ok_or_else(|| ALIGNMENT_FAILURE.to_string())?; - left += skip_left; - right += skip_right; - } - Ok(state.forward) -} - -struct Alignment<'a> { - authored: &'a [Lexeme], - generated: &'a [Lexeme], - authored_partners: Vec>, - generated_partners: Vec>, - forward: Vec>, - backward: Vec>, -} - -impl Alignment<'_> { - fn matches(&self, left: usize, right: usize) -> bool { - if !self.authored[left].same_key(&self.generated[right]) { - return false; - } - if !is_closer(&self.authored[left].raw) { - return true; - } - match (self.authored_partners[left], self.generated_partners[right]) { - (Some(opener), Some(generated_opener)) => match self.forward[opener] { - Some(aligned) => aligned == generated_opener, - None => self.backward[generated_opener].is_none(), - }, - (None, None) => true, - _ => false, - } - } - - fn resync(&self, left: usize, right: usize) -> Option<(usize, usize)> { - let remaining_left = self.authored.len() - left; - let remaining_right = self.generated.len() - right; - for cost in 1..=(2 * RESYNC_WINDOW) { - for skip_left in 0..=cost.min(RESYNC_WINDOW) { - let skip_right = cost - skip_left; - if skip_right > RESYNC_WINDOW { - continue; - } - if skip_left >= remaining_left || skip_right >= remaining_right { - if skip_left >= remaining_left && skip_right >= remaining_right { - return Some((remaining_left, remaining_right)); - } - continue; - } - if !self.matches(left + skip_left, right + skip_right) { - continue; - } - let anchor = &self.authored[left + skip_left]; - let confirmations = if matches!(anchor.key, LexemeKey::Punctuation(_)) { - RESYNC_CONFIRMATION - } else { - 0 - }; - let confirmed = (1..=confirmations).all(|offset| { - match ( - self.authored.get(left + skip_left + offset), - self.generated.get(right + skip_right + offset), - ) { - (Some(before), Some(after)) => before.same_key(after), - _ => true, - } - }); - if confirmed { - return Some((skip_left, skip_right)); - } - } - } - None - } -} - -fn is_closer(raw: &str) -> bool { - matches!(raw, ")" | "]" | "}") -} - -fn bracket_partners(lexemes: &[Lexeme]) -> Vec> { - let mut partners: Vec> = vec![None; lexemes.len()]; - let mut open: Vec = Vec::new(); - for (index, lexeme) in lexemes.iter().enumerate() { - match lexeme.raw.as_str() { - "(" | "[" | "{" => open.push(index), - ")" | "]" | "}" => { - if let Some(opener) = open.pop() { - partners[opener] = Some(index); - partners[index] = Some(opener); - } - } - _ => {} - } - } - partners -} - -fn separators( - authored: &[Lexeme], - generated: &[Lexeme], - alignment: &[Option], - oracle: &[char], -) -> Result, String> { - let mut separators: Vec = vec![String::new(); authored.len()]; - let anchors: Vec = (0..authored.len()) - .filter(|&index| alignment[index].is_some()) - .collect(); - if anchors.is_empty() { - return Err(ALIGNMENT_FAILURE.to_string()); - } - let gap = |left: usize, right: usize| -> String { - oracle[generated[left].end..generated[right].start] - .iter() - .collect() - }; - for index in 1..anchors[0] { - separators[index] = default_space(&authored[index - 1], &authored[index]).to_string(); - } - if anchors[0] > 0 { - separators[anchors[0]] = - default_space(&authored[anchors[0] - 1], &authored[anchors[0]]).to_string(); - } - for pair in anchors.windows(2) { - let (previous, next) = (pair[0], pair[1]); - let (previous_at, next_at) = ( - alignment[previous].ok_or(ALIGNMENT_FAILURE)?, - alignment[next].ok_or(ALIGNMENT_FAILURE)?, - ); - let merged = merged_gap( - &(previous_at..next_at) - .map(|at| gap(at, at + 1)) - .collect::>(), - ); - if previous + 1 == next { - separators[next] = merged; - continue; - } - for index in previous + 2..next { - separators[index] = default_space(&authored[index - 1], &authored[index]).to_string(); - } - let first = previous + 1; - let last = next - 1; - let substitution = next_at > previous_at + 1; - separators[first] = if default_space(&authored[previous], &authored[first]).is_empty() { - String::new() - } else if substitution { - gap(previous_at, previous_at + 1) - } else { - " ".to_string() - }; - separators[next] = if substitution { - let oracle_gap = (previous_at + 1..next_at) - .rev() - .find(|&at| generated[at].same_key(&authored[last])) - .map_or_else(|| gap(next_at - 1, next_at), |at| gap(at, at + 1)); - let unspaced = default_space(&authored[last], &authored[next]).is_empty(); - if unspaced && !oracle_gap.contains('\n') { - String::new() - } else { - oracle_gap - } - } else if merged.contains('\n') { - merged - } else { - default_space(&authored[last], &authored[next]).to_string() - }; - } - let last_anchor = anchors[anchors.len() - 1]; - for index in last_anchor + 1..authored.len() { - separators[index] = default_space(&authored[index - 1], &authored[index]).to_string(); - } - let last_at = alignment[last_anchor].ok_or(ALIGNMENT_FAILURE)?; - if last_anchor + 1 < authored.len() - && last_at + 1 < generated.len() - && !separators[last_anchor + 1].is_empty() - { - separators[last_anchor + 1] = gap(last_at, last_at + 1); - } - for index in 1..authored.len() { - let adjacent_in_source = authored[index - 1].end == authored[index].start; - let oracle_pair = matches!( - (alignment[index - 1], alignment[index]), - (Some(previous), Some(next)) if previous + 1 == next - ); - let structural = [&authored[index - 1], &authored[index]] - .iter() - .any(|lexeme| STRUCTURAL_PUNCTUATION.contains(&lexeme.raw.as_str())); - let call = - is_unquoted_word(&authored[index - 1].raw) && authored[index].raw == OPEN_PARENTHESIS; - let inside_parentheses = - authored[index - 1].raw == OPEN_PARENTHESIS || authored[index].raw == CLOSE_PARENTHESIS; - if (adjacent_in_source && !oracle_pair && (!structural || call)) - || (inside_parentheses && !separators[index].contains('\n')) - { - separators[index].clear(); - } else if separators[index].is_empty() - && !adjacent_in_source - && would_fuse(&authored[index - 1], &authored[index]) - { - separators[index] = " ".to_string(); - } - } - Ok(separators) -} - -fn merged_gap(gaps: &[String]) -> String { - if let [only] = gaps { - return only.clone(); - } - if let Some(newline) = gaps.iter().rev().find(|gap| gap.contains('\n')) { - return newline.clone(); - } - if gaps.iter().any(|gap| !gap.is_empty()) { - " ".to_string() - } else { - String::new() - } -} - -fn default_space(previous: &Lexeme, next: &Lexeme) -> &'static str { - if NO_SPACE_BEFORE.contains(&next.raw.as_str()) - || NO_SPACE_AFTER.contains(&previous.raw.as_str()) - { - "" - } else { - " " - } -} - -/// Return whether printing two separated tokens without a space could lex them differently. -fn would_fuse(previous: &Lexeme, next: &Lexeme) -> bool { - let (Some(left), Some(right)) = (previous.raw.chars().last(), next.raw.chars().next()) else { - return false; - }; - let word = |character: char| { - character.is_alphanumeric() || matches!(character, '_' | '$' | '\'' | '"' | '`' | '@') - }; - let operator = |character: char| "+-*/<>=!|&^%~:#?".contains(character); - (word(left) && word(right)) || (operator(left) && operator(right)) -} - -/// The authored text, recased only for keywords and built-in function names. -fn printed_text(authored: &Lexeme, generated: Option<&Lexeme>) -> String { - if let Some(generated) = generated.filter(|generated| confirms_keyword(authored, generated)) { - return generated.raw.clone(); - } - if !authored.foldable { - return authored.raw.clone(); - } - match generated { - Some(generated) if generated.raw.eq_ignore_ascii_case(&authored.raw) => { - generated.raw.clone() - } - Some(_) => authored.raw.clone(), - None => authored.raw.to_ascii_uppercase(), - } -} - -/// Whether the layout guide prints the aligned authored word as an upper-case keyword. -fn confirms_keyword(authored: &Lexeme, generated: &Lexeme) -> bool { - !authored.foldable - && authored.recasable - && generated.keyword - && generated.raw != authored.raw - && generated.raw == generated.raw.to_ascii_uppercase() - && generated.raw.eq_ignore_ascii_case(&authored.raw) -} - -fn attach_comments<'a>(tokens: &[Token], comments: &'a [Comment]) -> Vec> { - let mut attached: Vec> = (0..tokens.len()).map(|_| Attached::default()).collect(); - for comment in comments { - let next = tokens.partition_point(|token| token.span.start < comment.end); - let preceding = tokens.partition_point(|token| token.span.end <= comment.start); - let next = (next < tokens.len()).then_some(next); - let previous = preceding.checked_sub(1); - match (comment.leading, next, previous) { - (true, Some(next), _) | (false, Some(next), None) => { - attached[next].leading.push(comment); - } - (_, _, Some(previous)) => attached[previous].trailing.push(comment), - (_, None, None) => {} - } - } - attached -} - -fn current_indentation(output: &str) -> String { - let line = output.rsplit('\n').next().unwrap_or_default(); - line.chars() - .take_while(|character| *character == ' ' || *character == '\t') - .collect() -} diff --git a/crates/sqlclean/src/sql_lint/constants.rs b/crates/sqlclean/src/sql_lint/constants.rs index 35bb2aa..27fc585 100644 --- a/crates/sqlclean/src/sql_lint/constants.rs +++ b/crates/sqlclean/src/sql_lint/constants.rs @@ -1,10 +1,4 @@ -pub(crate) const CAST_TYPE_SEPARATOR_KEYWORD: &str = "AS"; -pub(crate) const CLOSE_PARENTHESIS: &str = ")"; pub const LINT_API_VERSION: u32 = 1; -pub(crate) const OPEN_PARENTHESIS: &str = "("; -pub(crate) const POSITION_ARGUMENT_SEPARATOR: &str = ","; -pub(crate) const CTE_MACRO_PLACEHOLDER_LITERAL: &str = "1"; -pub(crate) const POSTFIX_CAST_PREFIX_TOKEN_COUNT: usize = 2; /// The dialect name SQLQuack lays out as Snowflake. pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake"; /// The dialect name SQLQuack lays out as DuckDB. diff --git a/crates/sqlclean/src/sql_lint/models.rs b/crates/sqlclean/src/sql_lint/models.rs index 2f27e50..9f133cd 100644 --- a/crates/sqlclean/src/sql_lint/models.rs +++ b/crates/sqlclean/src/sql_lint/models.rs @@ -27,4 +27,7 @@ pub struct FormatResponse { pub formatted: bool, #[serde(skip_serializing_if = "Option::is_none")] pub reason: Option, + /// Set, with the SQL unchanged, for a dialect SQLQuack does not format yet; not a refusal. + #[serde(skip_serializing_if = "Option::is_none")] + pub unavailable: Option, } diff --git a/crates/sqlclean/src/sql_lint/tests.rs b/crates/sqlclean/src/sql_lint/tests.rs index dde04c7..cc7e710 100644 --- a/crates/sqlclean/src/sql_lint/tests.rs +++ b/crates/sqlclean/src/sql_lint/tests.rs @@ -1,5 +1,3 @@ -#[path = "tests/test_authored_tokens.rs"] -mod authored_tokens; #[path = "tests/test_batch_lint.rs"] mod batch_lint; #[path = "tests/test_comment_formatter.rs"] @@ -14,8 +12,6 @@ mod format_corpus; mod function_call_spacing; #[path = "tests/helpers.rs"] mod helpers; -#[path = "tests/test_idempotency_reprint.rs"] -mod idempotency_reprint; #[path = "tests/test_layout_rules.rs"] mod layout_rules; #[path = "tests/test_line_wrap.rs"] diff --git a/crates/sqlclean/src/sql_lint/tests/helpers.rs b/crates/sqlclean/src/sql_lint/tests/helpers.rs index 9c2f683..90b9fdd 100644 --- a/crates/sqlclean/src/sql_lint/tests/helpers.rs +++ b/crates/sqlclean/src/sql_lint/tests/helpers.rs @@ -1,16 +1,11 @@ -use std::str::FromStr; - -use polyglot_sql::{Dialect, DialectType}; use serde_json::{Map, Value, json}; +use sqlquack_fingerprint::{FingerprintDialect, canonical_tokens}; use crate::sql_lint::_helpers::formatter::format_requests; use crate::sql_lint::main::engine::lint_json; use crate::sql_lint::main::formatter::format_json; use crate::sql_lint::models::FormatRequest; use crate::sql_lint::tests::test_types::CrossJoinTestCase; -use crate::sql_tokens::constants::CALL_SYNTAX_FUNCTIONS; -use crate::sql_tokens::main::builtin_function_names::builtin_function_names; -use crate::sql_tokens::main::canonical_tokens::canonical_tokens; use crate::sql_tokens::main::query_fingerprint::query_fingerprint; /// The dependency sentinel the cross join cases name `order_policy`. @@ -143,21 +138,32 @@ pub(crate) fn check_dialect_corpus( /// Whether `printed` has the fingerprint tokens of `sql` except for words upper-cased by format. fn only_keyword_case_differs(sql: &str, printed: &str, dialect: &str) -> Result { - let parser = Dialect::get(DialectType::from_str(dialect).map_err(|error| error.to_string())?); - let left_tokens = parser.tokenize(sql).map_err(|error| error.to_string())?; - let right_tokens = parser - .tokenize(printed) - .map_err(|error| error.to_string())?; - let left = canonical_tokens(sql, &left_tokens, &parser)?; - let right = canonical_tokens(printed, &right_tokens, &parser)?; + let fingerprint = FingerprintDialect::from_name(dialect).map_err(|error| error.to_string())?; + let tokens = |sql: &str| canonical_tokens(sql, fingerprint).map_err(|error| error.to_string()); + let (left, right) = (tokens(sql)?, tokens(printed)?); Ok(left.len() == right.len() && left.iter().zip(&right).all(|(before, after)| { before == after - || (before.text.eq_ignore_ascii_case(&after.text) - && after.text == after.text.to_ascii_uppercase()) + || (before.eq_ignore_ascii_case(after) && *after == after.to_ascii_uppercase()) })) } +/// Check every corpus query of one dialect is left as written and reported unavailable; return how many. +pub(crate) fn check_unavailable_corpus(corpus: &[Value], dialect: &str) -> Result { + let mut count: usize = 0; + for entry in corpus.iter().filter(|entry| entry["dialect"] == dialect) { + let sql = entry["sql"].as_str().unwrap_or_default(); + let response = format_once(sql, dialect, Some(CORPUS_LINE_WIDTH))?; + assert_eq!(response["sql"], sql, "{dialect}: {sql}"); + assert_eq!(response["formatted"], false, "{dialect}: {sql}"); + assert_eq!(response["changed"], false, "{dialect}: {sql}"); + assert!(response.get("reason").is_none(), "{dialect}: {sql}"); + assert!(response["unavailable"].is_string(), "{dialect}: {sql}"); + count += 1; + } + Ok(count) +} + /// Re-format a layout of an earlier formatter release, which must keep its fingerprint. fn relaid_previous_layout(previous: &str, dialect: &str) -> Result { let relaid = format_once(previous, dialect, None)?; @@ -184,19 +190,39 @@ fn format_once(sql: &str, dialect: &str, line_width: Option) -> Result Result<(Vec, Vec), String> { - let dialect_type = DialectType::from_str(dialect).map_err(|error| error.to_string())?; - let mut names: Vec = builtin_function_names(dialect_type) - .into_iter() - .chain(KEYWORD_LIKE_CALLS.iter().map(|name| (*name).to_string())) - .filter(|name| name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')) - .filter(|name| !CALL_SYNTAX_FUNCTIONS.contains(&name.to_ascii_uppercase().as_str())) - .collect(); + let mut names: Vec = CALL_NAMES.iter().map(|name| (*name).to_string()).collect(); names.sort_unstable(); names.dedup(); let mut misplaced: Vec = Vec::new(); diff --git a/crates/sqlclean/src/sql_lint/tests/test_authored_tokens.rs b/crates/sqlclean/src/sql_lint/tests/test_authored_tokens.rs deleted file mode 100644 index dcc8d0a..0000000 --- a/crates/sqlclean/src/sql_lint/tests/test_authored_tokens.rs +++ /dev/null @@ -1,416 +0,0 @@ -use serde_json::{Value, json}; - -use std::collections::BTreeMap; - -use crate::sql_lint::_helpers::token_layout::{ - KeywordRecase, TokenTexts, tokenize, verify_printed_tokens, -}; -use crate::sql_lint::main::formatter::format_json; -use crate::sql_lint::tests::test_types; - -#[test] -fn given_generator_respellings_when_formatting_then_authored_spellings_are_kept() --> Result<(), String> { - let test_cases = [ - test_types::FormatTestCase { - description: "Snowflake STARTSWITH keeps its authored spelling and folds to upper case like every called name", - sql: "select startswith(order_code, 'EU') as is_eu_order from orders", - expected_sql: "SELECT STARTSWITH(order_code, 'EU') AS is_eu_order\nFROM orders", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "function synonyms keep their authored names", - sql: "select SUBSTR(order_code, 1, 2) as region, TIMESTAMPDIFF(day, ordered_at, shipped_at) as days_to_ship, TRY_TO_DECIMAL(amount_text, 10, 2) as amount from orders", - expected_sql: "SELECT\n SUBSTR(order_code, 1, 2) AS region,\n TIMESTAMPDIFF(day, ordered_at, shipped_at) AS days_to_ship,\n TRY_TO_DECIMAL(amount_text, 10, 2) AS amount\nFROM orders", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "operator and function spellings stay authored", - sql: "select IFNULL(status, 'open') as status from orders where quantity != 0", - expected_sql: "SELECT IFNULL(status, 'open') AS status\nFROM orders\nWHERE quantity != 0", - expected_changed: true, - }, - ]; - for test_case in &test_cases { - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": "snowflake"}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - assert_eq!( - payload["changed"], test_case.expected_changed, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_generator_restructuring_when_formatting_then_authored_tokens_are_laid_out() --> Result<(), String> { - let test_cases = [ - test_types::DialectFormatTestCase { - description: "PostgreSQL comma SUBSTRING arguments are not rewritten to FROM/FOR", - dialect: "postgres", - sql: "select substring(code, 1, 2) as region from orders", - expected_sql: "SELECT SUBSTRING(code, 1, 2) AS region\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "DuckDB adjacent string literals after a quoted-in-oracle alias keep comma spacing", - dialect: "duckdb", - sql: "select sku as replace, 'north' 'east' as region, sku from products", - expected_sql: "SELECT\n sku AS replace,\n 'north' 'east' AS region,\n sku\nFROM products", - }, - test_types::DialectFormatTestCase { - description: "DuckDB adjacent string literals after a plain column keep comma spacing", - dialect: "duckdb", - sql: "select sku, 'north' 'east' as region from products", - expected_sql: "SELECT\n sku,\n 'north' 'east' AS region\nFROM products", - }, - test_types::DialectFormatTestCase { - description: "DuckDB SELECT ALL keeps ALL", - dialect: "duckdb", - sql: "select all order_id from orders", - expected_sql: "SELECT ALL order_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "Snowflake SELECT ALL keeps ALL", - dialect: "snowflake", - sql: "select all order_id from orders", - expected_sql: "SELECT ALL order_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "PostgreSQL SELECT ALL keeps ALL", - dialect: "postgres", - sql: "select all order_id from orders", - expected_sql: "SELECT ALL order_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "BigQuery SELECT ALL keeps ALL", - dialect: "bigquery", - sql: "select all order_id from orders", - expected_sql: "SELECT ALL order_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "Databricks SELECT ALL keeps ALL", - dialect: "databricks", - sql: "select all order_id from orders", - expected_sql: "SELECT all order_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "T-SQL SELECT ALL keeps ALL", - dialect: "tsql", - sql: "select all order_id from orders", - expected_sql: "SELECT ALL order_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "implicit aliases stay implicit", - dialect: "duckdb", - sql: "select order_id id, amount from orders o", - expected_sql: "SELECT\n order_id id,\n amount\nFROM orders o", - }, - test_types::DialectFormatTestCase { - description: "a trailing select comma and statement terminator are kept", - dialect: "duckdb", - sql: "select order_id, amount, from orders;", - expected_sql: "SELECT\n order_id,\n amount,\nFROM orders;", - }, - test_types::DialectFormatTestCase { - description: "adjacent authored tokens are never split", - dialect: "duckdb", - sql: "select $name as v from orders where status != 1", - expected_sql: "SELECT $name AS v\nFROM orders\nWHERE status != 1", - }, - test_types::DialectFormatTestCase { - description: "BigQuery numeric-prefixed path parts stay joined", - dialect: "bigquery", - sql: "select * from orders.archive.25_", - expected_sql: "SELECT *\nFROM orders.archive.25_", - }, - ]; - for test_case in &test_cases { - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": test_case.dialect}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - let again = format_json( - &json!({"version": 1, "sql": test_case.expected_sql, "dialect": test_case.dialect}) - .to_string(), - )?; - let repeated: Value = serde_json::from_str(&again).map_err(|error| error.to_string())?; - assert_eq!(repeated["changed"], false, "{}", test_case.description); - } - Ok(()) -} - -#[test] -fn given_case_or_line_sensitive_tokens_when_formatting_then_they_are_kept_exactly() --> Result<(), String> { - let test_cases = [ - test_types::DialectFormatTestCase { - description: "DuckDB keyword-named aliases and qualified names keep their case", - dialect: "duckdb", - sql: "select pos as index, sku as replace, t.view from orders t", - expected_sql: "SELECT\n pos AS index,\n sku AS replace,\n t.view\nFROM orders t", - }, - test_types::DialectFormatTestCase { - description: "BigQuery keyword-named table names keep their case", - dialect: "bigquery", - sql: "select * from inventory.view", - expected_sql: "SELECT *\nFROM inventory.view", - }, - test_types::DialectFormatTestCase { - description: "Snowflake path keys keep their case", - dialect: "snowflake", - sql: "select payload:Customer.Name as name, payload:Date::string as day from orders", - expected_sql: "SELECT\n payload:Customer.Name AS name,\n payload:Date::string AS day\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "user-defined functions keep their case, built-ins are uppercased", - dialect: "bigquery", - sql: "select MyFunc(a), ds.MyUdf(b), coalesce(a, b) from orders", - expected_sql: "SELECT\n MyFunc(a),\n ds.MyUdf(b),\n COALESCE(a, b)\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "DuckDB non-reserved keyword operands keep their case before any operator", - dialect: "duckdb", - sql: "select a, Rows << 1, Rows == 1, Rows ^ 2 from t", - expected_sql: "SELECT\n a,\n Rows << 1,\n Rows == 1,\n Rows ^ 2\nFROM t", - }, - test_types::DialectFormatTestCase { - description: "BigQuery comma-joined keyword-named tables keep their case", - dialect: "bigquery", - sql: "select * from ds.a, View v", - expected_sql: "SELECT *\nFROM ds.a, View v", - }, - test_types::DialectFormatTestCase { - description: "DuckDB adjacent string literals keep their line break", - dialect: "duckdb", - sql: "select 'north'\n'east' as region from orders", - expected_sql: "SELECT\n 'north'\n 'east' AS region\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "PostgreSQL adjacent string literals keep their line break", - dialect: "postgres", - sql: "select 'north'\n 'east' as region from orders", - expected_sql: "SELECT\n 'north'\n 'east' AS region\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "Snowflake double-slash comments are kept", - dialect: "snowflake", - sql: "select order_id, // keep this note\n customer_id from orders", - expected_sql: "SELECT\n order_id, // keep this note\n customer_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "BigQuery hash comments are kept", - dialect: "bigquery", - sql: "select order_id, # keep this note\n customer_id from orders", - expected_sql: "SELECT\n order_id, # keep this note\n customer_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "T-SQL temporary table hashes are not comments", - dialect: "tsql", - sql: "select #orders.order_id from #orders", - expected_sql: "SELECT #orders.order_id\nFROM #orders", - }, - test_types::DialectFormatTestCase { - description: "DuckDB integer division and slashes inside strings are not comments", - dialect: "duckdb", - sql: "select 7 // 2 as half, 'https://example.com/a' as url from orders", - expected_sql: "SELECT\n 7 // 2 AS half,\n 'https://example.com/a' AS url\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "Snowflake double slashes inside strings are not comments", - dialect: "snowflake", - sql: "select 'https://example.com//a' as url from orders", - expected_sql: "SELECT 'https://example.com//a' AS url\nFROM orders", - }, - ]; - for test_case in &test_cases { - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": test_case.dialect}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - let again = format_json( - &json!({"version": 1, "sql": test_case.expected_sql, "dialect": test_case.dialect}) - .to_string(), - )?; - let repeated: Value = serde_json::from_str(&again).map_err(|error| error.to_string())?; - assert_eq!(repeated["changed"], false, "{}", test_case.description); - } - Ok(()) -} - -#[test] -fn given_formatted_candidates_when_checking_token_invariant_then_only_layout_may_differ() { - let test_cases = [ - test_types::TokenInvariantTestCase { - description: "whitespace and word case may change", - before: "select a,b from orders -- note", - after: "SELECT\n a,\n b\nFROM orders -- note", - recases: &[], - expected_error: None, - }, - test_types::TokenInvariantTestCase { - description: "blank lines between CTEs keep comments on their tokens", - before: "WITH orders AS (SELECT 1), -- note\n-- paid only\npaid AS (SELECT 2) SELECT * FROM paid", - after: "WITH orders AS (\n SELECT 1\n), -- note\n\n-- paid only\npaid AS (\n SELECT 2\n)\n\nSELECT *\nFROM paid", - recases: &[], - expected_error: None, - }, - test_types::TokenInvariantTestCase { - description: "a comment moved across a CTE blank line is refused", - before: "WITH orders AS (SELECT 1), -- note\npaid AS (SELECT 2) SELECT * FROM paid", - after: "WITH orders AS (\n SELECT 1\n)\n-- note\n,\n\npaid AS (\n SELECT 2\n)\n\nSELECT *\nFROM paid", - recases: &[], - expected_error: Some("native formatter could not preserve comment token attachments"), - }, - test_types::TokenInvariantTestCase { - description: "a renamed function is refused", - before: "select startswith(code, 'EU') from orders", - after: "SELECT STARTS_WITH(code, 'EU') FROM orders", - recases: &[], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "an added alias keyword is refused", - before: "select a b from orders", - after: "SELECT a AS b FROM orders", - recases: &[], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "string literal whitespace is refused", - before: "select 'a b' from orders", - after: "SELECT 'a b' FROM orders", - recases: &[], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "a recorded keyword recase from the layout guide is allowed", - before: "select a from orders order by a nulls last", - after: "SELECT a FROM orders ORDER BY a NULLS LAST", - recases: &[(7, "NULLS", true), (8, "LAST", true)], - expected_error: None, - }, - test_types::TokenInvariantTestCase { - description: "an unrecorded non-reserved keyword recase is refused", - before: "select a from orders order by a nulls last", - after: "SELECT a FROM orders ORDER BY a NULLS LAST", - recases: &[], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "a recase whose guide token is not a keyword is refused", - before: "select a from orders order by a nulls last", - after: "SELECT a FROM orders ORDER BY a NULLS LAST", - recases: &[(7, "NULLS", false), (8, "LAST", false)], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "an identifier case change is refused", - before: "select a from orders", - after: "SELECT a FROM Orders", - recases: &[], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "a path key case change is refused", - before: "select payload:customerId from orders", - after: "SELECT payload:CUSTOMERID FROM orders", - recases: &[], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "a built-in function may be recased", - before: "select coalesce(a, 0) from orders", - after: "SELECT COALESCE(a, 0) FROM orders", - recases: &[], - expected_error: None, - }, - test_types::TokenInvariantTestCase { - description: "joining adjacent string literals across a line break is refused", - before: "select 'a'\n'b' from orders", - after: "SELECT 'a' 'b' FROM orders", - recases: &[], - expected_error: Some( - "native formatter would join adjacent string literals across a line break", - ), - }, - test_types::TokenInvariantTestCase { - description: "a dropped double-slash comment is refused", - before: "select a // note\nfrom orders", - after: "SELECT a FROM orders", - recases: &[], - expected_error: Some("native formatter could not preserve comment token attachments"), - }, - test_types::TokenInvariantTestCase { - description: "a comment moved to another token is refused", - before: "select a, -- note\n b from orders", - after: "SELECT a, b -- note\nFROM orders", - recases: &[], - expected_error: Some("native formatter could not preserve comment token attachments"), - }, - ]; - let dialect = polyglot_sql::Dialect::get(polyglot_sql::DialectType::Snowflake); - for test_case in test_cases { - let recases: BTreeMap = test_case - .recases - .iter() - .map(|(index, generated, keyword)| { - ( - *index, - KeywordRecase { - generated: (*generated).to_string(), - keyword: *keyword, - }, - ) - }) - .collect(); - let before_tokens = tokenize(test_case.before, &dialect).expect("before tokenizes"); - let after_tokens = tokenize(test_case.after, &dialect).expect("after tokenizes"); - let before_texts = - TokenTexts::read(test_case.before, &before_tokens, &dialect).expect("before reads"); - let actual = verify_printed_tokens( - (test_case.before, &before_tokens, &before_texts), - (test_case.after, &after_tokens), - &dialect, - &recases, - ); - assert_eq!( - actual.err().as_deref(), - test_case.expected_error, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs b/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs index ebbd2bb..a4ce902 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs @@ -1,6 +1,6 @@ use serde_json::Value; -use crate::sql_lint::tests::helpers::check_dialect_corpus; +use crate::sql_lint::tests::helpers::{check_dialect_corpus, check_unavailable_corpus}; use crate::sql_lint::tests::test_types; const FORMAT_CORPUS: &str = include_str!("fixtures/format_corpus.json"); @@ -9,42 +9,18 @@ const FORMAT_CORPUS: &str = include_str!("fixtures/format_corpus.json"); fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stable() -> Result<(), String> { let test_cases = [ - test_types::FormatCorpusTestCase { - description: "BigQuery identity corpus", - dialect: "bigquery", - expected_formatted: 168, - expected_refused: 0, - }, - test_types::FormatCorpusTestCase { - description: "Databricks identity corpus", - dialect: "databricks", - expected_formatted: 62, - expected_refused: 0, - }, test_types::FormatCorpusTestCase { description: "DuckDB identity corpus", dialect: "duckdb", expected_formatted: 189, expected_refused: 0, }, - test_types::FormatCorpusTestCase { - description: "PostgreSQL identity corpus", - dialect: "postgres", - expected_formatted: 132, - expected_refused: 5, - }, test_types::FormatCorpusTestCase { description: "Snowflake identity corpus", dialect: "snowflake", expected_formatted: 427, expected_refused: 8, }, - test_types::FormatCorpusTestCase { - description: "T-SQL identity corpus", - dialect: "tsql", - expected_formatted: 75, - expected_refused: 0, - }, ]; let corpus: Vec = serde_json::from_str(FORMAT_CORPUS).map_err(|error| error.to_string())?; @@ -59,3 +35,25 @@ fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stab } Ok(()) } + +/// Dialects SQLQuack does not format yet are reported unavailable, with every query left as written. +#[test] +fn given_dialects_without_a_native_formatter_when_formatting_then_each_query_is_unavailable() +-> Result<(), String> { + let test_cases = [ + ("bigquery", 168), + ("databricks", 62), + ("postgres", 137), + ("tsql", 75), + ]; + let corpus: Vec = + serde_json::from_str(FORMAT_CORPUS).map_err(|error| error.to_string())?; + for (dialect, expected_unavailable) in test_cases { + assert_eq!( + check_unavailable_corpus(&corpus, dialect)?, + expected_unavailable, + "{dialect}" + ); + } + Ok(()) +} diff --git a/crates/sqlclean/src/sql_lint/tests/test_function_call_spacing.rs b/crates/sqlclean/src/sql_lint/tests/test_function_call_spacing.rs index 4acd30c..761a04c 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_function_call_spacing.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_function_call_spacing.rs @@ -32,76 +32,6 @@ fn given_builtin_call_names_when_formatting_then_no_space_enters_the_call() -> R argument: "order_amount_with_a_long_descriptive_name", expected_refused: &[], }, - test_types::FunctionCallSpacingTestCase { - description: "bigquery compact calls", - dialect: "bigquery", - line_width: 100, - argument: "a", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "bigquery breaking calls", - dialect: "bigquery", - line_width: 40, - argument: "order_amount_with_a_long_descriptive_name", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "postgres compact calls", - dialect: "postgres", - line_width: 100, - argument: "a", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "postgres breaking calls", - dialect: "postgres", - line_width: 40, - argument: "order_amount_with_a_long_descriptive_name", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "tsql compact calls", - dialect: "tsql", - line_width: 100, - argument: "a", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "tsql breaking calls", - dialect: "tsql", - line_width: 40, - argument: "order_amount_with_a_long_descriptive_name", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "databricks compact calls", - dialect: "databricks", - line_width: 100, - argument: "a", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "databricks breaking calls", - dialect: "databricks", - line_width: 40, - argument: "order_amount_with_a_long_descriptive_name", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "generic compact calls", - dialect: "generic", - line_width: 100, - argument: "a", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "generic breaking calls", - dialect: "generic", - line_width: 40, - argument: "order_amount_with_a_long_descriptive_name", - expected_refused: &[], - }, ]; for test_case in &test_cases { let (refused, misplaced) = diff --git a/crates/sqlclean/src/sql_lint/tests/test_idempotency_reprint.rs b/crates/sqlclean/src/sql_lint/tests/test_idempotency_reprint.rs deleted file mode 100644 index c3176dd..0000000 --- a/crates/sqlclean/src/sql_lint/tests/test_idempotency_reprint.rs +++ /dev/null @@ -1,48 +0,0 @@ -use serde_json::{Value, json}; - -use crate::sql_lint::_helpers::formatter::RELAYOUTS; -use crate::sql_lint::main::formatter::format_json; -use crate::sql_lint::tests::test_types; - -#[test] -fn given_formatted_and_unformatted_bodies_when_formatting_then_only_changed_bodies_are_reprinted() --> Result<(), String> { - let test_cases = [ - test_types::ReprintTestCase { - description: "an already formatted body is not re-printed", - sql: "SELECT\n order_id,\n amount\nFROM orders\nWHERE amount > 0", - expected_changed: false, - expected_reprints: 0, - }, - test_types::ReprintTestCase { - description: "a body the formatter changes is re-printed once", - sql: "select order_id, amount from orders where amount > 0", - expected_changed: true, - expected_reprints: 1, - }, - ]; - for test_case in &test_cases { - RELAYOUTS.with(|relayouts| relayouts.set(0)); - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": "postgres"}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["formatted"], true, - "{}: {payload}", - test_case.description - ); - assert_eq!( - payload["changed"], test_case.expected_changed, - "{}: {payload}", - test_case.description - ); - assert_eq!( - RELAYOUTS.with(std::cell::Cell::get), - test_case.expected_reprints, - "{}", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlclean/src/sql_lint/tests/test_layout_rules.rs b/crates/sqlclean/src/sql_lint/tests/test_layout_rules.rs index b746db4..07da408 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_layout_rules.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_layout_rules.rs @@ -29,20 +29,6 @@ fn given_clause_layout_rules_when_formatting_then_each_rule_applies_idempotently line_width: 100, expected_sql: "SELECT *\nFROM VALUES ('a', 'A'), ('b', 'B')", }, - test_types::LayoutRuleTestCase { - description: "a long parenthesized PostgreSQL VALUES moves under its parenthesis", - dialect: "postgres", - sql: "SELECT order_id, status FROM (VALUES (1, 'paid and shipped to customer'), (2, 'cancelled by customer')) AS orders(order_id, status)", - line_width: 60, - expected_sql: "SELECT\n order_id,\n status\nFROM (\n VALUES\n (1, 'paid and shipped to customer'),\n (2, 'cancelled by customer')\n) AS orders(order_id, status)", - }, - test_types::LayoutRuleTestCase { - description: "a long T-SQL VALUES relation keeps rows whole", - dialect: "tsql", - sql: "SELECT order_id FROM (VALUES (1, 'paid and shipped to customer'), (2, 'cancelled by customer')) AS orders(order_id, status)", - line_width: 60, - expected_sql: "SELECT order_id\nFROM (\n VALUES\n (1, 'paid and shipped to customer'),\n (2, 'cancelled by customer')\n) AS orders(order_id, status)", - }, test_types::LayoutRuleTestCase { description: "DuckDB INSERT VALUES rows stay whole when a row is over the width", dialect: "duckdb", @@ -64,27 +50,6 @@ fn given_clause_layout_rules_when_formatting_then_each_rule_applies_idempotently line_width: 50, expected_sql: "SELECT\n COALESCE(\n first_order_amount,\n second_order_amount\n )::DECIMAL(18, 4) AS amount\nFROM orders", }, - test_types::LayoutRuleTestCase { - description: "a BigQuery NUMERIC(38, 9) cast keeps its parameters", - dialect: "bigquery", - sql: "SELECT CAST(COALESCE(first_order_amount, second_order_amount) AS NUMERIC(38, 9)) AS amount FROM orders", - line_width: 50, - expected_sql: "SELECT\n CAST(\n COALESCE(\n first_order_amount,\n second_order_amount\n ) AS NUMERIC(38, 9)\n ) AS amount\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "a T-SQL VARCHAR(MAX) cast keeps its parameter", - dialect: "tsql", - sql: "SELECT CAST(COALESCE(first_order_note, second_order_note) AS VARCHAR(MAX)) AS note FROM orders", - line_width: 50, - expected_sql: "SELECT\n CAST(\n COALESCE(\n first_order_note,\n second_order_note\n ) AS VARCHAR(MAX)\n ) AS note\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "a Databricks DECIMAL(10, 2) cast keeps its parameters", - dialect: "databricks", - sql: "SELECT CAST(COALESCE(first_order_amount, second_order_amount) AS DECIMAL(10, 2)) AS amount FROM orders", - line_width: 50, - expected_sql: "SELECT\n CAST(\n COALESCE(\n first_order_amount,\n second_order_amount\n ) AS DECIMAL(10, 2)\n ) AS amount\nFROM orders", - }, test_types::LayoutRuleTestCase { description: "a long Snowflake DECODE puts each pair and the default on its own line", dialect: "snowflake", @@ -120,13 +85,6 @@ fn given_clause_layout_rules_when_formatting_then_each_rule_applies_idempotently line_width: 100, expected_sql: "SELECT\n CASE\n WHEN status = 'paid'\n THEN 'customer paid in full'\n WHEN status = 'open'\n THEN 'customer has not paid yet'\n ELSE 'other'\n END\n || CASE WHEN rush THEN '!' ELSE '' END AS label\nFROM orders", }, - test_types::LayoutRuleTestCase { - description: "AND after a multi-line subquery starts the next line", - dialect: "postgres", - sql: "SELECT order_id FROM orders WHERE customer_id IN (SELECT customer_id FROM customers WHERE region = 'north') AND EXISTS (SELECT 1 FROM shipments)", - line_width: 100, - expected_sql: "SELECT order_id\nFROM orders\nWHERE\n customer_id IN (\n SELECT customer_id\n FROM customers\n WHERE region = 'north'\n )\n AND EXISTS(\n SELECT 1\n FROM shipments\n )", - }, test_types::LayoutRuleTestCase { description: "BETWEEN bounds keep their AND", dialect: "snowflake", @@ -148,27 +106,6 @@ fn given_clause_layout_rules_when_formatting_then_each_rule_applies_idempotently line_width: 100, expected_sql: "WITH orders AS (\n SELECT *\n FROM raw_orders\n), -- raw\n\npaid AS (\n SELECT *\n FROM orders\n)\n\nSELECT *\nFROM paid", }, - test_types::LayoutRuleTestCase { - description: "a nested WITH is separated at its own indentation", - dialect: "postgres", - sql: "SELECT * FROM (WITH a AS MATERIALIZED (SELECT 1 AS x), b AS (SELECT x FROM a) SELECT x FROM b) AS nested", - line_width: 100, - expected_sql: "SELECT *\nFROM (\n WITH a AS MATERIALIZED (\n SELECT 1 AS x\n ),\n\n b AS (\n SELECT x\n FROM a\n )\n\n SELECT x\n FROM b\n) AS nested", - }, - test_types::LayoutRuleTestCase { - description: "BigQuery recursive CTEs with column lists are separated", - dialect: "bigquery", - sql: "WITH RECURSIVE numbers (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM numbers WHERE n < 3), doubled AS (SELECT n * 2 AS n FROM numbers) SELECT n FROM doubled", - line_width: 100, - expected_sql: "WITH RECURSIVE numbers (n) AS (\n SELECT 1\n UNION ALL\n SELECT n + 1\n FROM numbers\n WHERE n < 3\n),\n\ndoubled AS (\n SELECT n * 2 AS n\n FROM numbers\n)\n\nSELECT n\nFROM doubled", - }, - test_types::LayoutRuleTestCase { - description: "a T-SQL WITH is separated before INSERT", - dialect: "tsql", - sql: "WITH paid AS (SELECT order_id FROM orders) INSERT INTO paid_orders SELECT order_id FROM paid", - line_width: 100, - expected_sql: "WITH paid AS (\n SELECT order_id\n FROM orders\n)\n\nINSERT INTO paid_orders\nSELECT order_id\nFROM paid", - }, test_types::LayoutRuleTestCase { description: "a CTE macro placeholder is separated like a CTE", dialect: "duckdb", @@ -232,27 +169,6 @@ fn given_clause_layout_rules_when_formatting_then_each_rule_applies_idempotently line_width: 100, expected_sql: "SELECT DISTINCT order_id\nFROM orders\nWHERE\n amount > 0\n AND status = 'paid'\nUNION ALL\nSELECT\n order_id,\n customer_id\nFROM returns", }, - test_types::LayoutRuleTestCase { - description: "T-SQL TOP, HAVING and Snowflake-style single items share lines", - dialect: "tsql", - sql: "SELECT TOP 10 COUNT(*) AS n FROM orders GROUP BY customer_id HAVING COUNT(*) > 1", - line_width: 100, - expected_sql: "SELECT TOP 10 COUNT(*) AS n\nFROM orders\nGROUP BY\n customer_id\nHAVING COUNT(*) > 1", - }, - test_types::LayoutRuleTestCase { - description: "QUALIFY with one condition shares its line", - dialect: "databricks", - sql: "SELECT order_id, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) AS rn FROM orders QUALIFY rn = 1", - line_width: 100, - expected_sql: "SELECT\n order_id,\n ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) AS rn\nFROM orders\nQUALIFY rn = 1", - }, - test_types::LayoutRuleTestCase { - description: "a single item that does not fit stays under SELECT", - dialect: "bigquery", - sql: "SELECT COALESCE(first_order_amount, second_order_amount, third_order_amount) AS amount FROM orders WHERE first_order_amount + second_order_amount + third_order_amount > 100", - line_width: 60, - expected_sql: "SELECT\n COALESCE(\n first_order_amount,\n second_order_amount,\n third_order_amount\n ) AS amount\nFROM orders\nWHERE\n first_order_amount\n + second_order_amount\n + third_order_amount > 100", - }, test_types::LayoutRuleTestCase { description: "a short CASE item shares the SELECT line", dialect: "duckdb", diff --git a/crates/sqlclean/src/sql_lint/tests/test_set_operation_formatter.rs b/crates/sqlclean/src/sql_lint/tests/test_set_operation_formatter.rs index 9f8e7fb..e397a4f 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_set_operation_formatter.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_set_operation_formatter.rs @@ -3,14 +3,7 @@ use serde_json::{Value, json}; use crate::sql_lint::main::formatter::format_json; use crate::sql_lint::tests::test_types; -const DIALECTS: [&str; 6] = [ - "bigquery", - "databricks", - "duckdb", - "postgres", - "snowflake", - "tsql", -]; +const DIALECTS: [&str; 2] = ["duckdb", "snowflake"]; #[test] fn given_set_operation_after_scalar_subquery_operand_when_formatting_then_keyword_stays_outer() diff --git a/crates/sqlclean/src/sql_lint/tests/test_types.rs b/crates/sqlclean/src/sql_lint/tests/test_types.rs index bbe79a3..cbc3835 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_types.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_types.rs @@ -119,21 +119,6 @@ pub(crate) struct FormatCorpusTestCase { pub expected_refused: usize, } -pub(crate) struct DialectFormatTestCase { - pub description: &'static str, - pub dialect: &'static str, - pub sql: &'static str, - pub expected_sql: &'static str, -} - -pub(crate) struct TokenInvariantTestCase { - pub description: &'static str, - pub before: &'static str, - pub after: &'static str, - pub recases: &'static [(usize, &'static str, bool)], - pub expected_error: Option<&'static str>, -} - pub(crate) struct FormatTestCase { pub description: &'static str, pub sql: &'static str, @@ -141,13 +126,6 @@ pub(crate) struct FormatTestCase { pub expected_changed: bool, } -pub(crate) struct ReprintTestCase { - pub description: &'static str, - pub sql: &'static str, - pub expected_changed: bool, - pub expected_reprints: usize, -} - pub(crate) struct InvalidRuleTestCase { pub description: &'static str, pub rule: &'static str, diff --git a/crates/sqlclean/src/sql_tokens/constants.rs b/crates/sqlclean/src/sql_tokens/constants.rs index beee14b..4d12baf 100644 --- a/crates/sqlclean/src/sql_tokens/constants.rs +++ b/crates/sqlclean/src/sql_tokens/constants.rs @@ -34,5 +34,5 @@ pub(crate) const PROJECTION_ENDS: [TokenType; 13] = [ TokenType::Intersect, ]; /// Dialect names fingerprinted under another name, as SQLBuild resolves them. -pub(crate) const FINGERPRINT_DIALECT_ALIASES: [(&str, &str); 2] = - [("motherduck", "duckdb"), ("sqlserver", "tsql")]; +#[cfg(any(test, feature = "test-support"))] +pub(crate) const FINGERPRINT_DIALECT_ALIASES: [(&str, &str); 1] = [("motherduck", "duckdb")]; diff --git a/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs b/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs index b1452b4..73513f6 100644 --- a/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs +++ b/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs @@ -1,36 +1,16 @@ //! Whitespace-, comment- and keyword-case-insensitive SQL change fingerprints. -use std::str::FromStr; - -use polyglot_sql::{Dialect, DialectType}; -use sha2::{Digest, Sha256}; use sqlquack_fingerprint::FingerprintDialect; use crate::sql_tokens::constants::FINGERPRINT_DIALECT_ALIASES; -use crate::sql_tokens::main::canonical_tokens::canonical_tokens; -/// Return the fingerprint of `sql` under `dialect_name` that SQLBuild stores: SQLQuack's for the -/// dialects it fingerprints, and the Polyglot token stream's for the others, as SQLBuild does. +/// Return SQLQuack's fingerprint of `sql` under `dialect_name`, the one SQLBuild stores; a dialect +/// without its own SQLQuack tokenizer is fingerprinted with the generic tokens. pub fn query_fingerprint(sql: &str, dialect_name: &str) -> Result { let name: &str = FINGERPRINT_DIALECT_ALIASES .iter() .find(|(alias, _)| *alias == dialect_name) .map_or(dialect_name, |(_, name)| name); - if let Ok(dialect) = FingerprintDialect::from_name(name) { - return sqlquack_fingerprint::query_fingerprint(sql, dialect) - .map_err(|error| error.to_string()); - } - let dialect_type = DialectType::from_str(name) - .map_err(|_| format!("unknown SQL fingerprint dialect '{dialect_name}'"))?; - let dialect = Dialect::get(dialect_type); - let tokens = dialect - .tokenize(sql) - .map_err(|error| format!("SQL tokenization failed: {error}"))?; - let mut digest = Sha256::new(); - for token in canonical_tokens(sql, &tokens, &dialect)? { - digest.update(token.text.len().to_string().as_bytes()); - digest.update(b":"); - digest.update(token.text.as_bytes()); - } - Ok(format!("{:x}", digest.finalize())) + let dialect = FingerprintDialect::from_name(name).unwrap_or(FingerprintDialect::Generic); + sqlquack_fingerprint::query_fingerprint(sql, dialect).map_err(|error| error.to_string()) } diff --git a/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs b/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs index 72dc61f..c2150ce 100644 --- a/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs +++ b/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs @@ -1,6 +1,6 @@ use crate::sql_tokens::main::query_fingerprint::query_fingerprint; -/// SQLBuild's golden fingerprints for the dialects SQLQuack tokenizes; SQLBuild stores these +/// SQLBuild's golden fingerprints, generic tokens for dialects without their own; SQLBuild stores these /// hashes, so SQLClean must compute the same ones. #[test] fn given_sqlbuild_golden_queries_when_fingerprinting_then_hashes_match_sqlbuild() @@ -44,7 +44,7 @@ fn given_sqlbuild_golden_queries_when_fingerprinting_then_hashes_match_sqlbuild( ( "sqlserver", "SELECT TOP 10 [o].[id], N'order' AS kind FROM [dbo].[orders] AS o", - "f4fac3edb7682bbeddf4f73cc199493d45f1419bc8e2cd1d972fead2d67e6b71", + "f1525ee25924f9402677bc2cbd96f7066745948da73671d13fac2b8da03cbf0c", ), ]; for (dialect, sql, expected) in test_cases { diff --git a/website/recordings/README.md b/website/recordings/README.md index c80b098..c2fda38 100644 --- a/website/recordings/README.md +++ b/website/recordings/README.md @@ -16,7 +16,7 @@ The README uses `check-fix`, copied into `.github/demos/`. - DejaVu Sans Mono, the font the tapes use. - `sqlclean` on `PATH` with its Python package in the same environment, or `SQLCLEAN_BIN` pointing at a directory that contains the `sqlclean` to record. -- The `ast` extra (`pip install "sqlclean[ast]"`) for the `check-fix` custom Rule, or +- For the `check-fix` custom Rule, the `sqlclean` Python package in that environment, or `SQLCLEAN_PYTHON` pointing at an interpreter that has it. ## Record diff --git a/website/recordings/prepare.sh b/website/recordings/prepare.sh index 0d00ece..d388d5b 100755 --- a/website/recordings/prepare.sh +++ b/website/recordings/prepare.sh @@ -28,44 +28,31 @@ where c.email = 'Alice@example.com' and o.shipped_at = null SQL cat > sql_rules/team.py <<'PY' -import polyglot_sql as ast # pip install "sqlclean[ast]" +import re from sqlclean import Finding, RuleContext, Statement, rule EMAIL_COLUMNS = {"email", "billing_email"} # the columns this team treats as email addresses - - -def email_column(operand: ast.Expression) -> ast.Column | None: - """The designated email column an operand reads, written bare or as LOWER(column).""" - column = operand.this if isinstance(operand, ast.Lower) else operand - is_email = isinstance(column, ast.Column) and column.name.lower() in EMAIL_COLUMNS - return column if is_email else None - - -def lowered(operand: ast.Expression) -> bool: - """Whether an operand is LOWER(...) or a string literal with no upper-case letters.""" - literal = isinstance(operand, ast.Literal) and operand.is_string - return isinstance(operand, ast.Lower) or (literal and operand.name == operand.name.lower()) +EQUALITY = re.compile(r"\s*(=|<>|!=)\s*") @rule( code="XSQLCLEANTEAM001", message="Email comparisons must be case-insensitive", - remediation="Compare LOWER(email) with LOWER(...) instead.", + remediation="Compare LOWER(email) with a lower-case string instead.", enabled_by_default=True, ) def case_insensitive_email(*, statement: Statement, ctx: RuleContext) -> list[Finding]: - starts = { - column.to_dict()["column"]["span"]["start"] - for comparison in statement.polyglot_ast().find_all(ast.Eq, ast.Neq) - for column in map(email_column, (comparison.this, comparison.expression)) - if column is not None and not all(map(lowered, (comparison.this, comparison.expression))) - } - return [ - ctx.finding(subject=statement, node=column) - for column in statement.columns - if column.start - statement.start in starts - ] + literals = {literal.start: literal for literal in statement.literals if literal.kind == "string"} + findings = [] + for column in statement.columns: + if column.name.lower() not in EMAIL_COLUMNS: + continue + operator = EQUALITY.match(statement.text, column.end - statement.start) + literal = operator and literals.get(statement.start + operator.end()) + if literal is not None and literal.text != literal.text.lower(): + findings.append(ctx.finding(subject=statement, node=column)) + return findings PY ;; *) From 8d6499ca8222e534c26959e33eed1830759533a1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:20:20 +0000 Subject: [PATCH 10/21] wip(tokens): fold SQL tokens with SQLQuack's rules instead of Polyglot's catalogues --- Cargo.lock | 44 ----- Cargo.toml | 11 -- crates/sqlclean/Cargo.toml | 4 +- .../sql_tokens/_helpers/builtin_functions.rs | 151 ------------------ .../src/sql_tokens/_helpers/case_folding.rs | 23 +++ .../{main => _helpers}/is_unquoted_word.rs | 2 +- .../sqlclean/src/sql_tokens/_helpers/mod.rs | 4 +- .../{main => _helpers}/name_positions.rs | 2 +- .../sql_tokens/_helpers/reserved_keywords.rs | 29 ++-- .../src/sql_tokens/_helpers/token_context.rs | 66 ++++++-- crates/sqlclean/src/sql_tokens/constants.rs | 20 ++- .../sql_tokens/main/builtin_function_names.rs | 6 - .../src/sql_tokens/main/canonical_tokens.rs | 24 +-- .../src/sql_tokens/main/case_folding.rs | 23 --- crates/sqlclean/src/sql_tokens/main/mod.rs | 6 - .../src/sql_tokens/main/projection_spans.rs | 2 +- .../src/sql_tokens/main/token_texts.rs | 16 -- .../sql_tokens/reserved_keywords/duckdb.txt | 77 --------- .../sql_tokens/reserved_keywords/generic.txt | 28 ---- .../sql_tokens/reserved_keywords/postgres.txt | 80 ---------- .../reserved_keywords/snowflake.txt | 65 -------- crates/sqlclean/src/sql_tokens/tests.rs | 2 + .../sqlclean/src/sql_tokens/tests/helpers.rs | 12 ++ .../sql_tokens/tests/test_canonical_tokens.rs | 109 +++++++------ .../src/sql_tokens/tests/test_types.rs | 3 +- fensu.toml | 8 +- 26 files changed, 194 insertions(+), 623 deletions(-) delete mode 100644 crates/sqlclean/src/sql_tokens/_helpers/builtin_functions.rs create mode 100644 crates/sqlclean/src/sql_tokens/_helpers/case_folding.rs rename crates/sqlclean/src/sql_tokens/{main => _helpers}/is_unquoted_word.rs (89%) rename crates/sqlclean/src/sql_tokens/{main => _helpers}/name_positions.rs (90%) delete mode 100644 crates/sqlclean/src/sql_tokens/main/builtin_function_names.rs delete mode 100644 crates/sqlclean/src/sql_tokens/main/case_folding.rs delete mode 100644 crates/sqlclean/src/sql_tokens/main/token_texts.rs delete mode 100644 crates/sqlclean/src/sql_tokens/reserved_keywords/duckdb.txt delete mode 100644 crates/sqlclean/src/sql_tokens/reserved_keywords/generic.txt delete mode 100644 crates/sqlclean/src/sql_tokens/reserved_keywords/postgres.txt delete mode 100644 crates/sqlclean/src/sql_tokens/reserved_keywords/snowflake.txt create mode 100644 crates/sqlclean/src/sql_tokens/tests/helpers.rs diff --git a/Cargo.lock b/Cargo.lock index b094261..5d415db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -399,36 +399,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "polyglot-sql" -version = "0.13.9" -source = "git+https://github.com/sqlstack-io/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" -dependencies = [ - "polyglot-sql-ast-derive", - "polyglot-sql-function-catalogs", - "regex", - "serde", - "serde_json", - "stacker", - "thiserror", - "unicode-segmentation", -] - -[[package]] -name = "polyglot-sql-ast-derive" -version = "0.13.9" -source = "git+https://github.com/sqlstack-io/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "polyglot-sql-function-catalogs" -version = "0.13.9" -source = "git+https://github.com/sqlstack-io/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" - [[package]] name = "proc-macro2" version = "1.0.107" @@ -483,18 +453,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - [[package]] name = "regex-automata" version = "0.4.18" @@ -621,8 +579,6 @@ checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" name = "sqlclean" version = "0.4.3" dependencies = [ - "polyglot-sql", - "polyglot-sql-function-catalogs", "rayon", "rustc-hash", "serde", diff --git a/Cargo.toml b/Cargo.toml index ede7a57..b48d168 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,17 +21,6 @@ include_dir = "0.7" ignore = "0.4" tempfile = "3" toml = "0.9" -# The same Polyglot source and revision as SQLBuild, so a build that contains both compiles it once. -polyglot-sql = { git = "https://github.com/sqlstack-io/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ - "ast-tools", - "builder", - "all-dialects", - "semantic", - "stacker", -] } -polyglot-sql-function-catalogs = { git = "https://github.com/sqlstack-io/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ - "dialect-duckdb", -] } rayon = "1" regex = "1" rustc-hash = "2" diff --git a/crates/sqlclean/Cargo.toml b/crates/sqlclean/Cargo.toml index bd79287..31c92d0 100644 --- a/crates/sqlclean/Cargo.toml +++ b/crates/sqlclean/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "sqlclean" version.workspace = true -description = "SQL lint Rules, SQL quality Rules and deterministic formatting over Polyglot." +description = "SQL lint Rules, SQL quality Rules and deterministic formatting over SQLQuack." edition.workspace = true license.workspace = true publish.workspace = true @@ -14,8 +14,6 @@ default = [] test-support = [] [dependencies] -polyglot-sql.workspace = true -polyglot-sql-function-catalogs.workspace = true sqlquack-fingerprint.workspace = true sqlquack-format.workspace = true sqlquack-lint.workspace = true diff --git a/crates/sqlclean/src/sql_tokens/_helpers/builtin_functions.rs b/crates/sqlclean/src/sql_tokens/_helpers/builtin_functions.rs deleted file mode 100644 index 852563f..0000000 --- a/crates/sqlclean/src/sql_tokens/_helpers/builtin_functions.rs +++ /dev/null @@ -1,151 +0,0 @@ -//! Function names a dialect recognises as built-ins, from the SQL library's own metadata. - -use std::collections::{HashMap, HashSet}; -use std::sync::LazyLock; - -use crate::sql_tokens::constants::CALL_SYNTAX_FUNCTIONS; -use polyglot_sql::{Dialect, DialectType, Expression}; -use polyglot_sql_function_catalogs::types::{builtin_arity, type_signature}; -use polyglot_sql_function_catalogs::{ - CatalogSink, FunctionNameCase, FunctionSignature, register_enabled_catalogs, -}; - -const DUCKDB_CATALOG: &str = "duckdb"; - -static CATALOG_FUNCTIONS: LazyLock>> = LazyLock::new(|| { - let mut sink = NameSink::default(); - register_enabled_catalogs(&mut sink); - sink.names -}); -static SNOWFLAKE_TYPED: LazyLock> = - LazyLock::new(|| parser_typed_names(DialectType::Snowflake)); -static BIGQUERY_TYPED: LazyLock> = - LazyLock::new(|| parser_typed_names(DialectType::BigQuery)); -static POSTGRES_TYPED: LazyLock> = - LazyLock::new(|| parser_typed_names(DialectType::PostgreSQL)); -static DATABRICKS_TYPED: LazyLock> = - LazyLock::new(|| parser_typed_names(DialectType::Databricks)); -static TSQL_TYPED: LazyLock> = - LazyLock::new(|| parser_typed_names(DialectType::TSQL)); -static GENERIC_TYPED: LazyLock> = - LazyLock::new(|| parser_typed_names(DialectType::Generic)); - -/// Return whether `upper_name(...)` calls a function the dialect defines. -pub(crate) fn is_builtin_function(dialect: DialectType, upper_name: &str) -> bool { - if CALL_SYNTAX_FUNCTIONS.contains(&upper_name) { - return true; - } - let lower_name = upper_name.to_ascii_lowercase(); - let typed = match dialect { - DialectType::DuckDB => return catalogued(DUCKDB_CATALOG, &lower_name), - DialectType::Snowflake => &SNOWFLAKE_TYPED, - DialectType::BigQuery => &BIGQUERY_TYPED, - DialectType::PostgreSQL => &POSTGRES_TYPED, - DialectType::Databricks => &DATABRICKS_TYPED, - DialectType::TSQL => &TSQL_TYPED, - _ => &GENERIC_TYPED, - }; - typed.contains(&lower_name) || catalogued(catalog_key(dialect), &lower_name) -} - -/// Every catalogued or parser-typed built-in function name of `dialect`, lower-cased. -#[cfg(any(test, feature = "test-support"))] -pub(crate) fn builtin_function_names(dialect: DialectType) -> Vec { - let typed: &HashSet = match dialect { - DialectType::DuckDB => &HashSet::new(), - DialectType::Snowflake => &SNOWFLAKE_TYPED, - DialectType::BigQuery => &BIGQUERY_TYPED, - DialectType::PostgreSQL => &POSTGRES_TYPED, - DialectType::Databricks => &DATABRICKS_TYPED, - DialectType::TSQL => &TSQL_TYPED, - _ => &GENERIC_TYPED, - }; - let key = match dialect { - DialectType::DuckDB => DUCKDB_CATALOG, - _ => catalog_key(dialect), - }; - let catalogued = CATALOG_FUNCTIONS.get(key).into_iter().flatten(); - let mut names: Vec = typed.iter().chain(catalogued).cloned().collect(); - names.sort_unstable(); - names.dedup(); - names -} - -fn catalog_key(dialect: DialectType) -> &'static str { - match dialect { - DialectType::Snowflake => "snowflake", - DialectType::BigQuery => "bigquery", - DialectType::PostgreSQL => "postgres", - _ => "", - } -} - -fn catalogued(key: &str, lower_name: &str) -> bool { - CATALOG_FUNCTIONS - .get(key) - .is_some_and(|names| names.contains(lower_name)) - || builtin_arity(key, lower_name).is_some() - || type_signature(key, lower_name).is_some() -} - -/// Catalogued function names whose call the dialect's parser turns into a specific expression. -fn parser_typed_names(dialect: DialectType) -> HashSet { - let parser = Dialect::get(dialect); - let mut typed: HashSet = HashSet::new(); - for name in CATALOG_FUNCTIONS.get(DUCKDB_CATALOG).into_iter().flatten() { - if is_typed_call(&parser, name) { - typed.insert(name.clone()); - } - } - typed -} - -fn is_typed_call(parser: &Dialect, lower_name: &str) -> bool { - if !lower_name - .chars() - .all(|character| character.is_ascii_alphanumeric() || character == '_') - { - return false; - } - let Ok(statements) = parser.parse(&format!("SELECT {lower_name}(probe)")) else { - return false; - }; - let Some(Expression::Select(select)) = statements.first() else { - return false; - }; - select.expressions.first().is_some_and(|call| { - !matches!( - call, - Expression::Function(_) | Expression::Anonymous(_) | Expression::Column(_) - ) - }) -} - -#[derive(Default)] -struct NameSink { - names: HashMap<&'static str, HashSet>, -} - -impl CatalogSink for NameSink { - fn set_dialect_name_case(&mut self, _dialect: &'static str, _name_case: FunctionNameCase) {} - - fn set_function_name_case( - &mut self, - _dialect: &'static str, - _function_name: &str, - _name_case: FunctionNameCase, - ) { - } - - fn register( - &mut self, - dialect: &'static str, - function_name: &str, - _signatures: Vec, - ) { - self.names - .entry(dialect) - .or_default() - .insert(function_name.to_ascii_lowercase()); - } -} diff --git a/crates/sqlclean/src/sql_tokens/_helpers/case_folding.rs b/crates/sqlclean/src/sql_tokens/_helpers/case_folding.rs new file mode 100644 index 0000000..b43e43e --- /dev/null +++ b/crates/sqlclean/src/sql_tokens/_helpers/case_folding.rs @@ -0,0 +1,23 @@ +//! Which authored tokens are case-insensitive keywords or called function names. + +use crate::sql_tokens::_helpers::name_positions::name_positions; +use crate::sql_tokens::_helpers::token_context::{TokenContext, matching_closers}; + +/// For every token text, whether the formatter may recase it and the fingerprint folds it. +pub(crate) fn foldable_tokens(raws: &[String], dialect: &str) -> Vec { + let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); + let closers = matching_closers(&uppers); + name_positions(&uppers) + .into_iter() + .enumerate() + .map(|(index, name)| { + let context = TokenContext { + raws, + uppers: &uppers, + closers: &closers, + index, + }; + !name && context.is_foldable(dialect) + }) + .collect() +} diff --git a/crates/sqlclean/src/sql_tokens/main/is_unquoted_word.rs b/crates/sqlclean/src/sql_tokens/_helpers/is_unquoted_word.rs similarity index 89% rename from crates/sqlclean/src/sql_tokens/main/is_unquoted_word.rs rename to crates/sqlclean/src/sql_tokens/_helpers/is_unquoted_word.rs index 6ea92ef..c1c40c6 100644 --- a/crates/sqlclean/src/sql_tokens/main/is_unquoted_word.rs +++ b/crates/sqlclean/src/sql_tokens/_helpers/is_unquoted_word.rs @@ -1,7 +1,7 @@ //! Unquoted SQL word recognition shared by canonical tokens and the formatter. /// Return whether `text` is an unquoted ASCII word whose letter case carries no meaning. -pub fn is_unquoted_word(text: &str) -> bool { +pub(crate) fn is_unquoted_word(text: &str) -> bool { let mut characters = text.chars(); characters .next() diff --git a/crates/sqlclean/src/sql_tokens/_helpers/mod.rs b/crates/sqlclean/src/sql_tokens/_helpers/mod.rs index a2df8fb..8874215 100644 --- a/crates/sqlclean/src/sql_tokens/_helpers/mod.rs +++ b/crates/sqlclean/src/sql_tokens/_helpers/mod.rs @@ -1,3 +1,5 @@ -pub mod builtin_functions; +pub mod case_folding; +pub mod is_unquoted_word; +pub mod name_positions; pub mod reserved_keywords; pub mod token_context; diff --git a/crates/sqlclean/src/sql_tokens/main/name_positions.rs b/crates/sqlclean/src/sql_tokens/_helpers/name_positions.rs similarity index 90% rename from crates/sqlclean/src/sql_tokens/main/name_positions.rs rename to crates/sqlclean/src/sql_tokens/_helpers/name_positions.rs index b056a9b..91027ca 100644 --- a/crates/sqlclean/src/sql_tokens/main/name_positions.rs +++ b/crates/sqlclean/src/sql_tokens/_helpers/name_positions.rs @@ -3,7 +3,7 @@ use crate::sql_tokens::constants::{ALIAS_KEYWORD, NAME_PREFIXES}; /// Return, for every upper-cased token, whether it follows `.`, `:`, `@`, `$` or `AS`. -pub fn name_positions(uppers: &[String]) -> Vec { +pub(crate) fn name_positions(uppers: &[String]) -> Vec { let mut positions = Vec::with_capacity(uppers.len()); let mut previous: Option<&str> = None; for upper in uppers { diff --git a/crates/sqlclean/src/sql_tokens/_helpers/reserved_keywords.rs b/crates/sqlclean/src/sql_tokens/_helpers/reserved_keywords.rs index ccf9004..5fd64f8 100644 --- a/crates/sqlclean/src/sql_tokens/_helpers/reserved_keywords.rs +++ b/crates/sqlclean/src/sql_tokens/_helpers/reserved_keywords.rs @@ -3,33 +3,26 @@ use std::collections::HashSet; use std::sync::LazyLock; -use polyglot_sql::DialectType; +use sqlquack_fingerprint::FingerprintDialect; -static DUCKDB: LazyLock> = - LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/duckdb.txt"))); -static POSTGRES: LazyLock> = - LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/postgres.txt"))); static BIGQUERY: LazyLock> = LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/bigquery.txt"))); -static SNOWFLAKE: LazyLock> = - LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/snowflake.txt"))); static TSQL: LazyLock> = LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/tsql.txt"))); static DATABRICKS: LazyLock> = LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/databricks.txt"))); -static GENERIC: LazyLock> = - LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/generic.txt"))); -/// Return whether the upper-cased word is a reserved keyword of `dialect`. -pub(crate) fn is_reserved_keyword(dialect: DialectType, upper_word: &str) -> bool { +/// Whether the upper-cased word is reserved in `dialect`; unlisted dialects use SQLQuack's generic list. +pub(crate) fn is_reserved_keyword(dialect: &str, upper_word: &str) -> bool { let keywords: &HashSet = match dialect { - DialectType::DuckDB => &DUCKDB, - DialectType::PostgreSQL => &POSTGRES, - DialectType::BigQuery => &BIGQUERY, - DialectType::Snowflake => &SNOWFLAKE, - DialectType::TSQL => &TSQL, - DialectType::Databricks => &DATABRICKS, - _ => &GENERIC, + "bigquery" => &BIGQUERY, + "tsql" => &TSQL, + "databricks" => &DATABRICKS, + _ => { + let fingerprint = + FingerprintDialect::from_name(dialect).unwrap_or(FingerprintDialect::Generic); + return sqlquack_fingerprint::is_reserved_keyword(fingerprint, upper_word); + } }; keywords.contains(upper_word) } diff --git a/crates/sqlclean/src/sql_tokens/_helpers/token_context.rs b/crates/sqlclean/src/sql_tokens/_helpers/token_context.rs index fabb747..b85f1c4 100644 --- a/crates/sqlclean/src/sql_tokens/_helpers/token_context.rs +++ b/crates/sqlclean/src/sql_tokens/_helpers/token_context.rs @@ -1,30 +1,70 @@ //! Positional context of one authored token among its neighbours. -use polyglot_sql::DialectType; - -use crate::sql_tokens::_helpers::builtin_functions::is_builtin_function; +use crate::sql_tokens::_helpers::is_unquoted_word::is_unquoted_word; use crate::sql_tokens::_helpers::reserved_keywords::is_reserved_keyword; -use crate::sql_tokens::constants::CALL_OPENER; -use crate::sql_tokens::main::is_unquoted_word::is_unquoted_word; +use crate::sql_tokens::constants::{ + ALIAS_KEYWORD, CALL_CLOSER, CALL_OPENER, COLUMN_LIST_RELATION_KEYWORDS, CTE_BODY_OPENERS, + PLACEHOLDER_PREFIX, +}; pub(crate) struct TokenContext<'a> { pub(crate) raws: &'a [String], pub(crate) uppers: &'a [String], + pub(crate) closers: &'a [Option], pub(crate) index: usize, } impl TokenContext<'_> { - /// Whether the token is a reserved keyword or a built-in function name at a call site. - pub(crate) fn is_foldable(&self, dialect: DialectType) -> bool { - if !is_unquoted_word(&self.raws[self.index]) { + /// Whether the token is a reserved keyword or a called name, as SQLQuack's fingerprint folds. + pub(crate) fn is_foldable(&self, dialect: &str) -> bool { + let raw = self.raws[self.index].as_str(); + is_unquoted_word(raw) + && (is_reserved_keyword(dialect, &self.uppers[self.index]) || self.calls(raw)) + } + + /// Whether the word is followed by `(` and is neither a placeholder nor a relation's column list. + fn calls(&self, raw: &str) -> bool { + let index = self.index; + if self + .uppers + .get(index + 1) + .is_none_or(|next| next != CALL_OPENER) + || raw.starts_with(PLACEHOLDER_PREFIX) + { return false; } - let upper = self.uppers[self.index].as_str(); - is_reserved_keyword(dialect, upper) - || (self.next() == Some(CALL_OPENER) && is_builtin_function(dialect, upper)) + let previous: Option<&str> = index + .checked_sub(1) + .map(|before| self.uppers[before].as_str()); + if previous.is_some_and(|word| COLUMN_LIST_RELATION_KEYWORDS.contains(&word)) { + return false; + } + let Some(closer) = self.closers[index + 1] else { + return true; + }; + !(self + .uppers + .get(closer + 1) + .is_some_and(|next| next == ALIAS_KEYWORD) + && self + .uppers + .get(closer + 2) + .is_some_and(|next| CTE_BODY_OPENERS.contains(&next.as_str()))) } +} - fn next(&self) -> Option<&str> { - self.uppers.get(self.index + 1).map(String::as_str) +/// The index of the `)` closing each `(`, by the index of the `(`. +pub(crate) fn matching_closers(uppers: &[String]) -> Vec> { + let mut closers: Vec> = vec![None; uppers.len()]; + let mut open: Vec = Vec::new(); + for (index, token) in uppers.iter().enumerate() { + if token == CALL_OPENER { + open.push(index); + } else if token == CALL_CLOSER + && let Some(opener) = open.pop() + { + closers[opener] = Some(index); + } } + closers } diff --git a/crates/sqlclean/src/sql_tokens/constants.rs b/crates/sqlclean/src/sql_tokens/constants.rs index 4d12baf..a315b83 100644 --- a/crates/sqlclean/src/sql_tokens/constants.rs +++ b/crates/sqlclean/src/sql_tokens/constants.rs @@ -1,23 +1,21 @@ //! Token positions that decide which authored words may change letter case. -use polyglot_sql::tokens::TokenType; +use sqlquack_syntax::tokens::models::TokenType; /// Column aliases after `AS` may be any keyword in DuckDB and PostgreSQL. pub(crate) const ALIAS_KEYWORD: &str = "AS"; /// The token that turns a preceding name into a function call. pub(crate) const CALL_OPENER: &str = "("; +/// The token that closes a call's arguments or a column list. +pub(crate) const CALL_CLOSER: &str = ")"; +/// Keywords after which a word followed by `(` names a relation and its column list. +pub(crate) const COLUMN_LIST_RELATION_KEYWORDS: [&str; 2] = ["INTO", "TABLE"]; +/// The tokens that can follow `AS` after a CTE's column list. +pub(crate) const CTE_BODY_OPENERS: [&str; 3] = ["(", "MATERIALIZED", "NOT"]; +/// The prefix of placeholder calls such as SQLBuild's `__ref` and `__source`. +pub(crate) const PLACEHOLDER_PREFIX: &str = "__"; /// Field names after `.`, path keys after `:`, and parameter names after `@` or `$`. pub(crate) const NAME_PREFIXES: [&str; 4] = [".", ":", "@", "$"]; -/// Built-in calls with keyword syntax inside their parentheses, which catalogues do not list. -pub const CALL_SYNTAX_FUNCTIONS: [&str; 6] = [ - "CAST", - "TRY_CAST", - "SAFE_CAST", - "EXTRACT", - "CONVERT", - "TRY_CONVERT", -]; - pub(crate) const PROJECTION_ENDS: [TokenType; 13] = [ TokenType::From, TokenType::Where, diff --git a/crates/sqlclean/src/sql_tokens/main/builtin_function_names.rs b/crates/sqlclean/src/sql_tokens/main/builtin_function_names.rs deleted file mode 100644 index ec0c9e4..0000000 --- a/crates/sqlclean/src/sql_tokens/main/builtin_function_names.rs +++ /dev/null @@ -1,6 +0,0 @@ -use polyglot_sql::DialectType; - -/// Every catalogued or parser-typed built-in function name of `dialect`, lower-cased. -pub fn builtin_function_names(dialect: DialectType) -> Vec { - crate::sql_tokens::_helpers::builtin_functions::builtin_function_names(dialect) -} diff --git a/crates/sqlclean/src/sql_tokens/main/canonical_tokens.rs b/crates/sqlclean/src/sql_tokens/main/canonical_tokens.rs index d1bfb83..89df083 100644 --- a/crates/sqlclean/src/sql_tokens/main/canonical_tokens.rs +++ b/crates/sqlclean/src/sql_tokens/main/canonical_tokens.rs @@ -1,25 +1,15 @@ -//! Canonical token stream of SQL text under one dialect tokenizer. +//! Canonical token stream of authored token texts. -use polyglot_sql::Dialect; -use polyglot_sql::tokens::Token; - -use crate::sql_tokens::main::case_folding::foldable_tokens; -use crate::sql_tokens::main::token_texts::token_texts; +use crate::sql_tokens::_helpers::case_folding::foldable_tokens; use crate::sql_tokens::models::CanonicalToken; -/// Drop whitespace and comments from tokenized `sql` and upper-case case-insensitive tokens. -pub fn canonical_tokens( - sql: &str, - tokens: &[Token], - dialect: &Dialect, -) -> Result, String> { - let raws = token_texts(sql, tokens)?; - let foldable = foldable_tokens(&raws, tokens, dialect); - Ok(raws - .into_iter() +/// Upper-case the case-insensitive tokens among `raws`, the texts of the significant tokens. +pub fn canonical_tokens(raws: Vec, dialect: &str) -> Vec { + let foldable = foldable_tokens(&raws, dialect); + raws.into_iter() .zip(foldable) .map(|(raw, fold)| CanonicalToken { text: if fold { raw.to_ascii_uppercase() } else { raw }, }) - .collect()) + .collect() } diff --git a/crates/sqlclean/src/sql_tokens/main/case_folding.rs b/crates/sqlclean/src/sql_tokens/main/case_folding.rs deleted file mode 100644 index fc29290..0000000 --- a/crates/sqlclean/src/sql_tokens/main/case_folding.rs +++ /dev/null @@ -1,23 +0,0 @@ -//! Which authored tokens are case-insensitive keywords or built-in function names. - -use polyglot_sql::Dialect; -use polyglot_sql::tokens::Token; - -use crate::sql_tokens::_helpers::token_context::TokenContext; -use crate::sql_tokens::main::name_positions::name_positions; - -/// Return, for every token, whether the formatter may recase it and the fingerprint folds it. -pub fn foldable_tokens(raws: &[String], tokens: &[Token], dialect: &Dialect) -> Vec { - let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); - let names = name_positions(&uppers); - let mut foldable = Vec::with_capacity(tokens.len()); - for (index, name) in names.iter().enumerate().take(tokens.len()) { - let context = TokenContext { - raws, - uppers: &uppers, - index, - }; - foldable.push(!name && context.is_foldable(dialect.dialect_type())); - } - foldable -} diff --git a/crates/sqlclean/src/sql_tokens/main/mod.rs b/crates/sqlclean/src/sql_tokens/main/mod.rs index 02cb65b..66fbe56 100644 --- a/crates/sqlclean/src/sql_tokens/main/mod.rs +++ b/crates/sqlclean/src/sql_tokens/main/mod.rs @@ -1,10 +1,4 @@ -#[cfg(any(test, feature = "test-support"))] -pub mod builtin_function_names; pub mod canonical_tokens; -pub mod case_folding; -pub mod is_unquoted_word; -pub mod name_positions; pub mod projection_spans; #[cfg(any(test, feature = "test-support"))] pub mod query_fingerprint; -pub mod token_texts; diff --git a/crates/sqlclean/src/sql_tokens/main/projection_spans.rs b/crates/sqlclean/src/sql_tokens/main/projection_spans.rs index 5965bd7..3c29d1d 100644 --- a/crates/sqlclean/src/sql_tokens/main/projection_spans.rs +++ b/crates/sqlclean/src/sql_tokens/main/projection_spans.rs @@ -1,6 +1,6 @@ //! Authored spans of the top-level items of one select list. -use polyglot_sql::tokens::{Token, TokenType}; +use sqlquack_syntax::tokens::models::{Token, TokenType}; use crate::sql_tokens::constants::PROJECTION_ENDS; diff --git a/crates/sqlclean/src/sql_tokens/main/token_texts.rs b/crates/sqlclean/src/sql_tokens/main/token_texts.rs deleted file mode 100644 index 8467369..0000000 --- a/crates/sqlclean/src/sql_tokens/main/token_texts.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Authored text of tokens, taken from their character spans. - -use polyglot_sql::tokens::Token; - -/// Return the authored text of every token in `sql`. -pub fn token_texts(sql: &str, tokens: &[Token]) -> Result, String> { - let characters: Vec = sql.chars().collect(); - let mut texts = Vec::with_capacity(tokens.len()); - for token in tokens { - let slice = characters - .get(token.span.start..token.span.end) - .ok_or_else(|| "SQL tokenizer returned an invalid token span".to_string())?; - texts.push(slice.iter().collect()); - } - Ok(texts) -} diff --git a/crates/sqlclean/src/sql_tokens/reserved_keywords/duckdb.txt b/crates/sqlclean/src/sql_tokens/reserved_keywords/duckdb.txt deleted file mode 100644 index 4374c4e..0000000 --- a/crates/sqlclean/src/sql_tokens/reserved_keywords/duckdb.txt +++ /dev/null @@ -1,77 +0,0 @@ -# DuckDB reserved keywords: SELECT keyword_name FROM duckdb_keywords() -# WHERE keyword_category = 'reserved' (DuckDB 1.5.2). -all -analyse -analyze -and -any -array -as -asc -asymmetric -both -case -cast -check -collate -column -constraint -create -default -deferrable -desc -describe -distinct -do -else -end -except -false -fetch -for -foreign -from -group -having -in -initially -intersect -into -lambda -lateral -leading -limit -not -null -offset -on -only -or -order -pivot -pivot_longer -pivot_wider -placing -primary -qualify -references -returning -select -show -some -summarize -symmetric -table -then -to -trailing -true -union -unique -unpivot -using -variadic -when -where -window -with diff --git a/crates/sqlclean/src/sql_tokens/reserved_keywords/generic.txt b/crates/sqlclean/src/sql_tokens/reserved_keywords/generic.txt deleted file mode 100644 index 05f8ee4..0000000 --- a/crates/sqlclean/src/sql_tokens/reserved_keywords/generic.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Generic dialect: keywords reserved in every dialect file above except Databricks -# (DuckDB, PostgreSQL, BigQuery, Snowflake and T-SQL). -all -and -any -as -create -distinct -else -for -from -group -having -in -intersect -into -not -null -on -or -order -select -some -then -to -union -where -with diff --git a/crates/sqlclean/src/sql_tokens/reserved_keywords/postgres.txt b/crates/sqlclean/src/sql_tokens/reserved_keywords/postgres.txt deleted file mode 100644 index 7a57911..0000000 --- a/crates/sqlclean/src/sql_tokens/reserved_keywords/postgres.txt +++ /dev/null @@ -1,80 +0,0 @@ -# PostgreSQL reserved keywords (non-function): RESERVED_KEYWORD entries of -# src/include/parser/kwlist.h (REL_17_STABLE); Appendix C 'reserved'. -all -analyse -analyze -and -any -array -as -asc -asymmetric -both -case -cast -check -collate -column -constraint -create -current_catalog -current_date -current_role -current_time -current_timestamp -current_user -default -deferrable -desc -distinct -do -else -end -except -false -fetch -for -foreign -from -grant -group -having -in -initially -intersect -into -lateral -leading -limit -localtime -localtimestamp -not -null -offset -on -only -or -order -placing -primary -references -returning -select -session_user -some -symmetric -system_user -table -then -to -trailing -true -union -unique -user -using -variadic -when -where -window -with diff --git a/crates/sqlclean/src/sql_tokens/reserved_keywords/snowflake.txt b/crates/sqlclean/src/sql_tokens/reserved_keywords/snowflake.txt deleted file mode 100644 index 511b646..0000000 --- a/crates/sqlclean/src/sql_tokens/reserved_keywords/snowflake.txt +++ /dev/null @@ -1,65 +0,0 @@ -# Snowflake reserved keywords: entries marked 'Reserved by ANSI', 'Reserved by Snowflake' -# or 'Reserved by Snowflake and others' in https://docs.snowflake.com/en/sql-reference/reserved-keywords. -# Limited keywords (usable as identifiers in some positions) are excluded. -all -alter -and -any -as -between -by -check -column -connect -create -current -delete -distinct -drop -else -exists -following -for -from -grant -group -having -ilike -in -increment -insert -intersect -into -is -like -minus -not -null -of -on -or -order -qualify -regexp -revoke -rlike -row -rows -sample -select -set -some -start -table -tablesample -then -to -trigger -union -unique -update -values -whenever -where -window -with diff --git a/crates/sqlclean/src/sql_tokens/tests.rs b/crates/sqlclean/src/sql_tokens/tests.rs index b4e9827..7934a5e 100644 --- a/crates/sqlclean/src/sql_tokens/tests.rs +++ b/crates/sqlclean/src/sql_tokens/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_canonical_tokens.rs"] mod canonical_tokens; +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_query_fingerprint.rs"] mod query_fingerprint; #[path = "tests/test_types.rs"] diff --git a/crates/sqlclean/src/sql_tokens/tests/helpers.rs b/crates/sqlclean/src/sql_tokens/tests/helpers.rs new file mode 100644 index 0000000..b01abdf --- /dev/null +++ b/crates/sqlclean/src/sql_tokens/tests/helpers.rs @@ -0,0 +1,12 @@ +use sqlquack_syntax::lossless::models::TokenKind; +use sqlquack_syntax::tokens::models::Dialect; + +/// Significant token texts of `sql` under one of SQLQuack's tokenizers. +pub(crate) fn significant_texts(sql: &str, tokenizer: Dialect) -> Result, String> { + Ok(sqlquack_syntax::tokenize(sql, tokenizer) + .map_err(|error| error.to_string())? + .iter() + .filter(|token| matches!(token.kind, TokenKind::Significant(_))) + .map(|token| token.text(sql).to_owned()) + .collect()) +} diff --git a/crates/sqlclean/src/sql_tokens/tests/test_canonical_tokens.rs b/crates/sqlclean/src/sql_tokens/tests/test_canonical_tokens.rs index 2dec3bf..497f886 100644 --- a/crates/sqlclean/src/sql_tokens/tests/test_canonical_tokens.rs +++ b/crates/sqlclean/src/sql_tokens/tests/test_canonical_tokens.rs @@ -1,6 +1,7 @@ -use polyglot_sql::{Dialect, DialectType}; +use sqlquack_syntax::tokens::models::Dialect; use crate::sql_tokens::main::canonical_tokens::canonical_tokens; +use crate::sql_tokens::tests::helpers::significant_texts; use crate::sql_tokens::tests::test_types::CanonicalTokensTestCase; #[test] @@ -9,204 +10,222 @@ fn given_sql_pairs_when_canonicalizing_then_only_layout_comments_and_word_case_a let test_cases = [ CanonicalTokensTestCase { description: "whitespace between tokens is ignored", - dialect: DialectType::DuckDB, + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select a,b from orders", right: "SELECT\n a, b\nFROM orders", expected_equal: true, }, CanonicalTokensTestCase { description: "comments are ignored", - dialect: DialectType::Snowflake, + dialect: "snowflake", + tokenizer: Dialect::Snowflake, left: "select a -- note\nfrom orders /* block */", right: "select a from orders", expected_equal: true, }, CanonicalTokensTestCase { description: "keyword and built-in function case is ignored", - dialect: DialectType::Snowflake, + dialect: "snowflake", + tokenizer: Dialect::Snowflake, left: "select count(*) from orders", right: "SELECT COUNT(*) FROM orders", expected_equal: true, }, CanonicalTokensTestCase { description: "unquoted identifier case is significant", - dialect: DialectType::Snowflake, + dialect: "snowflake", + tokenizer: Dialect::Snowflake, left: "select count(*) from orders", right: "SELECT COUNT(*) FROM ORDERS", expected_equal: false, }, CanonicalTokensTestCase { description: "Snowflake path key case is significant", - dialect: DialectType::Snowflake, + dialect: "snowflake", + tokenizer: Dialect::Snowflake, left: "select payload:customerId from t", right: "select payload:customerid from t", expected_equal: false, }, CanonicalTokensTestCase { description: "BigQuery qualified table case is significant", - dialect: DialectType::BigQuery, + dialect: "bigquery", + tokenizer: Dialect::DuckDB, left: "select * from ds.Orders", right: "select * from ds.orders", expected_equal: false, }, CanonicalTokensTestCase { description: "DuckDB alias case is significant", - dialect: DialectType::DuckDB, + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select 1 as Foo", right: "select 1 as foo", expected_equal: false, }, CanonicalTokensTestCase { description: "keyword-named qualified name case is significant", - dialect: DialectType::BigQuery, + dialect: "bigquery", + tokenizer: Dialect::DuckDB, left: "select * from inventory.view", right: "select * from inventory.VIEW", expected_equal: false, }, CanonicalTokensTestCase { description: "keyword-named alias case is significant", - dialect: DialectType::DuckDB, + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select pos as index from t", right: "select pos AS INDEX from t", expected_equal: false, }, CanonicalTokensTestCase { - description: "user-defined function case is significant", - dialect: DialectType::DuckDB, + description: "a called name folds like SQLQuack's fingerprint, built-in or not", + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select my_udf(a) from t", right: "select MY_UDF(a) from t", - expected_equal: false, + expected_equal: true, }, CanonicalTokensTestCase { description: "built-in function case is ignored", - dialect: DialectType::DuckDB, + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select coalesce(a, 0) from t", right: "select COALESCE(a, 0) from t", expected_equal: true, }, CanonicalTokensTestCase { description: "Snowflake double-slash comments are ignored", - dialect: DialectType::Snowflake, + dialect: "snowflake", + tokenizer: Dialect::Snowflake, left: "select a // note\nfrom t", right: "select a from t", expected_equal: true, }, - CanonicalTokensTestCase { - description: "BigQuery hash comments are ignored", - dialect: DialectType::BigQuery, - left: "select a # note\nfrom t", - right: "select a from t", - expected_equal: true, - }, CanonicalTokensTestCase { description: "DuckDB struct-literal key case is significant", - dialect: DialectType::DuckDB, + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select {Key: 1}", right: "select {key: 1}", expected_equal: false, }, CanonicalTokensTestCase { description: "a non-reserved keyword operand before an operator keeps its case", - dialect: DialectType::DuckDB, + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select a, Rows ^ 2 from t", right: "select a, rows ^ 2 from t", expected_equal: false, }, CanonicalTokensTestCase { description: "a non-reserved keyword table in a comma join keeps its case", - dialect: DialectType::BigQuery, + dialect: "bigquery", + tokenizer: Dialect::DuckDB, left: "select * from ds.a, View v", right: "select * from ds.a, view v", expected_equal: false, }, CanonicalTokensTestCase { description: "a reserved word after a dot is a field name", - dialect: DialectType::BigQuery, + dialect: "bigquery", + tokenizer: Dialect::DuckDB, left: "select abc.Group from abc", right: "select abc.group from abc", expected_equal: false, }, CanonicalTokensTestCase { description: "a reserved keyword column alias keeps its case", - dialect: DialectType::DuckDB, + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select 1 as Select", right: "select 1 as select", expected_equal: false, }, CanonicalTokensTestCase { description: "reserved keyword case is ignored", - dialect: DialectType::DuckDB, + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select a from t where a in (1) limit 1", right: "SELECT a FROM t WHERE a IN (1) LIMIT 1", expected_equal: true, }, CanonicalTokensTestCase { description: "Databricks reserves no keywords", - dialect: DialectType::Databricks, + dialect: "databricks", + tokenizer: Dialect::DuckDB, left: "select a from t", right: "SELECT a FROM t", expected_equal: false, }, CanonicalTokensTestCase { description: "whitespace inside a string literal is significant", - dialect: DialectType::DuckDB, + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select 'a b' as code", right: "select 'a b' as code", expected_equal: false, }, CanonicalTokensTestCase { description: "string literal case is significant", - dialect: DialectType::DuckDB, + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select 'Open' as status", right: "select 'OPEN' as status", expected_equal: false, }, CanonicalTokensTestCase { description: "quoted identifier case is significant", - dialect: DialectType::Snowflake, + dialect: "snowflake", + tokenizer: Dialect::Snowflake, left: "select \"Order\" from orders", right: "select \"ORDER\" from orders", expected_equal: false, }, CanonicalTokensTestCase { description: "dollar-quoted literal content is significant", - dialect: DialectType::PostgreSQL, + dialect: "postgresql", + tokenizer: Dialect::PostgreSQL, left: "select $$a b$$ as note", right: "select $$a b$$ as note", expected_equal: false, }, CanonicalTokensTestCase { description: "number spelling is significant", - dialect: DialectType::DuckDB, + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select 1.0 as amount", right: "select 1.00 as amount", expected_equal: false, }, CanonicalTokensTestCase { description: "a renamed function is significant", - dialect: DialectType::Snowflake, + dialect: "snowflake", + tokenizer: Dialect::Snowflake, left: "select startswith(code, 'EU')", right: "select starts_with(code, 'EU')", expected_equal: false, }, CanonicalTokensTestCase { description: "an added alias keyword is significant", - dialect: DialectType::DuckDB, + dialect: "duckdb", + tokenizer: Dialect::DuckDB, left: "select a b from orders", right: "select a as b from orders", expected_equal: false, }, ]; for test_case in test_cases { - let dialect = Dialect::get(test_case.dialect); - let left_tokens = dialect - .tokenize(test_case.left) - .map_err(|error| error.to_string())?; - let right_tokens = dialect - .tokenize(test_case.right) - .map_err(|error| error.to_string())?; - let left = canonical_tokens(test_case.left, &left_tokens, &dialect)?; - let right = canonical_tokens(test_case.right, &right_tokens, &dialect)?; + let left = canonical_tokens( + significant_texts(test_case.left, test_case.tokenizer)?, + test_case.dialect, + ); + let right = canonical_tokens( + significant_texts(test_case.right, test_case.tokenizer)?, + test_case.dialect, + ); assert_eq!( left == right, test_case.expected_equal, diff --git a/crates/sqlclean/src/sql_tokens/tests/test_types.rs b/crates/sqlclean/src/sql_tokens/tests/test_types.rs index 2e7671d..74bbc3f 100644 --- a/crates/sqlclean/src/sql_tokens/tests/test_types.rs +++ b/crates/sqlclean/src/sql_tokens/tests/test_types.rs @@ -1,6 +1,7 @@ pub(crate) struct CanonicalTokensTestCase { pub(crate) description: &'static str, - pub(crate) dialect: polyglot_sql::DialectType, + pub(crate) dialect: &'static str, + pub(crate) tokenizer: sqlquack_syntax::tokens::models::Dialect, pub(crate) left: &'static str, pub(crate) right: &'static str, pub(crate) expected_equal: bool, diff --git a/fensu.toml b/fensu.toml index 72bf6fb..82c1d3d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -35,13 +35,13 @@ reason = "Integration surface: SQLBuild's Python bindings format through `format [[targets.rust.rule_exceptions]] rule = "FPRSL105" -path = "crates/sqlclean/src/sql_tokens/main/builtin_function_names.rs" -reason = "This entry exists for the sql_lint tests, which the analyzer does not count as importers." +path = "crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs" +reason = "This test-support entry checks that fixes preserve meaning in the sql_lint tests, which the analyzer does not count as importers." [[targets.rust.rule_exceptions]] rule = "FPRSL105" -path = "crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs" -reason = "This test-support entry checks that fixes preserve meaning in the sql_lint tests, which the analyzer does not count as importers." +path = "crates/sqlclean/src/sql_tokens/main/canonical_tokens.rs" +reason = "Integration surface: SQLBuild folds token texts for its fingerprints through this entry." [[targets.rust.rule_exceptions]] rule = "FPRSL105" From 3c3b182e3c2531cd96f45295e5defd3a7d9b6083 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:16:18 +0000 Subject: [PATCH 11/21] wip(cli): report files in a dialect SQLQuack does not parse instead of failing check lists each such file once as a warning, and as unavailable in JSON and SARIF, without counting it as a problem or running custom Rules on it. format and fix leave the file unchanged and do not fail. --- .../skill/references/docs/configuration.md | 9 ++++++ .../src/checking/_helpers/builtin.rs | 15 +++++++-- .../src/checking/_helpers/orchestration.rs | 18 +++++++++-- crates/sqlclean-cli/src/checking/constants.rs | 2 ++ crates/sqlclean-cli/src/checking/models.rs | 9 ++++++ .../src/command/_helpers/format_command.rs | 21 +++++++++--- .../src/fixing/_helpers/passes.rs | 4 ++- .../src/formatting/main/format_source.rs | 32 ++++++++++++------- crates/sqlclean-cli/src/formatting/mod.rs | 1 + crates/sqlclean-cli/src/formatting/models.rs | 8 +++++ .../src/reporting/_helpers/machine.rs | 12 ++++++- .../src/reporting/_helpers/text.rs | 3 ++ .../src/reporting/main/render_findings.rs | 2 +- tests/e2e/src/sqlclean/hosting/_test_types.py | 1 + .../sqlclean/hosting/test_sqlclean_command.py | 30 +++++++++++++++++ .../src/content/docs/docs/configuration.mdx | 9 ++++++ 16 files changed, 151 insertions(+), 25 deletions(-) create mode 100644 crates/sqlclean-cli/src/formatting/models.rs diff --git a/crates/sqlclean-cli/skill/references/docs/configuration.md b/crates/sqlclean-cli/skill/references/docs/configuration.md index 4e8d21f..ac2a08f 100644 --- a/crates/sqlclean-cli/skill/references/docs/configuration.md +++ b/crates/sqlclean-cli/skill/references/docs/configuration.md @@ -123,6 +123,15 @@ SELECT [order id] FROM orders error[SQLCLEAN999]: SQL could not be parsed as duckdb: Parse error at line 1, column 17: Expected RBracket, got Var ('id') near [SELECT [order id] FROM orders] ``` +SQLClean reads SQL with SQLQuack's parsers. A file in a dialect SQLQuack does not parse yet, such +as `mysql`, is not checked, fixed or formatted: `check` reports it once as a warning (in JSON, under +`unavailable`), and it never fails a command or counts as a problem. + +```text +warning: SQL lint is unavailable for the mysql dialect: SQLQuack does not parse it yet + --> reports/q.sql +``` + Supported dialects, case-insensitive: | Dialect | Also accepted | diff --git a/crates/sqlclean-cli/src/checking/_helpers/builtin.rs b/crates/sqlclean-cli/src/checking/_helpers/builtin.rs index 3ca621f..58c9b30 100644 --- a/crates/sqlclean-cli/src/checking/_helpers/builtin.rs +++ b/crates/sqlclean-cli/src/checking/_helpers/builtin.rs @@ -8,7 +8,8 @@ use sqlclean::sql_lint::main::lint::lint; use sqlclean::sql_lint::types::{LintDiagnostic, LintRequest}; use crate::checking::constants::{ - PARSE_ERROR_CODE, PARSE_ERROR_MESSAGE, PARSE_ERROR_REMEDIATION, UNSAFE_FIX_BUILTIN_RULES, + PARSE_ERROR_CODE, PARSE_ERROR_MESSAGE, PARSE_ERROR_REMEDIATION, UNAVAILABLE_CODE, + UNSAFE_FIX_BUILTIN_RULES, }; use crate::checking::models::{Finding, FixSafety}; use crate::config::constants::CONFIG_FILE_NAME; @@ -16,7 +17,7 @@ use crate::config::models::Thresholds; use crate::discovery::models::SourceFile; use crate::positions::models::Edit; -/// The built-in findings of `file`, or one parse-error finding when the engine rejects it. +/// The built-in findings of `file`, or its one parse-error finding or unavailable-dialect marker. pub(crate) fn builtin_findings( file: &SourceFile, codes: &BTreeSet, @@ -33,6 +34,16 @@ pub(crate) fn builtin_findings( ..LintRequest::default() }; match lint(request) { + Ok(response) if response.unavailable.is_some() => vec![Finding { + code: UNAVAILABLE_CODE.to_owned(), + path: file.path.clone(), + start: 0, + end: 0, + message: response.unavailable.unwrap_or_default(), + remediation: String::new(), + edits: Vec::new(), + fix: None, + }], Ok(response) => response .diagnostics .into_iter() diff --git a/crates/sqlclean-cli/src/checking/_helpers/orchestration.rs b/crates/sqlclean-cli/src/checking/_helpers/orchestration.rs index 2b708cc..0a2ee67 100644 --- a/crates/sqlclean-cli/src/checking/_helpers/orchestration.rs +++ b/crates/sqlclean-cli/src/checking/_helpers/orchestration.rs @@ -11,8 +11,8 @@ use crate::checking::_helpers::catalogue::{builtin_rules, custom_rules, custom_s use crate::checking::_helpers::custom::{CustomEvaluation, custom_findings}; use crate::checking::_helpers::exemptions::apply_exemptions; use crate::checking::_helpers::selection::selected_codes; -use crate::checking::constants::{CUSTOM_RULE_PREFIX, PARSE_ERROR_CODE}; -use crate::checking::models::{CheckOptions, CheckOutcome, Finding, RuleInfo}; +use crate::checking::constants::{CUSTOM_RULE_PREFIX, PARSE_ERROR_CODE, UNAVAILABLE_CODE}; +use crate::checking::models::{CheckOptions, CheckOutcome, Finding, RuleInfo, UnavailableFile}; use crate::config::models::Config; use crate::discovery::models::SourceFile; @@ -49,7 +49,7 @@ pub(crate) fn check_sources( .cloned() .collect(); let selected: BTreeSet = selected_codes(&all, select, &ignore)?; - let (mut findings, updated) = run_builtins( + let (builtin, updated) = run_builtins( &BuiltinRun { config, files, @@ -59,10 +59,21 @@ pub(crate) fn check_sources( cache, ); let mut cache: ResultCache = updated; + let (markers, mut findings): (Vec, Vec) = builtin + .into_iter() + .partition(|finding| finding.code == UNAVAILABLE_CODE); + let unavailable: Vec = markers + .into_iter() + .map(|marker| UnavailableFile { + path: marker.path, + reason: marker.message, + }) + .collect(); let unparsed: HashSet<&str> = findings .iter() .filter(|finding| finding.code == PARSE_ERROR_CODE) .map(|finding| finding.path.as_str()) + .chain(unavailable.iter().map(|file| file.path.as_str())) .collect(); let parsed: Vec<&SourceFile> = files .iter() @@ -97,6 +108,7 @@ pub(crate) fn check_sources( Ok(CheckOutcome { findings: kept, stale_exceptions, + unavailable, }) } diff --git a/crates/sqlclean-cli/src/checking/constants.rs b/crates/sqlclean-cli/src/checking/constants.rs index c35f2eb..797d18a 100644 --- a/crates/sqlclean-cli/src/checking/constants.rs +++ b/crates/sqlclean-cli/src/checking/constants.rs @@ -3,6 +3,8 @@ pub const PARSE_ERROR_CODE: &str = "SQLCLEAN999"; pub(crate) const PARSE_ERROR_MESSAGE: &str = "SQL could not be parsed"; pub(crate) const PARSE_ERROR_REMEDIATION: &str = "Fix the syntax error, or set this file's dialect in sqlclean.toml."; +/// Marks, in the built-in results and their cache, a file whose dialect SQLQuack does not parse. +pub(crate) const UNAVAILABLE_CODE: &str = "SQLCLEAN998"; pub(crate) const CROSS_JOIN_CODE: &str = "SQLCLEAN038"; /// Custom Rule codes start with this prefix. pub const CUSTOM_RULE_PREFIX: &str = "XSQLCLEAN"; diff --git a/crates/sqlclean-cli/src/checking/models.rs b/crates/sqlclean-cli/src/checking/models.rs index 85b95b5..e504392 100644 --- a/crates/sqlclean-cli/src/checking/models.rs +++ b/crates/sqlclean-cli/src/checking/models.rs @@ -59,11 +59,20 @@ pub struct StaleException { pub relation: Option, } +/// A file not checked because SQLQuack does not parse its dialect yet; a notice, not a problem. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct UnavailableFile { + pub path: String, + pub reason: String, +} + /// Every finding of one check, sorted by path, position and code. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] pub struct CheckOutcome { pub findings: Vec, pub stale_exceptions: Vec, + /// Files whose dialect has no SQLQuack parser, by path; they never fail the check. + pub unavailable: Vec, } /// What a check evaluates beyond the configuration. diff --git a/crates/sqlclean-cli/src/command/_helpers/format_command.rs b/crates/sqlclean-cli/src/command/_helpers/format_command.rs index 37cfe72..034f1b0 100644 --- a/crates/sqlclean-cli/src/command/_helpers/format_command.rs +++ b/crates/sqlclean-cli/src/command/_helpers/format_command.rs @@ -10,18 +10,24 @@ use crate::command::models::FormatArgs; use crate::config::models::Config; use crate::discovery::main::discover_files::discover_files; use crate::formatting::main::format_source::format_source; +use crate::formatting::models::Formatted; pub(crate) fn run(config: &Config, arguments: &FormatArgs) -> Result { let files = discover_files(config, &arguments.paths)?; let mut stderr = std::io::stderr(); - let (mut changed, mut refused): (usize, usize) = (0, 0); - let results: Vec> = files + let (mut changed, mut refused, mut unavailable): (usize, usize, usize) = (0, 0, 0); + let results: Vec> = files .par_iter() .map(|file| format_source(&file.text, &file.dialect, config.line_width, config.indent)) .collect(); for (file, result) in files.iter().zip(results) { let formatted: String = match result { - Ok(formatted) => formatted, + Ok(Formatted::Sql(formatted)) => formatted, + Ok(Formatted::Unavailable(reason)) => { + unavailable += 1; + let _ = writeln!(stderr, "warning: {}: {reason}", file.path); + continue; + } Err(reason) => { refused += 1; let _ = writeln!(stderr, "{}: could not format: {reason}", file.path); @@ -40,7 +46,7 @@ pub(crate) fn run(config: &Config, arguments: &FormatArgs) -> Result write_file(file, &formatted)?; } } - let unchanged: usize = files.len() - changed - refused; + let unchanged: usize = files.len() - changed - refused - unavailable; let verb: &str = if arguments.check || arguments.diff { "would be reformatted" } else { @@ -48,7 +54,12 @@ pub(crate) fn run(config: &Config, arguments: &FormatArgs) -> Result }; let _ = writeln!( stderr, - "{changed} file(s) {verb}, {unchanged} file(s) already formatted." + "{changed} file(s) {verb}, {unchanged} file(s) already formatted{}.", + if unavailable > 0 { + format!(", {unavailable} file(s) in a dialect SQLQuack does not format yet") + } else { + String::new() + } ); let failed: bool = refused > 0 || ((arguments.check || arguments.diff) && changed > 0); Ok(if failed { EXIT_FINDINGS } else { EXIT_SUCCESS }) diff --git a/crates/sqlclean-cli/src/fixing/_helpers/passes.rs b/crates/sqlclean-cli/src/fixing/_helpers/passes.rs index 4768da3..ae818de 100644 --- a/crates/sqlclean-cli/src/fixing/_helpers/passes.rs +++ b/crates/sqlclean-cli/src/fixing/_helpers/passes.rs @@ -12,6 +12,7 @@ use crate::fixing::_helpers::edit_selection::select_edits; use crate::fixing::constants::MAX_FIX_PASSES; use crate::fixing::models::FixOutcome; use crate::formatting::main::format_source::format_source; +use crate::formatting::models::Formatted; use crate::positions::main::apply_edits::apply_edits; use crate::positions::models::Edit; @@ -36,7 +37,8 @@ pub(crate) fn fix_sources( let mut format_refusals: Vec<(String, String)> = Vec::new(); for file in &mut files { match format_source(&file.text, &file.dialect, config.line_width, config.indent) { - Ok(formatted) => file.text = formatted, + Ok(Formatted::Sql(formatted)) => file.text = formatted, + Ok(Formatted::Unavailable(_)) => {} Err(reason) => format_refusals.push((file.path.clone(), reason)), } } diff --git a/crates/sqlclean-cli/src/formatting/main/format_source.rs b/crates/sqlclean-cli/src/formatting/main/format_source.rs index c235b0f..b59439d 100644 --- a/crates/sqlclean-cli/src/formatting/main/format_source.rs +++ b/crates/sqlclean-cli/src/formatting/main/format_source.rs @@ -6,18 +6,21 @@ use sqlclean::sql_lint::main::format_requests::format_requests; use sqlclean::sql_lint::models::{FormatRequest, FormatResponse}; use crate::formatting::constants::{FORMAT_REFUSAL as REFUSAL, STATEMENT_SEPARATOR}; +use crate::formatting::models::Formatted; -/// `text` in canonical layout ending in one newline, or why the formatter left it unchanged. +/// `text` in canonical layout ending in one newline, or unavailable, or why the formatter refused it. pub fn format_source( text: &str, dialect: &str, line_width: usize, indent: usize, -) -> Result { +) -> Result { let characters: Vec = text.chars().collect(); - let mut boundaries: Vec = statement_ends(text, dialect)?.ok_or_else(|| { - format!("SQL formatting is unavailable for the {dialect} dialect: SQLQuack does not parse it yet") - })?; + let Some(mut boundaries) = statement_ends(text, dialect)? else { + return Ok(Formatted::Unavailable(format!( + "SQL formatting is unavailable for the {dialect} dialect: SQLQuack does not parse it yet" + ))); + }; if let Some(last) = boundaries.last_mut() { *last = characters.len(); } @@ -26,20 +29,23 @@ pub fn format_source( for end in boundaries { let chunk: String = characters[start..end].iter().collect(); start = end; - let formatted: String = format_statement(chunk.trim(), dialect, line_width, indent)?; + let formatted: String = match format_statement(chunk.trim(), dialect, line_width, indent)? { + Formatted::Sql(sql) => sql, + unavailable @ Formatted::Unavailable(_) => return Ok(unavailable), + }; if !formatted.trim().is_empty() { chunks.push(formatted.trim_end().to_owned()); } } if chunks.is_empty() { let comments: &str = text.trim(); - return Ok(if comments.is_empty() { + return Ok(Formatted::Sql(if comments.is_empty() { String::new() } else { format!("{comments}\n") - }); + })); } - Ok(chunks.join(STATEMENT_SEPARATOR) + "\n") + Ok(Formatted::Sql(chunks.join(STATEMENT_SEPARATOR) + "\n")) } fn format_statement( @@ -47,7 +53,7 @@ fn format_statement( dialect: &str, line_width: usize, indent: usize, -) -> Result { +) -> Result { let request = FormatRequest { version: LINT_API_VERSION, sql: sql.to_owned(), @@ -59,8 +65,10 @@ fn format_statement( let response: FormatResponse = format_requests(vec![request]) .pop() .ok_or_else(|| REFUSAL.to_owned())?; - if response.formatted { - Ok(response.sql) + if let Some(reason) = response.unavailable { + Ok(Formatted::Unavailable(reason)) + } else if response.formatted { + Ok(Formatted::Sql(response.sql)) } else { Err(response.reason.unwrap_or_else(|| REFUSAL.to_owned())) } diff --git a/crates/sqlclean-cli/src/formatting/mod.rs b/crates/sqlclean-cli/src/formatting/mod.rs index 641a36d..26eac3a 100644 --- a/crates/sqlclean-cli/src/formatting/mod.rs +++ b/crates/sqlclean-cli/src/formatting/mod.rs @@ -2,3 +2,4 @@ pub mod constants; pub mod main; +pub mod models; diff --git a/crates/sqlclean-cli/src/formatting/models.rs b/crates/sqlclean-cli/src/formatting/models.rs new file mode 100644 index 0000000..43f8edd --- /dev/null +++ b/crates/sqlclean-cli/src/formatting/models.rs @@ -0,0 +1,8 @@ +//! What formatting one file yields. + +/// A file in canonical layout, or why its dialect cannot be formatted yet; a notice, not a refusal. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Formatted { + Sql(String), + Unavailable(String), +} diff --git a/crates/sqlclean-cli/src/reporting/_helpers/machine.rs b/crates/sqlclean-cli/src/reporting/_helpers/machine.rs index 7753ca0..2deef4f 100644 --- a/crates/sqlclean-cli/src/reporting/_helpers/machine.rs +++ b/crates/sqlclean-cli/src/reporting/_helpers/machine.rs @@ -14,16 +14,18 @@ pub(crate) fn render_json(outcome: &CheckOutcome, located: &[LocatedFinding]) -> let report: Value = json!({ "findings": located, "stale_exceptions": outcome.stale_exceptions, + "unavailable": outcome.unavailable, "summary": { "findings": located.len(), "fixable": located.iter().filter(|finding| finding.fix.is_some()).count(), "stale_exceptions": outcome.stale_exceptions.len(), + "unavailable": outcome.unavailable.len(), }, }); format!("{report:#}\n") } -pub(crate) fn render_sarif(located: &[LocatedFinding]) -> String { +pub(crate) fn render_sarif(outcome: &CheckOutcome, located: &[LocatedFinding]) -> String { let mut codes: Vec<&str> = located .iter() .map(|finding| finding.code.as_str()) @@ -62,6 +64,14 @@ pub(crate) fn render_sarif(located: &[LocatedFinding]) -> String { "rules": codes.iter().map(|code| json!({ "id": code })).collect::>(), }}, "results": results, + "invocations": [{ + "executionSuccessful": true, + "toolExecutionNotifications": outcome.unavailable.iter().map(|file| json!({ + "level": "warning", + "message": { "text": file.reason }, + "locations": [{ "physicalLocation": { "artifactLocation": { "uri": file.path } } }], + })).collect::>(), + }], }], }); format!("{report:#}\n") diff --git a/crates/sqlclean-cli/src/reporting/_helpers/text.rs b/crates/sqlclean-cli/src/reporting/_helpers/text.rs index 3e697f9..8cadf5b 100644 --- a/crates/sqlclean-cli/src/reporting/_helpers/text.rs +++ b/crates/sqlclean-cli/src/reporting/_helpers/text.rs @@ -28,6 +28,9 @@ pub(crate) fn render( stale.rule, stale.path ); } + for file in &outcome.unavailable { + let _ = writeln!(report, "warning: {}\n --> {}\n", file.reason, file.path); + } let safe: usize = fix_count(located, FixSafety::Safe); let unsafe_only: usize = fix_count(located, FixSafety::Unsafe); let problems: usize = located.len() + outcome.stale_exceptions.len(); diff --git a/crates/sqlclean-cli/src/reporting/main/render_findings.rs b/crates/sqlclean-cli/src/reporting/main/render_findings.rs index 00d9a85..060371a 100644 --- a/crates/sqlclean-cli/src/reporting/main/render_findings.rs +++ b/crates/sqlclean-cli/src/reporting/main/render_findings.rs @@ -15,6 +15,6 @@ pub fn render_findings( match format { OutputFormat::Text => crate::reporting::_helpers::text::render(outcome, &located, texts), OutputFormat::Json => crate::reporting::_helpers::machine::render_json(outcome, &located), - OutputFormat::Sarif => crate::reporting::_helpers::machine::render_sarif(&located), + OutputFormat::Sarif => crate::reporting::_helpers::machine::render_sarif(outcome, &located), } } diff --git a/tests/e2e/src/sqlclean/hosting/_test_types.py b/tests/e2e/src/sqlclean/hosting/_test_types.py index e47c068..484009e 100644 --- a/tests/e2e/src/sqlclean/hosting/_test_types.py +++ b/tests/e2e/src/sqlclean/hosting/_test_types.py @@ -16,6 +16,7 @@ class CheckTestCase: expected_exit: int expected_codes: tuple[str, ...] expected_stale: tuple[str, ...] = () + expected_unavailable: tuple[str, ...] = () command: str = "check" diff --git a/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py b/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py index 80e1c89..398f5e2 100644 --- a/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py +++ b/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py @@ -22,6 +22,7 @@ from tests.integration.src.sqlclean.rule_testing.helpers import SAMPLE_RULES DUCKDB: str = 'dialect = "duckdb"\n' +MYSQL_FILES: str = DUCKDB + '[[dialects]]\npaths = ["mysql/**"]\ndialect = "mysql"\n' CROSS_JOIN_ONLY: str = DUCKDB + '[rules]\nselect = ["SQLCLEAN038"]\n' CUSTOM: str = ( DUCKDB + '[rules]\nselect = ["XSQLCLEANNAME", "XSQLCLEANSTYLE"]\ncustom = ["sql_rules"]\n' @@ -118,6 +119,18 @@ expected_exit=1, expected_codes=("SQLCLEAN038",), ), + CheckTestCase( + description="a dialect SQLQuack does not parse is a notice that never fails the check", + config=MYSQL_FILES, + files={ + "mysql/q.sql": "select a.id from a where a.x = null\n", + "models/q.sql": "SELECT 1\n", + }, + arguments=(), + expected_exit=0, + expected_codes=(), + expected_unavailable=("mysql/q.sql",), + ), CheckTestCase( description="unparseable SQL is reported as a finding", config=CROSS_JOIN_ONLY, @@ -155,6 +168,7 @@ def test_given_project_when_checking_then_reports_findings_and_exit_status( assert ( tuple(stale["relation"] for stale in report["stale_exceptions"]) == test_case.expected_stale ) + assert tuple(file["path"] for file in report["unavailable"]) == test_case.expected_unavailable @pytest.mark.parametrize( @@ -199,6 +213,22 @@ def test_given_project_when_checking_then_reports_findings_and_exit_status( expected_exit=1, expected_files={"models/q.sql": "select a.id from a, b\n"}, ), + RewriteTestCase( + description="format leaves a dialect it cannot format unchanged and passes", + config=MYSQL_FILES, + files={"mysql/q.sql": "select 1;select 2\n"}, + arguments=("format", "--check"), + expected_exit=0, + expected_files={"mysql/q.sql": "select 1;select 2\n"}, + ), + RewriteTestCase( + description="fix leaves a dialect SQLQuack does not parse unchanged and passes", + config=MYSQL_FILES, + files={"mysql/q.sql": "select a.id from a, b\n"}, + arguments=("fix",), + expected_exit=0, + expected_files={"mysql/q.sql": "select a.id from a, b\n"}, + ), RewriteTestCase( description="format --check fails without writing", config=DUCKDB, diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx index b95d1c9..1d098fb 100644 --- a/website/src/content/docs/docs/configuration.mdx +++ b/website/src/content/docs/docs/configuration.mdx @@ -112,6 +112,15 @@ SELECT [order id] FROM orders error[SQLCLEAN999]: SQL could not be parsed as duckdb: Parse error at line 1, column 17: Expected RBracket, got Var ('id') near [SELECT [order id] FROM orders] ``` +SQLClean reads SQL with SQLQuack's parsers. A file in a dialect SQLQuack does not parse yet, such +as `mysql`, is not checked, fixed or formatted: `check` reports it once as a warning (in JSON, under +`unavailable`), and it never fails a command or counts as a problem. + +```text +warning: SQL lint is unavailable for the mysql dialect: SQLQuack does not parse it yet + --> reports/q.sql +``` + Supported dialects, case-insensitive: | Dialect | Also accepted | From 89073b63a748f2539a71671cff04f0cbd0562d06 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:41:08 +0000 Subject: [PATCH 12/21] wip(format): name every fingerprint and unavailable-corpus test case, and move the format cases beside the layout rules --- .../src/sql_lint/_helpers/cte_macros.rs | 9 +- .../src/sql_lint/_helpers/formatter.rs | 3 +- .../src/sql_lint/tests/test_format_corpus.rs | 33 ++- .../src/sql_lint/tests/test_layout_rules.rs | 123 ++++++++- .../src/sql_lint/tests/test_sql_lint.rs | 121 --------- .../sqlclean/src/sql_lint/tests/test_types.rs | 6 + .../src/sql_tokens/main/query_fingerprint.rs | 3 +- .../tests/test_query_fingerprint.rs | 234 ++++++++++-------- .../src/sql_tokens/tests/test_types.rs | 16 ++ 9 files changed, 304 insertions(+), 244 deletions(-) diff --git a/crates/sqlclean/src/sql_lint/_helpers/cte_macros.rs b/crates/sqlclean/src/sql_lint/_helpers/cte_macros.rs index 4a84384..de06f7a 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/cte_macros.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/cte_macros.rs @@ -7,6 +7,8 @@ use sqlquack_syntax::{Dialect, LosslessToken, TokenKind}; use crate::constants::INTERPOLATION_PLACEHOLDER_PREFIX; +/// The separator token taken out with a placeholder. +const COMMA: &str = ","; const RESTORE_FAILURE: &str = "native formatter could not restore CTE macro placeholders"; /// SQL with its CTE macro placeholders taken out, and the groups to put back. @@ -106,7 +108,6 @@ impl WithoutCteMacros { let gap_end = significant .get(group.anchor) .map_or(formatted.len(), |&anchor| tokens[anchor].start); - // After the comments that stood before the group; the token check below confirms it. let insert_at = match group.comments_before.checked_sub(1) { None => gap_start, Some(last) => tokens @@ -128,7 +129,7 @@ impl WithoutCteMacros { }; restored.push_str(&formatted[cursor..insert_at]); for text in &group.texts { - if text != "," { + if text != COMMA { restored.push_str(&separator); separator = format!("\n\n{indent}"); } @@ -142,7 +143,6 @@ impl WithoutCteMacros { restored.push_str(&formatted[cursor..]); let restored_tokens = sqlquack_syntax::tokenize(&restored, dialect) .map_err(|_| RESTORE_FAILURE.to_string())?; - // SQLQuack checked the recasing of the stripped SQL; here only the order and text matter. let restored_texts = token_texts(&restored, &restored_tokens); let same_tokens = restored_texts.len() == self.original.len() && restored_texts.iter().zip(&self.original).all( @@ -215,8 +215,7 @@ fn macro_positions(sql: &str, tokens: &[LosslessToken], significant: &[usize]) - positions } -/// The placeholders and the commas that go with them: each one's following comma, or, for a -/// group that ends the CTE list, the comma before the group. +/// The placeholders with each one's following comma, or the comma before a group ending the CTE list. fn removed_positions( tokens: &[LosslessToken], significant: &[usize], diff --git a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs index edf8e7c..a182d52 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs @@ -64,7 +64,7 @@ fn sqlquack_dialect(dialect: &str) -> Option { } } -/// Lay out `request` with SQLQuack's formatter, which checks token identity and idempotency itself. +/// Lay out `request` with SQLQuack; this re-checks idempotency only where placeholders were put back. fn format_with_sqlquack( request: FormatRequest, dialect: SqlQuackDialect, @@ -82,7 +82,6 @@ fn format_with_sqlquack( ..FormatOptions::new(dialect) }; let sql = format_around_cte_macros(&request.sql, &options)?; - // SQLQuack checks its own layout; putting placeholders back is checked here. if request.sql.contains(INTERPOLATION_PLACEHOLDER_PREFIX) && sql != request.sql && format_around_cte_macros(&sql, &options)? != sql diff --git a/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs b/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs index a4ce902..95ffa60 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs @@ -41,18 +41,35 @@ fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stab fn given_dialects_without_a_native_formatter_when_formatting_then_each_query_is_unavailable() -> Result<(), String> { let test_cases = [ - ("bigquery", 168), - ("databricks", 62), - ("postgres", 137), - ("tsql", 75), + test_types::UnavailableCorpusTestCase { + description: "BigQuery corpus", + dialect: "bigquery", + expected_unavailable: 168, + }, + test_types::UnavailableCorpusTestCase { + description: "Databricks corpus", + dialect: "databricks", + expected_unavailable: 62, + }, + test_types::UnavailableCorpusTestCase { + description: "PostgreSQL corpus", + dialect: "postgres", + expected_unavailable: 137, + }, + test_types::UnavailableCorpusTestCase { + description: "T-SQL corpus", + dialect: "tsql", + expected_unavailable: 75, + }, ]; let corpus: Vec = serde_json::from_str(FORMAT_CORPUS).map_err(|error| error.to_string())?; - for (dialect, expected_unavailable) in test_cases { + for test_case in &test_cases { assert_eq!( - check_unavailable_corpus(&corpus, dialect)?, - expected_unavailable, - "{dialect}" + check_unavailable_corpus(&corpus, test_case.dialect)?, + test_case.expected_unavailable, + "{}", + test_case.description ); } Ok(()) diff --git a/crates/sqlclean/src/sql_lint/tests/test_layout_rules.rs b/crates/sqlclean/src/sql_lint/tests/test_layout_rules.rs index 07da408..c6bf7f2 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_layout_rules.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_layout_rules.rs @@ -1,5 +1,6 @@ -use serde_json::Value; +use serde_json::{Value, json}; +use crate::sql_lint::main::formatter::format_json; use crate::sql_lint::tests::helpers::format_at_width; use crate::sql_lint::tests::test_types; use crate::sql_tokens::main::query_fingerprint::query_fingerprint; @@ -208,3 +209,123 @@ fn given_clause_layout_rules_when_formatting_then_each_rule_applies_idempotently } Ok(()) } + +#[test] +fn given_format_cases_when_formatting_then_output_matches() -> Result<(), String> { + let test_cases = [ + test_types::FormatTestCase { + description: "comment-free canonical SQL", + sql: "select a,b from items where a=1", + expected_sql: "SELECT\n a,\n b\nFROM items\nWHERE a = 1", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "an unaliased Snowflake select item keeps its authored text, which names its column", + sql: "select count(1) from items", + expected_sql: "SELECT count(1)\nFROM items", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "an aliased call folds to upper case", + sql: "select count(1) as n from items", + expected_sql: "SELECT COUNT(1) AS n\nFROM items", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "block comment attachment", + sql: "SELECT a, /* preserve */ b FROM items", + expected_sql: "SELECT\n a, /* preserve */\n b\nFROM items", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "line and leading comments", + sql: "-- lead\nselect a,b from items -- tail\nwhere a=1", + expected_sql: "-- lead\nSELECT\n a,\n b\nFROM items -- tail\nWHERE a = 1", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "standalone projection comments remain attached to following expressions", + sql: "SELECT\n -- first value\n a,\n -- second value\n b\nFROM items", + expected_sql: "SELECT\n -- first value\n a,\n -- second value\n b\nFROM items", + expected_changed: false, + }, + test_types::FormatTestCase { + description: "comment marker string", + sql: "select '-- not a comment' as value", + expected_sql: "SELECT '-- not a comment' AS value", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "doubled apostrophe string remains compiler compatible", + sql: "select 'Customer''s order' as value", + expected_sql: "SELECT 'Customer''s order' AS value", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "explicit null ordering remains explicit", + sql: "select id from items order by category nulls last, created_at desc nulls first", + expected_sql: "SELECT id\nFROM items\nORDER BY\n category NULLS LAST,\n created_at DESC NULLS FIRST", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "explicit cast type spellings remain authored", + sql: "select cast(label as TEXT) as label, cast(id as INTEGER) as id, cast(observed_at as TIMESTAMP_NTZ) as observed_at, cast(amount as NUMERIC(10, 2)) as amount from items", + expected_sql: "SELECT\n CAST(label AS TEXT) AS label,\n CAST(id AS INTEGER) AS id,\n CAST(observed_at AS TIMESTAMP_NTZ) AS observed_at,\n CAST(amount AS NUMERIC(10, 2)) AS amount\nFROM items", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "nested and try cast type spellings remain authored", + sql: "select cast(coalesce(try_cast(raw_id as INTEGER), 0) as NUMERIC(10, 2)) as amount, cast(tags as ARRAY) as tags from items", + expected_sql: "SELECT\n CAST(COALESCE(TRY_CAST(raw_id AS INTEGER), 0) AS NUMERIC(10, 2)) AS amount,\n CAST(tags AS ARRAY) AS tags\nFROM items", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "commented cast type remains idempotent", + sql: "select cast(amount as NUMERIC(/* precision */ 10, 2)) as amount from items", + expected_sql: "SELECT\n CAST(amount AS NUMERIC( /* precision */10, 2)) AS amount\nFROM items", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "cast-free token expansion remains supported", + sql: "select id label from items", + expected_sql: "SELECT id label\nFROM items", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "cast-free multiple statements remain supported", + sql: "select 1; select 2", + expected_sql: "SELECT 1;\nSELECT 2", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "trailing comment after a kept statement terminator", + sql: "select a from items; -- retained", + expected_sql: "SELECT a\nFROM items; -- retained", + expected_changed: true, + }, + test_types::FormatTestCase { + description: "unparenthesized values relation remains unparenthesized", + sql: "select column1::varchar as customer_key, column2::integer as order_count from values ('c1', 1), ('c2', 2)", + expected_sql: "SELECT\n column1::varchar AS customer_key,\n column2::integer AS order_count\nFROM VALUES ('c1', 1), ('c2', 2)", + expected_changed: true, + }, + ]; + + for test_case in &test_cases { + let response = format_json( + &json!({"version": 1, "sql": test_case.sql, "dialect": "snowflake"}).to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + assert_eq!( + payload["sql"], test_case.expected_sql, + "{}", + test_case.description + ); + assert_eq!( + payload["changed"], test_case.expected_changed, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlclean/src/sql_lint/tests/test_sql_lint.rs b/crates/sqlclean/src/sql_lint/tests/test_sql_lint.rs index 125f9e5..c4c76e4 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_sql_lint.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_sql_lint.rs @@ -1,7 +1,6 @@ use serde_json::{Value, json}; use crate::sql_lint::main::engine::lint_json; -use crate::sql_lint::main::formatter::format_json; use crate::sql_lint::tests::helpers::{fixture_harness, format_batch_json}; use crate::sql_lint::tests::{helpers, test_types}; @@ -1816,126 +1815,6 @@ fn given_tsql_bare_union_when_linting_then_diagnostic_has_no_invalid_fix() -> Re Ok(()) } -#[test] -fn given_format_cases_when_formatting_then_output_matches() -> Result<(), String> { - let test_cases = [ - test_types::FormatTestCase { - description: "comment-free canonical SQL", - sql: "select a,b from items where a=1", - expected_sql: "SELECT\n a,\n b\nFROM items\nWHERE a = 1", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "an unaliased Snowflake select item keeps its authored text, which names its column", - sql: "select count(1) from items", - expected_sql: "SELECT count(1)\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "an aliased call folds to upper case", - sql: "select count(1) as n from items", - expected_sql: "SELECT COUNT(1) AS n\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "block comment attachment", - sql: "SELECT a, /* preserve */ b FROM items", - expected_sql: "SELECT\n a, /* preserve */\n b\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "line and leading comments", - sql: "-- lead\nselect a,b from items -- tail\nwhere a=1", - expected_sql: "-- lead\nSELECT\n a,\n b\nFROM items -- tail\nWHERE a = 1", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "standalone projection comments remain attached to following expressions", - sql: "SELECT\n -- first value\n a,\n -- second value\n b\nFROM items", - expected_sql: "SELECT\n -- first value\n a,\n -- second value\n b\nFROM items", - expected_changed: false, - }, - test_types::FormatTestCase { - description: "comment marker string", - sql: "select '-- not a comment' as value", - expected_sql: "SELECT '-- not a comment' AS value", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "doubled apostrophe string remains compiler compatible", - sql: "select 'Customer''s order' as value", - expected_sql: "SELECT 'Customer''s order' AS value", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "explicit null ordering remains explicit", - sql: "select id from items order by category nulls last, created_at desc nulls first", - expected_sql: "SELECT id\nFROM items\nORDER BY\n category NULLS LAST,\n created_at DESC NULLS FIRST", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "explicit cast type spellings remain authored", - sql: "select cast(label as TEXT) as label, cast(id as INTEGER) as id, cast(observed_at as TIMESTAMP_NTZ) as observed_at, cast(amount as NUMERIC(10, 2)) as amount from items", - expected_sql: "SELECT\n CAST(label AS TEXT) AS label,\n CAST(id AS INTEGER) AS id,\n CAST(observed_at AS TIMESTAMP_NTZ) AS observed_at,\n CAST(amount AS NUMERIC(10, 2)) AS amount\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "nested and try cast type spellings remain authored", - sql: "select cast(coalesce(try_cast(raw_id as INTEGER), 0) as NUMERIC(10, 2)) as amount, cast(tags as ARRAY) as tags from items", - expected_sql: "SELECT\n CAST(COALESCE(TRY_CAST(raw_id AS INTEGER), 0) AS NUMERIC(10, 2)) AS amount,\n CAST(tags AS ARRAY) AS tags\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "commented cast type remains idempotent", - sql: "select cast(amount as NUMERIC(/* precision */ 10, 2)) as amount from items", - expected_sql: "SELECT\n CAST(amount AS NUMERIC( /* precision */10, 2)) AS amount\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "cast-free token expansion remains supported", - sql: "select id label from items", - expected_sql: "SELECT id label\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "cast-free multiple statements remain supported", - sql: "select 1; select 2", - expected_sql: "SELECT 1;\nSELECT 2", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "trailing comment after a kept statement terminator", - sql: "select a from items; -- retained", - expected_sql: "SELECT a\nFROM items; -- retained", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "unparenthesized values relation remains unparenthesized", - sql: "select column1::varchar as customer_key, column2::integer as order_count from values ('c1', 1), ('c2', 2)", - expected_sql: "SELECT\n column1::varchar AS customer_key,\n column2::integer AS order_count\nFROM VALUES ('c1', 1), ('c2', 2)", - expected_changed: true, - }, - ]; - - for test_case in &test_cases { - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": "snowflake"}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - assert_eq!( - payload["changed"], test_case.expected_changed, - "{}", - test_case.description - ); - } - Ok(()) -} - #[test] fn given_ordered_sql_batch_when_formatting_then_outputs_are_parallel_and_ordered() -> Result<(), String> { diff --git a/crates/sqlclean/src/sql_lint/tests/test_types.rs b/crates/sqlclean/src/sql_lint/tests/test_types.rs index cbc3835..a6c51e5 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_types.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_types.rs @@ -158,3 +158,9 @@ pub(crate) struct CrossJoinSubjectTestCase { pub message: &'static str, pub expected_subject: Option<(&'static str, Option<&'static str>)>, } + +pub(crate) struct UnavailableCorpusTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub expected_unavailable: usize, +} diff --git a/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs b/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs index 73513f6..5a92baf 100644 --- a/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs +++ b/crates/sqlclean/src/sql_tokens/main/query_fingerprint.rs @@ -4,8 +4,7 @@ use sqlquack_fingerprint::FingerprintDialect; use crate::sql_tokens::constants::FINGERPRINT_DIALECT_ALIASES; -/// Return SQLQuack's fingerprint of `sql` under `dialect_name`, the one SQLBuild stores; a dialect -/// without its own SQLQuack tokenizer is fingerprinted with the generic tokens. +/// SQLQuack's fingerprint of `sql`, as SQLBuild stores it; dialects without a tokenizer use generic tokens. pub fn query_fingerprint(sql: &str, dialect_name: &str) -> Result { let name: &str = FINGERPRINT_DIALECT_ALIASES .iter() diff --git a/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs b/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs index c2150ce..5957db8 100644 --- a/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs +++ b/crates/sqlclean/src/sql_tokens/tests/test_query_fingerprint.rs @@ -1,124 +1,148 @@ use crate::sql_tokens::main::query_fingerprint::query_fingerprint; +use crate::sql_tokens::tests::test_types; -/// SQLBuild's golden fingerprints, generic tokens for dialects without their own; SQLBuild stores these -/// hashes, so SQLClean must compute the same ones. +/// SQLBuild's stored golden fingerprints, which SQLClean must reproduce exactly. #[test] fn given_sqlbuild_golden_queries_when_fingerprinting_then_hashes_match_sqlbuild() -> Result<(), String> { - let test_cases: [(&str, &str, &str); 8] = [ - ( - "generic", - "SELECT id, name FROM orders", - "264bcb1f81ce5ba25394a052016edc265aefba85c0698f616d3037f9d872d2bd", - ), - ( - "duckdb", - "SELECT o.id, $$Customer's note$$ AS note, amount::DECIMAL(10, 2) AS amount FROM orders AS o", - "07786643d9593498ac2b2394e6801cb578b23ba0f9a0e70dbcdaf0d3a3cbf9db", - ), - ( - "postgresql", - "SELECT o.id, $tag$it's$tag$ AS note, amount::NUMERIC AS amount FROM orders AS o", - "8db865c6f239b990c917754df176bd3c9b2c0c322b9af1437298ec98b67241f9", - ), - ( - "snowflake", - "SELECT o.id, payload:customer.name::STRING AS customer FROM orders AS o QUALIFY ROW_NUMBER() OVER (PARTITION BY o.id ORDER BY o.id) = 1", - "c75f7b5931a18e1a074ffa7ab113c256fc838285af75a3f9feb62d61cc76d94d", - ), - ( - "duckdb", - "select o.id from __ref(\"orders\") as o where o.id > 0", - "fda88b9b0f26486bdd25b8e3bab5a982fbb387901ab229091b17772d044aebd0", - ), - ( - "bigquery", - "SELECT o.id, r'raw\\d' AS pattern FROM `project.dataset.orders` AS o", - "ce893f2fd9ab11ba453fc3c02e77b6690df6c3d1d8bc141fc19cf3891d9a61b4", - ), - ( - "databricks", - "SELECT /*+ BROADCAST(o) */ o.id FROM `catalog`.`orders` AS o", - "7cd96f42bcb764136a7f423181302ba13cead02bb6cfb7382502a3a2aa4107c3", - ), - ( - "sqlserver", - "SELECT TOP 10 [o].[id], N'order' AS kind FROM [dbo].[orders] AS o", - "f1525ee25924f9402677bc2cbd96f7066745948da73671d13fac2b8da03cbf0c", - ), + let test_cases = [ + test_types::FingerprintGoldenTestCase { + description: "generic token stream", + dialect: "generic", + sql: "SELECT id, name FROM orders", + expected_fingerprint: "264bcb1f81ce5ba25394a052016edc265aefba85c0698f616d3037f9d872d2bd", + }, + test_types::FingerprintGoldenTestCase { + description: "DuckDB dollar quote and cast", + dialect: "duckdb", + sql: "SELECT o.id, $$Customer's note$$ AS note, amount::DECIMAL(10, 2) AS amount FROM orders AS o", + expected_fingerprint: "07786643d9593498ac2b2394e6801cb578b23ba0f9a0e70dbcdaf0d3a3cbf9db", + }, + test_types::FingerprintGoldenTestCase { + description: "PostgreSQL tagged dollar quote", + dialect: "postgresql", + sql: "SELECT o.id, $tag$it's$tag$ AS note, amount::NUMERIC AS amount FROM orders AS o", + expected_fingerprint: "8db865c6f239b990c917754df176bd3c9b2c0c322b9af1437298ec98b67241f9", + }, + test_types::FingerprintGoldenTestCase { + description: "Snowflake path access and QUALIFY", + dialect: "snowflake", + sql: "SELECT o.id, payload:customer.name::STRING AS customer FROM orders AS o QUALIFY ROW_NUMBER() OVER (PARTITION BY o.id ORDER BY o.id) = 1", + expected_fingerprint: "c75f7b5931a18e1a074ffa7ab113c256fc838285af75a3f9feb62d61cc76d94d", + }, + test_types::FingerprintGoldenTestCase { + description: "DuckDB lower-case keywords and SQLBuild placeholder", + dialect: "duckdb", + sql: "select o.id from __ref(\"orders\") as o where o.id > 0", + expected_fingerprint: "fda88b9b0f26486bdd25b8e3bab5a982fbb387901ab229091b17772d044aebd0", + }, + test_types::FingerprintGoldenTestCase { + description: "BigQuery raw string and backtick path, as generic tokens", + dialect: "bigquery", + sql: "SELECT o.id, r'raw\\d' AS pattern FROM `project.dataset.orders` AS o", + expected_fingerprint: "ce893f2fd9ab11ba453fc3c02e77b6690df6c3d1d8bc141fc19cf3891d9a61b4", + }, + test_types::FingerprintGoldenTestCase { + description: "Databricks optimizer hint, as generic tokens", + dialect: "databricks", + sql: "SELECT /*+ BROADCAST(o) */ o.id FROM `catalog`.`orders` AS o", + expected_fingerprint: "7cd96f42bcb764136a7f423181302ba13cead02bb6cfb7382502a3a2aa4107c3", + }, + test_types::FingerprintGoldenTestCase { + description: "SQL Server bracket identifiers, as generic tokens", + dialect: "sqlserver", + sql: "SELECT TOP 10 [o].[id], N'order' AS kind FROM [dbo].[orders] AS o", + expected_fingerprint: "f1525ee25924f9402677bc2cbd96f7066745948da73671d13fac2b8da03cbf0c", + }, ]; - for (dialect, sql, expected) in test_cases { + for test_case in &test_cases { assert_eq!( - query_fingerprint(sql, dialect)?, - expected, - "{dialect}: {sql}" + query_fingerprint(test_case.sql, test_case.dialect)?, + test_case.expected_fingerprint, + "{}", + test_case.description ); } Ok(()) } -/// MotherDuck fingerprints as DuckDB; `qualify` is reserved in DuckDB but not generic SQL. +/// Which texts and dialect names share a fingerprint: call names and dialect aliases ignore case. #[test] -fn given_the_motherduck_alias_when_fingerprinting_then_it_hashes_as_duckdb() -> Result<(), String> { - let sql = "select a from t qualify b"; - assert_eq!( - query_fingerprint(sql, "motherduck")?, - query_fingerprint(sql, "duckdb")? - ); - assert_ne!( - query_fingerprint(sql, "duckdb")?, - query_fingerprint(sql, "generic")? - ); - Ok(()) -} - -/// Called names fold to upper case, except placeholders, quoted names and relation or CTE column lists. -#[test] -fn given_called_names_when_fingerprinting_then_only_unquoted_calls_ignore_case() +fn given_two_queries_when_fingerprinting_then_only_case_insensitive_differences_match() -> Result<(), String> { - let test_cases: [(&str, &str, &str, bool); 6] = [ - ( - "duckdb", - "SELECT MyFunc(id) FROM orders", - "SELECT myfunc(id) FROM orders", - true, - ), - ( - "snowflake", - "SELECT \"MyFunc\"(id) FROM orders", - "SELECT \"myfunc\"(id) FROM orders", - false, - ), - ( - "duckdb", - "SELECT id FROM __ref('orders')", - "SELECT id FROM __REF('orders')", - false, - ), - ( - "duckdb", - "WITH Tree(id) AS (SELECT 1) SELECT id FROM orders", - "WITH tree(id) AS (SELECT 1) SELECT id FROM orders", - false, - ), - ( - "postgres", - "INSERT INTO Paid(id) SELECT 1", - "INSERT INTO paid(id) SELECT 1", - false, - ), - ( - "sqlite", - "SELECT Upper(name) FROM orders", - "SELECT UPPER(name) FROM orders", - true, - ), + let test_cases = [ + test_types::FingerprintEqualityTestCase { + description: "an unquoted call name's case is ignored", + left_dialect: "duckdb", + left: "SELECT MyFunc(id) FROM orders", + right_dialect: "duckdb", + right: "SELECT myfunc(id) FROM orders", + expected_equal: true, + }, + test_types::FingerprintEqualityTestCase { + description: "a quoted call name's case is significant", + left_dialect: "snowflake", + left: "SELECT \"MyFunc\"(id) FROM orders", + right_dialect: "snowflake", + right: "SELECT \"myfunc\"(id) FROM orders", + expected_equal: false, + }, + test_types::FingerprintEqualityTestCase { + description: "a SQLBuild placeholder's case is significant", + left_dialect: "duckdb", + left: "SELECT id FROM __ref('orders')", + right_dialect: "duckdb", + right: "SELECT id FROM __REF('orders')", + expected_equal: false, + }, + test_types::FingerprintEqualityTestCase { + description: "a CTE name with a column list is not a call", + left_dialect: "duckdb", + left: "WITH Tree(id) AS (SELECT 1) SELECT id FROM orders", + right_dialect: "duckdb", + right: "WITH tree(id) AS (SELECT 1) SELECT id FROM orders", + expected_equal: false, + }, + test_types::FingerprintEqualityTestCase { + description: "an INSERT target with a column list is not a call", + left_dialect: "postgres", + left: "INSERT INTO Paid(id) SELECT 1", + right_dialect: "postgres", + right: "INSERT INTO paid(id) SELECT 1", + expected_equal: false, + }, + test_types::FingerprintEqualityTestCase { + description: "a SQLite call name's case is ignored", + left_dialect: "sqlite", + left: "SELECT Upper(name) FROM orders", + right_dialect: "sqlite", + right: "SELECT UPPER(name) FROM orders", + expected_equal: true, + }, + test_types::FingerprintEqualityTestCase { + description: "MotherDuck fingerprints as DuckDB", + left_dialect: "motherduck", + left: "select a from t qualify b", + right_dialect: "duckdb", + right: "select a from t qualify b", + expected_equal: true, + }, + test_types::FingerprintEqualityTestCase { + description: "DuckDB reserves QUALIFY and generic SQL does not", + left_dialect: "duckdb", + left: "select a from t qualify b", + right_dialect: "generic", + right: "select a from t qualify b", + expected_equal: false, + }, ]; - for (dialect, left, right, expected_equal) in test_cases { + for test_case in &test_cases { assert_eq!( - query_fingerprint(left, dialect)? == query_fingerprint(right, dialect)?, - expected_equal, - "{dialect}: {left} | {right}" + query_fingerprint(test_case.left, test_case.left_dialect)? + == query_fingerprint(test_case.right, test_case.right_dialect)?, + test_case.expected_equal, + "{}", + test_case.description ); } Ok(()) diff --git a/crates/sqlclean/src/sql_tokens/tests/test_types.rs b/crates/sqlclean/src/sql_tokens/tests/test_types.rs index 74bbc3f..fb34c24 100644 --- a/crates/sqlclean/src/sql_tokens/tests/test_types.rs +++ b/crates/sqlclean/src/sql_tokens/tests/test_types.rs @@ -6,3 +6,19 @@ pub(crate) struct CanonicalTokensTestCase { pub(crate) right: &'static str, pub(crate) expected_equal: bool, } + +pub(crate) struct FingerprintGoldenTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_fingerprint: &'static str, +} + +pub(crate) struct FingerprintEqualityTestCase { + pub(crate) description: &'static str, + pub(crate) left_dialect: &'static str, + pub(crate) left: &'static str, + pub(crate) right_dialect: &'static str, + pub(crate) right: &'static str, + pub(crate) expected_equal: bool, +} From e5ca8cb678f6bbff620a05186ccec5237be072f6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:44:21 +0000 Subject: [PATCH 13/21] wip(facts): name utility and DROP statement kinds from the native tree Maps the native utility_statement and drop_object kinds back to the statement kinds SQLClean reported before (describe, set_statement, transaction, drop_table and so on). Pins SQLQuack to the lint-postgres landing head provisionally; the shared rev is still to be agreed. --- Cargo.lock | 13 +++++---- Cargo.toml | 8 ++--- .../src/sql_facts/_helpers/collection.rs | 10 +++++-- crates/sqlclean/src/sql_facts/constants.rs | 29 +++++++++++++++++++ 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d415db..8333693 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -611,7 +611,7 @@ dependencies = [ [[package]] name = "sqlquack-fingerprint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=8cd264ab00390e1d64df2d30e7b25eb3a2d3b326#8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" dependencies = [ "sha2", "sqlquack-syntax", @@ -621,7 +621,7 @@ dependencies = [ [[package]] name = "sqlquack-format" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=8cd264ab00390e1d64df2d30e7b25eb3a2d3b326#8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" dependencies = [ "serde_json", "sqlquack-syntax", @@ -630,7 +630,7 @@ dependencies = [ [[package]] name = "sqlquack-lint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=8cd264ab00390e1d64df2d30e7b25eb3a2d3b326#8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" dependencies = [ "rayon", "rustc-hash", @@ -643,7 +643,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=8cd264ab00390e1d64df2d30e7b25eb3a2d3b326#8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" dependencies = [ "serde", "serde_json", @@ -656,7 +656,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-derive" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=8cd264ab00390e1d64df2d30e7b25eb3a2d3b326#8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" dependencies = [ "proc-macro2", "quote", @@ -666,9 +666,10 @@ dependencies = [ [[package]] name = "sqlquack-syntax-postgres" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=8cd264ab00390e1d64df2d30e7b25eb3a2d3b326#8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" dependencies = [ "sqlquack-syntax", + "stacker", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b48d168..653e50e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,10 +30,10 @@ sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } # SQLQuack's formatter and lint engine, and the tokens and trees they read, which replace Polyglot. -sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "6641c50e532f419f4da42ab5970edfccbe3c2dad" } -sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "6641c50e532f419f4da42ab5970edfccbe3c2dad" } -sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "6641c50e532f419f4da42ab5970edfccbe3c2dad" } -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "6641c50e532f419f4da42ab5970edfccbe3c2dad" } +sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" } +sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" } +sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sqlclean/src/sql_facts/_helpers/collection.rs b/crates/sqlclean/src/sql_facts/_helpers/collection.rs index 0434197..38e8ecc 100644 --- a/crates/sqlclean/src/sql_facts/_helpers/collection.rs +++ b/crates/sqlclean/src/sql_facts/_helpers/collection.rs @@ -10,7 +10,8 @@ use crate::sql_facts::_helpers::tokens::{ function_nodes, join_clause_span, literal_nodes, star_nodes, }; use crate::sql_facts::constants::{ - ALTER_OBJECT_KINDS, CREATE_OBJECT_KINDS, DATE_PART_CALLS, DATE_PART_NAMES, + ALTER_OBJECT_KINDS, CREATE_OBJECT_KINDS, DATE_PART_CALLS, DATE_PART_NAMES, DROP_OBJECT_KINDS, + UTILITY_KINDS, }; use crate::sql_facts::models::{ ColumnNode, FileFacts, JoinNode, NamedNode, StatementFacts, TableNode, @@ -333,11 +334,16 @@ fn authored(identifier: &Identifier, characters: &[char]) -> String { .unwrap_or_else(|| identifier.name.clone()) } -/// The statement's kind, naming a generic CREATE or ALTER by its object as Polyglot's tree did. +/// The statement's kind, naming a generic CREATE, ALTER, DROP or utility statement as Polyglot's tree did. fn statement_kind(statement: &Expression) -> String { let (kind, known): (&str, &[(&str, &str)]) = match statement { Expression::CreateObject(node) => (&node.kind, &CREATE_OBJECT_KINDS), Expression::AlterObject(node) => (&node.kind, &ALTER_OBJECT_KINDS), + Expression::DropObject(node) => (&node.kind, &DROP_OBJECT_KINDS), + Expression::UtilityStatement(node) => ( + node.kind.split(' ').next().unwrap_or_default(), + &UTILITY_KINDS, + ), _ => ("", &[]), }; known diff --git a/crates/sqlclean/src/sql_facts/constants.rs b/crates/sqlclean/src/sql_facts/constants.rs index aae0fa6..71cfc01 100644 --- a/crates/sqlclean/src/sql_facts/constants.rs +++ b/crates/sqlclean/src/sql_facts/constants.rs @@ -103,3 +103,32 @@ pub(crate) const ALTER_OBJECT_KINDS: [(&str, &str); 4] = [ ("SEQUENCE", "alter_sequence"), ("VIEW", "alter_view"), ]; +/// DROP statements the native PostgreSQL tree keeps as a generic object, by object kind, and their kind. +pub(crate) const DROP_OBJECT_KINDS: [(&str, &str); 12] = [ + ("TABLE", "drop_table"), + ("VIEW", "drop_view"), + ("MATERIALIZED VIEW", "drop_view"), + ("INDEX", "drop_index"), + ("FUNCTION", "drop_function"), + ("PROCEDURE", "drop_procedure"), + ("SCHEMA", "drop_schema"), + ("SEQUENCE", "drop_sequence"), + ("TRIGGER", "drop_trigger"), + ("TYPE", "drop_type"), + ("DOMAIN", "drop_type"), + ("DATABASE", "drop_database"), +]; +/// Utility statements of the native PostgreSQL tree, by their first keyword, and their kind. +pub(crate) const UTILITY_KINDS: [(&str, &str); 11] = [ + ("EXPLAIN", "describe"), + ("SET", "set_statement"), + ("BEGIN", "transaction"), + ("START", "transaction"), + ("COMMIT", "commit"), + ("END", "commit"), + ("ROLLBACK", "rollback"), + ("ANALYZE", "analyze"), + ("DECLARE", "declare"), + ("REFRESH", "refresh"), + ("SHOW", "show"), +]; From 791c93bd4d5a1a2ad4900431169e30256bfc307d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:36:52 +0000 Subject: [PATCH 14/21] wip(cli): read SQL without a leading byte order mark and write it back Editors such as SSMS save SQL with a UTF-8 byte order mark. check, format and fix now read the SQL after it, so every dialect lints and formats the file, and format and fix write the mark back unchanged. Not yet built or tested. --- .../src/command/_helpers/diffs.rs | 8 +++++- .../src/discovery/_helpers/walking.rs | 8 +++++- .../sqlclean-cli/src/discovery/constants.rs | 2 ++ crates/sqlclean-cli/src/discovery/mod.rs | 1 + crates/sqlclean-cli/src/discovery/models.rs | 3 +++ .../sqlclean/hosting/test_sqlclean_command.py | 26 +++++++++++++++++++ 6 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 crates/sqlclean-cli/src/discovery/constants.rs diff --git a/crates/sqlclean-cli/src/command/_helpers/diffs.rs b/crates/sqlclean-cli/src/command/_helpers/diffs.rs index e4447fe..e8f8982 100644 --- a/crates/sqlclean-cli/src/command/_helpers/diffs.rs +++ b/crates/sqlclean-cli/src/command/_helpers/diffs.rs @@ -5,6 +5,7 @@ use std::io::Write; use similar::TextDiff; +use crate::discovery::constants::UTF8_BOM; use crate::discovery::models::SourceFile; const CONTEXT_LINES: usize = 3; @@ -19,7 +20,12 @@ pub(crate) fn print_diff(file: &SourceFile, changed: &str) { let _ = std::io::stdout().write_all(rendered.as_bytes()); } +/// Write `changed` over `file`, with the byte order mark it was read with. pub(crate) fn write_file(file: &SourceFile, changed: &str) -> Result<(), String> { - fs::write(&file.absolute, changed) + let text: String = match file.bom { + true => format!("{UTF8_BOM}{changed}"), + false => changed.to_string(), + }; + fs::write(&file.absolute, text) .map_err(|error| format!("could not write {}: {error}", file.path)) } diff --git a/crates/sqlclean-cli/src/discovery/_helpers/walking.rs b/crates/sqlclean-cli/src/discovery/_helpers/walking.rs index 212893a..72016d9 100644 --- a/crates/sqlclean-cli/src/discovery/_helpers/walking.rs +++ b/crates/sqlclean-cli/src/discovery/_helpers/walking.rs @@ -8,6 +8,7 @@ use ignore::WalkBuilder; use crate::config::models::Config; use crate::discovery::_helpers::matching::PathMatchers; +use crate::discovery::constants::UTF8_BOM; use crate::discovery::models::SourceFile; pub(crate) fn discover_files( @@ -56,11 +57,16 @@ fn read_source( ) -> Result { let text: String = fs::read_to_string(&absolute).map_err(|error| format!("could not read {path}: {error}"))?; + let bom: bool = text.starts_with(UTF8_BOM); Ok(SourceFile { dialect: matchers.dialect(&path), path, absolute, - text, + bom, + text: match text.strip_prefix(UTF8_BOM).map(str::to_string) { + Some(sql) => sql, + None => text, + }, }) } diff --git a/crates/sqlclean-cli/src/discovery/constants.rs b/crates/sqlclean-cli/src/discovery/constants.rs new file mode 100644 index 0000000..e6cb154 --- /dev/null +++ b/crates/sqlclean-cli/src/discovery/constants.rs @@ -0,0 +1,2 @@ +/// The byte order mark some editors write at the start of a UTF-8 file, which is not SQL. +pub(crate) const UTF8_BOM: char = '\u{feff}'; diff --git a/crates/sqlclean-cli/src/discovery/mod.rs b/crates/sqlclean-cli/src/discovery/mod.rs index ecb90e4..87114c6 100644 --- a/crates/sqlclean-cli/src/discovery/mod.rs +++ b/crates/sqlclean-cli/src/discovery/mod.rs @@ -1,6 +1,7 @@ //! Which SQL files a command reads, and the dialect of each. pub mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlclean-cli/src/discovery/models.rs b/crates/sqlclean-cli/src/discovery/models.rs index e4de420..7f00e45 100644 --- a/crates/sqlclean-cli/src/discovery/models.rs +++ b/crates/sqlclean-cli/src/discovery/models.rs @@ -6,6 +6,9 @@ pub struct SourceFile { /// The path relative to the configuration root, with `/` separators. pub path: String, pub absolute: PathBuf, + /// The SQL, without the byte order mark when the file starts with one. pub text: String, + /// The file starts with a UTF-8 byte order mark, which a write keeps. + pub bom: bool, pub dialect: String, } diff --git a/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py b/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py index 398f5e2..da8d208 100644 --- a/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py +++ b/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py @@ -131,6 +131,14 @@ expected_codes=(), expected_unavailable=("mysql/q.sql",), ), + CheckTestCase( + description="a file saved with a byte order mark is checked as its SQL", + config=CROSS_JOIN_ONLY, + files={"models/q.sql": "\ufeffSELECT a.id FROM a CROSS JOIN b\n"}, + arguments=(), + expected_exit=1, + expected_codes=("SQLCLEAN038",), + ), CheckTestCase( description="unparseable SQL is reported as a finding", config=CROSS_JOIN_ONLY, @@ -229,6 +237,24 @@ def test_given_project_when_checking_then_reports_findings_and_exit_status( expected_exit=0, expected_files={"mysql/q.sql": "select a.id from a, b\n"}, ), + RewriteTestCase( + description="format keeps a file's byte order mark", + config=DUCKDB, + files={"models/q.sql": "\ufeffselect 1;select 2"}, + arguments=("format",), + expected_exit=0, + expected_files={"models/q.sql": "\ufeffSELECT 1;\n\nSELECT 2\n"}, + ), + RewriteTestCase( + description="fix keeps a file's byte order mark", + config=DUCKDB + '[rules]\nselect = ["SQLCLEAN001", "SQLCLEAN002"]\n', + files={"models/q.sql": "\ufeff" + NULL_AND_COMMA}, + arguments=("fix",), + expected_exit=1, + expected_files={ + "models/q.sql": "\ufeffSELECT a.id\nFROM a\nCROSS JOIN b\nWHERE a.x = NULL\n" + }, + ), RewriteTestCase( description="format --check fails without writing", config=DUCKDB, From b325490ab1cdc697db0ff4249e37752892a5ea6a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:22:33 +0000 Subject: [PATCH 15/21] wip(facts): leave the SQLQuack pin to the release head The statement-kind mapping builds on the shared SQLQuack rev already pinned here, so the provisional pin bump is dropped. --- Cargo.lock | 13 ++++++------- Cargo.toml | 8 ++++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8333693..5d415db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -611,7 +611,7 @@ dependencies = [ [[package]] name = "sqlquack-fingerprint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=8cd264ab00390e1d64df2d30e7b25eb3a2d3b326#8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" dependencies = [ "sha2", "sqlquack-syntax", @@ -621,7 +621,7 @@ dependencies = [ [[package]] name = "sqlquack-format" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=8cd264ab00390e1d64df2d30e7b25eb3a2d3b326#8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" dependencies = [ "serde_json", "sqlquack-syntax", @@ -630,7 +630,7 @@ dependencies = [ [[package]] name = "sqlquack-lint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=8cd264ab00390e1d64df2d30e7b25eb3a2d3b326#8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" dependencies = [ "rayon", "rustc-hash", @@ -643,7 +643,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=8cd264ab00390e1d64df2d30e7b25eb3a2d3b326#8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" dependencies = [ "serde", "serde_json", @@ -656,7 +656,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-derive" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=8cd264ab00390e1d64df2d30e7b25eb3a2d3b326#8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" dependencies = [ "proc-macro2", "quote", @@ -666,10 +666,9 @@ dependencies = [ [[package]] name = "sqlquack-syntax-postgres" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=8cd264ab00390e1d64df2d30e7b25eb3a2d3b326#8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" dependencies = [ "sqlquack-syntax", - "stacker", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 653e50e..b48d168 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,10 +30,10 @@ sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } # SQLQuack's formatter and lint engine, and the tokens and trees they read, which replace Polyglot. -sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" } -sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" } -sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" } -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "8cd264ab00390e1d64df2d30e7b25eb3a2d3b326" } +sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "6641c50e532f419f4da42ab5970edfccbe3c2dad" } +sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "6641c50e532f419f4da42ab5970edfccbe3c2dad" } +sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "6641c50e532f419f4da42ab5970edfccbe3c2dad" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "6641c50e532f419f4da42ab5970edfccbe3c2dad" } [workspace.lints.rust] unsafe_code = "forbid" From 9f160bcae28f4cf1afff9c89f798964a3ca84aca Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:13:20 +0000 Subject: [PATCH 16/21] wip(deps): trial pin of SQLQuack at the train tip 19c51fa2 Not releasable: 19c51fa2 is not on SQLQuack main yet. Two test updates follow SQLQuack's Snowflake grammar checks: the format corpus expects UNNEST refused (Snowflake has no UNNEST), and the no-harness empty-fixture case calls the relation through TABLE( ) as Snowflake requires. --- Cargo.lock | 13 +++++++------ Cargo.toml | 8 ++++---- .../tests/fixtures/sqlquack_departures.json | 7 +++++++ .../src/sql_lint/tests/test_empty_fixture_lint.rs | 2 +- .../src/sql_lint/tests/test_format_corpus.rs | 4 ++-- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d415db..fc1726f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -611,7 +611,7 @@ dependencies = [ [[package]] name = "sqlquack-fingerprint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" dependencies = [ "sha2", "sqlquack-syntax", @@ -621,7 +621,7 @@ dependencies = [ [[package]] name = "sqlquack-format" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" dependencies = [ "serde_json", "sqlquack-syntax", @@ -630,7 +630,7 @@ dependencies = [ [[package]] name = "sqlquack-lint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" dependencies = [ "rayon", "rustc-hash", @@ -643,7 +643,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" dependencies = [ "serde", "serde_json", @@ -656,7 +656,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-derive" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" dependencies = [ "proc-macro2", "quote", @@ -666,9 +666,10 @@ dependencies = [ [[package]] name = "sqlquack-syntax-postgres" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=6641c50e532f419f4da42ab5970edfccbe3c2dad#6641c50e532f419f4da42ab5970edfccbe3c2dad" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" dependencies = [ "sqlquack-syntax", + "stacker", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b48d168..6577aa2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,10 +30,10 @@ sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } # SQLQuack's formatter and lint engine, and the tokens and trees they read, which replace Polyglot. -sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "6641c50e532f419f4da42ab5970edfccbe3c2dad" } -sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "6641c50e532f419f4da42ab5970edfccbe3c2dad" } -sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "6641c50e532f419f4da42ab5970edfccbe3c2dad" } -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "6641c50e532f419f4da42ab5970edfccbe3c2dad" } +sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } +sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } +sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json b/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json index 4f2cfa4..2ab6f2f 100644 --- a/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json +++ b/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json @@ -118,6 +118,13 @@ "reason": "Snowflake rejects a quoted date part in the FROM form of DATE_PART, so SQLQuack's parser refuses it", "receipt": "sf.functions.date-part.date-part-from.k10b598e047b5" }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM t, UNNEST(x) WITH ORDINALITY", + "formatted": null, + "reason": "Snowflake has no UNNEST, so SQLQuack's parser refuses it", + "receipt": "sf.query.format-probe.k47586a1689b7" + }, { "dialect": "snowflake", "sql": "WITH t AS (SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data) SELECT data: level1 : level2 : level3::VARIANT FROM t", diff --git a/crates/sqlclean/src/sql_lint/tests/test_empty_fixture_lint.rs b/crates/sqlclean/src/sql_lint/tests/test_empty_fixture_lint.rs index 11a3aec..bc97262 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_empty_fixture_lint.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_empty_fixture_lint.rs @@ -22,7 +22,7 @@ fn given_empty_fixture_context_when_linting_projected_star_then_only_canonical_i }, test_types::EmptyFixtureLintTestCase { description: "ordinary SQL has no intrinsic exemption", - sql: "SELECT * FROM __empty_fixture()", + sql: "SELECT * FROM TABLE(__empty_fixture())", in_fixture_harness: false, expected_diagnostic_count: 1, }, diff --git a/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs b/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs index 95ffa60..8cb4910 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs @@ -18,8 +18,8 @@ fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stab test_types::FormatCorpusTestCase { description: "Snowflake identity corpus", dialect: "snowflake", - expected_formatted: 427, - expected_refused: 8, + expected_formatted: 426, + expected_refused: 9, }, ]; let corpus: Vec = From 1c7d9155f02ba862c2d57c5ba9f4df3e68249232 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:07:16 +0000 Subject: [PATCH 17/21] wip(format): pass SQLQuack its formatter dialect SQLQuack's FormatOptions now takes a FormatDialect; SQLClean keeps its syntax dialect for the CTE macro pass and maps it for the layout. --- Cargo.lock | 22 +++++++++++---- Cargo.toml | 8 +++--- .../src/sql_lint/_helpers/formatter.rs | 28 +++++++++++++------ 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fc1726f..f8b3dd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -611,7 +611,7 @@ dependencies = [ [[package]] name = "sqlquack-fingerprint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" dependencies = [ "sha2", "sqlquack-syntax", @@ -621,16 +621,17 @@ dependencies = [ [[package]] name = "sqlquack-format" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" dependencies = [ "serde_json", "sqlquack-syntax", + "sqlquack-syntax-sqlserver", ] [[package]] name = "sqlquack-lint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" dependencies = [ "rayon", "rustc-hash", @@ -643,7 +644,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" dependencies = [ "serde", "serde_json", @@ -656,7 +657,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-derive" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" dependencies = [ "proc-macro2", "quote", @@ -666,7 +667,16 @@ dependencies = [ [[package]] name = "sqlquack-syntax-postgres" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" +dependencies = [ + "sqlquack-syntax", + "stacker", +] + +[[package]] +name = "sqlquack-syntax-sqlserver" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" dependencies = [ "sqlquack-syntax", "stacker", diff --git a/Cargo.toml b/Cargo.toml index 6577aa2..1ffe702 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,10 +30,10 @@ sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } # SQLQuack's formatter and lint engine, and the tokens and trees they read, which replace Polyglot. -sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } -sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } -sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } +sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "d9a53958e592123a4fc5efae8421752e369e312c" } +sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "d9a53958e592123a4fc5efae8421752e369e312c" } +sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "d9a53958e592123a4fc5efae8421752e369e312c" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "d9a53958e592123a4fc5efae8421752e369e312c" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs index a182d52..59907fd 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs @@ -1,5 +1,5 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator}; -use sqlquack_format::FormatOptions; +use sqlquack_format::{FormatDialect, FormatOptions}; use sqlquack_syntax::Dialect as SqlQuackDialect; use crate::constants::INTERPOLATION_PLACEHOLDER_PREFIX; @@ -64,6 +64,14 @@ fn sqlquack_dialect(dialect: &str) -> Option { } } +/// SQLQuack's formatter dialect for a dialect it formats natively. +fn format_dialect(dialect: SqlQuackDialect) -> FormatDialect { + match dialect { + SqlQuackDialect::DuckDB => FormatDialect::DuckDB, + _ => FormatDialect::Snowflake, + } +} + /// Lay out `request` with SQLQuack; this re-checks idempotency only where placeholders were put back. fn format_with_sqlquack( request: FormatRequest, @@ -79,12 +87,12 @@ fn format_with_sqlquack( ..Default::default() }), }, - ..FormatOptions::new(dialect) + ..FormatOptions::new(format_dialect(dialect)) }; - let sql = format_around_cte_macros(&request.sql, &options)?; + let sql = format_around_cte_macros(&request.sql, dialect, &options)?; if request.sql.contains(INTERPOLATION_PLACEHOLDER_PREFIX) && sql != request.sql - && format_around_cte_macros(&sql, &options)? != sql + && format_around_cte_macros(&sql, dialect, &options)? != sql { return Err(NOT_IDEMPOTENT_FAILURE.to_string()); } @@ -106,11 +114,13 @@ fn sqlquack_layout(sql: &str, options: &FormatOptions) -> Result } /// Lay out `sql` with SQLQuack, taking CTE macro placeholders out first and putting them back. -fn format_around_cte_macros(sql: &str, options: &FormatOptions) -> Result { - match take_out_cte_macros(sql, options.dialect)? { +fn format_around_cte_macros( + sql: &str, + dialect: SqlQuackDialect, + options: &FormatOptions, +) -> Result { + match take_out_cte_macros(sql, dialect)? { None => sqlquack_layout(sql, options), - Some(without) => { - without.put_back(&sqlquack_layout(&without.sql, options)?, options.dialect) - } + Some(without) => without.put_back(&sqlquack_layout(&without.sql, options)?, dialect), } } From 338e82dc4eb50b01b04bfbf3ff42bb6059fbae9f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:10:16 +0000 Subject: [PATCH 18/21] wip(format): lay out T-SQL with SQLQuack's SQL Server formatter T-SQL requests go to SQLQuack's SQL Server formatter; the CTE macro pass, which needs the shared tokenizer, is skipped, so a placeholder SQLQuack cannot parse is refused. The corpus test gates T-SQL at 52 formatted and 23 refused, with SQLQuack's named departures. --- .../src/sql_lint/_helpers/formatter.rs | 41 ++-- crates/sqlclean/src/sql_lint/constants.rs | 2 + .../tests/fixtures/sqlquack_departures.json | 182 ++++++++++++++++++ crates/sqlclean/src/sql_lint/tests/helpers.rs | 2 +- .../src/sql_lint/tests/test_format_corpus.rs | 11 +- 5 files changed, 211 insertions(+), 27 deletions(-) diff --git a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs index 59907fd..552dae8 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs @@ -4,7 +4,9 @@ use sqlquack_syntax::Dialect as SqlQuackDialect; use crate::constants::INTERPOLATION_PLACEHOLDER_PREFIX; use crate::sql_lint::_helpers::cte_macros::take_out_cte_macros; -use crate::sql_lint::constants::{DUCKDB_DIALECT, LINT_API_VERSION, SNOWFLAKE_DIALECT}; +use crate::sql_lint::constants::{ + DUCKDB_DIALECT, LINT_API_VERSION, SNOWFLAKE_DIALECT, TSQL_DIALECT, +}; use crate::sql_lint::models::{FormatRequest, FormatResponse}; /// Spaces per indent step when a request does not choose. @@ -41,7 +43,7 @@ fn format_sql(request: FormatRequest) -> FormatResponse { ); return declined(request.sql, Some(reason), None); } - let Some(dialect) = sqlquack_dialect(&request.dialect) else { + let Some((dialect, macros)) = sqlquack_dialect(&request.dialect) else { let unavailable = format!( "SQL formatting is unavailable for the {} dialect: SQLQuack does not format it yet", request.dialect @@ -49,33 +51,27 @@ fn format_sql(request: FormatRequest) -> FormatResponse { return declined(request.sql, None, Some(unavailable)); }; let original = request.sql.clone(); - match format_with_sqlquack(request, dialect) { + match format_with_sqlquack(request, dialect, macros) { Ok(response) => response, Err(reason) => declined(original, Some(reason), None), } } -/// The SQLQuack dialect that lays out `dialect`, for the dialects SQLQuack formats natively. -fn sqlquack_dialect(dialect: &str) -> Option { +/// The SQLQuack dialect that lays out `dialect`, for the dialects SQLQuack formats natively, with the shared tokenizer's dialect that takes out CTE macros where it reads that dialect. +fn sqlquack_dialect(dialect: &str) -> Option<(FormatDialect, Option)> { match dialect.to_ascii_lowercase().as_str() { - SNOWFLAKE_DIALECT => Some(SqlQuackDialect::Snowflake), - DUCKDB_DIALECT => Some(SqlQuackDialect::DuckDB), + SNOWFLAKE_DIALECT => Some((FormatDialect::Snowflake, Some(SqlQuackDialect::Snowflake))), + DUCKDB_DIALECT => Some((FormatDialect::DuckDB, Some(SqlQuackDialect::DuckDB))), + TSQL_DIALECT => Some((FormatDialect::SqlServer, None)), _ => None, } } -/// SQLQuack's formatter dialect for a dialect it formats natively. -fn format_dialect(dialect: SqlQuackDialect) -> FormatDialect { - match dialect { - SqlQuackDialect::DuckDB => FormatDialect::DuckDB, - _ => FormatDialect::Snowflake, - } -} - /// Lay out `request` with SQLQuack; this re-checks idempotency only where placeholders were put back. fn format_with_sqlquack( request: FormatRequest, - dialect: SqlQuackDialect, + dialect: FormatDialect, + macros: Option, ) -> Result { let options = FormatOptions { line_width: request.line_width, @@ -87,12 +83,12 @@ fn format_with_sqlquack( ..Default::default() }), }, - ..FormatOptions::new(format_dialect(dialect)) + ..FormatOptions::new(dialect) }; - let sql = format_around_cte_macros(&request.sql, dialect, &options)?; + let sql = format_around_cte_macros(&request.sql, macros, &options)?; if request.sql.contains(INTERPOLATION_PLACEHOLDER_PREFIX) && sql != request.sql - && format_around_cte_macros(&sql, dialect, &options)? != sql + && format_around_cte_macros(&sql, macros, &options)? != sql { return Err(NOT_IDEMPOTENT_FAILURE.to_string()); } @@ -113,12 +109,15 @@ fn sqlquack_layout(sql: &str, options: &FormatOptions) -> Result .map_err(|refusal| refusal.reason) } -/// Lay out `sql` with SQLQuack, taking CTE macro placeholders out first and putting them back. +/// Lay out `sql` with SQLQuack, taking CTE macro placeholders out first and putting them back where `macros` names the shared tokenizer's dialect; otherwise SQLQuack refuses a placeholder it cannot parse. fn format_around_cte_macros( sql: &str, - dialect: SqlQuackDialect, + macros: Option, options: &FormatOptions, ) -> Result { + let Some(dialect) = macros else { + return sqlquack_layout(sql, options); + }; match take_out_cte_macros(sql, dialect)? { None => sqlquack_layout(sql, options), Some(without) => without.put_back(&sqlquack_layout(&without.sql, options)?, dialect), diff --git a/crates/sqlclean/src/sql_lint/constants.rs b/crates/sqlclean/src/sql_lint/constants.rs index 27fc585..9691fab 100644 --- a/crates/sqlclean/src/sql_lint/constants.rs +++ b/crates/sqlclean/src/sql_lint/constants.rs @@ -3,3 +3,5 @@ pub const LINT_API_VERSION: u32 = 1; pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake"; /// The dialect name SQLQuack lays out as DuckDB. pub(crate) const DUCKDB_DIALECT: &str = "duckdb"; +/// The dialect name SQLQuack lays out as SQL Server. +pub(crate) const TSQL_DIALECT: &str = "tsql"; diff --git a/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json b/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json index 2ab6f2f..aa8e8c2 100644 --- a/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json +++ b/crates/sqlclean/src/sql_lint/tests/fixtures/sqlquack_departures.json @@ -201,5 +201,187 @@ "formatted": "SELECT\n SEARCH(line, 'king', SEARCH_MODE => 'AND', ANALYZER => 'PATTERN_ANALYZER')", "reason": "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", "receipt": "cases/relational/unaliased_output_names.toml#unaliased_expressions" + }, + { + "dialect": "tsql", + "sql": "SELECT val FROM (VALUES ((TRUE), (FALSE), (NULL))) AS t(val)", + "formatted": "SELECT val\nFROM (\n VALUES\n ((\n TRUE\n), (\n FALSE\n), (\n NULL\n))\n) AS t(val)", + "reason": "The shared layout of a parenthesised value in a VALUES row, the same as Snowflake's; SQLClean broke after the values instead", + "receipt": null + }, + { + "dialect": "tsql", + "sql": "WITH t1 AS (SELECT 1 AS a), t2 AS (SELECT 1 AS a) SELECT COUNT(*) FROM (SELECT TOP 10 a FROM t1 UNION ALL SELECT TOP 10 a FROM t2 ORDER BY a DESC) AS t", + "formatted": "WITH t1 AS (\n SELECT 1 AS a\n),\n\nt2 AS (\n SELECT 1 AS a\n)\n\nSELECT COUNT(*)\nFROM (\n SELECT TOP 10 a\n FROM t1\n UNION ALL\n SELECT TOP 10 a\n FROM t2\n ORDER BY\n a DESC\n) AS t", + "reason": "A set operation's ORDER BY sits at the depth of its set operation with its items on the next line; SQLClean outdented it to the enclosing parenthesis and kept the items on its line", + "receipt": null + }, + { + "dialect": "tsql", + "sql": "WITH t1 AS (SELECT 1 AS a), t2 AS (SELECT 1 AS a) SELECT TOP 10 a FROM t1 UNION ALL SELECT TOP 10 a FROM t2 ORDER BY a DESC", + "formatted": "WITH t1 AS (\n SELECT 1 AS a\n),\n\nt2 AS (\n SELECT 1 AS a\n)\n\nSELECT TOP 10 a\nFROM t1\nUNION ALL\nSELECT TOP 10 a\nFROM t2\nORDER BY\n a DESC", + "reason": "A set operation's ORDER BY puts its items on the next line, as in every other dialect and SQL Server's own `ORDER BY\\n 1`; SQLClean kept them on the ORDER BY line", + "receipt": null + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t TABLESAMPLE (10 PERCENT)", + "formatted": null, + "reason": "SQL Server accepts it; SQLQuack's parser keeps a TABLESAMPLE clause opaque (gap `tablesample`), so the formatter refuses rather than guess; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t TABLESAMPLE (20 ROWS)", + "formatted": null, + "reason": "SQL Server accepts it; SQLQuack's parser keeps a TABLESAMPLE clause opaque (gap `tablesample`), so the formatter refuses rather than guess; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t TABLESAMPLE (10 PERCENT) REPEATABLE (123)", + "formatted": null, + "reason": "SQL Server accepts it; SQLQuack's parser keeps a TABLESAMPLE clause opaque (gap `tablesample`), so the formatter refuses rather than guess; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t WHERE NOT c", + "formatted": null, + "reason": "SQL Server rejects it: `An expression of non-boolean type specified in a context where a condition is expected, near 'c'.` (4145); SQLQuack's parser refuses it (`An expression of non-boolean type specified in a context where a condition is expected.`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t WITH (TABLOCK, INDEX(myindex))", + "formatted": null, + "reason": "SQL Server accepts it; SQLQuack's parser keeps a table hint list opaque (gap `table_hint`), so the formatter refuses rather than guess; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT Employee_ID, Department_ID FROM @MyTableVar", + "formatted": null, + "reason": "SQL Server rejects it: `Must declare the table variable \"@MyTableVar\".` (1087); SQLQuack's parser keeps a table variable source opaque (gap `table_variable`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT x FROM @MyTableVar AS m JOIN Employee ON m.EmployeeID = Employee.EmployeeID", + "formatted": null, + "reason": "SQL Server rejects it: `Must declare the table variable \"@MyTableVar\".` (1087); SQLQuack's parser keeps a table variable source opaque (gap `table_variable`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t FOR XML PATH, BINARY BASE64, ELEMENTS XSINIL", + "formatted": null, + "reason": "SQL Server accepts it; SQLQuack's parser keeps a FOR XML, FOR JSON or FOR BROWSE clause opaque (gap `for_xml_json`), so the formatter refuses rather than guess; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT (SELECT TOP 5 a, b FROM t ORDER BY b DESC FOR JSON PATH) AS j FROM x", + "formatted": null, + "reason": "SQL Server accepts it; SQLQuack's parser keeps a FOR XML, FOR JSON or FOR BROWSE clause opaque (gap `for_xml_json`) on FOR JSON in a scalar subquery, a parser false reject; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT j FROM (SELECT a AS j FROM t FOR JSON PATH) AS x", + "formatted": null, + "reason": "SQL Server rejects it at binding: `Invalid column name 'j'.` (207, then 8155), as a FOR JSON derived table needs a column list; SQLQuack's parser keeps a FOR XML, FOR JSON or FOR BROWSE clause opaque (gap `for_xml_json`), even with one; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t FOR JSON PATH, ROOT('Root'), INCLUDE_NULL_VALUES", + "formatted": null, + "reason": "SQL Server accepts it; SQLQuack's parser keeps a FOR XML, FOR JSON or FOR BROWSE clause opaque (gap `for_xml_json`), so the formatter refuses rather than guess; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t FOR BROWSE", + "formatted": null, + "reason": "SQL Server accepts it; SQLQuack's parser keeps a FOR XML, FOR JSON or FOR BROWSE clause opaque (gap `for_xml_json`), so the formatter refuses rather than guess; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT TOP 10 PERCENT", + "formatted": null, + "reason": "SQL Server rejects it: `Incorrect syntax near 'PERCENT'.` (102); SQLQuack's parser refuses it (`Incorrect syntax near the end of the input.`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT TOP 10 PERCENT WITH TIES", + "formatted": null, + "reason": "SQL Server rejects it: `Incorrect syntax near 'TIES'.` (102); SQLQuack's parser refuses it (`Incorrect syntax near the end of the input.`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME AS OF 'foo'", + "formatted": null, + "reason": "SQL Server accepts it; SQLQuack's parser keeps a FOR SYSTEM_TIME clause opaque (gap `system_time`), so the formatter refuses rather than guess; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME AS OF 'foo' AS alias", + "formatted": null, + "reason": "SQL Server accepts it; SQLQuack's parser keeps a FOR SYSTEM_TIME clause opaque (gap `system_time`), so the formatter refuses rather than guess; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME FROM c TO d", + "formatted": null, + "reason": "SQL Server rejects it: `Incorrect syntax near 'c'.` (102); SQLQuack's parser keeps a FOR SYSTEM_TIME clause opaque (gap `system_time`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME BETWEEN c AND d", + "formatted": null, + "reason": "SQL Server rejects it: `Incorrect syntax near 'c'.` (102); SQLQuack's parser keeps a FOR SYSTEM_TIME clause opaque (gap `system_time`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME CONTAINED IN (c, d)", + "formatted": null, + "reason": "SQL Server rejects it: `Incorrect syntax near 'c'.` (102); SQLQuack's parser keeps a FOR SYSTEM_TIME clause opaque (gap `system_time`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME ALL AS alias", + "formatted": null, + "reason": "SQL Server accepts it; SQLQuack's parser keeps a FOR SYSTEM_TIME clause opaque (gap `system_time`), so the formatter refuses rather than guess; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT x FROM a INNER LOOP JOIN b ON b.id = a.id", + "formatted": null, + "reason": "SQL Server accepts it; SQLQuack's parser keeps a join hint opaque (gap `join_hint`), so the formatter refuses rather than guess; SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT NEXT VALUE FOR db.schema.sequence_name OVER (ORDER BY foo), col", + "formatted": null, + "reason": "SQL Server rejects it: `Incorrect syntax near the keyword 'schema'.` (156); SQLQuack's parser refuses it (`Incorrect syntax near 'schema'.`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM @x", + "formatted": null, + "reason": "SQL Server rejects it: `Must declare the table variable \"@x\".` (1087); SQLQuack's parser keeps a table variable source opaque (gap `table_variable`); SQLClean formatted it", + "receipt": "crates/sqlquack-format/src/format/tests/fixtures/sqlserver_receipts.jsonl" } ] \ No newline at end of file diff --git a/crates/sqlclean/src/sql_lint/tests/helpers.rs b/crates/sqlclean/src/sql_lint/tests/helpers.rs index 90b9fdd..c3d1654 100644 --- a/crates/sqlclean/src/sql_lint/tests/helpers.rs +++ b/crates/sqlclean/src/sql_lint/tests/helpers.rs @@ -138,7 +138,7 @@ pub(crate) fn check_dialect_corpus( /// Whether `printed` has the fingerprint tokens of `sql` except for words upper-cased by format. fn only_keyword_case_differs(sql: &str, printed: &str, dialect: &str) -> Result { - let fingerprint = FingerprintDialect::from_name(dialect).map_err(|error| error.to_string())?; + let fingerprint = FingerprintDialect::from_name(dialect).unwrap_or(FingerprintDialect::Generic); let tokens = |sql: &str| canonical_tokens(sql, fingerprint).map_err(|error| error.to_string()); let (left, right) = (tokens(sql)?, tokens(printed)?); Ok(left.len() == right.len() diff --git a/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs b/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs index 8cb4910..5053447 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_format_corpus.rs @@ -21,6 +21,12 @@ fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stab expected_formatted: 426, expected_refused: 9, }, + test_types::FormatCorpusTestCase { + description: "T-SQL identity corpus", + dialect: "tsql", + expected_formatted: 52, + expected_refused: 23, + }, ]; let corpus: Vec = serde_json::from_str(FORMAT_CORPUS).map_err(|error| error.to_string())?; @@ -56,11 +62,6 @@ fn given_dialects_without_a_native_formatter_when_formatting_then_each_query_is_ dialect: "postgres", expected_unavailable: 137, }, - test_types::UnavailableCorpusTestCase { - description: "T-SQL corpus", - dialect: "tsql", - expected_unavailable: 75, - }, ]; let corpus: Vec = serde_json::from_str(FORMAT_CORPUS).map_err(|error| error.to_string())?; From 44e0a9ea31607b54b6f3ca91ec8d0595eeb3bc8c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:25:09 +0000 Subject: [PATCH 19/21] fix(facts): report comma and ON-nested joins as Polyglot did The native tree keeps a comma relation that follows an explicit join in the FROM list, and a join nested by its ON clauses inside the outer join's relation, so facts lost both. A comma relation after the first explicit join is now an IMPLICIT join, and the joins of an unparenthesized nested join are reported, with the outer join named by its first table. A parenthesized join group still reports only the join to it, as before. --- .../src/sql_facts/_helpers/collection.rs | 117 ++++++++++++++++-- .../src/sql_facts/tests/test_file_facts.rs | 65 +++++++++- .../src/sql_facts/tests/test_types.rs | 6 + 3 files changed, 180 insertions(+), 8 deletions(-) diff --git a/crates/sqlclean/src/sql_facts/_helpers/collection.rs b/crates/sqlclean/src/sql_facts/_helpers/collection.rs index 38e8ecc..a9a0216 100644 --- a/crates/sqlclean/src/sql_facts/_helpers/collection.rs +++ b/crates/sqlclean/src/sql_facts/_helpers/collection.rs @@ -1,7 +1,7 @@ //! Walk each parsed statement once and collect its structural facts. use sqlquack_syntax::expressions::names::models::{Column, Identifier, TableRef}; -use sqlquack_syntax::expressions::queries::models::{Cte, Join, With}; +use sqlquack_syntax::expressions::queries::models::{Cte, Join, JoinedTable, With}; use sqlquack_syntax::tokens::models::{Span, Token, TokenType}; use sqlquack_syntax::{Expression, ExpressionWalk}; @@ -149,8 +149,12 @@ fn statement_facts(source: &StatementSource<'_>, characters: &[char]) -> Stateme joins: sorted( nodes .iter() - .flat_map(|node| select_joins(node)) - .filter_map(|join| join_node(join, tokens, characters)) + .flat_map(|node| { + node_joins(node, tokens) + .into_iter() + .filter_map(|join| join_node(join, tokens, characters)) + .chain(implicit_join_nodes(node, tokens, characters)) + }) .collect(), ), stars: star_nodes(tokens), @@ -198,11 +202,106 @@ fn with_ctes(node: &Expression) -> &[Cte] { with.map_or(&[], |with| with.ctes.as_slice()) } -fn select_joins(node: &Expression) -> &[Join] { - match node { - Expression::Select(select) => select.joins.as_slice(), - _ => &[], +/// The joins of a query, with those of a join nested by its ON clauses, such as +/// `t2 LEFT JOIN t3 FULL JOIN t4 ON ... ON ...`; Polyglot left a parenthesized join group unread. +fn node_joins<'a>(node: &'a Expression, tokens: &[Token]) -> Vec<&'a Join> { + let Expression::Select(select) = node else { + return Vec::new(); + }; + let mut joins: Vec<&Join> = Vec::new(); + let mut pending: Vec<&Join> = select.joins.iter().rev().collect(); + while let Some(join) = pending.pop() { + joins.push(join); + if let Expression::JoinedTable(joined) = &join.this + && !parenthesized(joined, tokens) + { + pending.extend(joined.joins.iter().rev()); + } } + joins +} + +/// Whether a join group is written in parentheses, `JOIN (t2 LEFT JOIN t3 ON ...)`: the `(` after +/// its JOIN keyword closes after everything the group names. +fn parenthesized(joined: &JoinedTable, tokens: &[Token]) -> bool { + let spans: Vec = located_spans(&joined.left) + .chain(joined.joins.iter().flat_map(|join| { + located_spans(&join.this).chain(join.on.iter().flat_map(located_spans)) + })) + .collect(); + let (Some(first), Some(last)) = ( + spans.iter().map(|span| span.start).min(), + spans.iter().map(|span| span.end).max(), + ) else { + return false; + }; + let Some(keyword) = tokens + .iter() + .rposition(|token| token.span.end <= first && token.token_type == TokenType::Join) + else { + return false; + }; + let Some(open) = tokens[keyword + 1..] + .iter() + .position(|token| token.token_type != TokenType::Lateral) + .map(|offset| keyword + 1 + offset) + .filter(|&open| tokens[open].token_type == TokenType::LParen) + else { + return false; + }; + let mut depth: usize = 0; + for token in &tokens[open..] { + match token.token_type { + TokenType::LParen => depth += 1, + TokenType::RParen => { + depth -= 1; + if depth == 0 { + return last <= token.span.start; + } + } + _ => {} + } + } + false +} + +/// The spans of the tables and columns `expression` names. +fn located_spans(expression: &Expression) -> impl Iterator + '_ { + expression.dfs().filter_map(|node| match node { + Expression::Table(table) => table.span.or(table.name.span), + Expression::Column(column) => column.span, + _ => None, + }) +} + +/// The comma relations after a query's first explicit join, which Polyglot read as IMPLICIT joins. +fn implicit_join_nodes(node: &Expression, tokens: &[Token], characters: &[char]) -> Vec { + let Expression::Select(select) = node else { + return Vec::new(); + }; + let (Some(from), Some(first_join)) = ( + select.from.as_ref(), + select.joins.first().and_then(|join| first_span(&join.this)), + ) else { + return Vec::new(); + }; + from.expressions + .iter() + .filter_map(|relation| { + let span: Span = first_span(relation).filter(|span| span.start > first_join.start)?; + let (start, end) = join_clause_span(tokens, span.start)?; + Some(JoinNode { + start, + end, + kind: "IMPLICIT".to_owned(), + relation: match relation { + Expression::Table(table) => Some(authored(&table.name, characters)), + _ => None, + }, + has_condition: false, + }) + }) + .collect() } /// The CTE located at its name; the parser leaves CTE names unlocated, so read `name AS (`. @@ -259,6 +358,10 @@ fn table_node(node: &Expression, characters: &[char]) -> Option { fn join_node(join: &Join, tokens: &[Token], characters: &[char]) -> Option { let relation: Option<&TableRef> = match &join.this { Expression::Table(table) => Some(table), + Expression::JoinedTable(joined) if !parenthesized(joined, tokens) => match &joined.left { + Expression::Table(table) => Some(table), + _ => None, + }, _ => None, }; let (start, end) = join_clause_span(tokens, first_span(&join.this)?.start)?; diff --git a/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs b/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs index 5334436..aa30ce2 100644 --- a/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs +++ b/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs @@ -3,7 +3,8 @@ use crate::sql_facts::main::statement_ends::statement_ends; use crate::sql_facts::models::{FileFacts, StatementFacts}; use crate::sql_facts::tests::helpers::text; use crate::sql_facts::tests::test_types::{ - ParseOutcomeTestCase, StatementEndsTestCase, StatementFactsTestCase, UnavailableFactsTestCase, + JoinFactsTestCase, ParseOutcomeTestCase, StatementEndsTestCase, StatementFactsTestCase, + UnavailableFactsTestCase, }; const TWO_STATEMENTS: &str = "WITH recent AS (SELECT o.order_id, o.* FROM shop.orders AS o WHERE o.total > 10)\n\ @@ -207,6 +208,68 @@ FROM recent AS r LEFT JOIN customers AS c ON r.customer_id = c.customer_id", Ok(()) } +#[test] +fn given_comma_and_nested_joins_when_collecting_postgres_facts_then_joins_read_as_polyglot_did() +-> Result<(), String> { + let test_cases = [ + JoinFactsTestCase { + description: "a comma relation after an explicit join is an implicit join", + sql: "SELECT * FROM a FULL JOIN b USING (x), c AS cc", + expected_joins: &[ + ("FULL", "b", true, "FULL JOIN b USING (x)"), + ("IMPLICIT", "c", false, "c AS cc"), + ], + }, + JoinFactsTestCase { + description: "a comma relation before the first join is not", + sql: "SELECT * FROM a, b JOIN c ON true", + expected_joins: &[("INNER", "c", true, "JOIN c ON true")], + }, + JoinFactsTestCase { + description: "a join nested by its ON clauses reports both joins", + sql: "SELECT * FROM t2 LEFT JOIN t3 FULL JOIN t4 ON false ON false", + expected_joins: &[ + ("LEFT", "t3", true, "LEFT JOIN t3"), + ("FULL", "t4", true, "FULL JOIN t4 ON false ON false"), + ], + }, + JoinFactsTestCase { + description: "a parenthesized join group reports only the join to it", + sql: "SELECT * FROM t1 LEFT JOIN (t2 LEFT JOIN t3 ON true) ON true", + expected_joins: &[( + "LEFT", + "", + true, + "LEFT JOIN (t2 LEFT JOIN t3 ON true) ON true", + )], + }, + ]; + for test_case in &test_cases { + let facts: FileFacts = file_facts(test_case.sql, "postgres")?; + let joins: Vec<(&str, &str, bool, String)> = facts.statements[0] + .joins + .iter() + .map(|join| { + ( + join.kind.as_str(), + join.relation.as_deref().unwrap_or(""), + join.has_condition, + text(test_case.sql, join.start, join.end), + ) + }) + .collect(); + let expected: Vec<(&str, &str, bool, String)> = test_case + .expected_joins + .iter() + .map(|(kind, relation, condition, text)| { + (*kind, *relation, *condition, (*text).to_owned()) + }) + .collect(); + assert_eq!(joins, expected, "{}", test_case.description); + } + Ok(()) +} + #[test] fn given_sql_text_when_collecting_facts_then_only_unparseable_sql_is_an_error() { let test_cases = [ diff --git a/crates/sqlclean/src/sql_facts/tests/test_types.rs b/crates/sqlclean/src/sql_facts/tests/test_types.rs index f06b5fd..841dabe 100644 --- a/crates/sqlclean/src/sql_facts/tests/test_types.rs +++ b/crates/sqlclean/src/sql_facts/tests/test_types.rs @@ -31,3 +31,9 @@ pub(crate) struct StatementEndsTestCase { pub(crate) sql: &'static str, pub(crate) expected_statements: Option<&'static [&'static str]>, } + +pub(crate) struct JoinFactsTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_joins: &'static [(&'static str, &'static str, bool, &'static str)], +} From f3e56a7f996f81dc75c106f165816dc15054c930 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:35:08 +0000 Subject: [PATCH 20/21] fix(facts): collect joins in plain loops and assert the expected joins directly --- .../src/sql_facts/_helpers/collection.rs | 67 ++++++++++--------- .../src/sql_facts/tests/test_file_facts.rs | 13 ++-- 2 files changed, 42 insertions(+), 38 deletions(-) diff --git a/crates/sqlclean/src/sql_facts/_helpers/collection.rs b/crates/sqlclean/src/sql_facts/_helpers/collection.rs index a9a0216..d75af14 100644 --- a/crates/sqlclean/src/sql_facts/_helpers/collection.rs +++ b/crates/sqlclean/src/sql_facts/_helpers/collection.rs @@ -146,17 +146,7 @@ fn statement_facts(source: &StatementSource<'_>, characters: &[char]) -> Stateme .filter_map(|node| table_node(node, characters)) .collect(), ), - joins: sorted( - nodes - .iter() - .flat_map(|node| { - node_joins(node, tokens) - .into_iter() - .filter_map(|join| join_node(join, tokens, characters)) - .chain(implicit_join_nodes(node, tokens, characters)) - }) - .collect(), - ), + joins: sorted(statement_joins(&nodes, tokens, characters)), stars: star_nodes(tokens), functions: function_nodes(tokens), columns: sorted( @@ -202,8 +192,19 @@ fn with_ctes(node: &Expression) -> &[Cte] { with.map_or(&[], |with| with.ctes.as_slice()) } -/// The joins of a query, with those of a join nested by its ON clauses, such as -/// `t2 LEFT JOIN t3 FULL JOIN t4 ON ... ON ...`; Polyglot left a parenthesized join group unread. +/// The explicit and IMPLICIT joins of every query among `nodes`. +fn statement_joins(nodes: &[&Expression], tokens: &[Token], characters: &[char]) -> Vec { + let mut joins: Vec = Vec::new(); + for node in nodes { + for join in node_joins(node, tokens) { + joins.extend(join_node(join, tokens, characters)); + } + joins.extend(implicit_join_nodes(node, tokens, characters)); + } + joins +} + +/// A query's joins, with those nested by ON clauses but not those inside a parenthesized group. fn node_joins<'a>(node: &'a Expression, tokens: &[Token]) -> Vec<&'a Join> { let Expression::Select(select) = node else { return Vec::new(); @@ -221,8 +222,7 @@ fn node_joins<'a>(node: &'a Expression, tokens: &[Token]) -> Vec<&'a Join> { joins } -/// Whether a join group is written in parentheses, `JOIN (t2 LEFT JOIN t3 ON ...)`: the `(` after -/// its JOIN keyword closes after everything the group names. +/// Whether the `(` after a join group's JOIN keyword closes after everything the group names. fn parenthesized(joined: &JoinedTable, tokens: &[Token]) -> bool { let spans: Vec = located_spans(&joined.left) .chain(joined.joins.iter().flat_map(|join| { @@ -285,23 +285,26 @@ fn implicit_join_nodes(node: &Expression, tokens: &[Token], characters: &[char]) ) else { return Vec::new(); }; - from.expressions - .iter() - .filter_map(|relation| { - let span: Span = first_span(relation).filter(|span| span.start > first_join.start)?; - let (start, end) = join_clause_span(tokens, span.start)?; - Some(JoinNode { - start, - end, - kind: "IMPLICIT".to_owned(), - relation: match relation { - Expression::Table(table) => Some(authored(&table.name, characters)), - _ => None, - }, - has_condition: false, - }) - }) - .collect() + let mut joins: Vec = Vec::new(); + for relation in &from.expressions { + let Some(span) = first_span(relation).filter(|span| span.start > first_join.start) else { + continue; + }; + let Some((start, end)) = join_clause_span(tokens, span.start) else { + continue; + }; + joins.push(JoinNode { + start, + end, + kind: "IMPLICIT".to_owned(), + relation: match relation { + Expression::Table(table) => Some(authored(&table.name, characters)), + _ => None, + }, + has_condition: false, + }); + } + joins } /// The CTE located at its name; the parser leaves CTE names unlocated, so read `name AS (`. diff --git a/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs b/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs index aa30ce2..774d943 100644 --- a/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs +++ b/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs @@ -258,14 +258,15 @@ fn given_comma_and_nested_joins_when_collecting_postgres_facts_then_joins_read_a ) }) .collect(); - let expected: Vec<(&str, &str, bool, String)> = test_case - .expected_joins + let located: Vec<(&str, &str, bool, &str)> = joins .iter() - .map(|(kind, relation, condition, text)| { - (*kind, *relation, *condition, (*text).to_owned()) - }) + .map(|(kind, relation, condition, text)| (*kind, *relation, *condition, text.as_str())) .collect(); - assert_eq!(joins, expected, "{}", test_case.description); + assert_eq!( + located, test_case.expected_joins, + "{}", + test_case.description + ); } Ok(()) } From 3020a075102a395a8cfc1ae8d3b11d9ed578eb3f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:36:12 +0000 Subject: [PATCH 21/21] build(deps): pin SQLQuack at main edfa30b2 Every SQLQuack crate resolves to the one main commit edfa30b26a0bab86661f96490b86a4f24b92db0a. --- Cargo.lock | 14 +++++++------- Cargo.toml | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8b3dd0..abbbd22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -611,7 +611,7 @@ dependencies = [ [[package]] name = "sqlquack-fingerprint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" dependencies = [ "sha2", "sqlquack-syntax", @@ -621,7 +621,7 @@ dependencies = [ [[package]] name = "sqlquack-format" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" dependencies = [ "serde_json", "sqlquack-syntax", @@ -631,7 +631,7 @@ dependencies = [ [[package]] name = "sqlquack-lint" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" dependencies = [ "rayon", "rustc-hash", @@ -644,7 +644,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" dependencies = [ "serde", "serde_json", @@ -657,7 +657,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-derive" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" dependencies = [ "proc-macro2", "quote", @@ -667,7 +667,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-postgres" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" dependencies = [ "sqlquack-syntax", "stacker", @@ -676,7 +676,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-sqlserver" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=d9a53958e592123a4fc5efae8421752e369e312c#d9a53958e592123a4fc5efae8421752e369e312c" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=edfa30b26a0bab86661f96490b86a4f24b92db0a#edfa30b26a0bab86661f96490b86a4f24b92db0a" dependencies = [ "sqlquack-syntax", "stacker", diff --git a/Cargo.toml b/Cargo.toml index 1ffe702..cfcb627 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,10 +30,10 @@ sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } # SQLQuack's formatter and lint engine, and the tokens and trees they read, which replace Polyglot. -sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "d9a53958e592123a4fc5efae8421752e369e312c" } -sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "d9a53958e592123a4fc5efae8421752e369e312c" } -sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "d9a53958e592123a4fc5efae8421752e369e312c" } -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "d9a53958e592123a4fc5efae8421752e369e312c" } +sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "edfa30b26a0bab86661f96490b86a4f24b92db0a" } +sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "edfa30b26a0bab86661f96490b86a4f24b92db0a" } +sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "edfa30b26a0bab86661f96490b86a4f24b92db0a" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "edfa30b26a0bab86661f96490b86a4f24b92db0a" } [workspace.lints.rust] unsafe_code = "forbid"